Changeset View
Standalone View
sys/dev/safexcel/safexcel.c
- This file was added.
/*- | |||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD | |||||
* | |||||
* Copyright (c) 2020 Rubicon Communications, LLC (Netgate) | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | |||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | |||||
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | |||||
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | |||||
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
*/ | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD$"); | |||||
#include <sys/param.h> | |||||
#include <sys/bus.h> | |||||
#include <sys/endian.h> | |||||
#include <sys/kernel.h> | |||||
#include <sys/lock.h> | |||||
#include <sys/malloc.h> | |||||
#include <sys/module.h> | |||||
#include <sys/mutex.h> | |||||
#include <sys/rman.h> | |||||
#include <sys/sglist.h> | |||||
#include <sys/sysctl.h> | |||||
#include <machine/atomic.h> | |||||
#include <machine/bus.h> | |||||
#include <machine/resource.h> | |||||
cem: We get this one already via sys/rman.h.
It seems odd to me, but we don't get machine/bus.h via… | |||||
#include <crypto/rijndael/rijndael.h> | |||||
#include <opencrypto/cryptodev.h> | |||||
#include <opencrypto/xform.h> | |||||
#include <dev/ofw/ofw_bus.h> | |||||
#include <dev/ofw/ofw_bus_subr.h> | |||||
#include "cryptodev_if.h" | |||||
#include "safexcel_reg.h" | |||||
#include "safexcel_var.h" | |||||
static MALLOC_DEFINE(M_SAFEXCEL, "safexcel_req", "safexcel request buffers"); | |||||
/* | |||||
* We only support the EIP97 for now. | |||||
*/ | |||||
static struct ofw_compat_data safexcel_compat[] = { | |||||
{ "inside-secure,safexcel-eip97ies", (uintptr_t)97 }, | |||||
{ "inside-secure,safexcel-eip97", (uintptr_t)97 }, | |||||
{ "marvell,safexcel-eip97", (uintptr_t)97 }, | |||||
andrewUnsubmitted Done Inline ActionsWhen is this used? I don't see it in the Linux driver or bindings documentation. andrew: When is this used? I don't see it in the Linux driver or bindings documentation. | |||||
{ NULL, 0 } | |||||
}; | |||||
const struct safexcel_reg_offsets eip97_regs_offset = { | |||||
.hia_aic = SAFEXCEL_EIP97_HIA_AIC_BASE, | |||||
.hia_aic_g = SAFEXCEL_EIP97_HIA_AIC_G_BASE, | |||||
.hia_aic_r = SAFEXCEL_EIP97_HIA_AIC_R_BASE, | |||||
.hia_aic_xdr = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE, | |||||
.hia_dfe = SAFEXCEL_EIP97_HIA_DFE_BASE, | |||||
.hia_dfe_thr = SAFEXCEL_EIP97_HIA_DFE_THR_BASE, | |||||
.hia_dse = SAFEXCEL_EIP97_HIA_DSE_BASE, | |||||
.hia_dse_thr = SAFEXCEL_EIP97_HIA_DSE_THR_BASE, | |||||
.hia_gen_cfg = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE, | |||||
.pe = SAFEXCEL_EIP97_PE_BASE, | |||||
}; | |||||
const struct safexcel_reg_offsets eip197_regs_offset = { | |||||
.hia_aic = SAFEXCEL_EIP197_HIA_AIC_BASE, | |||||
.hia_aic_g = SAFEXCEL_EIP197_HIA_AIC_G_BASE, | |||||
.hia_aic_r = SAFEXCEL_EIP197_HIA_AIC_R_BASE, | |||||
.hia_aic_xdr = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE, | |||||
.hia_dfe = SAFEXCEL_EIP197_HIA_DFE_BASE, | |||||
.hia_dfe_thr = SAFEXCEL_EIP197_HIA_DFE_THR_BASE, | |||||
.hia_dse = SAFEXCEL_EIP197_HIA_DSE_BASE, | |||||
.hia_dse_thr = SAFEXCEL_EIP197_HIA_DSE_THR_BASE, | |||||
.hia_gen_cfg = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE, | |||||
.pe = SAFEXCEL_EIP197_PE_BASE, | |||||
}; | |||||
static struct safexcel_command_desc * | |||||
safexcel_cdesc_next_r(struct safexcel_cdesc_ring *ring) | |||||
{ | |||||
struct safexcel_command_desc *cdesc; | |||||
if (ring->write == ring->read) | |||||
return (NULL); | |||||
cdesc = &ring->desc[ring->read]; | |||||
ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; | |||||
return (cdesc); | |||||
} | |||||
static struct safexcel_result_desc * | |||||
safexcel_rdesc_next_r(struct safexcel_rdesc_ring *ring) | |||||
{ | |||||
struct safexcel_result_desc *rdesc; | |||||
if (ring->write == ring->read) | |||||
return (NULL); | |||||
rdesc = &ring->desc[ring->read]; | |||||
ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE; | |||||
return (rdesc); | |||||
} | |||||
static struct safexcel_request * | |||||
safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring) | |||||
{ | |||||
struct safexcel_request *req; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
if ((req = STAILQ_FIRST(&ring->free_requests)) != NULL) | |||||
STAILQ_REMOVE_HEAD(&ring->free_requests, link); | |||||
return (req); | |||||
} | |||||
static void | |||||
safexcel_free_request(struct safexcel_softc *sc, struct safexcel_ring *ring, | |||||
struct safexcel_request *req) | |||||
{ | |||||
struct safexcel_context_record *ctx; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
if (req->dmap_loaded) { | |||||
bus_dmamap_unload(sc->sc_data_dtag, req->dmap); | |||||
req->dmap_loaded = false; | |||||
} | |||||
ctx = (struct safexcel_context_record *)req->ctx.vaddr; | |||||
explicit_bzero(ctx->data, sizeof(ctx->data)); | |||||
explicit_bzero(req->iv, sizeof(req->iv)); | |||||
STAILQ_INSERT_TAIL(&ring->free_requests, req, link); | |||||
} | |||||
static void | |||||
safexcel_enqueue_request(struct safexcel_softc *sc, struct safexcel_ring *ring, | |||||
struct safexcel_request *req) | |||||
{ | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
STAILQ_INSERT_TAIL(&ring->ready_requests, req, link); | |||||
} | |||||
static void | |||||
safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx) | |||||
{ | |||||
struct safexcel_command_desc *cdesc; | |||||
struct safexcel_result_desc *rdesc; | |||||
struct safexcel_request *req; | |||||
struct safexcel_ring *ring; | |||||
uint32_t error, i, ncdescs, nrdescs, nreqs; | |||||
ring = &sc->sc_ring[ringidx]; | |||||
nreqs = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT); | |||||
nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET; | |||||
nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK; | |||||
if (nreqs == 0) { | |||||
mtx_lock(&ring->mtx); | |||||
goto out; | |||||
} | |||||
ring = &sc->sc_ring[ringidx]; | |||||
bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
ncdescs = nrdescs = 0; | |||||
mtx_lock(&ring->mtx); | |||||
for (i = 0; i < nreqs; i++) { | |||||
req = STAILQ_FIRST(&ring->queued_requests); | |||||
if (req == NULL) | |||||
Done Inline ActionsShould this be a KASSERT rather than a plain panic()? gnn: Should this be a KASSERT rather than a plain panic()? | |||||
panic("expected %d pending requests", nreqs); | |||||
STAILQ_REMOVE_HEAD(&ring->queued_requests, link); | |||||
mtx_unlock(&ring->mtx); | |||||
bus_dmamap_sync(req->ctx.tag, req->ctx.map, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
bus_dmamap_sync(sc->sc_data_dtag, req->dmap, | |||||
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | |||||
ncdescs += req->cdescs; | |||||
while (req->cdescs-- > 0) { | |||||
cdesc = safexcel_cdesc_next_r(&ring->cdr); | |||||
KASSERT(cdesc != NULL, | |||||
("%s: missing control descriptor", __func__)); | |||||
} | |||||
nrdescs += req->rdescs; | |||||
while (req->rdescs-- > 0) { | |||||
rdesc = safexcel_rdesc_next_r(&ring->rdr); | |||||
error = rdesc->result_data.error_code; | |||||
if (error != 0) { | |||||
SAFEXCEL_DPRINTF(sc, 1, "error code %#x\n", | |||||
error); | |||||
if ((error & RESULT_TOKEN_ERR_AUTH_FAILED) == 0) | |||||
Done Inline ActionsI would perhaps invert this condition so it's 'If this specific case, use EBADMSG, else use EIO for generic errors'. Also, what happens if you have multiple errors for a single crp? Right now you use the last one. That is probably ok since it probably never happens anyway? jhb: I would perhaps invert this condition so it's 'If this specific case, use EBADMSG, else use… | |||||
Done Inline ActionsI changed it so that we set EBADMSG only if AUTH_FAILED is set, which I should have been doing anyway. That error flag is set if digest verification fails, so it is the only flag we expect to see. Other error flags can be set as a result of programming errors in the driver (I triggered that during development), but I don't expect to see them otherwise. markj: I changed it so that we set EBADMSG only if AUTH_FAILED is set, which I should have been doing… | |||||
req->crp->crp_etype = EIO; | |||||
else | |||||
req->crp->crp_etype = EBADMSG; | |||||
} | |||||
} | |||||
crypto_done(req->crp); | |||||
mtx_lock(&ring->mtx); | |||||
safexcel_free_request(sc, ring, req); | |||||
} | |||||
if (nreqs != 0) { | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | | |||||
(sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t))); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_xD_PKT(nreqs) | | |||||
(sc->sc_config.cd_offset * ncdescs * sizeof(uint32_t))); | |||||
} | |||||
out: | |||||
if (!STAILQ_EMPTY(&ring->queued_requests)) { | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, | |||||
SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | 1); | |||||
} | |||||
mtx_unlock(&ring->mtx); | |||||
} | |||||
static void | |||||
safexcel_ring_intr(void *arg) | |||||
{ | |||||
struct safexcel_softc *sc; | |||||
struct safexcel_intr_handle *ih; | |||||
uint32_t status, stat; | |||||
int ring; | |||||
bool blocked, rdrpending; | |||||
ih = arg; | |||||
sc = ih->sc; | |||||
ring = ih->ring; | |||||
status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + | |||||
SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring)); | |||||
/* CDR interrupts */ | |||||
if (status & SAFEXCEL_CDR_IRQ(ring)) { | |||||
stat = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, | |||||
stat & SAFEXCEL_CDR_INTR_MASK); | |||||
} | |||||
/* RDR interrupts */ | |||||
rdrpending = false; | |||||
if (status & SAFEXCEL_RDR_IRQ(ring)) { | |||||
stat = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT); | |||||
if ((stat & SAFEXCEL_xDR_ERR) == 0) | |||||
rdrpending = true; | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT, | |||||
stat & SAFEXCEL_RDR_INTR_MASK); | |||||
} | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring), | |||||
status); | |||||
if (rdrpending) | |||||
safexcel_rdr_intr(sc, ring); | |||||
mtx_lock(&sc->sc_mtx); | |||||
blocked = sc->sc_blocked; | |||||
sc->sc_blocked = 0; | |||||
mtx_unlock(&sc->sc_mtx); | |||||
if (blocked) | |||||
crypto_unblock(sc->sc_cid, blocked); | |||||
} | |||||
static int | |||||
safexcel_configure(struct safexcel_softc *sc) | |||||
{ | |||||
uint32_t i, mask, pemask, reg; | |||||
device_t dev; | |||||
if (sc->sc_type == 197) { | |||||
sc->sc_offsets = eip197_regs_offset; | |||||
pemask = SAFEXCEL_N_PES_MASK; | |||||
} else { | |||||
sc->sc_offsets = eip97_regs_offset; | |||||
pemask = EIP97_N_PES_MASK; | |||||
} | |||||
dev = sc->sc_dev; | |||||
/* Scan for valid ring interrupt controllers. */ | |||||
for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) { | |||||
reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) + | |||||
SAFEXCEL_HIA_AIC_R_VERSION(i)); | |||||
if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE) | |||||
break; | |||||
} | |||||
sc->sc_config.aic_rings = i; | |||||
if (sc->sc_config.aic_rings == 0) | |||||
return (-1); | |||||
reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS); | |||||
/* Check for 64bit addressing. */ | |||||
if ((reg & SAFEXCEL_OPT_ADDR_64) == 0) | |||||
return (-1); | |||||
/* Check alignment constraints (which we do not support). */ | |||||
if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >> | |||||
SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0) | |||||
return (-1); | |||||
sc->sc_config.hdw = | |||||
(reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET; | |||||
mask = (1 << sc->sc_config.hdw) - 1; | |||||
sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK; | |||||
/* Limit the number of rings to the number of the AIC Rings. */ | |||||
sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings); | |||||
sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET; | |||||
sc->sc_config.cd_size = | |||||
sizeof(struct safexcel_command_desc) / sizeof(uint32_t); | |||||
sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask; | |||||
sc->sc_config.rd_size = | |||||
sizeof(struct safexcel_result_desc) / sizeof(uint32_t); | |||||
sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask; | |||||
sc->sc_config.atok_offset = | |||||
(SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) & | |||||
~mask; | |||||
return (0); | |||||
} | |||||
static void | |||||
safexcel_init_hia_bus_access(struct safexcel_softc *sc) | |||||
{ | |||||
uint32_t version, val; | |||||
/* Determine endianness and configure byte swap. */ | |||||
version = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION); | |||||
val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); | |||||
if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) { | |||||
val = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL); | |||||
val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL, | |||||
val); | |||||
} | |||||
/* Configure wr/rd cache values. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL, | |||||
SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) | | |||||
SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS)); | |||||
} | |||||
static void | |||||
safexcel_disable_global_interrupts(struct safexcel_softc *sc) | |||||
{ | |||||
/* Disable and clear pending interrupts. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, | |||||
SAFEXCEL_AIC_G_ACK_ALL_MASK); | |||||
} | |||||
/* | |||||
* Configure the data fetch engine. This component parses command descriptors | |||||
* and sets up DMA transfers from host memory to the corresponding processing | |||||
* engine. | |||||
*/ | |||||
static void | |||||
safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe) | |||||
{ | |||||
/* Reset all DFE threads. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), | |||||
SAFEXCEL_DxE_THR_CTRL_RESET_PE); | |||||
/* Deassert the DFE reset. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0); | |||||
/* DMA transfer size to use. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe), | |||||
SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG | | |||||
SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) | | |||||
SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) | | |||||
SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) | | |||||
SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) | | |||||
SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) | | |||||
SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS)); | |||||
/* Configure the PE DMA transfer thresholds. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe), | |||||
SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | | |||||
SAFEXCEL_PE_IN_xBUF_THRES_MAX(9)); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe), | |||||
SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) | | |||||
SAFEXCEL_PE_IN_xBUF_THRES_MAX(7)); | |||||
} | |||||
/* | |||||
* Configure the data store engine. This component parses result descriptors | |||||
* and sets up DMA transfers from the processing engine to host memory. | |||||
*/ | |||||
static int | |||||
safexcel_configure_dse_engine(struct safexcel_softc *sc, int pe) | |||||
{ | |||||
uint32_t val; | |||||
int count; | |||||
/* Disable and reset all DSE threads. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), | |||||
SAFEXCEL_DxE_THR_CTRL_RESET_PE); | |||||
/* Wait for threads to go idle. */ | |||||
for (count = 0;;) { | |||||
val = SAFEXCEL_READ(sc, | |||||
SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe)); | |||||
if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) == | |||||
SAFEXCEL_DSE_THR_RDR_ID_MASK) | |||||
break; | |||||
if (count++ > 10000) { | |||||
device_printf(sc->sc_dev, "DSE reset timeout\n"); | |||||
return (-1); | |||||
} | |||||
DELAY(100); | |||||
Done Inline ActionsIf you're going to keep this delay can you leave a comment as to why it's 100? Also if you want to keep it can you #define it to something so it's not a magic number? gnn: If you're going to keep this delay can you leave a comment as to why it's 100?
Also if you… | |||||
} | |||||
/* Exit the reset state. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0); | |||||
/* DMA transfer size to use */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe), | |||||
SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG | | |||||
SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) | | |||||
SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) | | |||||
SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) | | |||||
SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE); | |||||
/* Configure the procesing engine thresholds */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe), | |||||
SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) | | |||||
SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8)); | |||||
return (0); | |||||
} | |||||
static void | |||||
safexcel_hw_prepare_rings(struct safexcel_softc *sc) | |||||
{ | |||||
int i; | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
/* | |||||
* Command descriptors. | |||||
*/ | |||||
/* Clear interrupts for this ring. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), | |||||
SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK); | |||||
/* Disable external triggering. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); | |||||
/* Clear the pending prepared counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
SAFEXCEL_xDR_PREP_CLR_COUNT); | |||||
/* Clear the pending processed counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_CLR_COUNT); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, | |||||
SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset * | |||||
sizeof(uint32_t)); | |||||
/* | |||||
* Result descriptors. | |||||
*/ | |||||
/* Disable external triggering. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0); | |||||
/* Clear the pending prepared counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
SAFEXCEL_xDR_PREP_CLR_COUNT); | |||||
/* Clear the pending processed counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_CLR_COUNT); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); | |||||
/* Ring size. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, | |||||
SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset * | |||||
sizeof(uint32_t)); | |||||
} | |||||
} | |||||
static void | |||||
safexcel_hw_setup_rings(struct safexcel_softc *sc) | |||||
{ | |||||
struct safexcel_ring *ring; | |||||
uint32_t cd_size_rnd, mask, rd_size_rnd, val; | |||||
int i; | |||||
mask = (1 << sc->sc_config.hdw) - 1; | |||||
cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw; | |||||
val = (sizeof(struct safexcel_result_desc) - | |||||
sizeof(struct result_data_desc)) / sizeof(uint32_t); | |||||
rd_size_rnd = (val + mask) >> sc->sc_config.hdw; | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
ring = &sc->sc_ring[i]; | |||||
/* | |||||
* Command descriptors. | |||||
*/ | |||||
/* Ring base address. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, | |||||
SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr)); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, | |||||
SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr)); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, | |||||
SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP | | |||||
(sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | | |||||
sc->sc_config.cd_size); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, | |||||
((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) << | |||||
SAFEXCEL_xDR_xD_FETCH_THRESH) | | |||||
(SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset)); | |||||
/* Configure DMA tx control. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, | |||||
SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | | |||||
SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS)); | |||||
/* Clear any pending interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, | |||||
SAFEXCEL_CDR_INTR_MASK); | |||||
/* | |||||
* Result descriptors. | |||||
*/ | |||||
/* Ring base address. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, | |||||
SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr)); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, | |||||
SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr)); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE, | |||||
SAFEXCEL_xDR_DESC_MODE_64BIT | | |||||
(sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) | | |||||
sc->sc_config.rd_size); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, | |||||
((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) << | |||||
SAFEXCEL_xDR_xD_FETCH_THRESH) | | |||||
(SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset)); | |||||
/* Configure DMA tx control. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG, | |||||
SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) | | |||||
SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) | | |||||
SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF); | |||||
/* Clear any pending interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, | |||||
SAFEXCEL_RDR_INTR_MASK); | |||||
/* Enable ring interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i), | |||||
SAFEXCEL_RDR_IRQ(i)); | |||||
} | |||||
} | |||||
/* Reset the command and result descriptor rings. */ | |||||
static void | |||||
safexcel_hw_reset_rings(struct safexcel_softc *sc) | |||||
{ | |||||
int i; | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
/* | |||||
* Result descriptor ring operations. | |||||
*/ | |||||
/* Reset ring base address. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); | |||||
/* Clear the pending prepared counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
SAFEXCEL_xDR_PREP_CLR_COUNT); | |||||
/* Clear the pending processed counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_CLR_COUNT); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); | |||||
/* Clear any pending interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, | |||||
SAFEXCEL_RDR_INTR_MASK); | |||||
/* Disable ring interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i), | |||||
SAFEXCEL_RDR_IRQ(i)); | |||||
/* | |||||
* Command descriptor ring operations. | |||||
*/ | |||||
/* Reset ring base address. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) + | |||||
SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0); | |||||
/* Clear the pending prepared counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
SAFEXCEL_xDR_PREP_CLR_COUNT); | |||||
/* Clear the pending processed counter. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT, | |||||
SAFEXCEL_xDR_PROC_CLR_COUNT); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0); | |||||
/* Clear any pending interrupt. */ | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT, | |||||
SAFEXCEL_CDR_INTR_MASK); | |||||
} | |||||
} | |||||
static void | |||||
safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe) | |||||
{ | |||||
int i, ring_mask; | |||||
for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) { | |||||
ring_mask <<= 1; | |||||
ring_mask |= 1; | |||||
} | |||||
/* Enable command descriptor rings. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), | |||||
SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); | |||||
/* Enable result descriptor rings. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), | |||||
SAFEXCEL_DxE_THR_CTRL_EN | ring_mask); | |||||
/* Clear any HIA interrupt. */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK, | |||||
SAFEXCEL_AIC_G_ACK_HIA_MASK); | |||||
} | |||||
static void | |||||
safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring, | |||||
struct safexcel_request *req) | |||||
{ | |||||
uint32_t ncdescs, nrdescs, nreqs; | |||||
int ringidx; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
ringidx = req->sess->ringidx; | |||||
if (STAILQ_EMPTY(&ring->ready_requests)) | |||||
return; | |||||
ncdescs = nrdescs = nreqs = 0; | |||||
while ((req = STAILQ_FIRST(&ring->ready_requests)) != NULL && | |||||
req->cdescs + ncdescs <= SAFEXCEL_MAX_BATCH_SIZE && | |||||
req->rdescs + nrdescs <= SAFEXCEL_MAX_BATCH_SIZE) { | |||||
STAILQ_REMOVE_HEAD(&ring->ready_requests, link); | |||||
STAILQ_INSERT_TAIL(&ring->queued_requests, req, link); | |||||
ncdescs += req->cdescs; | |||||
nrdescs += req->rdescs; | |||||
nreqs++; | |||||
} | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH, | |||||
SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | nreqs); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
nrdescs * sc->sc_config.rd_offset * sizeof(uint32_t)); | |||||
SAFEXCEL_WRITE(sc, | |||||
SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT, | |||||
ncdescs * sc->sc_config.cd_offset * sizeof(uint32_t)); | |||||
} | |||||
static void | |||||
safexcel_init_rings(struct safexcel_softc *sc) | |||||
{ | |||||
struct safexcel_command_desc *cdesc; | |||||
struct safexcel_ring *ring; | |||||
char buf[32]; | |||||
Not Done Inline ActionsPlease give that magic number a #define'd name. gnn: Please give that magic number a #define'd name. | |||||
uint64_t atok; | |||||
int i, j; | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
ring = &sc->sc_ring[i]; | |||||
snprintf(buf, sizeof(buf), "safexcel_ring%d", i); | |||||
mtx_init(&ring->mtx, buf, NULL, MTX_DEF); | |||||
STAILQ_INIT(&ring->free_requests); | |||||
STAILQ_INIT(&ring->ready_requests); | |||||
STAILQ_INIT(&ring->queued_requests); | |||||
ring->cdr.read = ring->cdr.write = 0; | |||||
ring->rdr.read = ring->rdr.write = 0; | |||||
for (j = 0; j < SAFEXCEL_RING_SIZE; j++) { | |||||
cdesc = &ring->cdr.desc[j]; | |||||
atok = ring->dma_atok.paddr + | |||||
sc->sc_config.atok_offset * j; | |||||
cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok); | |||||
cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok); | |||||
} | |||||
} | |||||
} | |||||
static void | |||||
safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, | |||||
int error) | |||||
{ | |||||
struct safexcel_dma_mem *sdm; | |||||
if (error != 0) | |||||
return; | |||||
KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); | |||||
sdm = arg; | |||||
sdm->paddr = segs->ds_addr; | |||||
} | |||||
static int | |||||
safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm, | |||||
bus_size_t size) | |||||
{ | |||||
int error; | |||||
KASSERT(sdm->vaddr == NULL, | |||||
("%s: DMA memory descriptor in use.", __func__)); | |||||
error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ | |||||
PAGE_SIZE, 0, /* alignment, boundary */ | |||||
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filtfunc, filtfuncarg */ | |||||
size, 1, /* maxsize, nsegments */ | |||||
size, BUS_DMA_COHERENT, /* maxsegsz, flags */ | |||||
NULL, NULL, /* lockfunc, lockfuncarg */ | |||||
&sdm->tag); /* dmat */ | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"failed to allocate busdma tag, error %d\n", error); | |||||
goto err1; | |||||
} | |||||
error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr, | |||||
BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map); | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"failed to allocate DMA safe memory, error %d\n", error); | |||||
goto err2; | |||||
} | |||||
error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size, | |||||
safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT); | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"cannot get address of the DMA memory, error %d\n", error); | |||||
goto err3; | |||||
} | |||||
return (0); | |||||
err3: | |||||
bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); | |||||
err2: | |||||
bus_dma_tag_destroy(sdm->tag); | |||||
err1: | |||||
sdm->vaddr = NULL; | |||||
return (error); | |||||
} | |||||
static void | |||||
safexcel_dma_free_mem(struct safexcel_dma_mem *sdm) | |||||
{ | |||||
bus_dmamap_unload(sdm->tag, sdm->map); | |||||
bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map); | |||||
bus_dma_tag_destroy(sdm->tag); | |||||
} | |||||
static void | |||||
safexcel_dma_free_rings(struct safexcel_softc *sc) | |||||
{ | |||||
struct safexcel_ring *ring; | |||||
int i; | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
ring = &sc->sc_ring[i]; | |||||
safexcel_dma_free_mem(&ring->cdr.dma); | |||||
safexcel_dma_free_mem(&ring->dma_atok); | |||||
safexcel_dma_free_mem(&ring->rdr.dma); | |||||
mtx_destroy(&ring->mtx); | |||||
} | |||||
} | |||||
static int | |||||
safexcel_dma_init(struct safexcel_softc *sc) | |||||
{ | |||||
struct safexcel_ring *ring; | |||||
int error, i, size; | |||||
error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ | |||||
1, 0, /* alignment, boundary */ | |||||
BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ | |||||
BUS_SPACE_MAXADDR, /* highaddr */ | |||||
NULL, NULL, /* filtfunc, filtfuncarg */ | |||||
SAFEXCEL_MAX_REQUEST_SIZE, /* maxsize */ | |||||
SAFEXCEL_MAX_FRAGMENTS, /* nsegments */ | |||||
SAFEXCEL_MAX_REQUEST_SIZE, /* maxsegsz */ | |||||
BUS_DMA_COHERENT, /* flags */ | |||||
NULL, NULL, /* lockfunc, lockfuncarg */ | |||||
&sc->sc_data_dtag); /* dmat */ | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"bus_dma_tag_create main failed; error %d\n", error); | |||||
return (error); | |||||
} | |||||
sc->sc_ring = malloc(sizeof(struct safexcel_ring) * sc->sc_config.rings, | |||||
M_SAFEXCEL, M_WAITOK | M_ZERO); | |||||
for (i = 0; i < sc->sc_config.rings; i++) { | |||||
ring = &sc->sc_ring[i]; | |||||
size = sizeof(uint32_t) * sc->sc_config.cd_offset * | |||||
SAFEXCEL_RING_SIZE; | |||||
error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size); | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"failed to allocate CDR DMA memory, error %d\n", | |||||
error); | |||||
goto err; | |||||
} | |||||
ring->cdr.desc = | |||||
(struct safexcel_command_desc *)ring->cdr.dma.vaddr; | |||||
/* Allocate additional CDR token memory. */ | |||||
error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, | |||||
sc->sc_config.atok_offset * SAFEXCEL_RING_SIZE); | |||||
if (error != 0) { | |||||
device_printf(sc->sc_dev, | |||||
"failed to allocate atoken DMA memory, error %d\n", | |||||
error); | |||||
goto err; | |||||
} | |||||
size = sizeof(uint32_t) * sc->sc_config.rd_offset * | |||||
SAFEXCEL_RING_SIZE; | |||||
error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size); | |||||
if (error) { | |||||
device_printf(sc->sc_dev, | |||||
"failed to allocate RDR DMA memory, error %d\n", | |||||
error); | |||||
goto err; | |||||
} | |||||
ring->rdr.desc = | |||||
(struct safexcel_result_desc *)ring->rdr.dma.vaddr; | |||||
} | |||||
return (0); | |||||
err: | |||||
safexcel_dma_free_rings(sc); | |||||
free(sc->sc_ring, M_SAFEXCEL); | |||||
bus_dma_tag_destroy(sc->sc_data_dtag); | |||||
return (error); | |||||
} | |||||
static void | |||||
safexcel_deinit_hw(struct safexcel_softc *sc) | |||||
{ | |||||
safexcel_hw_reset_rings(sc); | |||||
safexcel_dma_free_rings(sc); | |||||
free(sc->sc_ring, M_SAFEXCEL); | |||||
bus_dma_tag_destroy(sc->sc_data_dtag); | |||||
} | |||||
static int | |||||
safexcel_init_hw(struct safexcel_softc *sc) | |||||
{ | |||||
int pe; | |||||
/* 23.3.7 Initialization */ | |||||
if (safexcel_configure(sc) != 0) | |||||
return (EINVAL); | |||||
if (safexcel_dma_init(sc) != 0) | |||||
return (ENOMEM); | |||||
safexcel_init_rings(sc); | |||||
safexcel_init_hia_bus_access(sc); | |||||
/* 23.3.7.2 Disable EIP-97 global Interrupts */ | |||||
safexcel_disable_global_interrupts(sc); | |||||
for (pe = 0; pe < sc->sc_config.pes; pe++) { | |||||
/* 23.3.7.3 Configure Data Fetch Engine */ | |||||
safexcel_configure_dfe_engine(sc, pe); | |||||
/* 23.3.7.4 Configure Data Store Engine */ | |||||
if (safexcel_configure_dse_engine(sc, pe)) { | |||||
safexcel_deinit_hw(sc); | |||||
return (-1); | |||||
} | |||||
/* 23.3.7.5 1. Protocol enables */ | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe), | |||||
0xffffffff); | |||||
SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe), | |||||
0xffffffff); | |||||
} | |||||
safexcel_hw_prepare_rings(sc); | |||||
/* 23.3.7.5 Configure the Processing Engine(s). */ | |||||
for (pe = 0; pe < sc->sc_config.pes; pe++) | |||||
safexcel_enable_pe_engine(sc, pe); | |||||
safexcel_hw_setup_rings(sc); | |||||
return (0); | |||||
} | |||||
static int | |||||
safexcel_setup_dev_interrupts(struct safexcel_softc *sc) | |||||
{ | |||||
int i, j; | |||||
for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) { | |||||
sc->sc_ih[i].sc = sc; | |||||
sc->sc_ih[i].ring = i; | |||||
if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i], | |||||
INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr, | |||||
&sc->sc_ih[i], &sc->sc_ih[i].handle)) { | |||||
device_printf(sc->sc_dev, | |||||
"couldn't setup interrupt %d\n", i); | |||||
goto err; | |||||
} | |||||
} | |||||
return (0); | |||||
err: | |||||
for (j = 0; j < i; j++) | |||||
bus_teardown_intr(sc->sc_dev, sc->sc_intr[j], | |||||
sc->sc_ih[j].handle); | |||||
return (ENXIO); | |||||
} | |||||
static void | |||||
safexcel_teardown_dev_interrupts(struct safexcel_softc *sc) | |||||
{ | |||||
int i; | |||||
for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) | |||||
bus_teardown_intr(sc->sc_dev, sc->sc_intr[i], | |||||
sc->sc_ih[i].handle); | |||||
} | |||||
static int | |||||
safexcel_alloc_dev_resources(struct safexcel_softc *sc) | |||||
{ | |||||
char name[16]; | |||||
device_t dev; | |||||
phandle_t node; | |||||
int error, i, rid; | |||||
dev = sc->sc_dev; | |||||
node = ofw_bus_get_node(dev); | |||||
rid = 0; | |||||
sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, | |||||
RF_ACTIVE); | |||||
if (sc->sc_res == NULL) { | |||||
device_printf(dev, "couldn't allocate memory resources\n"); | |||||
return (ENXIO); | |||||
} | |||||
for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) { | |||||
(void)snprintf(name, sizeof(name), "ring%d", i); | |||||
error = ofw_bus_find_string_index(node, "interrupt-names", name, | |||||
&rid); | |||||
if (error != 0) | |||||
break; | |||||
sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, | |||||
RF_ACTIVE | RF_SHAREABLE); | |||||
if (sc->sc_intr[i] == NULL) { | |||||
error = ENXIO; | |||||
goto out; | |||||
} | |||||
} | |||||
if (i == 0) { | |||||
device_printf(dev, "couldn't allocate interrupt resources\n"); | |||||
error = ENXIO; | |||||
goto out; | |||||
} | |||||
mtx_init(&sc->sc_mtx, "safexcel softc", NULL, MTX_DEF); | |||||
sc->sc_bt = rman_get_bustag(sc->sc_res); | |||||
sc->sc_bh = rman_get_bushandle(sc->sc_res); | |||||
return (0); | |||||
out: | |||||
for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) | |||||
bus_release_resource(dev, SYS_RES_IRQ, | |||||
rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); | |||||
bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res), | |||||
sc->sc_res); | |||||
return (error); | |||||
} | |||||
static void | |||||
safexcel_free_dev_resources(struct safexcel_softc *sc) | |||||
{ | |||||
int i; | |||||
mtx_destroy(&sc->sc_mtx); | |||||
for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) | |||||
bus_release_resource(sc->sc_dev, SYS_RES_IRQ, | |||||
rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]); | |||||
if (sc->sc_res != NULL) | |||||
bus_release_resource(sc->sc_dev, SYS_RES_MEMORY, | |||||
rman_get_rid(sc->sc_res), sc->sc_res); | |||||
} | |||||
static int | |||||
safexcel_probe(device_t dev) | |||||
{ | |||||
struct safexcel_softc *sc; | |||||
if (!ofw_bus_status_okay(dev)) | |||||
return (ENXIO); | |||||
sc = device_get_softc(dev); | |||||
sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data; | |||||
if (sc->sc_type == 0) | |||||
return (ENXIO); | |||||
device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator"); | |||||
return (BUS_PROBE_DEFAULT); | |||||
} | |||||
static int | |||||
safexcel_attach(device_t dev) | |||||
{ | |||||
struct sysctl_ctx_list *sctx; | |||||
struct safexcel_softc *sc; | |||||
struct safexcel_request *req; | |||||
struct safexcel_ring *ring; | |||||
int i, j, ringidx; | |||||
sc = device_get_softc(dev); | |||||
sc->sc_dev = dev; | |||||
sc->sc_blocked = 0; | |||||
sc->sc_cid = -1; | |||||
if (safexcel_alloc_dev_resources(sc)) | |||||
goto err; | |||||
if (safexcel_setup_dev_interrupts(sc)) | |||||
goto err1; | |||||
if (safexcel_init_hw(sc)) | |||||
goto err2; | |||||
for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { | |||||
ring = &sc->sc_ring[ringidx]; | |||||
ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); | |||||
ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK); | |||||
ring->requests = mallocarray(SAFEXCEL_REQUESTS_PER_RING, | |||||
sizeof(struct safexcel_request), M_SAFEXCEL, | |||||
M_WAITOK | M_ZERO); | |||||
for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { | |||||
req = &ring->requests[i]; | |||||
if (bus_dmamap_create(sc->sc_data_dtag, | |||||
BUS_DMA_COHERENT, &req->dmap) != 0) { | |||||
for (j = 0; j < i; j++) | |||||
bus_dmamap_destroy(sc->sc_data_dtag, | |||||
ring->requests[j].dmap); | |||||
goto err2; | |||||
} | |||||
if (safexcel_dma_alloc_mem(sc, &req->ctx, | |||||
sizeof(struct safexcel_context_record)) != 0) { | |||||
for (j = 0; j < i; j++) { | |||||
bus_dmamap_destroy(sc->sc_data_dtag, | |||||
ring->requests[j].dmap); | |||||
safexcel_dma_free_mem( | |||||
&ring->requests[j].ctx); | |||||
} | |||||
goto err2; | |||||
} | |||||
STAILQ_INSERT_TAIL(&ring->free_requests, req, link); | |||||
} | |||||
} | |||||
sctx = device_get_sysctl_ctx(dev); | |||||
SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), | |||||
OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0, | |||||
"Debug message verbosity"); | |||||
sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session), | |||||
CRYPTOCAP_F_HARDWARE); | |||||
if (sc->sc_cid < 0) | |||||
goto err2; | |||||
return (bus_generic_attach(dev)); | |||||
Done Inline ActionsThis seems somewhat curious as you haven't added any children devices so bus_generic_attach(dev) should be a no-op? I would normally only use this in bus drivers and just return (0) in a leaf driver. jhb: This seems somewhat curious as you haven't added any children devices so `bus_generic_attach… | |||||
Done Inline ActionsIndeed, we can just return 0 here and below. markj: Indeed, we can just return 0 here and below. | |||||
err2: | |||||
safexcel_teardown_dev_interrupts(sc); | |||||
err1: | |||||
safexcel_free_dev_resources(sc); | |||||
err: | |||||
return (ENXIO); | |||||
} | |||||
static int | |||||
safexcel_detach(device_t dev) | |||||
{ | |||||
struct safexcel_ring *ring; | |||||
struct safexcel_softc *sc; | |||||
int i, ringidx; | |||||
sc = device_get_softc(dev); | |||||
if (sc->sc_cid >= 0) | |||||
crypto_unregister_all(sc->sc_cid); | |||||
for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) { | |||||
ring = &sc->sc_ring[ringidx]; | |||||
for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) { | |||||
bus_dmamap_destroy(sc->sc_data_dtag, | |||||
ring->requests[i].dmap); | |||||
safexcel_dma_free_mem(&ring->requests[i].ctx); | |||||
} | |||||
free(ring->requests, M_SAFEXCEL); | |||||
sglist_free(ring->cmd_data); | |||||
sglist_free(ring->res_data); | |||||
} | |||||
safexcel_deinit_hw(sc); | |||||
safexcel_teardown_dev_interrupts(sc); | |||||
safexcel_free_dev_resources(sc); | |||||
return (bus_generic_detach(dev)); | |||||
Done Inline ActionsLikewise. (Plus, normally you want to detach children devices first so they can veto the attach early) jhb: Likewise. (Plus, normally you want to detach children devices first so they can veto the… | |||||
} | |||||
/* | |||||
* Populate the request's context record with pre-computed key material. | |||||
*/ | |||||
static int | |||||
safexcel_set_context(struct safexcel_request *req) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct cryptop *crp; | |||||
struct safexcel_context_record *ctx; | |||||
struct safexcel_session *sess; | |||||
uint8_t *data; | |||||
int off; | |||||
crp = req->crp; | |||||
csp = crypto_get_params(crp->crp_session); | |||||
sess = req->sess; | |||||
ctx = (struct safexcel_context_record *)req->ctx.vaddr; | |||||
data = (uint8_t *)ctx->data; | |||||
if (csp->csp_cipher_alg != 0) { | |||||
if (crp->crp_cipher_key != NULL) | |||||
memcpy(data, crp->crp_cipher_key, sess->klen); | |||||
else | |||||
memcpy(data, csp->csp_cipher_key, sess->klen); | |||||
off = sess->klen; | |||||
} else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { | |||||
if (crp->crp_auth_key != NULL) | |||||
memcpy(data, crp->crp_auth_key, sess->klen); | |||||
else | |||||
memcpy(data, csp->csp_auth_key, sess->klen); | |||||
off = sess->klen; | |||||
} else { | |||||
off = 0; | |||||
} | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); | |||||
off += GMAC_BLOCK_LEN; | |||||
break; | |||||
Not Done Inline ActionsI tend to favor parens for this type of equation, even when not strictly necessary. gnn: I tend to favor parens for this type of equation, even when not strictly necessary. | |||||
case CRYPTO_AES_CCM_16: | |||||
memcpy(data + off, sess->xcbc_key, | |||||
AES_BLOCK_LEN * 2 + sess->klen); | |||||
off += AES_BLOCK_LEN * 2 + sess->klen; | |||||
break; | |||||
Not Done Inline ActionsIs the OCF klen in XTS context actually correct, or is half? cem: Is the OCF `klen` in XTS context actually correct, or is half? | |||||
Done Inline ActionsNote that sess->klen is half of the klen given by OCF (see the XTS case in safexcel_setkey()). So here, sess->klen is the block cipher key length. markj: Note that sess->klen is half of the klen given by OCF (see the XTS case in safexcel_setkey()). | |||||
case CRYPTO_AES_XTS: | |||||
memcpy(data + off, sess->tweak_key, sess->klen); | |||||
off += sess->klen; | |||||
break; | |||||
} | |||||
switch (csp->csp_auth_alg) { | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN); | |||||
off += GMAC_BLOCK_LEN; | |||||
break; | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
memcpy(data + off, sess->hmac_ipad, sess->statelen); | |||||
off += sess->statelen; | |||||
memcpy(data + off, sess->hmac_opad, sess->statelen); | |||||
off += sess->statelen; | |||||
break; | |||||
} | |||||
return (off); | |||||
} | |||||
/* | |||||
* Populate fields in the first command descriptor of the chain used to encode | |||||
* the specified request. These fields indicate the algorithms used, the size | |||||
* of the key material stored in the associated context record, the primitive | |||||
* operations to be performed on input data, and the location of the IV if any. | |||||
*/ | |||||
static void | |||||
safexcel_set_command(struct safexcel_request *req, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct cryptop *crp; | |||||
struct safexcel_session *sess; | |||||
uint32_t ctrl0, ctrl1, ctxr_len; | |||||
int alg; | |||||
crp = req->crp; | |||||
csp = crypto_get_params(crp->crp_session); | |||||
sess = req->sess; | |||||
ctrl0 = sess->alg | sess->digest | sess->hash; | |||||
ctrl1 = sess->mode; | |||||
ctxr_len = safexcel_set_context(req) / sizeof(uint32_t); | |||||
ctrl0 |= CONTEXT_CONTROL_SIZE(ctxr_len); | |||||
alg = csp->csp_cipher_alg; | |||||
if (alg == 0) | |||||
alg = csp->csp_auth_alg; | |||||
switch (alg) { | |||||
case CRYPTO_AES_CCM_16: | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT | | |||||
CONTEXT_CONTROL_KEY_EN; | |||||
} else { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN | | |||||
CONTEXT_CONTROL_KEY_EN; | |||||
} | |||||
ctrl1 |= CONTEXT_CONTROL_IV0 | CONTEXT_CONTROL_IV1 | | |||||
CONTEXT_CONTROL_IV2 | CONTEXT_CONTROL_IV3; | |||||
break; | |||||
case CRYPTO_AES_CBC: | |||||
case CRYPTO_AES_ICM: | |||||
case CRYPTO_AES_XTS: | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT | | |||||
CONTEXT_CONTROL_KEY_EN; | |||||
if (csp->csp_auth_alg != 0) | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; | |||||
} else { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN | | |||||
CONTEXT_CONTROL_KEY_EN; | |||||
if (csp->csp_auth_alg != 0) | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; | |||||
} | |||||
break; | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) || | |||||
csp->csp_auth_alg != 0) { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT | | |||||
CONTEXT_CONTROL_KEY_EN | | |||||
CONTEXT_CONTROL_TYPE_HASH_OUT; | |||||
} else { | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN | | |||||
CONTEXT_CONTROL_KEY_EN | | |||||
CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; | |||||
} | |||||
if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) { | |||||
ctrl1 |= CONTEXT_CONTROL_COUNTER_MODE | | |||||
CONTEXT_CONTROL_IV0 | CONTEXT_CONTROL_IV1 | | |||||
CONTEXT_CONTROL_IV2; | |||||
} | |||||
break; | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_512: | |||||
ctrl0 |= CONTEXT_CONTROL_RESTART_HASH; | |||||
/* FALLTHROUGH */ | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
ctrl0 |= CONTEXT_CONTROL_TYPE_HASH_OUT; | |||||
break; | |||||
} | |||||
cdesc->control_data.control0 = ctrl0; | |||||
cdesc->control_data.control1 = ctrl1; | |||||
} | |||||
/* | |||||
* Handle a request for an unauthenticated block cipher. | |||||
*/ | |||||
static void | |||||
safexcel_instr_cipher(struct safexcel_request *req, | |||||
struct safexcel_instr *instr, struct safexcel_command_desc *cdesc) | |||||
{ | |||||
struct cryptop *crp; | |||||
crp = req->crp; | |||||
/* Insert the payload. */ | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET | | |||||
SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_INS_LAST | | |||||
SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT; | |||||
cdesc->additional_cdata_size = 1; | |||||
} | |||||
static void | |||||
safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct cryptop *crp; | |||||
struct safexcel_instr *start; | |||||
crp = req->crp; | |||||
csp = crypto_get_params(crp->crp_session); | |||||
start = instr; | |||||
/* Insert the AAD. */ | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_aad_length; | |||||
instr->status = crp->crp_payload_length == 0 ? | |||||
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; | |||||
instr->instructions = SAFEXCEL_INSTR_INS_LAST | | |||||
SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
/* Encrypt any data left in the request. */ | |||||
if (crp->crp_payload_length > 0) { | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_INS_LAST | | |||||
SAFEXCEL_INSTR_DEST_CRYPTO | | |||||
SAFEXCEL_INSTR_DEST_HASH | | |||||
SAFEXCEL_INSTR_DEST_OUTPUT; | |||||
instr++; | |||||
} | |||||
/* | |||||
* Compute the digest, or extract it and place it in the output stream. | |||||
*/ | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) | |||||
safexcel_instr_insert_digest(&instr, req->sess->digestlen); | |||||
else | |||||
safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); | |||||
cdesc->additional_cdata_size = instr - start; | |||||
} | |||||
static void | |||||
safexcel_instr_sha_hash(struct safexcel_request *req, | |||||
struct safexcel_instr *instr) | |||||
{ | |||||
struct cryptop *crp; | |||||
struct safexcel_instr *start; | |||||
crp = req->crp; | |||||
start = instr; | |||||
/* Pass the input data to the hash engine. */ | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
/* Insert the hash result into the output stream. */ | |||||
safexcel_instr_insert_digest(&instr, req->sess->digestlen); | |||||
/* Pad the rest of the inline instruction space. */ | |||||
while (instr != start + SAFEXCEL_MAX_ITOKENS) | |||||
safexcel_instr_nop(&instr); | |||||
} | |||||
static void | |||||
safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
struct cryptop *crp; | |||||
struct safexcel_instr *start; | |||||
uint8_t *a0, *b0, *alenp; | |||||
int aalign; | |||||
crp = req->crp; | |||||
start = instr; | |||||
/* | |||||
* Construct two blocks, A0 and B0, used in encryption and | |||||
* authentication, respectively. A0 is embedded in the token | |||||
* descriptor, and B0 is inserted directly into the data stream using | |||||
* instructions below. | |||||
* | |||||
* OCF seems to assume a 12-byte IV, fixing L (the payload length size) | |||||
Not Done Inline ActionsThis is true of existing OCF consumers and it's ok to rely on that for now (your existing validation of csp_ivlen in your probesession hook will protect you if this changes). I may eventually add a new ioctl to /dev/crypto that permits setting the IV length for a given session. The only use case currently is to permit running all of the NIST KAT vectors for CCM and GCM. Some of the test vectors have IV lengths and tag lengths other than 12 and 16, respectively. jhb: This is true of existing OCF consumers and it's ok to rely on that for now (your existing… | |||||
Done Inline ActionsI was surprised that OCF isn't more flexible here. It shouldn't be a problem to generalize this code once the framework supports multiple IV lengths. markj: I was surprised that OCF isn't more flexible here. It shouldn't be a problem to generalize this… | |||||
Not Done Inline ActionsWell, prior to the refactor the IV length was hardcoded. The refactor added csp_ivlen which makes it possible to support this. However, existing in-tree uses of GCM and CCM (IPsec, TLS) all use an IV length of 12. As I said, I need to extend /dev/crypto's interface to support setting the value of csp_ivlen. The current ioctl doesn't include an IV length so it always uses 12. jhb: Well, prior to the refactor the IV length was hardcoded. The refactor added `csp_ivlen` which… | |||||
Done Inline ActionsOk, I can update the driver and test once you're able to update OCF and cryptotest to handle alternate IV lengths. markj: Ok, I can update the driver and test once you're able to update OCF and cryptotest to handle… | |||||
Not Done Inline ActionsYes. In fact, I looked at that last week and concluded that I probably need to adjust this anyway to have the IV length be per-op instead of per-session. The only algorithms that support variable length IVs currently (GCM and CCM) don't have any cached session state that is contingent on the IV length, and the existing /dev/crypto ioctl has a per-op length. It may be a while before I get to this though. The 'mlen' for auth should likely also be per-op. I don't think any drivers in the tree depend on that being a constant in the session but always have to either cache the session value or go get it out of the csp for each operation. I will probably adjust both of those at the same time when I do it. No reason to wait for that for this driver though. jhb: Yes. In fact, I looked at that last week and concluded that I probably need to adjust this… | |||||
* at 3 bytes due to the layout of B0. This is fine since the driver | |||||
* has a maximum of 65535 bytes anyway. | |||||
*/ | |||||
Done Inline Actions0x2 is...? Name it please. gnn: 0x2 is...? Name it please. | |||||
a0 = (uint8_t *)&cdesc->control_data.token[0]; | |||||
memset(a0, 0, AES_BLOCK_LEN); | |||||
a0[0] = 0x2; /* L - 1 */ | |||||
memcpy(&a0[1], req->iv, AES_CCM_IV_LEN); | |||||
/* | |||||
* Insert B0 and the AAD length into the input stream. | |||||
*/ | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; | |||||
instr->length = AES_BLOCK_LEN + (crp->crp_aad_length > 0 ? 2 : 0); | |||||
instr->status = 0; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_HASH | | |||||
SAFEXCEL_INSTR_INSERT_IMMEDIATE; | |||||
instr++; | |||||
b0 = (uint8_t *)instr; | |||||
memset(b0, 0, AES_BLOCK_LEN); | |||||
b0[0] = | |||||
0x2 /* L - 1 */ | | |||||
((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ | | |||||
(crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */; | |||||
memcpy(&b0[1], req->iv, AES_CCM_IV_LEN); | |||||
b0[14] = crp->crp_payload_length >> 8; | |||||
b0[15] = crp->crp_payload_length & 0xff; | |||||
instr += AES_BLOCK_LEN / sizeof(*instr); | |||||
/* Insert the AAD length and data into the input stream. */ | |||||
if (crp->crp_aad_length > 0) { | |||||
alenp = (uint8_t *)instr; | |||||
alenp[0] = crp->crp_aad_length >> 8; | |||||
alenp[1] = crp->crp_aad_length & 0xff; | |||||
alenp[2] = 0; | |||||
alenp[3] = 0; | |||||
instr++; | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_aad_length; | |||||
instr->status = 0; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
/* Insert zero padding. */ | |||||
aalign = (crp->crp_aad_length + 2) & 15; | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; | |||||
instr->length = aalign == 0 ? 0 : | |||||
16 - ((crp->crp_aad_length + 2) & 15); | |||||
instr->status = crp->crp_payload_length == 0 ? | |||||
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
} | |||||
safexcel_instr_temp_aes_block(&instr); | |||||
/* Insert the cipher payload into the input stream. */ | |||||
if (crp->crp_payload_length > 0) { | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = (crp->crp_payload_length & 15) == 0 ? | |||||
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | | |||||
SAFEXCEL_INSTR_DEST_CRYPTO | | |||||
SAFEXCEL_INSTR_DEST_HASH | | |||||
SAFEXCEL_INSTR_INS_LAST; | |||||
instr++; | |||||
Done Inline ActionsMore magic numbers. gnn: More magic numbers. | |||||
/* Insert zero padding. */ | |||||
if (crp->crp_payload_length & 15) { | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT; | |||||
instr->length = 16 - (crp->crp_payload_length & 15); | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
} | |||||
} | |||||
/* | |||||
* Compute the digest, or extract it and place it in the output stream. | |||||
*/ | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) | |||||
safexcel_instr_insert_digest(&instr, req->sess->digestlen); | |||||
else | |||||
safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); | |||||
cdesc->additional_cdata_size = instr - start; | |||||
} | |||||
static void | |||||
safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
struct cryptop *crp; | |||||
struct safexcel_instr *start; | |||||
memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); | |||||
cdesc->control_data.token[3] = htobe32(1); | |||||
crp = req->crp; | |||||
start = instr; | |||||
/* Insert the AAD into the input stream. */ | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_aad_length; | |||||
instr->status = crp->crp_payload_length == 0 ? | |||||
SAFEXCEL_INSTR_STATUS_LAST_HASH : 0; | |||||
instr->instructions = SAFEXCEL_INSTR_INS_LAST | | |||||
SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
safexcel_instr_temp_aes_block(&instr); | |||||
/* Insert the cipher payload into the input stream. */ | |||||
if (crp->crp_payload_length > 0) { | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT | | |||||
SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH | | |||||
SAFEXCEL_INSTR_INS_LAST; | |||||
instr++; | |||||
} | |||||
/* | |||||
* Compute the digest, or extract it and place it in the output stream. | |||||
*/ | |||||
if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) | |||||
safexcel_instr_insert_digest(&instr, req->sess->digestlen); | |||||
else | |||||
safexcel_instr_retrieve_digest(&instr, req->sess->digestlen); | |||||
cdesc->additional_cdata_size = instr - start; | |||||
} | |||||
static void | |||||
safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
struct cryptop *crp; | |||||
struct safexcel_instr *start; | |||||
memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN); | |||||
cdesc->control_data.token[3] = htobe32(1); | |||||
crp = req->crp; | |||||
start = instr; | |||||
instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION; | |||||
instr->length = crp->crp_payload_length; | |||||
instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH; | |||||
instr->instructions = SAFEXCEL_INSTR_INS_LAST | | |||||
SAFEXCEL_INSTR_DEST_HASH; | |||||
instr++; | |||||
safexcel_instr_temp_aes_block(&instr); | |||||
safexcel_instr_insert_digest(&instr, req->sess->digestlen); | |||||
cdesc->additional_cdata_size = instr - start; | |||||
} | |||||
static void | |||||
safexcel_set_token(struct safexcel_request *req, | |||||
struct safexcel_command_desc *cdesc) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct safexcel_instr *instr; | |||||
struct safexcel_softc *sc; | |||||
int ringidx; | |||||
csp = crypto_get_params(req->crp->crp_session); | |||||
sc = req->sc; | |||||
ringidx = req->sess->ringidx; | |||||
safexcel_set_command(req, cdesc); | |||||
/* | |||||
* For keyless hash operations, the token instructions can be embedded | |||||
* in the token itself. Otherwise we use an additional token descriptor | |||||
* and the embedded instruction space is used to store the IV. | |||||
*/ | |||||
if (csp->csp_cipher_alg == 0 && | |||||
csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) { | |||||
instr = (void *)cdesc->control_data.token; | |||||
} else { | |||||
instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr + | |||||
sc->sc_config.atok_offset * | |||||
(cdesc - sc->sc_ring[ringidx].cdr.desc)); | |||||
cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD; | |||||
} | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
safexcel_instr_gcm(req, instr, cdesc); | |||||
break; | |||||
case CRYPTO_AES_CCM_16: | |||||
safexcel_instr_ccm(req, instr, cdesc); | |||||
break; | |||||
case CRYPTO_AES_XTS: | |||||
memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN); | |||||
memset(cdesc->control_data.token + | |||||
AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN); | |||||
safexcel_instr_cipher(req, instr, cdesc); | |||||
break; | |||||
case CRYPTO_AES_CBC: | |||||
case CRYPTO_AES_ICM: | |||||
memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN); | |||||
if (csp->csp_auth_alg != 0) | |||||
safexcel_instr_eta(req, instr, cdesc); | |||||
else | |||||
Done Inline ActionsPlease name this even if it is 0. gnn: Please name this even if it is 0. | |||||
safexcel_instr_cipher(req, instr, cdesc); | |||||
break; | |||||
case 0: | |||||
switch (csp->csp_auth_alg) { | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
safexcel_instr_sha_hash(req, instr); | |||||
break; | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
safexcel_instr_gmac(req, instr, cdesc); | |||||
break; | |||||
default: | |||||
panic("unhandled auth request %d", csp->csp_auth_alg); | |||||
} | |||||
break; | |||||
default: | |||||
panic("unhandled cipher request %d", csp->csp_cipher_alg); | |||||
} | |||||
} | |||||
static struct safexcel_result_desc * | |||||
safexcel_add_rdesc(struct safexcel_ring *ring, bool first, bool last, | |||||
bus_addr_t data, uint32_t len) | |||||
{ | |||||
struct safexcel_result_desc *rdesc; | |||||
struct safexcel_rdesc_ring *rring; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
rring = &ring->rdr; | |||||
if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read) | |||||
return (NULL); | |||||
rdesc = &rring->desc[rring->write]; | |||||
rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE; | |||||
rdesc->particle_size = len; | |||||
rdesc->rsvd0 = 0; | |||||
rdesc->descriptor_overflow = 0; | |||||
rdesc->buffer_overflow = 0; | |||||
rdesc->last_seg = last; | |||||
rdesc->first_seg = first; | |||||
rdesc->result_size = sizeof(struct result_data_desc) / sizeof(uint32_t); | |||||
rdesc->rsvd1 = 0; | |||||
rdesc->data_lo = SAFEXCEL_ADDR_LO(data); | |||||
rdesc->data_hi = SAFEXCEL_ADDR_HI(data); | |||||
if (first) { | |||||
rdesc->result_data.packet_length = 0; | |||||
rdesc->result_data.error_code = 0; | |||||
} | |||||
return (rdesc); | |||||
} | |||||
static struct safexcel_command_desc * | |||||
safexcel_add_cdesc(struct safexcel_ring *ring, bool first, bool last, | |||||
bus_addr_t data, uint32_t data_len, uint32_t full_data_len, | |||||
bus_addr_t context) | |||||
{ | |||||
struct safexcel_command_desc *cdesc; | |||||
struct safexcel_cdesc_ring *cring; | |||||
KASSERT(full_data_len <= SAFEXCEL_MAX_REQUEST_SIZE, | |||||
("%s: request length %u too long", __func__, full_data_len)); | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
cring = &ring->cdr; | |||||
if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read) | |||||
return (NULL); | |||||
cdesc = &cring->desc[cring->write]; | |||||
cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE; | |||||
cdesc->particle_size = data_len; | |||||
cdesc->rsvd0 = 0; | |||||
cdesc->last_seg = last; | |||||
cdesc->first_seg = first; | |||||
cdesc->additional_cdata_size = 0; | |||||
cdesc->rsvd1 = 0; | |||||
cdesc->data_lo = SAFEXCEL_ADDR_LO(data); | |||||
cdesc->data_hi = SAFEXCEL_ADDR_HI(data); | |||||
if (first) { | |||||
cdesc->control_data.packet_length = full_data_len; | |||||
cdesc->control_data.options = SAFEXCEL_OPTION_MAGIC_VALUE | | |||||
SAFEXCEL_OPTION_64BIT_CTX | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD | | |||||
SAFEXCEL_OPTION_RC_AUTO; | |||||
cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS; | |||||
cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) | | |||||
SAFEXCEL_CONTEXT_SMALL; | |||||
cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context); | |||||
} | |||||
return (cdesc); | |||||
} | |||||
static void | |||||
safexcel_rollback_cdesc(struct safexcel_ring *ring, int count) | |||||
{ | |||||
struct safexcel_cdesc_ring *cring; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
cring = &ring->cdr; | |||||
cring->write -= count; | |||||
if (cring->write < 0) | |||||
cring->write += SAFEXCEL_RING_SIZE; | |||||
} | |||||
static void | |||||
safexcel_rollback_rdesc(struct safexcel_ring *ring, int count) | |||||
{ | |||||
struct safexcel_rdesc_ring *rring; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
rring = &ring->rdr; | |||||
rring->write -= count; | |||||
if (rring->write < 0) | |||||
rring->write += SAFEXCEL_RING_SIZE; | |||||
} | |||||
static void | |||||
safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg, | |||||
int start, int len) | |||||
{ | |||||
bus_dma_segment_t *seg; | |||||
size_t seglen; | |||||
int error, i; | |||||
for (i = 0; i < nseg && len > 0; i++) { | |||||
seg = &segs[i]; | |||||
if (seg->ds_len <= start) { | |||||
start -= seg->ds_len; | |||||
continue; | |||||
} | |||||
seglen = MIN(len, seg->ds_len - start); | |||||
error = sglist_append_phys(sg, seg->ds_addr + start, seglen); | |||||
if (error != 0) | |||||
panic("%s: ran out of segments: %d", __func__, error); | |||||
len -= seglen; | |||||
start = 0; | |||||
} | |||||
} | |||||
static void | |||||
safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, | |||||
int error) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct cryptop *crp; | |||||
struct safexcel_command_desc *cdesc, *first_cdesc; | |||||
struct safexcel_request *req; | |||||
struct safexcel_ring *ring; | |||||
struct safexcel_session *sess; | |||||
struct sglist *sg; | |||||
size_t inlen; | |||||
int i; | |||||
req = (struct safexcel_request *)arg; | |||||
if (error != 0) { | |||||
req->error = error; | |||||
return; | |||||
} | |||||
crp = req->crp; | |||||
csp = crypto_get_params(crp->crp_session); | |||||
sess = req->sess; | |||||
ring = &req->sc->sc_ring[sess->ringidx]; | |||||
mtx_assert(&ring->mtx, MA_OWNED); | |||||
/* | |||||
* Set up descriptors for input and output data. | |||||
* | |||||
* The processing engine programs require that any AAD comes first, | |||||
* followed by the cipher plaintext, followed by the digest. Some | |||||
* consumers place the digest first in the input buffer, in which case | |||||
* we have to create an extra descriptor. | |||||
* | |||||
* As an optimization, unmodified data is not passed to the output | |||||
* stream. | |||||
*/ | |||||
sglist_reset(ring->cmd_data); | |||||
sglist_reset(ring->res_data); | |||||
if (crp->crp_aad_length != 0) { | |||||
Not Done Inline ActionsThis doesn't appear to handle the recently added CSP_F_SEPARATE_AAD. For that, you wound want to append the equivalent of sglist_append(sg, crp->crp_aad, crp->crp_aad_length) when crp->crp_aad != NULL. IPsec will eventually use this for AES-GCM when using ESN. jhb: This doesn't appear to handle the recently added CSP_F_SEPARATE_AAD. For that, you wound want… | |||||
safexcel_append_segs(segs, nseg, ring->cmd_data, | |||||
crp->crp_aad_start, crp->crp_aad_length); | |||||
} | |||||
safexcel_append_segs(segs, nseg, ring->cmd_data, | |||||
crp->crp_payload_start, crp->crp_payload_length); | |||||
if (csp->csp_cipher_alg != 0) { | |||||
safexcel_append_segs(segs, nseg, ring->res_data, | |||||
Not Done Inline ActionsTo support CSP_F_SEPARATE_OUTPUT I think you would just need to adjust this to use segs from the output buffer and start at crp_payload_output_start when CRYPTO_HAS_OUTPUT_BUFFER() is true. jhb: To support CSP_F_SEPARATE_OUTPUT I think you would just need to adjust this to use segs from… | |||||
Done Inline ActionsHmm, how is this supposed to work with bus_dmamap_load_crp()? Don't I need a function that will load crp->crp_obuf? markj: Hmm, how is this supposed to work with `bus_dmamap_load_crp()`? Don't I need a function that… | |||||
Not Done Inline ActionsThere is now a bus_dmamap_load_crp_buf(). bus_dmamap_load_crp(crp, ...) is now just bus_dmamap_load_crp_buf(&crp->crp_buf, ...). When you have an output buffer you would use bus_dmamap_load_crp_buf() with crp->crp_obuf, but you would need to deal with the fact that you have two sglists. jhb: There is now a `bus_dmamap_load_crp_buf()`. `bus_dmamap_load_crp(crp, ...)` is now just… | |||||
Done Inline ActionsI see, thanks. This still seems pretty awkward. To support both separate AAD and separate output buffers I need to use up to three dmamaps per request. As far as I can see, to load both crp->crp_buf and crp->crp_obuf I need two back-to-back calls to bus_dmamap_load_crp_buffer(), and I don't think they can share a dmamap_t. Similarly I need a third dmamap_load for separate AAD. I have it implemented and working, but it feels more complicated than it should be. markj: I see, thanks. This still seems pretty awkward. To support both separate AAD and separate… | |||||
Not Done Inline ActionsThree separate maps is what it would take, yes. You can avoid adding these for now since your use case is IPsec. At some point separate AAD will probably get used for ESN with GCM in IPsec. KTLS uses these features, but nothing else currently. jhb: Three separate maps is what it would take, yes. You can avoid adding these for now since your… | |||||
Done Inline ActionsOk, thanks for confirming that I'm not missing something. I will commit the required support as a follow-up change, since I'm working on porting this driver to stable/12 and having support for separate buffers will just complicate things a bit at this point. markj: Ok, thanks for confirming that I'm not missing something. I will commit the required support as… | |||||
crp->crp_payload_start, crp->crp_payload_length); | |||||
} | |||||
if (sess->digestlen > 0) { | |||||
if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { | |||||
safexcel_append_segs(segs, nseg, ring->cmd_data, | |||||
crp->crp_digest_start, sess->digestlen); | |||||
} else { | |||||
safexcel_append_segs(segs, nseg, ring->res_data, | |||||
crp->crp_digest_start, sess->digestlen); | |||||
} | |||||
} | |||||
sg = ring->cmd_data; | |||||
for (i = 0, inlen = 0; i < sg->sg_nseg; i++) | |||||
inlen += sg->sg_segs[i].ss_len; | |||||
for (i = 0; i < sg->sg_nseg; i++) { | |||||
cdesc = safexcel_add_cdesc(ring, i == 0, i == sg->sg_nseg - 1, | |||||
sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len, | |||||
(uint32_t)inlen, req->ctx.paddr); | |||||
if (cdesc == NULL) { | |||||
safexcel_rollback_cdesc(ring, i); | |||||
req->error = EAGAIN; | |||||
return; | |||||
} | |||||
if (i == 0) | |||||
first_cdesc = cdesc; | |||||
} | |||||
req->cdescs = sg->sg_nseg; | |||||
sg = ring->res_data; | |||||
for (i = 0; i < sg->sg_nseg; i++) { | |||||
if (safexcel_add_rdesc(ring, i == 0, i == sg->sg_nseg - 1, | |||||
sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) { | |||||
safexcel_rollback_cdesc(ring, ring->cmd_data->sg_nseg); | |||||
safexcel_rollback_rdesc(ring, i); | |||||
req->error = EAGAIN; | |||||
return; | |||||
} | |||||
} | |||||
if (i != 0) { | |||||
req->rdescs = sg->sg_nseg; | |||||
} else { | |||||
/* | |||||
* Add a dummy result descriptor in the case where the output | |||||
* stream will be empty. | |||||
*/ | |||||
if (safexcel_add_rdesc(ring, true, true, 0, 0) == NULL) { | |||||
safexcel_rollback_cdesc(ring, ring->cmd_data->sg_nseg); | |||||
req->error = EAGAIN; | |||||
return; | |||||
} | |||||
req->rdescs = 1; | |||||
} | |||||
/* | |||||
* Finally, populate the request descriptor with algorithm-specific | |||||
* fields. | |||||
*/ | |||||
safexcel_set_token(req, first_cdesc); | |||||
} | |||||
static int | |||||
safexcel_create_chain(struct safexcel_softc *sc, struct safexcel_request *req) | |||||
{ | |||||
int error; | |||||
req->sc = sc; | |||||
req->error = 0; | |||||
req->cdescs = req->rdescs = 0; | |||||
error = bus_dmamap_load_crp(sc->sc_data_dtag, req->dmap, req->crp, | |||||
safexcel_create_chain_cb, req, BUS_DMA_NOWAIT); | |||||
if (error == 0) | |||||
req->dmap_loaded = true; | |||||
if (req->error != 0) | |||||
error = req->error; | |||||
return (error); | |||||
} | |||||
static bool | |||||
safexcel_probe_cipher(const struct crypto_session_params *csp) | |||||
{ | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_CBC: | |||||
case CRYPTO_AES_ICM: | |||||
if (csp->csp_ivlen != AES_BLOCK_LEN) | |||||
return (false); | |||||
break; | |||||
case CRYPTO_AES_XTS: | |||||
if (csp->csp_ivlen != AES_XTS_IV_LEN) | |||||
return (false); | |||||
break; | |||||
default: | |||||
return (false); | |||||
} | |||||
return (true); | |||||
} | |||||
/* | |||||
* Determine whether the driver can implement a session with the requested | |||||
* parameters. | |||||
*/ | |||||
static int | |||||
safexcel_probesession(device_t dev, const struct crypto_session_params *csp) | |||||
{ | |||||
switch (csp->csp_mode) { | |||||
case CSP_MODE_CIPHER: | |||||
if (!safexcel_probe_cipher(csp)) | |||||
return (EINVAL); | |||||
break; | |||||
case CSP_MODE_DIGEST: | |||||
switch (csp->csp_auth_alg) { | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
if (csp->csp_ivlen != AES_GCM_IV_LEN) | |||||
return (EINVAL); | |||||
break; | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
break; | |||||
case CSP_MODE_AEAD: | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
if (csp->csp_ivlen != AES_GCM_IV_LEN) | |||||
return (EINVAL); | |||||
break; | |||||
case CRYPTO_AES_CCM_16: | |||||
if (csp->csp_ivlen != AES_CCM_IV_LEN) | |||||
return (EINVAL); | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
break; | |||||
case CSP_MODE_ETA: | |||||
if (!safexcel_probe_cipher(csp)) | |||||
return (EINVAL); | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_CBC: | |||||
case CRYPTO_AES_ICM: | |||||
/* | |||||
* The EIP-97 does not support combining AES-XTS with | |||||
* hash operations. | |||||
*/ | |||||
if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC && | |||||
csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC && | |||||
csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC && | |||||
csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC && | |||||
csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC) | |||||
return (EINVAL); | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
break; | |||||
default: | |||||
return (EINVAL); | |||||
} | |||||
return (CRYPTODEV_PROBE_HARDWARE); | |||||
} | |||||
/* | |||||
* Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted | |||||
* using the cipher key. | |||||
*/ | |||||
static void | |||||
safexcel_setkey_ghash(struct safexcel_session *sess, const uint8_t *key, | |||||
int klen) | |||||
{ | |||||
uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; | |||||
uint8_t zeros[AES_BLOCK_LEN]; | |||||
int i, rounds; | |||||
memset(zeros, 0, sizeof(zeros)); | |||||
rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); | |||||
rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)sess->ghash_key); | |||||
Not Done Inline ActionsAES key scheduling and encryption in software is generally pretty slow. Is it slower to chain this operation through the device? cem: AES key scheduling and encryption in software is generally pretty slow. Is it slower to chain… | |||||
Done Inline ActionsIn principle it is possible to offload this operation to the device, but typically this is going to be done as part of session initialization, so the cost will be amortized over all operations using the session. For IPSec the cost should be negligible. markj: In principle it is possible to offload this operation to the device, but typically this is… | |||||
Not Done Inline ActionsWhile this is true and ccr(4) takes the same approach, it does occur to me that it would be nice to have a little wrapper around AESNI or the like that one could use for quick AES bits. It could fallback to the existing C software on platforms without some kind of MD software AES. Probably not very urgent though. jhb: While this is true and ccr(4) takes the same approach, it does occur to me that it would be… | |||||
for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++) | |||||
sess->ghash_key[i] = htobe32(sess->ghash_key[i]); | |||||
explicit_bzero(ks, sizeof(ks)); | |||||
} | |||||
/* | |||||
* Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3 | |||||
* in the hardware implementation. K1 is the cipher key and comes last in the | |||||
* buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN. For now XCBC-MAC | |||||
* is not implemented so K2 and K3 are fixed. | |||||
*/ | |||||
static void | |||||
safexcel_setkey_xcbcmac(struct safexcel_session *sess, const uint8_t *key, | |||||
int klen) | |||||
{ | |||||
int i, off; | |||||
memset(sess->xcbc_key, 0, sizeof(sess->xcbc_key)); | |||||
off = 2 * AES_BLOCK_LEN / sizeof(uint32_t); | |||||
for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4) | |||||
sess->xcbc_key[i + off] = htobe32(le32dec(key)); | |||||
} | |||||
static void | |||||
safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx, | |||||
char *buf) | |||||
{ | |||||
int hashwords, i; | |||||
ahash->Init(ctx); | |||||
ahash->Update(ctx, buf, ahash->blocksize); | |||||
switch (ahash->type) { | |||||
case CRYPTO_SHA1_HMAC: | |||||
hashwords = ahash->hashsize / sizeof(uint32_t); | |||||
for (i = 0; i < hashwords; i++) | |||||
((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]); | |||||
break; | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t); | |||||
for (i = 0; i < hashwords; i++) | |||||
((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]); | |||||
break; | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
hashwords = ahash->hashsize / sizeof(uint32_t); | |||||
for (i = 0; i < ahash->hashsize / sizeof(uint32_t); i++) | |||||
((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]); | |||||
break; | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t); | |||||
for (i = 0; i < hashwords; i++) | |||||
((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]); | |||||
Done Inline Actions/ sizeof(uint64_t)? cem: `/ sizeof(uint64_t)`? | |||||
Done Inline ActionsThanks. markj: Thanks. | |||||
break; | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
hashwords = ahash->hashsize / sizeof(uint32_t); | |||||
for (i = 0; i < hashwords; i++) | |||||
((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]); | |||||
break; | |||||
} | |||||
} | |||||
/* | |||||
* Pre-compute the inner and outer digests used in the HMAC algorithm. | |||||
*/ | |||||
static void | |||||
safexcel_setkey_hmac(struct safexcel_session *sess, int alg, const uint8_t *key, | |||||
int klen) | |||||
{ | |||||
union authctx ctx; | |||||
struct auth_hash *ahash; | |||||
int i; | |||||
switch (alg) { | |||||
Done Inline ActionsThis can use ahash = crypto_auth_hash(csp) if you pass the csp to this routine. jhb: This can use `ahash = crypto_auth_hash(csp)` if you pass the `csp` to this routine. | |||||
case CRYPTO_SHA1_HMAC: | |||||
ahash = &auth_hash_hmac_sha1; | |||||
break; | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
ahash = &auth_hash_hmac_sha2_224; | |||||
break; | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
ahash = &auth_hash_hmac_sha2_256; | |||||
break; | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
ahash = &auth_hash_hmac_sha2_384; | |||||
break; | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
ahash = &auth_hash_hmac_sha2_512; | |||||
break; | |||||
default: | |||||
__assert_unreachable(); | |||||
} | |||||
KASSERT(sizeof(sess->hmac_ipad) >= ahash->blocksize, | |||||
("%s: hash algorithm %s has invalid block size %d", | |||||
__func__, ahash->name, ahash->blocksize)); | |||||
KASSERT(sizeof(sess->hmac_ipad) >= ahash->hashsize, | |||||
("%s: hash algorithm %s has invalid hash size %d", | |||||
__func__, ahash->name, ahash->hashsize)); | |||||
memset(sess->hmac_ipad, 0, sizeof(sess->hmac_ipad)); | |||||
Done Inline ActionsYou could perhaps use hmac_init_ipad and hmac_init_opad here. To do that, you would change setkey_hmac_digest to only copy the bits out of the ctx. If you combine that with passing in the csp from the caller this function becomes: { union authctx ctx; ahash = crypto_auth_hash(csp); hmac_init_ipad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, ses->hmac_ipad); hmac_init_opad(ahash, key, klen, &ctx); safexcel_setkey_hmac_digest(ahash, &ctx, ses->hmac_opad); explicit_bzero(&ctx, sizeof(ctx)); } jhb: You could perhaps use `hmac_init_ipad` and `hmac_init_opad` here. To do that, you would change… | |||||
Done Inline ActionsThanks, that is much nicer. It is a bit unfortunate though that we compute the same hash twice that way. markj: Thanks, that is much nicer. It is a bit unfortunate though that we compute the same hash twice… | |||||
Not Done Inline ActionsSimilar to the ghash case above, this isn't typically a hot path. I could provide a hmac_init_pads() perhaps that would take in two contexts and to avoid some of the duplication, but you really only duplicate work in the case of a key longer than the digest size (which OCF didn't even support consistently prior to the refactor, so isn't common anyway). The Init method of these transforms is pretty simple, and you always have to do the hash on the xored key twice. jhb: Similar to the ghash case above, this isn't typically a hot path. I could provide a… | |||||
Done Inline ActionsRight, the only reason I hadn't used hmac_init_ipad/opad() was that I didn't know about them. markj: Right, the only reason I hadn't used hmac_init_ipad/opad() was that I didn't know about them. | |||||
if (klen > ahash->blocksize) { | |||||
ahash->Init(&ctx); | |||||
ahash->Update(&ctx, key, klen); | |||||
ahash->Final(sess->hmac_ipad, &ctx); | |||||
explicit_bzero(&ctx, ahash->ctxsize); | |||||
} else { | |||||
memcpy(sess->hmac_ipad, key, klen); | |||||
} | |||||
memcpy(sess->hmac_opad, sess->hmac_ipad, ahash->blocksize); | |||||
for (i = 0; i < ahash->blocksize; i++) { | |||||
sess->hmac_ipad[i] ^= HMAC_IPAD_VAL; | |||||
sess->hmac_opad[i] ^= HMAC_OPAD_VAL; | |||||
} | |||||
safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_ipad); | |||||
safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_opad); | |||||
explicit_bzero(&ctx, ahash->ctxsize); | |||||
} | |||||
static void | |||||
safexcel_setkey_xts(struct safexcel_session *sess, const uint8_t *key, int klen) | |||||
{ | |||||
memcpy(sess->tweak_key, key + klen / 2, klen / 2); | |||||
} | |||||
static void | |||||
safexcel_setkey(struct safexcel_session *sess, | |||||
const struct crypto_session_params *csp, struct cryptop *crp) | |||||
{ | |||||
const uint8_t *akey, *ckey; | |||||
int aklen, cklen; | |||||
aklen = csp->csp_auth_klen; | |||||
cklen = csp->csp_cipher_klen; | |||||
akey = ckey = NULL; | |||||
if (crp != NULL) { | |||||
akey = crp->crp_auth_key; | |||||
ckey = crp->crp_cipher_key; | |||||
} | |||||
if (akey == NULL) | |||||
akey = csp->csp_auth_key; | |||||
if (ckey == NULL) | |||||
ckey = csp->csp_cipher_key; | |||||
sess->klen = cklen; | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
safexcel_setkey_ghash(sess, ckey, cklen); | |||||
break; | |||||
case CRYPTO_AES_CCM_16: | |||||
safexcel_setkey_xcbcmac(sess, ckey, cklen); | |||||
break; | |||||
case CRYPTO_AES_XTS: | |||||
safexcel_setkey_xts(sess, ckey, cklen); | |||||
sess->klen /= 2; | |||||
break; | |||||
} | |||||
switch (csp->csp_auth_alg) { | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
safexcel_setkey_hmac(sess, csp->csp_auth_alg, akey, aklen); | |||||
break; | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
sess->klen = aklen; | |||||
safexcel_setkey_ghash(sess, akey, aklen); | |||||
break; | |||||
} | |||||
} | |||||
static uint32_t | |||||
safexcel_aes_algid(int keylen) | |||||
{ | |||||
switch (keylen) { | |||||
case 16: | |||||
return (CONTEXT_CONTROL_CRYPTO_ALG_AES128); | |||||
case 24: | |||||
return (CONTEXT_CONTROL_CRYPTO_ALG_AES192); | |||||
case 32: | |||||
return (CONTEXT_CONTROL_CRYPTO_ALG_AES256); | |||||
default: | |||||
panic("invalid AES key length %d", keylen); | |||||
} | |||||
} | |||||
static uint32_t | |||||
safexcel_aes_ccm_hashid(int keylen) | |||||
{ | |||||
switch (keylen) { | |||||
case 16: | |||||
return (CONTEXT_CONTROL_HASH_ALG_XCBC128); | |||||
case 24: | |||||
return (CONTEXT_CONTROL_HASH_ALG_XCBC192); | |||||
case 32: | |||||
return (CONTEXT_CONTROL_HASH_ALG_XCBC256); | |||||
default: | |||||
panic("invalid AES key length %d", keylen); | |||||
} | |||||
} | |||||
static uint32_t | |||||
safexcel_sha_hashid(int alg) | |||||
{ | |||||
switch (alg) { | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA1_HMAC: | |||||
return (CONTEXT_CONTROL_HASH_ALG_SHA1); | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
return (CONTEXT_CONTROL_HASH_ALG_SHA224); | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
return (CONTEXT_CONTROL_HASH_ALG_SHA256); | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
return (CONTEXT_CONTROL_HASH_ALG_SHA384); | |||||
case CRYPTO_SHA2_512: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
return (CONTEXT_CONTROL_HASH_ALG_SHA512); | |||||
default: | |||||
__assert_unreachable(); | |||||
} | |||||
} | |||||
static int | |||||
safexcel_sha_hashlen(int alg) | |||||
{ | |||||
switch (alg) { | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA1_HMAC: | |||||
return (SHA1_HASH_LEN); | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
return (SHA2_224_HASH_LEN); | |||||
Done Inline ActionsDon't we guarantee the session is pre-zeroed? Or is that some other object. I might misremember. cem: Don't we guarantee the session is pre-zeroed? Or is that some other object. I might… | |||||
Done Inline ActionsYou are right, crypto_newsession() allocates a zeroed session softc structure. markj: You are right, crypto_newsession() allocates a zeroed session softc structure. | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
return (SHA2_256_HASH_LEN); | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
return (SHA2_384_HASH_LEN); | |||||
case CRYPTO_SHA2_512: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
return (SHA2_512_HASH_LEN); | |||||
default: | |||||
__assert_unreachable(); | |||||
} | |||||
} | |||||
static int | |||||
safexcel_sha_statelen(int alg) | |||||
{ | |||||
switch (alg) { | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA1_HMAC: | |||||
return (SHA1_HASH_LEN); | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
return (SHA2_256_HASH_LEN); | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
return (SHA2_512_HASH_LEN); | |||||
default: | |||||
__assert_unreachable(); | |||||
} | |||||
} | |||||
static int | |||||
safexcel_newsession(device_t dev, crypto_session_t cses, | |||||
const struct crypto_session_params *csp) | |||||
{ | |||||
struct safexcel_session *sess; | |||||
struct safexcel_softc *sc; | |||||
sc = device_get_softc(dev); | |||||
sess = crypto_get_driver_session(cses); | |||||
memset(sess, 0, sizeof(*sess)); | |||||
switch (csp->csp_auth_alg) { | |||||
case CRYPTO_SHA1: | |||||
case CRYPTO_SHA2_224: | |||||
case CRYPTO_SHA2_256: | |||||
case CRYPTO_SHA2_384: | |||||
case CRYPTO_SHA2_512: | |||||
sess->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED; | |||||
sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); | |||||
sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); | |||||
sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); | |||||
break; | |||||
case CRYPTO_SHA1_HMAC: | |||||
case CRYPTO_SHA2_224_HMAC: | |||||
case CRYPTO_SHA2_256_HMAC: | |||||
case CRYPTO_SHA2_384_HMAC: | |||||
case CRYPTO_SHA2_512_HMAC: | |||||
sess->digest = CONTEXT_CONTROL_DIGEST_HMAC; | |||||
sess->hash = safexcel_sha_hashid(csp->csp_auth_alg); | |||||
sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg); | |||||
sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg); | |||||
Not Done Inline ActionsIt does eliminate concurrency if the same session is used from many threads. I think GELI does this. Perhaps it shouldn't. cem: It does eliminate concurrency if the same session is used from many threads. I think GELI does… | |||||
Done Inline ActionsEach GELI worker thread gets a unique session, from my reading of the code. markj: Each GELI worker thread gets a unique session, from my reading of the code. | |||||
Not Done Inline ActionsIt does, but it's also true that right now there is nothing in OCF that implicitly says sessions are single-threaded or can only have a single request in flight. For co-processors you probably want to support multiple sessions in flight (and in theory IPsec should be able to trigger that since multiple flows can share a single SA). Most drivers permit multiple in-flight operations for a single session (ccp(4) is the one driver I know that isn't safe to do this, and it is also missing the checks to defer requests to avoid doing the wrong thing in this scenario). I believe most drivers also permit concurrent operations though they may use a lock to single thread them. jhb: It does, but it's also true that right now there is nothing in OCF that implicitly says… | |||||
Done Inline ActionsThere is nothing here preventing having multiple in-flight operations on a single session, the mutex simply serializes setup of the request. The session just seemed like a natural serialization point, which is why I went with this scheme. I doubt it will matter much in practice since the driver is going to be used on SOCs with low core counts anyway, but I could revisit this if IPSec ends up triggering a lot of ring lock contention. The only other option I can see is to have a global or per-session round-robin iterator which gets incremented for each request. markj: There is nothing here preventing having multiple in-flight operations on a single session, the… | |||||
Not Done Inline ActionsOk, that sounds sane. ccr(4) uses the same approach of locking just for setup / completion. jhb: Ok, that sounds sane. ccr(4) uses the same approach of locking just for setup / completion. | |||||
break; | |||||
case CRYPTO_AES_NIST_GMAC: | |||||
sess->digest = CONTEXT_CONTROL_DIGEST_GMAC; | |||||
sess->digestlen = GMAC_DIGEST_LEN; | |||||
sess->hash = CONTEXT_CONTROL_HASH_ALG_GHASH; | |||||
sess->alg = safexcel_aes_algid(csp->csp_auth_klen); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_GCM; | |||||
break; | |||||
} | |||||
switch (csp->csp_cipher_alg) { | |||||
case CRYPTO_AES_NIST_GCM_16: | |||||
sess->digest = CONTEXT_CONTROL_DIGEST_GMAC; | |||||
sess->digestlen = GMAC_DIGEST_LEN; | |||||
sess->hash = CONTEXT_CONTROL_HASH_ALG_GHASH; | |||||
sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_GCM; | |||||
break; | |||||
case CRYPTO_AES_CCM_16: | |||||
sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen); | |||||
sess->digest = CONTEXT_CONTROL_DIGEST_CCM; | |||||
sess->digestlen = CCM_CBC_MAX_DIGEST_LEN; | |||||
sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_CCM; | |||||
break; | |||||
case CRYPTO_AES_CBC: | |||||
sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; | |||||
break; | |||||
case CRYPTO_AES_ICM: | |||||
sess->alg = safexcel_aes_algid(csp->csp_cipher_klen); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR; | |||||
break; | |||||
case CRYPTO_AES_XTS: | |||||
sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2); | |||||
sess->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; | |||||
break; | |||||
} | |||||
if (csp->csp_auth_mlen != 0) | |||||
sess->digestlen = csp->csp_auth_mlen; | |||||
safexcel_setkey(sess, csp, NULL); | |||||
/* Bind each session to a fixed ring to minimize lock contention. */ | |||||
sess->ringidx = atomic_fetchadd_int(&sc->sc_ringidx, 1); | |||||
sess->ringidx %= sc->sc_config.rings; | |||||
return (0); | |||||
} | |||||
static int | |||||
safexcel_process(device_t dev, struct cryptop *crp, int hint) | |||||
{ | |||||
const struct crypto_session_params *csp; | |||||
struct safexcel_request *req; | |||||
struct safexcel_ring *ring; | |||||
struct safexcel_session *sess; | |||||
struct safexcel_softc *sc; | |||||
int error; | |||||
sc = device_get_softc(dev); | |||||
sess = crypto_get_driver_session(crp->crp_session); | |||||
csp = crypto_get_params(crp->crp_session); | |||||
if (__predict_false(crypto_buffer_len(&crp->crp_buf) > | |||||
SAFEXCEL_MAX_REQUEST_SIZE)) { | |||||
crp->crp_etype = E2BIG; | |||||
crypto_done(crp); | |||||
return (0); | |||||
} | |||||
if (__predict_false(crp->crp_aad_length == 0 && | |||||
crp->crp_payload_length == 0)) { | |||||
crp->crp_etype = EINVAL; | |||||
crypto_done(crp); | |||||
return (0); | |||||
} | |||||
safexcel_setkey(sess, csp, crp); | |||||
ring = &sc->sc_ring[sess->ringidx]; | |||||
mtx_lock(&ring->mtx); | |||||
req = safexcel_alloc_request(sc, ring); | |||||
if (__predict_false(req == NULL)) { | |||||
mtx_unlock(&ring->mtx); | |||||
mtx_lock(&sc->sc_mtx); | |||||
sc->sc_blocked = CRYPTO_SYMQ; | |||||
mtx_unlock(&sc->sc_mtx); | |||||
return (ERESTART); | |||||
} | |||||
req->crp = crp; | |||||
req->sess = sess; | |||||
crypto_read_iv(crp, req->iv); | |||||
error = safexcel_create_chain(sc, req); | |||||
if (__predict_false(error != 0)) { | |||||
safexcel_free_request(sc, ring, req); | |||||
mtx_unlock(&ring->mtx); | |||||
crp->crp_etype = error; | |||||
crypto_done(crp); | |||||
return (0); | |||||
} | |||||
bus_dmamap_sync(sc->sc_data_dtag, req->dmap, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
bus_dmamap_sync(req->ctx.tag, req->ctx.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map, | |||||
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | |||||
safexcel_enqueue_request(sc, ring, req); | |||||
if ((hint & CRYPTO_HINT_MORE) == 0) | |||||
safexcel_execute(sc, ring, req); | |||||
mtx_unlock(&ring->mtx); | |||||
return (0); | |||||
} | |||||
static device_method_t safexcel_methods[] = { | |||||
/* Device interface */ | |||||
DEVMETHOD(device_probe, safexcel_probe), | |||||
DEVMETHOD(device_attach, safexcel_attach), | |||||
DEVMETHOD(device_detach, safexcel_detach), | |||||
/* Cryptodev interface */ | |||||
DEVMETHOD(cryptodev_probesession, safexcel_probesession), | |||||
DEVMETHOD(cryptodev_newsession, safexcel_newsession), | |||||
DEVMETHOD(cryptodev_process, safexcel_process), | |||||
DEVMETHOD_END | |||||
}; | |||||
static devclass_t safexcel_devclass; | |||||
static driver_t safexcel_driver = { | |||||
.name = "safexcel", | |||||
.methods = safexcel_methods, | |||||
.size = sizeof(struct safexcel_softc), | |||||
}; | |||||
DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0); | |||||
MODULE_VERSION(safexcel, 1); | |||||
MODULE_DEPEND(safexcel, crypto, 1, 1, 1); |
We get this one already via sys/rman.h.
It seems odd to me, but we don't get machine/bus.h via sys/bus.h.