Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -232,6 +232,7 @@ dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_acpi.c optional pci acpi dev/pci/pci_host_generic_fdt.c optional pci fdt +dev/pdc/pdc.c optional fdt pdc | spum dev/psci/psci.c standard dev/psci/psci_arm64.S standard dev/psci/smccc.c standard Index: sys/dev/pdc/pdc.h =================================================================== --- /dev/null +++ sys/dev/pdc/pdc.h @@ -0,0 +1,57 @@ +/*- + * Copyright (c) 2019 Juniper Networks, Inc. + * Copyright (c) 2019 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PDC_H +#define PDC_H + +#include + +#define PDC_MAX_HEADER_SIZE 512 + +struct pdc_request { + int flags; + + uint8_t *header; + size_t header_len; + size_t resp_header_len; + + void *data; + size_t data_len; + + uint8_t *footer; + size_t footer_len; + size_t resp_footer_len; + + void *arg; + + void (*pdc_callback)(struct pdc_request *); +}; + +int pdc_process_request(struct pdc_request *req); +struct pdc_request* pdc_allocate(void); +void pdc_free(struct pdc_request *req); + +#endif /* PDC_H */ Index: sys/dev/pdc/pdc.c =================================================================== --- /dev/null +++ sys/dev/pdc/pdc.c @@ -0,0 +1,1045 @@ +/*- + * Copyright (c) 2019 Juniper Networks, Inc. + * Copyright (c) 2019 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_bus.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "pdc.h" +#include "pdc_regs.h" + +//#define PDC_DEBUG + +static int pdc_count; +static struct pdc_sc *pdc_sc[4]; +static uint32_t pdc_id; + +static inline uint32_t RD4(struct pdc_sc *sc, bus_size_t off); +static inline void WR4(struct pdc_sc *sc, bus_size_t off, uint32_t val); +static inline void AND4(struct pdc_sc *sc, bus_size_t off, uint32_t val); +static inline void OR4(struct pdc_sc *sc, bus_size_t off, uint32_t val); + +static void pdc_dma_sync_desc(struct pdc_sc *sc, bus_dmasync_op_t op); +static void pdc_fill_desc(struct pdc_dma_desc *desc, bus_dma_segment_t seg, + size_t size, int flags); + +#ifdef PDC_DEBUG +static void pdc_dump_desc(struct pdc_dma_desc *desc); +static void pdc_dump_regs(struct pdc_sc *sc); +#endif + +static void pdc_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error); +static void pdc_dma_load_cb2(void *arg, bus_dma_segment_t *segs, int nseg, + bus_size_t size, int error); + +static int pdc_allocate_request(struct pdc_request_ctx *ctx); +static int pdc_enqueue_one(struct pdc_sc *sc, struct pdc_channel *channel, + struct pdc_request_ctx *ctx); +static void pdc_enqueue(struct pdc_sc *sc); + +static int pdc_alloc_static_dma_mem(bus_dma_tag_t tag, + struct pdc_dma_mem *mem, size_t size); +static void pdc_free_static_dma_mem(bus_dma_tag_t tag, struct pdc_dma_mem *mem); +static void pdc_free_req(struct pdc_sc *sc, struct pdc_request_ctx *ctx); +static void pdc_free_resources(struct pdc_sc *sc); + +static int pdc_setup_desc(struct pdc_sc *sc); +static void pdc_setup_hw(struct pdc_sc *sc); +static int pdc_setup_intr(struct pdc_sc *sc); + +static void pdc_intr_handler(void *arg); + +static void pdc_receive(struct pdc_sc *sc); +static void pdc_process_response(struct pdc_sc *sc, struct pdc_request_ctx *ctx); + +static int pdc_attach(device_t dev); +static int pdc_detach(device_t dev); +static int pdc_probe(device_t dev); + +static inline uint32_t +RD4(struct pdc_sc *sc, bus_size_t off) +{ + + return (bus_read_4(sc->mem_res, off)); +} + +static inline void +WR4(struct pdc_sc *sc, bus_size_t off, uint32_t val) +{ + + bus_write_4(sc->mem_res, off, val); +} + +static inline void +AND4(struct pdc_sc *sc, bus_size_t off, uint32_t val) +{ + + WR4(sc, off, RD4(sc, off) & val); +} + +static inline void +OR4(struct pdc_sc *sc, bus_size_t off, uint32_t val) +{ + + WR4(sc, off, RD4(sc, off) | val); +} + +static void +pdc_dma_sync_desc(struct pdc_sc *sc, bus_dmasync_op_t op) +{ + + bus_dmamap_sync(sc->desc_dma_tag, + sc->channels[0].rx.mem.map, + op); + bus_dmamap_sync(sc->desc_dma_tag, + sc->channels[0].tx.mem.map, + op); +} + +static void +pdc_fill_desc(struct pdc_dma_desc *desc, bus_dma_segment_t seg, + size_t size, int flags) +{ + + desc->addr_low = htole32(seg.ds_addr & (uint32_t)~0); + desc->addr_high = htole32(seg.ds_addr >> 32); + desc->ctrl1 = htole32(flags); + desc->ctrl2 = htole32(size); + +#ifdef PDC_DEBUG + pdc_dump_desc(desc); +#endif +} + +#ifdef PDC_DEBUG +static void +pdc_dump_desc(struct pdc_dma_desc *desc) +{ + + printf("%s: desc: %p, addr_low: 0x%08x, addr_high: 0x%08x, " + "size: %u, flags: 0x%04x\n", + __func__, desc, desc->addr_low, desc->addr_high, + desc->ctrl2, desc->ctrl1); +} + +static void +pdc_dump_regs(struct pdc_sc *sc) +{ + + device_printf(sc->dev, "DUMPING REGISTERS:\n***********************\n"); + device_printf(sc->dev, "PDC_DEVCONTROL - 0x%08x\n", + RD4(sc, PDC_DEVCONTROL)); + device_printf(sc->dev, "PDC_DEVSTATUS - 0x%08x\n", + RD4(sc, PDC_DEVSTATUS)); + device_printf(sc->dev, "PDC_BISTSTATUS - 0x%08x\n", + RD4(sc, PDC_BISTSTATUS)); + device_printf(sc->dev, "--------------------------\n"); + device_printf(sc->dev, "PDC_INTSTATUS - 0x%08x\n", + RD4(sc, PDC_INTSTATUS)); + device_printf(sc->dev, "PDC_INTMASK - 0x%08x\n", + RD4(sc, PDC_INTMASK)); + device_printf(sc->dev, "--------------------------\n"); + device_printf(sc->dev, "TX_RING_ADDR_LOW - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_ADDR_LOW)); + device_printf(sc->dev, "TX_RING_ADDR_HIGH - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_ADDR_HIGH)); + device_printf(sc->dev, "TX_RING_CTRL - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_CTRL)); + device_printf(sc->dev, "TX_RING_PTR - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_PTR)); + device_printf(sc->dev, "TX_RING_STATUS0 - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_STATUS0)); + device_printf(sc->dev, "TX_RING_STATUS1 - 0x%08x\n", + RD4(sc, PDC_TX_RING + PDC_DMA_STATUS1)); + device_printf(sc->dev, "--------------------------\n"); + device_printf(sc->dev, "RX_RING_ADDR_LOW - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_ADDR_LOW)); + device_printf(sc->dev, "RX_RING_ADDR_HIGH - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_ADDR_HIGH)); + device_printf(sc->dev, "RX_RING_CTRL - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_CTRL)); + device_printf(sc->dev, "RX_RING_PTR - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_PTR)); + device_printf(sc->dev, "RX_RING_STATUS0 - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_STATUS0)); + device_printf(sc->dev, "RX_RING_STATUS1 - 0x%08x\n", + RD4(sc, PDC_RX_RING + PDC_DMA_STATUS1)); + device_printf(sc->dev, "************************\n\n"); +} +#endif + +static void +pdc_dma_load_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct pdc_dma_mem *obj = (struct pdc_dma_mem*)arg; + + if (error) + return; + + obj->nseg = nseg; + memcpy(obj->segs, segs, nseg * sizeof(bus_dma_segment_t)); +} + +static void +pdc_dma_load_cb2(void *arg, bus_dma_segment_t *segs, int nseg, + bus_size_t size, int error) +{ + + pdc_dma_load_cb(arg, segs, nseg, error); +} + +static int +pdc_allocate_request(struct pdc_request_ctx *ctx) +{ + struct pdc_sc *sc = ctx->sc; + int rc; + + ctx->data.vaddr = ctx->req.data; + ctx->data.size = ctx->req.data_len; + + if (ctx->req.flags & CRYPTO_F_IMBUF) + rc = bus_dmamap_load_mbuf_sg(sc->data_dma_tag, + ctx->data.map, + (struct mbuf*)ctx->data.vaddr, + ctx->data.segs, + &ctx->data.nseg, + BUS_DMA_NOWAIT); + else if (ctx->req.flags & CRYPTO_F_IOV) + rc = bus_dmamap_load_uio(sc->data_dma_tag, + ctx->data.map, + (struct uio*)ctx->data.vaddr, + pdc_dma_load_cb2, + &ctx->data, + BUS_DMA_NOWAIT); + else + rc = bus_dmamap_load(sc->data_dma_tag, + ctx->data.map, + ctx->data.vaddr, ctx->data.size, + pdc_dma_load_cb, + &ctx->data, + BUS_DMA_NOWAIT); + if (rc != 0) + goto fail; + +#ifdef PDC_DEBUG + bus_dmamap_sync(sc->header_dma_tag, + ctx->pdc_header.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); +#endif + + bus_dmamap_sync(sc->header_dma_tag, + ctx->header.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(sc->data_dma_tag, + ctx->data.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(sc->header_dma_tag, + ctx->footer.map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + mtx_lock(&sc->pending_requests_mtx); + STAILQ_INSERT_TAIL(&sc->pending_requests, ctx, pdc_ctx_stq); + mtx_unlock(&sc->pending_requests_mtx); + + return (0); + +fail: + mtx_lock(&sc->free_requests_mtx); + STAILQ_INSERT_HEAD(&sc->free_requests, ctx, pdc_ctx_stq); + mtx_unlock(&sc->free_requests_mtx); + device_printf(sc->dev, "bus_dmamap_load failed with: %d\n", rc); + return (rc); +} + +static int +pdc_enqueue_one(struct pdc_sc *sc, struct pdc_channel *channel, + struct pdc_request_ctx *ctx) +{ + uint32_t rx_desc_id, tx_desc_id; + int i, rx_flags, tx_flags; + /* + * Data buffer size in mbuf/uio struct can be bigger + * than the one declared by cryptodev. + * Make sure that descriptors point to the right size. + */ + size_t real_data_size = ctx->req.data_len; + + ctx->tx_desc_id = tx_desc_id = channel->tx.free_desc_id; + ctx->rx_desc_id = rx_desc_id = channel->rx.free_desc_id; + /* Data + Header + Footer */ + ctx->tx_desc_count = ctx->data.nseg + 2; + /* TX + PDC Header */ + ctx->rx_desc_count = ctx->tx_desc_count + 1; + + if (channel->rx.free_desc_count < ctx->rx_desc_count || + channel->tx.free_desc_count < ctx->tx_desc_count) + return (ENOSPC); + + /* + * Fill RX/TX descriptors. + * First goes an obligatory PDC RX header. + * We use it for debugging purposes only. + */ + rx_flags = PDC_DESC_CTRL1_SOF; + if (rx_desc_id == PDC_RING_MASK) + rx_flags |= PDC_DESC_CTRL1_EOT; + + pdc_fill_desc(&channel->rx.desc[rx_desc_id], + ctx->pdc_header.segs[0], ctx->pdc_header.size, + rx_flags); + rx_desc_id = (rx_desc_id + 1) & PDC_RING_MASK; + + /* + * We use the same memory for RX/TX header. + * They usually have different sizes though. + */ + tx_flags = PDC_DESC_CTRL1_SOF; + rx_flags = 0; + if (tx_desc_id == PDC_RING_MASK) + tx_flags |= PDC_DESC_CTRL1_EOT; + if (rx_desc_id == PDC_RING_MASK) + rx_flags |= PDC_DESC_CTRL1_EOT; + + pdc_fill_desc(&channel->tx.desc[tx_desc_id], + ctx->header.segs[0], ctx->req.header_len, + tx_flags); + pdc_fill_desc(&channel->rx.desc[rx_desc_id], + ctx->header.segs[0], ctx->req.resp_header_len, + rx_flags); + + tx_desc_id = (tx_desc_id + 1) & PDC_RING_MASK; + rx_desc_id = (rx_desc_id + 1) & PDC_RING_MASK; + + /* + * Now goes the data buffer. + * Again we use the same memory for RX/TX. + */ + for (i = 0; i < ctx->data.nseg; i++) { + rx_flags = 0; + tx_flags = 0; + if (rx_desc_id == PDC_RING_MASK) + rx_flags |= PDC_DESC_CTRL1_EOT; + if (tx_desc_id == PDC_RING_MASK) + tx_flags |= PDC_DESC_CTRL1_EOT; + + pdc_fill_desc(&channel->tx.desc[tx_desc_id], + ctx->data.segs[i], + MIN(ctx->data.segs[i].ds_len, real_data_size), + tx_flags); + pdc_fill_desc(&channel->rx.desc[rx_desc_id], + ctx->data.segs[i], + MIN(ctx->data.segs[i].ds_len, real_data_size), + rx_flags); + + real_data_size -= MIN(ctx->data.segs[i].ds_len, real_data_size); + + rx_desc_id = (rx_desc_id + 1) & PDC_RING_MASK; + tx_desc_id = (tx_desc_id + 1) & PDC_RING_MASK; + + if (real_data_size == 0) { + ctx->rx_desc_count -= (ctx->data.nseg - 1 - i); + ctx->tx_desc_count -= (ctx->data.nseg - 1 - i); + break; + } + } + + if (real_data_size != 0) + panic("Data buffer is to small size:%zd vs expected:%zd", + ctx->req.data_len - real_data_size, ctx->req.data_len); + + /* + * The last descriptor contains footer. + * Same buffer for RX/TX is used. + */ + rx_flags = 0; + if (rx_desc_id == PDC_RING_MASK) + rx_flags |= PDC_DESC_CTRL1_EOT; + + tx_flags = PDC_DESC_CTRL1_EOF; + if (tx_desc_id == PDC_RING_MASK) + tx_flags |= PDC_DESC_CTRL1_EOT; + + pdc_fill_desc(&channel->rx.desc[rx_desc_id], + ctx->footer.segs[0], ctx->req.resp_footer_len, + rx_flags); + pdc_fill_desc(&channel->tx.desc[tx_desc_id], + ctx->footer.segs[0], ctx->req.footer_len, + tx_flags); + + rx_desc_id = (rx_desc_id + 1) & PDC_RING_MASK; + tx_desc_id = (tx_desc_id + 1) & PDC_RING_MASK; + + /* Reserve descriptors used for this request. */ + channel->rx.free_desc_id = rx_desc_id; + channel->tx.free_desc_id = tx_desc_id; + channel->rx.free_desc_count -= ctx->rx_desc_count; + channel->tx.free_desc_count -= ctx->tx_desc_count; + + return (0); +} + +static void +pdc_enqueue(struct pdc_sc *sc) +{ + struct pdc_request_ctx *ctx; + struct pdc_channel *channel = &sc->channels[0]; + int rc; + int count = 0; + + if (STAILQ_EMPTY(&sc->pending_requests)) + return; + + mtx_lock(&channel->channel_mtx); + + for (;;) { + mtx_lock(&sc->pending_requests_mtx); + ctx = STAILQ_FIRST(&sc->pending_requests); + if (ctx == NULL) { + mtx_unlock(&sc->pending_requests_mtx); + break; + } + STAILQ_REMOVE_HEAD(&sc->pending_requests, pdc_ctx_stq); + mtx_unlock(&sc->pending_requests_mtx); + + rc = pdc_enqueue_one(sc, channel, ctx); + if (rc != 0) { + mtx_lock(&sc->pending_requests_mtx); + STAILQ_INSERT_HEAD(&sc->pending_requests, ctx, + pdc_ctx_stq); + mtx_unlock(&sc->pending_requests_mtx); + break; + } + + STAILQ_INSERT_TAIL(&channel->enqueued_requests, ctx, + pdc_ctx_stq); + count++; + } + + if (count == 0) { + mtx_unlock(&channel->channel_mtx); + return; + } + + /* Do a full sync before ringing the bell. */ + pdc_dma_sync_desc(sc, BUS_DMASYNC_PREWRITE); + + WR4(sc, PDC_RX_RING + PDC_DMA_PTR, + (channel->rx.free_desc_id << 4)); + WR4(sc, PDC_TX_RING + PDC_DMA_PTR, + (channel->tx.free_desc_id << 4)); + +#ifdef PDC_DEBUG + device_printf(sc->dev, + "FIRING UP TX: 0x%08x, RX: 0x%08x\n", + channel->tx.free_desc_id << 4, + channel->rx.free_desc_id << 4); +#endif + + mtx_unlock(&channel->channel_mtx); +} + +struct pdc_request* +pdc_allocate() +{ + struct pdc_sc *sc; + struct pdc_request_ctx *ctx; + int id = atomic_fetchadd_32(&pdc_id, 1) % pdc_count; + int i; + + for (i = 0; i < pdc_count; i++) { + sc = pdc_sc[(id + i) % pdc_count]; + + mtx_lock(&sc->free_requests_mtx); + + ctx = STAILQ_FIRST(&sc->free_requests); + if (ctx != NULL) { + STAILQ_REMOVE_HEAD(&sc->free_requests, pdc_ctx_stq); + mtx_unlock(&sc->free_requests_mtx); + ctx->sc = sc; + return (&ctx->req); + } + mtx_unlock(&sc->free_requests_mtx); + } + + return (NULL); +} + +void +pdc_free(struct pdc_request *req) +{ + struct pdc_request_ctx *ctx = (struct pdc_request_ctx*)req; + + mtx_lock(&ctx->sc->free_requests_mtx); + STAILQ_INSERT_HEAD(&ctx->sc->free_requests, ctx, pdc_ctx_stq); + mtx_unlock(&ctx->sc->free_requests_mtx); +} + +int +pdc_process_request(struct pdc_request *req) +{ + struct pdc_request_ctx *ctx = (struct pdc_request_ctx*)req; + int rc; + + rc = pdc_allocate_request(ctx); + if (rc != 0) + return (rc); + + pdc_enqueue(ctx->sc); + + return (0); +} + +static int +pdc_alloc_static_dma_mem(bus_dma_tag_t tag, struct pdc_dma_mem *mem, size_t size) +{ + int rc; + + mem->vaddr = NULL; + + rc = bus_dmamem_alloc(tag, + &mem->vaddr, + BUS_DMA_COHERENT | BUS_DMA_NOWAIT | BUS_DMA_ZERO, + &mem->map); + if (rc != 0) + return (rc); + + rc = bus_dmamap_load(tag, + mem->map, + mem->vaddr, size, + pdc_dma_load_cb, mem, + BUS_DMA_NOWAIT); + if (rc != 0) + goto fail; + + mem->size = size; + return (0); + +fail: + bus_dmamem_free(tag, mem->vaddr, mem->map); + mem->vaddr = NULL; + return (rc); + +} + +static void +pdc_free_static_dma_mem(bus_dma_tag_t tag, struct pdc_dma_mem *mem) +{ + + if (mem->vaddr != NULL) { + bus_dmamap_unload(tag, mem->map); + bus_dmamem_free(tag, mem->vaddr, mem->map); + } + mem->vaddr = NULL; +} + +static void +pdc_free_req(struct pdc_sc *sc, struct pdc_request_ctx *ctx) +{ + + bus_dmamap_unload(sc->data_dma_tag, ctx->data.map); + bus_dmamap_destroy(sc->data_dma_tag, ctx->data.map); + pdc_free_static_dma_mem(sc->header_dma_tag, &ctx->pdc_header); + pdc_free_static_dma_mem(sc->header_dma_tag, &ctx->header); + pdc_free_static_dma_mem(sc->header_dma_tag, &ctx->footer); +} + +static void +pdc_free_resources(struct pdc_sc *sc) +{ + struct pdc_request_ctx *ctx; + + if (sc->channels[0].rx.mem.vaddr != NULL) + pdc_free_static_dma_mem(sc->desc_dma_tag, + &sc->channels[0].rx.mem); + + if (sc->channels[0].tx.mem.vaddr != NULL) + pdc_free_static_dma_mem(sc->desc_dma_tag, + &sc->channels[0].tx.mem); + + STAILQ_FOREACH(ctx, &sc->pending_requests, pdc_ctx_stq) + pdc_free_req(sc, ctx); + + STAILQ_FOREACH(ctx, &sc->channels[0].enqueued_requests, pdc_ctx_stq) + pdc_free_req(sc, ctx); + + STAILQ_FOREACH(ctx, &sc->free_requests, pdc_ctx_stq) + pdc_free_req(sc, ctx); + + if (sc->desc_dma_tag != NULL) + bus_dma_tag_destroy(sc->desc_dma_tag); + + if (sc->data_dma_tag != NULL) + bus_dma_tag_destroy(sc->data_dma_tag); + + if (sc->header_dma_tag != NULL) + bus_dma_tag_destroy(sc->header_dma_tag); +} + +static int +pdc_setup_desc(struct pdc_sc *sc) +{ + int rc, i; + + mtx_init(&sc->free_requests_mtx, device_get_nameunit(sc->dev), + "PDC free requests pool lock", MTX_DEF); + mtx_init(&sc->pending_requests_mtx, device_get_nameunit(sc->dev), + "PDC pending requests pool lock", MTX_DEF); + mtx_init(&sc->channels[0].channel_mtx, device_get_nameunit(sc->dev), + "PDC DMA channel lock", MTX_DEF); + + STAILQ_INIT(&sc->free_requests); + STAILQ_INIT(&sc->pending_requests); + STAILQ_INIT(&sc->channels[0].enqueued_requests); + + /* DMA tag for RX/TX descriptors. */ + rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), + RING_ALIGMENT, 0, + BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, + NULL, NULL, + PDC_MAX_RING_ENTRIES * PDC_RING_ENTRY_SIZE, + 1, + PDC_MAX_RING_ENTRIES * PDC_RING_ENTRY_SIZE, + BUS_DMA_COHERENT | BUS_DMA_ALLOCNOW, NULL, NULL, + &sc->desc_dma_tag); + if (rc != 0) { + device_printf(sc->dev, + "Failed to allocate DMA tag for ring memory, rc: %d\n", + rc); + return (rc); + } + + /* DMA tag for request headers. */ + rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), + DATA_ALIGMENT, 0, + BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, + NULL, NULL, + MAX(PDC_MAX_HEADER_SIZE, sc->rx_header_len), + 1, + MAX(PDC_MAX_HEADER_SIZE, sc->rx_header_len), + BUS_DMA_COHERENT, NULL, NULL, + &sc->header_dma_tag); + if (rc != 0) { + device_printf(sc->dev, + "Failed to allocate DMA tag, rc: %d\n", + rc); + return (rc); + } + + /* DMA tag for data buffers. */ + rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), + DATA_ALIGMENT, 0, + BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, + NULL, NULL, + PDC_MAX_REQUEST_SIZE, + PDC_MAX_RING_ENTRIES / 2, + PDC_MAX_DESC_SIZE, + BUS_DMA_COHERENT, NULL, NULL, + &sc->data_dma_tag); + if (rc != 0) { + device_printf(sc->dev, + "Failed to allocate DMA tag, rc: %d\n", + rc); + return (rc); + } + + /* Allocate memory for descriptors. */ + rc = pdc_alloc_static_dma_mem(sc->desc_dma_tag, + &sc->channels[0].tx.mem, + PDC_MAX_RING_ENTRIES * PDC_RING_ENTRY_SIZE); + if (rc != 0) + return (rc); + + rc = pdc_alloc_static_dma_mem(sc->desc_dma_tag, + &sc->channels[0].rx.mem, + PDC_MAX_RING_ENTRIES * PDC_RING_ENTRY_SIZE); + if (rc != 0) + return (rc); + + sc->channels[0].rx.desc = + (struct pdc_dma_desc *)sc->channels[0].rx.mem.vaddr; + sc->channels[0].tx.desc = + (struct pdc_dma_desc *)sc->channels[0].tx.mem.vaddr; + + for (i = 0; i < PDC_MAX_RING_ENTRIES / 2; i++) { + rc = bus_dmamap_create(sc->data_dma_tag, BUS_DMA_COHERENT, + &sc->requests[i].data.map); + if (rc != 0) + goto loop_fail; + + rc = pdc_alloc_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].header, + PDC_MAX_HEADER_SIZE); + if (rc != 0) { + bus_dmamap_destroy(sc->data_dma_tag, + sc->requests[i].data.map); + goto loop_fail; + } + + rc = pdc_alloc_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].footer, + PDC_MAX_HEADER_SIZE); + if (rc != 0) { + bus_dmamap_destroy(sc->data_dma_tag, + sc->requests[i].data.map); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].header); + goto loop_fail; + } + + rc = pdc_alloc_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].pdc_header, + sc->rx_header_len); + if (rc != 0) { + bus_dmamap_destroy(sc->data_dma_tag, + sc->requests[i].data.map); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].header); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].footer); + goto loop_fail; + } + + sc->requests[i].req.header = sc->requests[i].header.vaddr; + sc->requests[i].req.footer = sc->requests[i].footer.vaddr; + + STAILQ_INSERT_TAIL(&sc->free_requests, &sc->requests[i], + pdc_ctx_stq); + } + + return (0); + +loop_fail: + while (i-- > 0) { + bus_dmamap_destroy(sc->data_dma_tag, + sc->requests[i].data.map); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].header); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].footer); + pdc_free_static_dma_mem(sc->header_dma_tag, + &sc->requests[i].pdc_header); + } + return (rc); +} + +static void +pdc_setup_hw(struct pdc_sc *sc) +{ + + /* Set DMA flags */ + WR4(sc, PDC_TX_RING + PDC_DMA_CTRL, + (3 << PDC_DMA_CTRL_BURST_LEN_OFFSET) | + PDC_DMA_CTRL_DISABLE_PARITY_CHECK); + WR4(sc, PDC_RX_RING + PDC_DMA_CTRL, + (3 << PDC_DMA_CTRL_BURST_LEN_OFFSET) | + PDC_DMA_CTRL_DISABLE_PARITY_CHECK | + PDC_DMA_CTRL_RX_OFLOWCONTINUE | + PDC_DMA_CTRL_RX_SEPRXHD | + (sc->rx_status_len << 1)); + + /* Point to the beginning of rings. */ + WR4(sc, PDC_TX_RING + PDC_DMA_PTR, 0); + WR4(sc, PDC_RX_RING + PDC_DMA_PTR, 0); + + /* Set RX/TX ring addresses. */ + WR4(sc, PDC_TX_RING + PDC_DMA_ADDR_LOW, + (sc->channels[0].tx.mem.segs[0].ds_addr & (uint32_t)(~0))); + WR4(sc, PDC_TX_RING + PDC_DMA_ADDR_HIGH, + (sc->channels[0].tx.mem.segs[0].ds_addr >> 32)); + WR4(sc, PDC_RX_RING + PDC_DMA_ADDR_LOW, + (sc->channels[0].rx.mem.segs[0].ds_addr & (uint32_t)(~0))); + WR4(sc, PDC_RX_RING + PDC_DMA_ADDR_HIGH, + (sc->channels[0].rx.mem.segs[0].ds_addr >> 32)); + + /* Enable DMA */ + OR4(sc, PDC_RX_RING + PDC_DMA_CTRL, PDC_DMA_CTRL_ENABLE); + OR4(sc, PDC_TX_RING + PDC_DMA_CTRL, PDC_DMA_CTRL_ENABLE); +} + +static int +pdc_setup_intr(struct pdc_sc *sc) +{ + int rc; + + sc->irq_rid = 0; + sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->irq_res == NULL) + return (ENXIO); + + rc = bus_setup_intr(sc->dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE, + NULL, pdc_intr_handler, sc, &sc->intr_cookie); + if (rc != 0) + return (rc); + + WR4(sc, PDC_INTMASK, PDC_INTMASK_RXINT); + WR4(sc, PDC_INTRCVLAZY0, PDC_INTRCVLAZY_VALUE); + + return (0); +} + +static void +pdc_intr_handler(void *arg) +{ + struct pdc_sc *sc = (struct pdc_sc *)arg; + uint32_t status; + + status = RD4(sc, PDC_INTSTATUS); + WR4(sc, PDC_INTMASK, 0); + WR4(sc, PDC_INTSTATUS, status); + +#ifdef PDC_DEBUG + device_printf(sc->dev, "Received an interrupt\n"); + pdc_dump_regs(sc); +#endif + pdc_receive(sc); + + /* Re-enable interrupts. */ + WR4(sc, PDC_INTMASK, PDC_INTMASK_RXINT); + return; +} + +static void +pdc_receive(struct pdc_sc *sc) +{ + struct pdc_channel *channel = &sc->channels[0]; + struct pdc_request_ctx *ctx; + uint16_t ready_desc_count; + + while (1) { + mtx_lock(&channel->channel_mtx); + ctx = STAILQ_FIRST(&channel->enqueued_requests); + if (ctx == NULL) { + mtx_unlock(&channel->channel_mtx); + break; + } + + ready_desc_count = RD4(sc, PDC_RX_RING + PDC_DMA_STATUS0); + ready_desc_count = ((ready_desc_count & PDC_DESCID_MASK) >> 4); + ready_desc_count = (ctx->rx_desc_id - ready_desc_count) & + PDC_RING_MASK; + + /* Check if request represented by ctx has been processed. */ + if (ready_desc_count < ctx->rx_desc_count) { + mtx_unlock(&channel->channel_mtx); + break; + } + + STAILQ_REMOVE_HEAD(&channel->enqueued_requests, pdc_ctx_stq); + channel->rx.free_desc_count += ctx->rx_desc_count; + channel->tx.free_desc_count += ctx->tx_desc_count; + mtx_unlock(&channel->channel_mtx); + + pdc_process_response(sc, ctx); + } +#ifdef PDC_DEBUG + device_printf(sc->dev, + "Got %d RX and %d TX free descriptors \n", + channel->rx.free_desc_count, + channel->tx.free_desc_count); + pdc_dump_regs(sc); +#endif + /* + * Since we should have some free descriptors now + * try to enqueue requests. + */ + pdc_enqueue(sc); +} + +static void +pdc_process_response(struct pdc_sc *sc, struct pdc_request_ctx *ctx) +{ + + /* First sync all DMA memory used in this transaction. */ + bus_dmamap_sync(sc->header_dma_tag, + ctx->header.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(sc->data_dma_tag, + ctx->data.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(sc->header_dma_tag, + ctx->footer.map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + /* + * PDC header is used only for debugging, + * don't sync it if we don't have to. + */ +#ifdef PDC_DEBUG + bus_dmamap_sync(sc->header_dma_tag, + ctx->pdc_header.map, + BUS_DMASYNC_POSTWRITE); + + printf("Dumping PDC header:\n"); + hexdump(ctx->pdc_header.vaddr, sc->rx_status_len, NULL, 0); +#endif + + ctx->req.pdc_callback(&ctx->req); + + /* Data buffer is the only one allocated dynamically. */ + bus_dmamap_unload(sc->data_dma_tag, ctx->data.map); + + mtx_lock(&sc->free_requests_mtx); + STAILQ_INSERT_TAIL(&sc->free_requests, ctx, pdc_ctx_stq); + mtx_unlock(&sc->free_requests_mtx); +} + +static int +pdc_attach(device_t dev) +{ + struct pdc_sc *sc; + pcell_t pcell; + int rc; + + sc = device_get_softc(dev); + + memset(sc, 0, sizeof(struct pdc_sc)); + sc->dev = dev; + + rc = OF_getencprop(ofw_bus_get_node(dev), "brcm,rx-status-len", + &pcell, sizeof(pcell_t)); + if (rc < 0) { + return (-rc); + } + sc->rx_status_len = sc->rx_header_len = pcell; + if (ofw_bus_has_prop(dev, "brcm,use-bcm-hdr")) + sc->rx_header_len += BCM_HDR_LEN; + + sc->mem_rid = 0; + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, + RF_ACTIVE); + if (sc->mem_res == NULL) + return (ENXIO); + + rc = pdc_setup_desc(sc); + if (rc != 0) { + pdc_detach(sc->dev); + return (rc); + } + + rc = pdc_setup_intr(sc); + if (rc != 0) { + pdc_detach(sc->dev); + return (rc); + } + + pdc_setup_hw(sc); + +#ifdef PDC_DEBUG + pdc_dump_regs(sc); +#endif + pdc_sc[atomic_fetchadd_32(&pdc_count, 1)] = sc; + sc->channels[0].rx.free_desc_id = 0; + sc->channels[0].rx.free_desc_count = PDC_MAX_RING_ENTRIES; + sc->channels[0].tx.free_desc_id = 0; + sc->channels[0].tx.free_desc_count = PDC_MAX_RING_ENTRIES; + + return (0); + +} + +static int +pdc_detach(device_t dev) +{ + struct pdc_sc *sc; + + sc = device_get_softc(dev); + /* + * First disable DMA channel and all interrupts. + * We are not interested in processing any requests at this point. + */ + WR4(sc, PDC_INTMASK, 0); + AND4(sc, PDC_RX_RING + PDC_DMA_CTRL, ~PDC_DMA_CTRL_ENABLE); + AND4(sc, PDC_TX_RING + PDC_DMA_CTRL, ~PDC_DMA_CTRL_ENABLE); + + pdc_free_resources(sc); + + if (sc->intr_cookie != NULL) + bus_teardown_intr(dev, sc->irq_res, sc->intr_cookie); + + if (sc->irq_res != NULL) + bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid, sc->irq_res); + + if (sc->mem_res != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); + + mtx_destroy(&sc->pending_requests_mtx); + mtx_destroy(&sc->free_requests_mtx); + mtx_destroy(&sc->channels[0].channel_mtx); + + return (0); +} + +static int +pdc_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "brcm,iproc-pdc-mbox")) + return (ENXIO); + + device_set_desc(dev, "Broadcom PDC Controller"); + return (BUS_PROBE_DEFAULT); +} + +static device_method_t pdc_methods[] = { + DEVMETHOD(device_probe, pdc_probe), + DEVMETHOD(device_attach, pdc_attach), + DEVMETHOD(device_detach, pdc_detach), + + DEVMETHOD_END +}; + +static driver_t pdc_driver = { + .name = "pdc", + .methods = pdc_methods, + .size = sizeof(struct pdc_sc), +}; + +static devclass_t pdc_devclass; + +MODULE_VERSION(pdc, 1); +DRIVER_MODULE(pdc, simplebus, pdc_driver, pdc_devclass, 0, 0); Index: sys/dev/pdc/pdc_regs.h =================================================================== --- /dev/null +++ sys/dev/pdc/pdc_regs.h @@ -0,0 +1,182 @@ +/*- + * Copyright (c) 2019 Juniper Networks, Inc. + * Copyright (c) 2019 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PDC_REGS_H +#define PDC_REGS_H + +#include + +#include "pdc.h" + +#define BIT(x) (1 << (x)) + +/* There are four rings, but we support only the first one currently. */ +#define PDC_RING_COUNT 1 +#define PDC_MAX_RING_ENTRIES 512 + +#define PDC_MAX_DESC_SIZE 16384 +#define PDC_MAX_REQUEST_SIZE (BIT(16) - 1) + +#define PDC_RING_MASK (PDC_MAX_RING_ENTRIES-1) + +#define BCM_HDR_LEN 8 + +#define PDC_RING_ENTRY_SIZE sizeof(struct pdc_dma_desc) + +#define RING_ALIGMENT BIT(13) +#define DATA_ALIGMENT BIT(5) + +#define PDC_DEVCONTROL 0x0 +#define PDC_DEVSTATUS 0x4 +#define PDC_BISTSTATUS 0xC +#define PDC_INTSTATUS 0x20 +#define PDC_INTMASK 0x24 +#define PDC_GPTIMER 0x28 +#define PDC_INTRCVLAZY0 0x30 +#define PDC_INTRCVLAZY1 0x34 +#define PDC_INTRCVLAZY2 0x38 +#define PDC_INTRCVLAZY3 0x3C +#define PDC_FLOWCNTL_TH 0x104 +#define PDC_TXARB_WRR_TH 0x108 +#define PDC_GMACIDLE_CNT_TH 0x10C + +/* We only support Ring 0. */ +#define PDC_TX_RING 0x200 +#define PDC_RX_RING 0x220 + +#define PDC_DMA_CTRL 0x0 +#define PDC_DMA_PTR 0x4 +#define PDC_DMA_ADDR_LOW 0x8 +#define PDC_DMA_ADDR_HIGH 0xC +#define PDC_DMA_STATUS0 0x10 +#define PDC_DMA_STATUS1 0x14 + +#define PDC_DESCID_MASK 0x1FFF + +#define PDC_DMA_CTRL_ENABLE BIT(0) +#define PDC_DMA_CTRL_BURST_LEN_OFFSET 18 +#define PDC_DMA_CTRL_DISABLE_PARITY_CHECK BIT(11) + +#define PDC_DMA_CTRL_RX_RCVOFFSET_OFFSET 1 +#define PDC_DMA_CTRL_RX_SEPRXHD BIT(9) +#define PDC_DMA_CTRL_RX_OFLOWCONTINUE BIT(10) +#define PDC_DMA_CTRL_RX_WAITFORCOMPLETE BIT(12) + +#define PDC_INTMASK_RXINT BIT(16) +#define PDC_INTMASK_TXINT BIT(24) +#define PDC_INTRCVLAZY_VALUE ((1 << 24) | 10000) // (Framecount) | Timeout + +#define PDC_DESC_CTRL1_EOT BIT(28) +#define PDC_DESC_CTRL1_IOC BIT(29) +#define PDC_DESC_CTRL1_EOF BIT(30) +#define PDC_DESC_CTRL1_SOF BIT(31) + +#define PDC_INTSTATUS_RX0 BIT(16) +#define PDC_INTSTATUS_TX0 BIT(24) +#define PDC_INTSTATUS_DESCPROTERR BIT(12) + +MALLOC_DECLARE(M_PDC); +MALLOC_DEFINE(M_PDC, "pdc_memory", "memory used by Broadcom PDC"); + +struct pdc_dma_mem { + bus_dma_segment_t segs[PDC_MAX_RING_ENTRIES / 2]; + bus_dmamap_t map; + size_t size; + int nseg; + void *vaddr; +}; + +struct pdc_request_ctx { + struct pdc_request req; + struct pdc_sc *sc; + + struct pdc_dma_mem pdc_header; + struct pdc_dma_mem header; + struct pdc_dma_mem data; + struct pdc_dma_mem footer; + + uint32_t rx_desc_id; + uint32_t rx_desc_count; + uint32_t tx_desc_id; + uint32_t tx_desc_count; + + STAILQ_ENTRY(pdc_request_ctx) pdc_ctx_stq; +}; + +struct pdc_dma_desc { + uint32_t ctrl1; + uint32_t ctrl2; + uint32_t addr_low; + uint32_t addr_high; +} __packed; + +struct pdc_ring { + struct pdc_dma_mem mem; + struct pdc_dma_desc *desc; + + uint32_t free_desc_id; + uint32_t free_desc_count; +}; + +struct pdc_channel { + struct mtx channel_mtx; + struct pdc_ring rx; + struct pdc_ring tx; + + /* Requests that are currently being processed by hw. */ + STAILQ_HEAD(, pdc_request_ctx) enqueued_requests; +}; + +struct pdc_sc { + device_t dev; + + struct resource *mem_res; + int mem_rid; + struct resource *irq_res; + int irq_rid; + + void *intr_cookie; + + uint32_t rx_header_len; + uint32_t rx_status_len; + + bus_dma_tag_t data_dma_tag; + bus_dma_tag_t desc_dma_tag; + bus_dma_tag_t header_dma_tag; + + struct pdc_channel channels[PDC_RING_COUNT]; + + struct pdc_request_ctx requests[PDC_MAX_RING_ENTRIES / 2]; + + /* Pool of requests to use */ + struct mtx free_requests_mtx; + STAILQ_HEAD(, pdc_request_ctx) free_requests; + + /* Requests that haven't been enqueued yet. */ + struct mtx pending_requests_mtx; + STAILQ_HEAD(, pdc_request_ctx) pending_requests; +}; +#endif /* PDC_REGS_H */