diff --git a/sys/dev/qat/qat.c b/sys/dev/qat/qat.c index 1e533d1bebcc..acfe0c6a1e07 100644 --- a/sys/dev/qat/qat.c +++ b/sys/dev/qat/qat.c @@ -1,2152 +1,2309 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $"); #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cryptodev_if.h" #include #include #include "qatreg.h" #include "qatvar.h" #include "qat_aevar.h" extern struct qat_hw qat_hw_c2xxx; extern struct qat_hw qat_hw_c3xxx; extern struct qat_hw qat_hw_c62x; extern struct qat_hw qat_hw_d15xx; extern struct qat_hw qat_hw_dh895xcc; #define PCI_VENDOR_INTEL 0x8086 #define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18 #define PCI_PRODUCT_INTEL_C3K_QAT 0x19e2 #define PCI_PRODUCT_INTEL_C3K_QAT_VF 0x19e3 #define PCI_PRODUCT_INTEL_C620_QAT 0x37c8 #define PCI_PRODUCT_INTEL_C620_QAT_VF 0x37c9 #define PCI_PRODUCT_INTEL_XEOND_QAT 0x6f54 #define PCI_PRODUCT_INTEL_XEOND_QAT_VF 0x6f55 #define PCI_PRODUCT_INTEL_DH895XCC_QAT 0x0435 #define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF 0x0443 static const struct qat_product { uint16_t qatp_vendor; uint16_t qatp_product; const char *qatp_name; enum qat_chip_type qatp_chip; const struct qat_hw *qatp_hw; } qat_products[] = { { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS, "Intel C2000 QuickAssist PF", QAT_CHIP_C2XXX, &qat_hw_c2xxx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C3K_QAT, "Intel C3000 QuickAssist PF", QAT_CHIP_C3XXX, &qat_hw_c3xxx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C620_QAT, "Intel C620/Xeon D-2100 QuickAssist PF", QAT_CHIP_C62X, &qat_hw_c62x }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XEOND_QAT, "Intel Xeon D-1500 QuickAssist PF", QAT_CHIP_D15XX, &qat_hw_d15xx }, { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH895XCC_QAT, "Intel 8950 QuickAssist PCIe Adapter PF", QAT_CHIP_DH895XCC, &qat_hw_dh895xcc }, { 0, 0, NULL, 0, NULL }, }; /* Hash Algorithm specific structure */ /* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = { 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0 }; /* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19 }; /* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = { 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 }; /* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = { 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 }; static const struct qat_sym_hash_alg_info sha1_info = { .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE, .qshai_init_state = sha1_initial_state, .qshai_sah = &auth_hash_hmac_sha1, .qshai_state_offset = 0, .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha256_info = { .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE, .qshai_init_state = sha256_initial_state, .qshai_sah = &auth_hash_hmac_sha2_256, .qshai_state_offset = offsetof(SHA256_CTX, state), .qshai_state_word = 4, }; static const struct qat_sym_hash_alg_info sha384_info = { .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE, .qshai_init_state = sha384_initial_state, .qshai_sah = &auth_hash_hmac_sha2_384, .qshai_state_offset = offsetof(SHA384_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info sha512_info = { .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE, .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE, .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE, .qshai_init_state = sha512_initial_state, .qshai_sah = &auth_hash_hmac_sha2_512, .qshai_state_offset = offsetof(SHA512_CTX, state), .qshai_state_word = 8, }; static const struct qat_sym_hash_alg_info aes_gcm_info = { .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE, .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE, .qshai_sah = &auth_hash_nist_gmac_aes_128, }; /* Hash QAT specific structures */ static const struct qat_sym_hash_qat_info sha1_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA1, .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE, .qshqi_state1_len = HW_SHA1_STATE1_SZ, .qshqi_state2_len = HW_SHA1_STATE2_SZ, }; static const struct qat_sym_hash_qat_info sha256_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA256, .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE, .qshqi_state1_len = HW_SHA256_STATE1_SZ, .qshqi_state2_len = HW_SHA256_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha384_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA384, .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE, .qshqi_state1_len = HW_SHA384_STATE1_SZ, .qshqi_state2_len = HW_SHA384_STATE2_SZ }; static const struct qat_sym_hash_qat_info sha512_config = { .qshqi_algo_enc = HW_AUTH_ALGO_SHA512, .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE, .qshqi_state1_len = HW_SHA512_STATE1_SZ, .qshqi_state2_len = HW_SHA512_STATE2_SZ }; static const struct qat_sym_hash_qat_info aes_gcm_config = { .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128, .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE, .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ, .qshqi_state2_len = HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ, }; static const struct qat_sym_hash_def qat_sym_hash_defs[] = { [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config }, [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config }, [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config }, [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config }, [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config }, }; static const struct qat_product *qat_lookup(device_t); static int qat_probe(device_t); static int qat_attach(device_t); static int qat_init(struct device *); static int qat_start(struct device *); static int qat_detach(device_t); static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp); static void qat_freesession(device_t dev, crypto_session_t cses); static int qat_setup_msix_intr(struct qat_softc *); static void qat_etr_init(struct qat_softc *); static void qat_etr_deinit(struct qat_softc *); static void qat_etr_bank_init(struct qat_softc *, int); static void qat_etr_bank_deinit(struct qat_softc *sc, int); static void qat_etr_ap_bank_init(struct qat_softc *); static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int); static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *, uint32_t, int); static void qat_etr_ap_bank_setup_ring(struct qat_softc *, struct qat_ring *); static int qat_etr_verify_ring_size(uint32_t, uint32_t); static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *, struct qat_ring *); static void qat_etr_bank_intr(void *); static void qat_arb_update(struct qat_softc *, struct qat_bank *); static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie( struct qat_crypto_bank *); static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *, struct qat_sym_cookie *); static int qat_crypto_setup_ring(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_bank_init(struct qat_softc *, struct qat_crypto_bank *); static int qat_crypto_init(struct qat_softc *); static void qat_crypto_deinit(struct qat_softc *); static int qat_crypto_start(struct qat_softc *); static void qat_crypto_stop(struct qat_softc *); static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *); static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver"); static const struct qat_product * qat_lookup(device_t dev) { const struct qat_product *qatp; for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) { if (pci_get_vendor(dev) == qatp->qatp_vendor && pci_get_device(dev) == qatp->qatp_product) return qatp; } return NULL; } static int qat_probe(device_t dev) { const struct qat_product *prod; prod = qat_lookup(dev); if (prod != NULL) { device_set_desc(dev, prod->qatp_name); return BUS_PROBE_DEFAULT; } return ENXIO; } static int qat_attach(device_t dev) { struct qat_softc *sc = device_get_softc(dev); const struct qat_product *qatp; bus_size_t msixtbl_offset; int bar, count, error, i, msixoff, msixtbl_bar; sc->sc_dev = dev; sc->sc_rev = pci_get_revid(dev); sc->sc_crypto.qcy_cid = -1; qatp = qat_lookup(dev); memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw)); /* Determine active accelerators and engines */ sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc); sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc); sc->sc_accel_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) { if (sc->sc_accel_mask & (1 << i)) sc->sc_accel_num++; } sc->sc_ae_num = 0; for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) { if (sc->sc_ae_mask & (1 << i)) sc->sc_ae_num++; } if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) { device_printf(sc->sc_dev, "couldn't find acceleration"); goto fail; } MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL); MPASS(sc->sc_ae_num <= MAX_NUM_AE); /* Determine SKU and capabilities */ sc->sc_sku = sc->sc_hw.qhw_get_sku(sc); sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc); sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc); /* Map BARs */ msixtbl_bar = 0; msixtbl_offset = 0; if (pci_find_cap(dev, PCIY_MSIX, &msixoff) == 0) { uint32_t msixtbl; msixtbl = pci_read_config(dev, msixoff + PCIR_MSIX_TABLE, 4); msixtbl_offset = msixtbl & ~PCIM_MSIX_BIR_MASK; msixtbl_bar = PCIR_BAR(msixtbl & PCIM_MSIX_BIR_MASK); } i = 0; if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) { MPASS(sc->sc_hw.qhw_sram_bar_id == 0); uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4); /* Skip SRAM BAR */ i = (fusectl & FUSECTL_MASK) ? 1 : 0; } for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) { uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4); if (val == 0 || !PCI_BAR_MEM(val)) continue; sc->sc_rid[i] = PCIR_BAR(bar); sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid[i], RF_ACTIVE); if (sc->sc_res[i] == NULL) { device_printf(dev, "couldn't map BAR %d\n", bar); goto fail; } sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]); sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]); i++; if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) bar++; } pci_enable_busmaster(dev); count = sc->sc_hw.qhw_num_banks + 1; if (pci_msix_count(dev) < count) { device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n", pci_msix_count(dev), count); goto fail; } error = pci_alloc_msix(dev, &count); if (error != 0) { device_printf(dev, "failed to allocate MSI-X vectors\n"); goto fail; } error = qat_init(dev); if (error == 0) return 0; fail: qat_detach(dev); return ENXIO; } static int qat_init(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; qat_etr_init(sc); if (sc->sc_hw.qhw_init_admin_comms != NULL && (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize admin comms: %d\n", error); return error; } if (sc->sc_hw.qhw_init_arb != NULL && (error = sc->sc_hw.qhw_init_arb(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize hw arbiter: %d\n", error); return error; } error = qat_ae_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize Acceleration Engine: %d\n", error); return error; } error = qat_aefw_load(sc); if (error) { device_printf(sc->sc_dev, "Could not load firmware: %d\n", error); return error; } error = qat_setup_msix_intr(sc); if (error) { device_printf(sc->sc_dev, "Could not setup interrupts: %d\n", error); return error; } sc->sc_hw.qhw_enable_intr(sc); error = qat_crypto_init(sc); if (error) { device_printf(sc->sc_dev, "Could not initialize service: %d\n", error); return error; } if (sc->sc_hw.qhw_enable_error_correction != NULL) sc->sc_hw.qhw_enable_error_correction(sc); if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL && (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) { device_printf(sc->sc_dev, "Could not initialize watchdog timer: %d\n", error); return error; } error = qat_start(dev); if (error) { device_printf(sc->sc_dev, "Could not start: %d\n", error); return error; } return 0; } static int qat_start(device_t dev) { struct qat_softc *sc = device_get_softc(dev); int error; error = qat_ae_start(sc); if (error) return error; if (sc->sc_hw.qhw_send_admin_init != NULL && (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) { return error; } error = qat_crypto_start(sc); if (error) return error; return 0; } static int qat_detach(device_t dev) { struct qat_softc *sc; int bar, i; sc = device_get_softc(dev); qat_crypto_stop(sc); qat_crypto_deinit(sc); qat_aefw_unload(sc); if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i]; if (qb->qb_ih_cookie != NULL) (void)bus_teardown_intr(dev, qb->qb_ih, qb->qb_ih_cookie); if (qb->qb_ih != NULL) (void)bus_release_resource(dev, SYS_RES_IRQ, i + 1, qb->qb_ih); } } if (sc->sc_ih_cookie != NULL) { (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie); sc->sc_ih_cookie = NULL; } if (sc->sc_ih != NULL) { (void)bus_release_resource(dev, SYS_RES_IRQ, sc->sc_hw.qhw_num_banks + 1, sc->sc_ih); sc->sc_ih = NULL; } pci_release_msi(dev); qat_etr_deinit(sc); for (bar = 0; bar < MAX_BARS; bar++) { if (sc->sc_res[bar] != NULL) { (void)bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid[bar], sc->sc_res[bar]); sc->sc_res[bar] = NULL; } } return 0; } void * qat_alloc_mem(size_t size) { return (malloc(size, M_QAT, M_WAITOK | M_ZERO)); } void qat_free_mem(void *ptr) { free(ptr, M_QAT); } static void qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct qat_dmamem *qdm; if (error != 0) return; KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); qdm = arg; qdm->qdm_dma_seg = segs[0]; } int qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm, int nseg, bus_size_t size, bus_size_t alignment) { int error; KASSERT(qdm->qdm_dma_vaddr == NULL, ("%s: DMA memory descriptor in use", __func__)); error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), alignment, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ nseg, /* nsegments */ size, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &qdm->qdm_dma_tag); if (error != 0) return error; error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &qdm->qdm_dma_map); if (error != 0) { device_printf(sc->sc_dev, "couldn't allocate dmamem, error = %d\n", error); goto fail_0; } error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map, qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm, BUS_DMA_NOWAIT); if (error) { device_printf(sc->sc_dev, "couldn't load dmamem map, error = %d\n", error); goto fail_1; } return 0; fail_1: bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); fail_0: bus_dma_tag_destroy(qdm->qdm_dma_tag); return error; } void qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm) { if (qdm->qdm_dma_tag != NULL) { bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map); bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); bus_dma_tag_destroy(qdm->qdm_dma_tag); explicit_bzero(qdm, sizeof(*qdm)); } } static int qat_setup_msix_intr(struct qat_softc *sc) { device_t dev; int error, i, rid; dev = sc->sc_dev; for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) { struct qat_bank *qb = &sc->sc_etr_banks[i - 1]; rid = i; qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (qb->qb_ih == NULL) { device_printf(dev, "failed to allocate bank intr resource\n"); return ENXIO; } error = bus_setup_intr(dev, qb->qb_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb, &qb->qb_ih_cookie); if (error != 0) { device_printf(dev, "failed to set up bank intr\n"); return error; } error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus); if (error != 0) device_printf(dev, "failed to bind intr %d\n", i); } rid = i; sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_ih == NULL) return ENXIO; error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie); return error; } static void qat_etr_init(struct qat_softc *sc) { int i; sc->sc_etr_banks = qat_alloc_mem( sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks); for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_init(sc, i); if (sc->sc_hw.qhw_num_ap_banks) { sc->sc_etr_ap_banks = qat_alloc_mem( sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks); qat_etr_ap_bank_init(sc); } } static void qat_etr_deinit(struct qat_softc *sc) { int i; if (sc->sc_etr_banks != NULL) { for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) qat_etr_bank_deinit(sc, i); qat_free_mem(sc->sc_etr_banks); sc->sc_etr_banks = NULL; } if (sc->sc_etr_ap_banks != NULL) { qat_free_mem(sc->sc_etr_ap_banks); sc->sc_etr_ap_banks = NULL; } } static void qat_etr_bank_init(struct qat_softc *sc, int bank) { struct qat_bank *qb = &sc->sc_etr_banks[bank]; int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap; MPASS(bank < sc->sc_hw.qhw_num_banks); mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF); qb->qb_sc = sc; qb->qb_bank = bank; qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT; /* Clean CSRs for all rings within the bank */ for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { struct qat_ring *qr = &qb->qb_et_rings[i]; qat_etr_bank_ring_write_4(sc, bank, i, ETR_RING_CONFIG, 0); qat_etr_bank_ring_base_write_8(sc, bank, i, 0); if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t)); } else if (sc->sc_hw.qhw_tx_rings_mask & (1 << (i - tx_rx_gap))) { /* Share inflight counter with rx and tx */ qr->qr_inflight = qb->qb_et_rings[i - tx_rx_gap].qr_inflight; } } if (sc->sc_hw.qhw_init_etr_intr != NULL) { sc->sc_hw.qhw_init_etr_intr(sc, bank); } else { /* common code in qat 1.7 */ qat_etr_bank_write_4(sc, bank, ETR_INT_REG, ETR_INT_REG_CLEAR_MASK); for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank / ETR_RINGS_PER_INT_SRCSEL; i++) { qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL + (i * ETR_INT_SRCSEL_NEXT_OFFSET), ETR_INT_SRCSEL_MASK); } } } static void qat_etr_bank_deinit(struct qat_softc *sc, int bank) { struct qat_bank *qb; struct qat_ring *qr; int i; qb = &sc->sc_etr_banks[bank]; for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { qr = &qb->qb_et_rings[i]; qat_free_mem(qr->qr_inflight); } } } static void qat_etr_ap_bank_init(struct qat_softc *sc) { int ap_bank; for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) { struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank]; qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK, ETR_AP_NF_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK, ETR_AP_NE_MASK_INIT); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0); memset(qab, 0, sizeof(*qab)); } } static void qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask) { if (set_mask) *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); else *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); } static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest, uint32_t ring, int set_dest) { uint32_t ae_mask; uint8_t mailbox, ae, nae; uint8_t *dest = (uint8_t *)ap_dest; mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring); nae = 0; ae_mask = sc->sc_ae_mask; for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) { if ((ae_mask & (1 << ae)) == 0) continue; if (set_dest) { dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) | __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) | ETR_AP_DEST_ENABLE; } else { dest[nae] = 0; } nae++; if (nae == ETR_MAX_AE_PER_MAILBOX) break; } } static void qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr) { struct qat_ap_bank *qab; int ap_bank; if (sc->sc_hw.qhw_num_ap_banks == 0) return; ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring); MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks); qab = &sc->sc_etr_ap_banks[ap_bank]; if (qr->qr_cb == NULL) { qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1); if (!qab->qab_ne_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, qab->qab_ne_dest); } } else { qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1); if (!qab->qab_nf_dest) { qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest, qr->qr_ring, 1); qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, qab->qab_nf_dest); } } } static int qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs) { int i = QAT_MIN_RING_SIZE; for (; i <= QAT_MAX_RING_SIZE; i++) if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i)) return i; return QAT_DEFAULT_RING_SIZE; } int qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring, uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg, const char *name, struct qat_ring **rqr) { struct qat_bank *qb; struct qat_ring *qr = NULL; int error; uint32_t ring_size_bytes, ring_config; uint64_t ring_base; uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512; uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0; MPASS(bank < sc->sc_hw.qhw_num_banks); /* Allocate a ring from specified bank */ qb = &sc->sc_etr_banks[bank]; if (ring >= sc->sc_hw.qhw_num_rings_per_bank) return EINVAL; if (qb->qb_allocated_rings & (1 << ring)) return ENOENT; qr = &qb->qb_et_rings[ring]; qb->qb_allocated_rings |= 1 << ring; /* Initialize allocated ring */ qr->qr_ring = ring; qr->qr_bank = bank; qr->qr_name = name; qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring; qr->qr_ring_mask = (1 << ring); qr->qr_cb = cb; qr->qr_cb_arg = cb_arg; /* Setup the shadow variables */ qr->qr_head = 0; qr->qr_tail = 0; qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size); qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs); /* * To make sure that ring is alligned to ring size allocate * at least 4k and then tell the user it is smaller. */ ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size); ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes); error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes, ring_size_bytes); if (error) return error; qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr; qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr; memset(qr->qr_ring_vaddr, QAT_RING_PATTERN, qr->qr_dma.qdm_dma_seg.ds_len); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (cb == NULL) { ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size); } else { ring_config = ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne); } qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config); ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size); qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base); if (sc->sc_hw.qhw_init_arb != NULL) qat_arb_update(sc, qb); mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF); qat_etr_ap_bank_setup_ring(sc, qr); if (cb != NULL) { uint32_t intr_mask; qb->qb_intr_mask |= qr->qr_ring_mask; intr_mask = qb->qb_intr_mask; qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask); qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); } *rqr = qr; return 0; } static inline u_int qat_modulo(u_int data, u_int shift) { u_int div = data >> shift; u_int mult = div << shift; return data - mult; } int qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg) { uint32_t inflight; uint32_t *addr; mtx_lock(&qr->qr_ring_mtx); inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1; if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) { atomic_subtract_32(qr->qr_inflight, 1); qr->qr_need_wakeup = true; mtx_unlock(&qr->qr_ring_mtx); counter_u64_add(sc->sc_ring_full_restarts, 1); return ERESTART; } addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail); memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size)); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE); qr->qr_tail = qat_modulo(qr->qr_tail + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_TAIL_OFFSET, qr->qr_tail); mtx_unlock(&qr->qr_ring_mtx); return 0; } static int qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb, struct qat_ring *qr) { uint32_t *msg, nmsg = 0; int handled = 0; bool blocked = false; mtx_lock(&qr->qr_ring_mtx); msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) { atomic_subtract_32(qr->qr_inflight, 1); if (qr->qr_cb != NULL) { mtx_unlock(&qr->qr_ring_mtx); handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg); mtx_lock(&qr->qr_ring_mtx); } atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG); qr->qr_head = qat_modulo(qr->qr_head + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), QAT_RING_SIZE_MODULO(qr->qr_ring_size)); nmsg++; msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); } bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); if (nmsg > 0) { qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, ETR_RING_HEAD_OFFSET, qr->qr_head); if (qr->qr_need_wakeup) { blocked = true; qr->qr_need_wakeup = false; } } mtx_unlock(&qr->qr_ring_mtx); if (blocked) crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ); return handled; } static void qat_etr_bank_intr(void *arg) { struct qat_bank *qb = arg; struct qat_softc *sc = qb->qb_sc; uint32_t estat; int i, handled = 0; mtx_lock(&qb->qb_bank_mtx); qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0); /* Now handle all the responses */ estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT); estat &= qb->qb_intr_mask; qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); mtx_unlock(&qb->qb_bank_mtx); while ((i = ffs(estat)) != 0) { struct qat_ring *qr = &qb->qb_et_rings[--i]; estat &= ~(1 << i); handled |= qat_etr_ring_intr(sc, qb, qr); } } void qat_arb_update(struct qat_softc *sc, struct qat_bank *qb) { qat_arb_ringsrvarben_write_4(sc, qb->qb_bank, qb->qb_allocated_rings & 0xff); } static struct qat_sym_cookie * qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb) { struct qat_sym_cookie *qsc; mtx_lock(&qcb->qcb_bank_mtx); if (qcb->qcb_symck_free_count == 0) { mtx_unlock(&qcb->qcb_bank_mtx); return NULL; } qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count]; mtx_unlock(&qcb->qcb_bank_mtx); return qsc; } static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb, struct qat_sym_cookie *qsc) { - - explicit_bzero(qsc->qsc_iv_buf, sizeof(qsc->qsc_iv_buf)); - explicit_bzero(qsc->qsc_auth_res, sizeof(qsc->qsc_auth_res)); + explicit_bzero(qsc->qsc_iv_buf, EALG_MAX_BLOCK_LEN); + explicit_bzero(qsc->qsc_auth_res, QAT_SYM_HASH_BUFFER_LEN); mtx_lock(&qcb->qcb_bank_mtx); qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc; mtx_unlock(&qcb->qcb_bank_mtx); } void qat_memcpy_htobe64(void *dst, const void *src, size_t len) { uint64_t *dst0 = dst; const uint64_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe64(*(src0 + i)); } void qat_memcpy_htobe32(void *dst, const void *src, size_t len) { uint32_t *dst0 = dst; const uint32_t *src0 = src; size_t i; MPASS(len % sizeof(*dst0) == 0); for (i = 0; i < len / sizeof(*dst0); i++) *(dst0 + i) = htobe32(*(src0 + i)); } void qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte) { switch (wordbyte) { case 4: qat_memcpy_htobe32(dst, src, len); break; case 8: qat_memcpy_htobe64(dst, src, len); break; default: panic("invalid word size %u", wordbyte); } } void qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state) { uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; char zeros[AES_BLOCK_LEN]; int rounds; memset(zeros, 0, sizeof(zeros)); rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); rijndaelEncrypt(ks, rounds, zeros, state); explicit_bzero(ks, sizeof(ks)); } void qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc, const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, uint8_t *state1, uint8_t *state2) { union authctx ctx; const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah; uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset; uint32_t state_size = hash_def->qshd_alg->qshai_state_size; uint32_t state_word = hash_def->qshd_alg->qshai_state_word; hmac_init_ipad(sah, key, klen, &ctx); qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size, state_word); hmac_init_opad(sah, key, klen, &ctx); qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size, state_word); explicit_bzero(&ctx, sizeof(ctx)); } static enum hw_cipher_algo qat_aes_cipher_algo(int klen) { switch (klen) { case HW_AES_128_KEY_SZ: return HW_CIPHER_ALGO_AES128; case HW_AES_192_KEY_SZ: return HW_CIPHER_ALGO_AES192; case HW_AES_256_KEY_SZ: return HW_CIPHER_ALGO_AES256; default: panic("invalid key length %d", klen); } } uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc, const struct qat_session *qs) { enum hw_cipher_algo algo; enum hw_cipher_dir dir; enum hw_cipher_convert key_convert; enum hw_cipher_mode mode; dir = desc->qcd_cipher_dir; key_convert = HW_CIPHER_NO_CONVERT; mode = qs->qs_cipher_mode; switch (mode) { case HW_CIPHER_CBC_MODE: case HW_CIPHER_XTS_MODE: algo = qs->qs_cipher_algo; /* * AES decrypt key needs to be reversed. * Instead of reversing the key at session registration, * it is instead reversed on-the-fly by setting the KEY_CONVERT * bit here. */ if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT) key_convert = HW_CIPHER_KEY_CONVERT; break; case HW_CIPHER_CTR_MODE: algo = qs->qs_cipher_algo; dir = HW_CIPHER_ENCRYPT; break; default: panic("unhandled cipher mode %d", mode); break; } return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir); } uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct qat_sym_hash_def **hash_def) { enum qat_sym_hash_algorithm algo; switch (qs->qs_auth_algo) { case HW_AUTH_ALGO_SHA1: algo = QAT_SYM_HASH_SHA1; break; case HW_AUTH_ALGO_SHA256: algo = QAT_SYM_HASH_SHA256; break; case HW_AUTH_ALGO_SHA384: algo = QAT_SYM_HASH_SHA384; break; case HW_AUTH_ALGO_SHA512: algo = QAT_SYM_HASH_SHA512; break; case HW_AUTH_ALGO_GALOIS_128: algo = QAT_SYM_HASH_AES_GCM; break; default: panic("unhandled auth algorithm %d", qs->qs_auth_algo); break; } *hash_def = &qat_sym_hash_defs[algo]; return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode, (*hash_def)->qshd_qat->qshqi_algo_enc, (*hash_def)->qshd_alg->qshai_digest_len); } struct qat_crypto_load_cb_arg { struct qat_session *qs; struct qat_sym_cookie *qsc; struct cryptop *crp; int error; }; +static int +qat_crypto_populate_buf_list(struct buffer_list_desc *buffers, + bus_dma_segment_t *segs, int niseg, int noseg, int skip) +{ + struct flat_buffer_desc *flatbuf; + bus_addr_t addr; + bus_size_t len; + int iseg, oseg; + + for (iseg = 0, oseg = noseg; iseg < niseg && oseg < QAT_MAXSEG; + iseg++) { + addr = segs[iseg].ds_addr; + len = segs[iseg].ds_len; + + if (skip > 0) { + if (skip < len) { + addr += skip; + len -= skip; + skip = 0; + } else { + skip -= len; + continue; + } + } + + flatbuf = &buffers->flat_bufs[oseg++]; + flatbuf->data_len_in_bytes = (uint32_t)len; + flatbuf->phy_buffer = (uint64_t)addr; + } + buffers->num_buffers = oseg; + return iseg < niseg ? E2BIG : 0; +} + static void -qat_crypto_load_cb(void *_arg, bus_dma_segment_t *segs, int nseg, +qat_crypto_load_aadbuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct qat_crypto_load_cb_arg *arg; + struct qat_sym_cookie *qsc; + + arg = _arg; + if (error != 0) { + arg->error = error; + return; + } + + qsc = arg->qsc; + arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, + nseg, 0, 0); +} + +static void +qat_crypto_load_buf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, int error) { struct cryptop *crp; - struct flat_buffer_desc *flatbuf; struct qat_crypto_load_cb_arg *arg; struct qat_session *qs; struct qat_sym_cookie *qsc; - bus_addr_t addr; - bus_size_t len; - int iseg, oseg, skip; + int noseg, skip; arg = _arg; if (error != 0) { arg->error = error; return; } crp = arg->crp; qs = arg->qs; qsc = arg->qsc; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { - /* - * The firmware expects AAD to be in a contiguous buffer and - * padded to a multiple of 16 bytes. To satisfy these - * constraints we bounce the AAD into a per-request buffer. - */ - crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, - qsc->qsc_gcm_aad); - memset(qsc->qsc_gcm_aad + crp->crp_aad_length, 0, - roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) - - crp->crp_aad_length); + /* AAD was handled in qat_crypto_load(). */ skip = crp->crp_payload_start; - } else if (crp->crp_aad_length > 0) { + noseg = 0; + } else if (crp->crp_aad == NULL && crp->crp_aad_length > 0) { skip = crp->crp_aad_start; + noseg = 0; } else { skip = crp->crp_payload_start; + noseg = crp->crp_aad == NULL ? + 0 : qsc->qsc_buf_list.num_buffers; } + arg->error = qat_crypto_populate_buf_list(&qsc->qsc_buf_list, segs, + nseg, noseg, skip); +} - for (iseg = oseg = 0; iseg < nseg; iseg++) { - addr = segs[iseg].ds_addr; - len = segs[iseg].ds_len; +static void +qat_crypto_load_obuf_cb(void *_arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct buffer_list_desc *ibufs, *obufs; + struct flat_buffer_desc *ibuf, *obuf; + struct cryptop *crp; + struct qat_crypto_load_cb_arg *arg; + struct qat_session *qs; + struct qat_sym_cookie *qsc; + int buflen, osegs, tocopy; - if (skip > 0) { - if (skip < len) { - addr += skip; - len -= skip; - skip = 0; - } else { - skip -= len; - continue; - } - } + arg = _arg; + if (error != 0) { + arg->error = error; + return; + } - flatbuf = &qsc->qsc_flat_bufs[oseg++]; - flatbuf->data_len_in_bytes = (uint32_t)len; - flatbuf->phy_buffer = (uint64_t)addr; + crp = arg->crp; + qs = arg->qs; + qsc = arg->qsc; + + /* + * The payload must start at the same offset in the output SG list as in + * the input SG list. Copy over SG entries from the input corresponding + * to the AAD buffer. + */ + osegs = 0; + if (qs->qs_auth_algo != HW_AUTH_ALGO_GALOIS_128 && + crp->crp_aad_length > 0) { + tocopy = crp->crp_aad == NULL ? + crp->crp_payload_start - crp->crp_aad_start : + crp->crp_aad_length; + + ibufs = &qsc->qsc_buf_list; + obufs = &qsc->qsc_obuf_list; + for (; osegs < ibufs->num_buffers && tocopy > 0; osegs++) { + ibuf = &ibufs->flat_bufs[osegs]; + obuf = &obufs->flat_bufs[osegs]; + + obuf->phy_buffer = ibuf->phy_buffer; + buflen = imin(ibuf->data_len_in_bytes, tocopy); + obuf->data_len_in_bytes = buflen; + tocopy -= buflen; + } } - qsc->qsc_buf_list.num_buffers = oseg; + + arg->error = qat_crypto_populate_buf_list(&qsc->qsc_obuf_list, segs, + nseg, osegs, crp->crp_payload_output_start); } static int qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc, struct qat_crypto_desc const *desc, struct cryptop *crp) { struct qat_crypto_load_cb_arg arg; int error; crypto_read_iv(crp, qsc->qsc_iv_buf); arg.crp = crp; arg.qs = qs; arg.qsc = qsc; arg.error = 0; - error = bus_dmamap_load_crp(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, - crp, qat_crypto_load_cb, &arg, BUS_DMA_NOWAIT); - if (error == 0) - error = arg.error; + + error = 0; + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128 && + crp->crp_aad_length > 0) { + /* + * The firmware expects AAD to be in a contiguous buffer and + * padded to a multiple of 16 bytes. To satisfy these + * constraints we bounce the AAD into a per-request buffer. + * There is a small limit on the AAD size so this is not too + * onerous. + */ + memset(qsc->qsc_gcm_aad, 0, QAT_GCM_AAD_SIZE_MAX); + if (crp->crp_aad == NULL) { + crypto_copydata(crp, crp->crp_aad_start, + crp->crp_aad_length, qsc->qsc_gcm_aad); + } else { + memcpy(qsc->qsc_gcm_aad, crp->crp_aad, + crp->crp_aad_length); + } + } else if (crp->crp_aad != NULL) { + error = bus_dmamap_load( + qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, + crp->crp_aad, crp->crp_aad_length, + qat_crypto_load_aadbuf_cb, &arg, BUS_DMA_NOWAIT); + if (error == 0) + error = arg.error; + } + if (error == 0) { + error = bus_dmamap_load_crp_buffer( + qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, + &crp->crp_buf, qat_crypto_load_buf_cb, &arg, + BUS_DMA_NOWAIT); + if (error == 0) + error = arg.error; + } + if (error == 0 && CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + error = bus_dmamap_load_crp_buffer( + qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, + &crp->crp_obuf, qat_crypto_load_obuf_cb, &arg, + BUS_DMA_NOWAIT); + if (error == 0) + error = arg.error; + } return error; } static inline struct qat_crypto_bank * qat_crypto_select_bank(struct qat_crypto *qcy) { u_int cpuid = PCPU_GET(cpuid); return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks]; } static int qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb) { - int error, i, bank; - int curname = 0; char *name; + int bank, curname, error, i, j; bank = qcb->qcb_bank; + curname = 0; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size, NULL, NULL, name, &qcb->qcb_sym_tx); if (error) return error; name = qcb->qcb_ring_names[curname++]; snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank); error = qat_etr_setup_ring(sc, qcb->qcb_bank, sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size, qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx); if (error) return error; for (i = 0; i < QAT_NSYMCOOKIE; i++) { struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i]; struct qat_sym_cookie *qsc; error = qat_alloc_dmamem(sc, qdm, 1, sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN); if (error) return error; qsc = qdm->qdm_dma_vaddr; qsc->qsc_self_dmamap = qdm->qdm_dma_map; qsc->qsc_self_dma_tag = qdm->qdm_dma_tag; qsc->qsc_bulk_req_params_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, - u.qsc_bulk_cookie.qsbc_req_params_buf); + qsc_bulk_cookie.qsbc_req_params_buf); qsc->qsc_buffer_list_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_buf_list); + qsc->qsc_obuffer_list_desc_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_obuf_list); + qsc->qsc_obuffer_list_desc_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_obuf_list); qsc->qsc_iv_buf_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_iv_buf); qsc->qsc_auth_res_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_auth_res); qsc->qsc_gcm_aad_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_gcm_aad); qsc->qsc_content_desc_paddr = qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, qsc_content_desc); qcb->qcb_symck_free[i] = qsc; qcb->qcb_symck_free_count++; - error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), - 1, 0, /* alignment, boundary */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - QAT_MAXLEN, /* maxsize */ - QAT_MAXSEG, /* nsegments */ - QAT_MAXLEN, /* maxsegsize */ - BUS_DMA_COHERENT, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &qsc->qsc_buf_dma_tag); - if (error != 0) - return error; - - error = bus_dmamap_create(qsc->qsc_buf_dma_tag, - BUS_DMA_COHERENT, &qsc->qsc_buf_dmamap); - if (error) - return error; + for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { + error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + QAT_MAXLEN, /* maxsize */ + QAT_MAXSEG, /* nsegments */ + QAT_MAXLEN, /* maxsegsize */ + BUS_DMA_COHERENT, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &qsc->qsc_dma[j].qsd_dma_tag); + if (error != 0) + return error; + error = bus_dmamap_create(qsc->qsc_dma[j].qsd_dma_tag, + BUS_DMA_COHERENT, &qsc->qsc_dma[j].qsd_dmamap); + if (error != 0) + return error; + } } return 0; } static int qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb) { mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF); return qat_crypto_setup_ring(sc, qcb); } static void qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb) { struct qat_dmamem *qdm; - int i; + struct qat_sym_cookie *qsc; + int i, j; for (i = 0; i < QAT_NSYMCOOKIE; i++) { qdm = &qcb->qcb_symck_dmamems[i]; + qsc = qcb->qcb_symck_free[i]; + for (j = 0; j < QAT_SYM_DMA_COUNT; j++) { + bus_dmamap_destroy(qsc->qsc_dma[j].qsd_dma_tag, + qsc->qsc_dma[j].qsd_dmamap); + bus_dma_tag_destroy(qsc->qsc_dma[j].qsd_dma_tag); + } qat_free_dmamem(sc, qdm); } qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma); qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma); mtx_destroy(&qcb->qcb_bank_mtx); } static int qat_crypto_init(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct sysctl_ctx_list *ctx; struct sysctl_oid *oid; struct sysctl_oid_list *children; int bank, error, num_banks; qcy->qcy_sc = sc; if (sc->sc_hw.qhw_init_arb != NULL) num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks); else num_banks = sc->sc_ae_num; qcy->qcy_num_banks = num_banks; qcy->qcy_banks = qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks); for (bank = 0; bank < num_banks; bank++) { struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank]; qcb->qcb_bank = bank; error = qat_crypto_bank_init(sc, qcb); if (error) return error; } mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF); ctx = device_get_sysctl_ctx(sc->sc_dev); oid = device_get_sysctl_tree(sc->sc_dev); children = SYSCTL_CHILDREN(oid); oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); children = SYSCTL_CHILDREN(oid); sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts", CTLFLAG_RD, &sc->sc_gcm_aad_restarts, "GCM requests deferred due to AAD size change"); sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates", CTLFLAG_RD, &sc->sc_gcm_aad_updates, "GCM requests that required session state update"); sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full", CTLFLAG_RD, &sc->sc_ring_full_restarts, "Requests deferred due to in-flight max reached"); sc->sc_sym_alloc_failures = counter_u64_alloc(M_WAITOK); SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "sym_alloc_failures", CTLFLAG_RD, &sc->sc_sym_alloc_failures, "Request allocation failures"); return 0; } static void qat_crypto_deinit(struct qat_softc *sc) { struct qat_crypto *qcy = &sc->sc_crypto; struct qat_crypto_bank *qcb; int bank; counter_u64_free(sc->sc_sym_alloc_failures); counter_u64_free(sc->sc_ring_full_restarts); counter_u64_free(sc->sc_gcm_aad_updates); counter_u64_free(sc->sc_gcm_aad_restarts); if (qcy->qcy_banks != NULL) { for (bank = 0; bank < qcy->qcy_num_banks; bank++) { qcb = &qcy->qcy_banks[bank]; qat_crypto_bank_deinit(sc, qcb); } qat_free_mem(qcy->qcy_banks); mtx_destroy(&qcy->qcy_crypto_mtx); } } static int qat_crypto_start(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; qcy->qcy_cid = crypto_get_driverid(sc->sc_dev, sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE); if (qcy->qcy_cid < 0) { device_printf(sc->sc_dev, "could not get opencrypto driver id\n"); return ENOENT; } return 0; } static void qat_crypto_stop(struct qat_softc *sc) { struct qat_crypto *qcy; qcy = &sc->sc_crypto; if (qcy->qcy_cid >= 0) (void)crypto_unregister_all(qcy->qcy_cid); } +static void +qat_crypto_sym_dma_unload(struct qat_sym_cookie *qsc, enum qat_sym_dma i) +{ + bus_dmamap_sync(qsc->qsc_dma[i].qsd_dma_tag, qsc->qsc_dma[i].qsd_dmamap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(qsc->qsc_dma[i].qsd_dma_tag, + qsc->qsc_dma[i].qsd_dmamap); +} + static int qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg) { char icv[QAT_SYM_HASH_BUFFER_LEN]; struct qat_crypto_bank *qcb = arg; struct qat_crypto *qcy; struct qat_session *qs; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; struct cryptop *crp; int error; uint16_t auth_sz; bool blocked; qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset); - qsbc = &qsc->u.qsc_bulk_cookie; + qsbc = &qsc->qsc_bulk_cookie; qcy = qsbc->qsbc_crypto; qs = qsbc->qsbc_session; crp = qsbc->qsbc_cb_tag; bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap); + + if (crp->crp_aad != NULL) + qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_AADBUF); + qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_BUF); + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) + qat_crypto_sym_dma_unload(qsc, QAT_SYM_DMA_OBUF); error = 0; if ((auth_sz = qs->qs_auth_mlen) != 0) { if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { crypto_copydata(crp, crp->crp_digest_start, auth_sz, icv); if (timingsafe_bcmp(icv, qsc->qsc_auth_res, auth_sz) != 0) { error = EBADMSG; } } else { crypto_copyback(crp, crp->crp_digest_start, auth_sz, qsc->qsc_auth_res); } } qat_crypto_free_sym_cookie(qcb, qsc); blocked = false; mtx_lock(&qs->qs_session_mtx); MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE); qs->qs_inflight--; if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) { blocked = true; qs->qs_need_wakeup = false; } mtx_unlock(&qs->qs_session_mtx); crp->crp_etype = error; crypto_done(crp); if (blocked) crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ); return 1; } static int qat_probesession(device_t dev, const struct crypto_session_params *csp) { + if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != + 0) + return EINVAL; + if (csp->csp_cipher_alg == CRYPTO_AES_XTS && qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) { /* * AES-XTS is not supported by the NanoQAT. */ return EINVAL; } switch (csp->csp_mode) { case CSP_MODE_CIPHER: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_DIGEST: switch (csp->csp_auth_alg) { case CRYPTO_SHA1: case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512: case CRYPTO_SHA2_512_HMAC: break; case CRYPTO_AES_NIST_GMAC: if (csp->csp_ivlen != AES_GCM_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { case CRYPTO_AES_NIST_GCM_16: if (csp->csp_ivlen != AES_GCM_IV_LEN) return EINVAL; break; default: return EINVAL; } break; case CSP_MODE_ETA: switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: case CRYPTO_SHA2_256_HMAC: case CRYPTO_SHA2_384_HMAC: case CRYPTO_SHA2_512_HMAC: switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: case CRYPTO_AES_ICM: if (csp->csp_ivlen != AES_BLOCK_LEN) return EINVAL; break; case CRYPTO_AES_XTS: if (csp->csp_ivlen != AES_XTS_IV_LEN) return EINVAL; break; default: return EINVAL; } break; default: return EINVAL; } break; default: return EINVAL; } return CRYPTODEV_PROBE_HARDWARE; } static int qat_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct qat_crypto *qcy; struct qat_dmamem *qdm; struct qat_session *qs; struct qat_softc *sc; struct qat_crypto_desc *ddesc, *edesc; int error, slices; sc = device_get_softc(dev); qs = crypto_get_driver_session(cses); qcy = &sc->sc_crypto; qdm = &qs->qs_desc_mem; error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG, sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN); if (error != 0) return error; mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF); qs->qs_aad_length = -1; qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr; qs->qs_enc_desc = edesc = ddesc + 1; ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr; ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr + sizeof(struct qat_crypto_desc); edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); qs->qs_status = QAT_SESSION_STATUS_ACTIVE; qs->qs_inflight = 0; qs->qs_cipher_key = csp->csp_cipher_key; qs->qs_cipher_klen = csp->csp_cipher_klen; qs->qs_auth_key = csp->csp_auth_key; qs->qs_auth_klen = csp->csp_auth_klen; switch (csp->csp_cipher_alg) { case CRYPTO_AES_CBC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CBC_MODE; break; case CRYPTO_AES_ICM: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; break; case CRYPTO_AES_XTS: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen / 2); qs->qs_cipher_mode = HW_CIPHER_XTS_MODE; break; case CRYPTO_AES_NIST_GCM_16: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; break; case 0: break; default: panic("%s: unhandled cipher algorithm %d", __func__, csp->csp_cipher_alg); } switch (csp->csp_auth_alg) { case CRYPTO_SHA1_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA1: qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_256_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_256: qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_384_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_384: qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_SHA2_512_HMAC: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE1; break; case CRYPTO_SHA2_512: qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; qs->qs_auth_mode = HW_AUTH_MODE0; break; case CRYPTO_AES_NIST_GMAC: qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen); qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; qs->qs_auth_mode = HW_AUTH_MODE1; qs->qs_cipher_key = qs->qs_auth_key; qs->qs_cipher_klen = qs->qs_auth_klen; break; case 0: break; default: panic("%s: unhandled auth algorithm %d", __func__, csp->csp_auth_alg); } slices = 0; switch (csp->csp_mode) { case CSP_MODE_AEAD: case CSP_MODE_ETA: /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; break; case CSP_MODE_CIPHER: /* decrypt */ ddesc->qcd_slices[0] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER; /* encrypt */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER; slices = 1; break; case CSP_MODE_DIGEST: if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { /* auth then decrypt */ ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_slices[1] = FW_SLICE_CIPHER; ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; /* encrypt then auth */ edesc->qcd_slices[0] = FW_SLICE_CIPHER; edesc->qcd_slices[1] = FW_SLICE_AUTH; edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; slices = 2; } else { ddesc->qcd_slices[0] = FW_SLICE_AUTH; ddesc->qcd_cmd_id = FW_LA_CMD_AUTH; edesc->qcd_slices[0] = FW_SLICE_AUTH; edesc->qcd_cmd_id = FW_LA_CMD_AUTH; slices = 1; } break; default: panic("%s: unhandled crypto algorithm %d, %d", __func__, csp->csp_cipher_alg, csp->csp_auth_alg); } ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc); qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc); if (csp->csp_auth_mlen != 0) qs->qs_auth_mlen = csp->csp_auth_mlen; else qs->qs_auth_mlen = edesc->qcd_auth_sz; /* Compute the GMAC by specifying a null cipher payload. */ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH; return 0; } static void qat_crypto_clear_desc(struct qat_crypto_desc *desc) { explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc)); explicit_bzero(desc->qcd_hash_state_prefix_buf, sizeof(desc->qcd_hash_state_prefix_buf)); explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache)); } static void qat_freesession(device_t dev, crypto_session_t cses) { struct qat_session *qs; qs = crypto_get_driver_session(cses); KASSERT(qs->qs_inflight == 0, ("%s: session %p has requests in flight", __func__, qs)); qat_crypto_clear_desc(qs->qs_enc_desc); qat_crypto_clear_desc(qs->qs_dec_desc); qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem); mtx_destroy(&qs->qs_session_mtx); } static int qat_process(device_t dev, struct cryptop *crp, int hint) { struct qat_crypto *qcy; struct qat_crypto_bank *qcb; struct qat_crypto_desc const *desc; struct qat_session *qs; struct qat_softc *sc; struct qat_sym_cookie *qsc; struct qat_sym_bulk_cookie *qsbc; int error; sc = device_get_softc(dev); qcy = &sc->sc_crypto; qs = crypto_get_driver_session(crp->crp_session); qsc = NULL; if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) { error = E2BIG; goto fail1; } mtx_lock(&qs->qs_session_mtx); if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) { error = E2BIG; mtx_unlock(&qs->qs_session_mtx); goto fail1; } /* * The firmware interface for GCM annoyingly requires the AAD * size to be stored in the session's content descriptor, which * is not really meant to be updated after session * initialization. For IPSec the AAD size is fixed so this is * not much of a problem in practice, but we have to catch AAD * size updates here so that the device code can safely update * the session's recorded AAD size. */ if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) { if (qs->qs_inflight == 0) { if (qs->qs_aad_length != -1) { counter_u64_add(sc->sc_gcm_aad_updates, 1); } qs->qs_aad_length = crp->crp_aad_length; } else { qs->qs_need_wakeup = true; mtx_unlock(&qs->qs_session_mtx); counter_u64_add(sc->sc_gcm_aad_restarts, 1); error = ERESTART; goto fail1; } } } qs->qs_inflight++; mtx_unlock(&qs->qs_session_mtx); qcb = qat_crypto_select_bank(qcy); qsc = qat_crypto_alloc_sym_cookie(qcb); if (qsc == NULL) { counter_u64_add(sc->sc_sym_alloc_failures, 1); error = ENOBUFS; goto fail2; } if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) desc = qs->qs_enc_desc; else desc = qs->qs_dec_desc; error = qat_crypto_load(qs, qsc, desc, crp); if (error != 0) goto fail2; - qsbc = &qsc->u.qsc_bulk_cookie; + qsbc = &qsc->qsc_bulk_cookie; qsbc->qsbc_crypto = qcy; qsbc->qsbc_session = qs; qsbc->qsbc_cb_tag = crp; sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp); - bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, + if (crp->crp_aad != NULL) { + bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_AADBUF].qsd_dmamap, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + } + bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_BUF].qsd_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + bus_dmamap_sync(qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dma_tag, + qsc->qsc_dma[QAT_SYM_DMA_OBUF].qsd_dmamap, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + } bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); error = qat_etr_put_msg(sc, qcb->qcb_sym_tx, (uint32_t *)qsbc->qsbc_msg); if (error) goto fail2; return 0; fail2: if (qsc) qat_crypto_free_sym_cookie(qcb, qsc); mtx_lock(&qs->qs_session_mtx); qs->qs_inflight--; mtx_unlock(&qs->qs_session_mtx); fail1: crp->crp_etype = error; crypto_done(crp); return 0; } static device_method_t qat_methods[] = { /* Device interface */ DEVMETHOD(device_probe, qat_probe), DEVMETHOD(device_attach, qat_attach), DEVMETHOD(device_detach, qat_detach), /* Cryptodev interface */ DEVMETHOD(cryptodev_probesession, qat_probesession), DEVMETHOD(cryptodev_newsession, qat_newsession), DEVMETHOD(cryptodev_freesession, qat_freesession), DEVMETHOD(cryptodev_process, qat_process), DEVMETHOD_END }; static devclass_t qat_devclass; static driver_t qat_driver = { .name = "qat", .methods = qat_methods, .size = sizeof(struct qat_softc), }; DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0); MODULE_VERSION(qat, 1); MODULE_DEPEND(qat, crypto, 1, 1, 1); MODULE_DEPEND(qat, pci, 1, 1, 1); diff --git a/sys/dev/qat/qat_hw15.c b/sys/dev/qat/qat_hw15.c index 4f823a0a9ae4..aab5be9b647d 100644 --- a/sys/dev/qat/qat_hw15.c +++ b/sys/dev/qat/qat_hw15.c @@ -1,953 +1,965 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); #endif #include #include #include #include #include #include #include #include #include #include "qatreg.h" #include "qat_hw15reg.h" #include "qatvar.h" #include "qat_hw15var.h" static int qat_adm_ring_init_ring_table(struct qat_softc *); static void qat_adm_ring_build_slice_mask(uint16_t *, uint32_t, uint32_t); static void qat_adm_ring_build_shram_mask(uint64_t *, uint32_t, uint32_t); static int qat_adm_ring_build_ring_table(struct qat_softc *, uint32_t); static int qat_adm_ring_build_init_msg(struct qat_softc *, struct fw_init_req *, enum fw_init_cmd_id, uint32_t, struct qat_accel_init_cb *); static int qat_adm_ring_send_init_msg_sync(struct qat_softc *, enum fw_init_cmd_id, uint32_t); static int qat_adm_ring_send_init_msg(struct qat_softc *, enum fw_init_cmd_id); static int qat_adm_ring_intr(struct qat_softc *, void *, void *); void qat_msg_req_type_populate(struct arch_if_req_hdr *msg, enum arch_if_req type, uint32_t rxring) { memset(msg, 0, sizeof(struct arch_if_req_hdr)); msg->flags = ARCH_IF_FLAGS_VALID_FLAG | ARCH_IF_FLAGS_RESP_RING_TYPE_ET | ARCH_IF_FLAGS_RESP_TYPE_S; msg->req_type = type; msg->resp_pipe_id = rxring; } void qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *msg, bus_addr_t desc_paddr, uint8_t hdrsz, uint8_t hwblksz, uint16_t comn_req_flags, uint32_t flow_id) { struct fw_comn_req_hdr *hdr = &msg->comn_hdr; hdr->comn_req_flags = comn_req_flags; hdr->content_desc_params_sz = hwblksz; hdr->content_desc_hdr_sz = hdrsz; hdr->content_desc_addr = desc_paddr; msg->flow_id = flow_id; } void qat_msg_service_cmd_populate(struct fw_la_bulk_req *msg, enum fw_la_cmd_id cmdid, uint16_t cmd_flags) { msg->comn_la_req.la_cmd_id = cmdid; msg->comn_la_req.u.la_flags = cmd_flags; } void qat_msg_cmn_mid_populate(struct fw_comn_req_mid *msg, void *cookie, uint64_t src, uint64_t dst) { msg->opaque_data = (uint64_t)(uintptr_t)cookie; msg->src_data_addr = src; if (dst == 0) msg->dest_data_addr = src; else msg->dest_data_addr = dst; } void qat_msg_req_params_populate(struct fw_la_bulk_req *msg, bus_addr_t req_params_paddr, uint8_t req_params_sz) { msg->req_params_addr = req_params_paddr; msg->comn_la_req.u1.req_params_blk_sz = req_params_sz / 8; } void qat_msg_cmn_footer_populate(union fw_comn_req_ftr *msg, uint64_t next_addr) { msg->next_request_addr = next_addr; } void qat_msg_params_populate(struct fw_la_bulk_req *msg, struct qat_crypto_desc *desc, uint8_t req_params_sz, uint16_t service_cmd_flags, uint16_t comn_req_flags) { qat_msg_cmn_hdr_populate(msg, desc->qcd_desc_paddr, desc->qcd_hdr_sz, desc->qcd_hw_blk_sz, comn_req_flags, 0); qat_msg_service_cmd_populate(msg, desc->qcd_cmd_id, service_cmd_flags); qat_msg_cmn_mid_populate(&msg->comn_mid, NULL, 0, 0); qat_msg_req_params_populate(msg, 0, req_params_sz); qat_msg_cmn_footer_populate(&msg->comn_ftr, 0); } static int qat_adm_ring_init_ring_table(struct qat_softc *sc) { struct qat_admin_rings *qadr = &sc->sc_admin_rings; if (sc->sc_ae_num == 1) { qadr->qadr_cya_ring_tbl = &qadr->qadr_master_ring_tbl[0]; qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; } else if (sc->sc_ae_num == 2 || sc->sc_ae_num == 4) { qadr->qadr_cya_ring_tbl = &qadr->qadr_master_ring_tbl[0]; qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; qadr->qadr_cyb_ring_tbl = &qadr->qadr_master_ring_tbl[1]; qadr->qadr_srv_mask[1] = QAT_SERVICE_CRYPTO_B; } return 0; } int qat_adm_ring_init(struct qat_softc *sc) { struct qat_admin_rings *qadr = &sc->sc_admin_rings; int error, i, j; error = qat_alloc_dmamem(sc, &qadr->qadr_dma, 1, PAGE_SIZE, PAGE_SIZE); if (error) return error; qadr->qadr_master_ring_tbl = qadr->qadr_dma.qdm_dma_vaddr; MPASS(sc->sc_ae_num * sizeof(struct fw_init_ring_table) <= PAGE_SIZE); /* Initialize the Master Ring Table */ for (i = 0; i < sc->sc_ae_num; i++) { struct fw_init_ring_table *firt = &qadr->qadr_master_ring_tbl[i]; for (j = 0; j < INIT_RING_TABLE_SZ; j++) { struct fw_init_ring_params *firp = &firt->firt_bulk_rings[j]; firp->firp_reserved = 0; firp->firp_curr_weight = QAT_DEFAULT_RING_WEIGHT; firp->firp_init_weight = QAT_DEFAULT_RING_WEIGHT; firp->firp_ring_pvl = QAT_DEFAULT_PVL; } memset(firt->firt_ring_mask, 0, sizeof(firt->firt_ring_mask)); } error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_TX, ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_req_size, NULL, NULL, "admin_tx", &qadr->qadr_admin_tx); if (error) return error; error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_RX, ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_resp_size, qat_adm_ring_intr, qadr, "admin_rx", &qadr->qadr_admin_rx); if (error) return error; /* * Finally set up the service indices into the Master Ring Table * and convenient ring table pointers for each service enabled. * Only the Admin rings are initialized. */ error = qat_adm_ring_init_ring_table(sc); if (error) return error; /* * Calculate the number of active AEs per QAT * needed for Shram partitioning. */ for (i = 0; i < sc->sc_ae_num; i++) { if (qadr->qadr_srv_mask[i]) qadr->qadr_active_aes_per_accel++; } return 0; } static void qat_adm_ring_build_slice_mask(uint16_t *slice_mask, uint32_t srv_mask, uint32_t init_shram) { uint16_t shram = 0, comn_req = 0; if (init_shram) shram = COMN_REQ_SHRAM_INIT_REQUIRED; if (srv_mask & QAT_SERVICE_CRYPTO_A) comn_req |= COMN_REQ_CY0_ONLY(shram); if (srv_mask & QAT_SERVICE_CRYPTO_B) comn_req |= COMN_REQ_CY1_ONLY(shram); *slice_mask = comn_req; } static void qat_adm_ring_build_shram_mask(uint64_t *shram_mask, uint32_t active_aes, uint32_t ae) { *shram_mask = 0; if (active_aes == 1) { *shram_mask = ~(*shram_mask); } else if (active_aes == 2) { if (ae == 1) *shram_mask = ((~(*shram_mask)) & 0xffffffff); else *shram_mask = ((~(*shram_mask)) & 0xffffffff00000000ull); } else if (active_aes == 3) { if (ae == 0) *shram_mask = ((~(*shram_mask)) & 0x7fffff); else if (ae == 1) *shram_mask = ((~(*shram_mask)) & 0x3fffff800000ull); else *shram_mask = ((~(*shram_mask)) & 0xffffc00000000000ull); } else { panic("Only three services are supported in current version"); } } static int qat_adm_ring_build_ring_table(struct qat_softc *sc, uint32_t ae) { struct qat_admin_rings *qadr = &sc->sc_admin_rings; struct fw_init_ring_table *tbl; struct fw_init_ring_params *param; uint8_t srv_mask = sc->sc_admin_rings.qadr_srv_mask[ae]; if ((srv_mask & QAT_SERVICE_CRYPTO_A)) { tbl = qadr->qadr_cya_ring_tbl; } else if ((srv_mask & QAT_SERVICE_CRYPTO_B)) { tbl = qadr->qadr_cyb_ring_tbl; } else { device_printf(sc->sc_dev, "Invalid execution engine %d\n", ae); return EINVAL; } param = &tbl->firt_bulk_rings[sc->sc_hw.qhw_ring_sym_tx]; param->firp_curr_weight = QAT_HI_PRIO_RING_WEIGHT; param->firp_init_weight = QAT_HI_PRIO_RING_WEIGHT; FW_INIT_RING_MASK_SET(tbl, sc->sc_hw.qhw_ring_sym_tx); return 0; } static int qat_adm_ring_build_init_msg(struct qat_softc *sc, struct fw_init_req *initmsg, enum fw_init_cmd_id cmd, uint32_t ae, struct qat_accel_init_cb *cb) { struct fw_init_set_ae_info_hdr *aehdr; struct fw_init_set_ae_info *aeinfo; struct fw_init_set_ring_info_hdr *ringhdr; struct fw_init_set_ring_info *ringinfo; int init_shram = 0, tgt_id, cluster_id; uint32_t srv_mask; srv_mask = sc->sc_admin_rings.qadr_srv_mask[ ae % sc->sc_ae_num]; memset(initmsg, 0, sizeof(struct fw_init_req)); qat_msg_req_type_populate(&initmsg->comn_hdr.arch_if, ARCH_IF_REQ_QAT_FW_INIT, sc->sc_admin_rings.qadr_admin_rx->qr_ring_id); qat_msg_cmn_mid_populate(&initmsg->comn_mid, cb, 0, 0); switch (cmd) { case FW_INIT_CMD_SET_AE_INFO: if (ae % sc->sc_ae_num == 0) init_shram = 1; if (ae >= sc->sc_ae_num) { tgt_id = 1; cluster_id = 1; } else { cluster_id = 0; if (sc->sc_ae_mask) tgt_id = 0; else tgt_id = 1; } aehdr = &initmsg->u.set_ae_info; aeinfo = &initmsg->u1.set_ae_info; aehdr->init_cmd_id = cmd; /* XXX that does not support sparse ae_mask */ aehdr->init_trgt_id = ae; aehdr->init_ring_cluster_id = cluster_id; aehdr->init_qat_id = tgt_id; qat_adm_ring_build_slice_mask(&aehdr->init_slice_mask, srv_mask, init_shram); qat_adm_ring_build_shram_mask(&aeinfo->init_shram_mask, sc->sc_admin_rings.qadr_active_aes_per_accel, ae % sc->sc_ae_num); break; case FW_INIT_CMD_SET_RING_INFO: ringhdr = &initmsg->u.set_ring_info; ringinfo = &initmsg->u1.set_ring_info; ringhdr->init_cmd_id = cmd; /* XXX that does not support sparse ae_mask */ ringhdr->init_trgt_id = ae; /* XXX */ qat_adm_ring_build_ring_table(sc, ae % sc->sc_ae_num); ringhdr->init_ring_tbl_sz = sizeof(struct fw_init_ring_table); ringinfo->init_ring_table_ptr = sc->sc_admin_rings.qadr_dma.qdm_dma_seg.ds_addr + ((ae % sc->sc_ae_num) * sizeof(struct fw_init_ring_table)); break; default: return ENOTSUP; } return 0; } static int qat_adm_ring_send_init_msg_sync(struct qat_softc *sc, enum fw_init_cmd_id cmd, uint32_t ae) { struct fw_init_req initmsg; struct qat_accel_init_cb cb; int error; error = qat_adm_ring_build_init_msg(sc, &initmsg, cmd, ae, &cb); if (error) return error; error = qat_etr_put_msg(sc, sc->sc_admin_rings.qadr_admin_tx, (uint32_t *)&initmsg); if (error) return error; error = tsleep(&cb, PZERO, "qat_init", hz * 3 / 2); if (error) { device_printf(sc->sc_dev, "Timed out initialization firmware: %d\n", error); return error; } if (cb.qaic_status) { device_printf(sc->sc_dev, "Failed to initialize firmware\n"); return EIO; } return error; } static int qat_adm_ring_send_init_msg(struct qat_softc *sc, enum fw_init_cmd_id cmd) { struct qat_admin_rings *qadr = &sc->sc_admin_rings; uint32_t error, ae; for (ae = 0; ae < sc->sc_ae_num; ae++) { uint8_t srv_mask = qadr->qadr_srv_mask[ae]; switch (cmd) { case FW_INIT_CMD_SET_AE_INFO: case FW_INIT_CMD_SET_RING_INFO: if (!srv_mask) continue; break; case FW_INIT_CMD_TRNG_ENABLE: case FW_INIT_CMD_TRNG_DISABLE: if (!(srv_mask & QAT_SERVICE_CRYPTO_A)) continue; break; default: return ENOTSUP; } error = qat_adm_ring_send_init_msg_sync(sc, cmd, ae); if (error) return error; } return 0; } int qat_adm_ring_send_init(struct qat_softc *sc) { int error; error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_AE_INFO); if (error) return error; error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_RING_INFO); if (error) return error; return 0; } static int qat_adm_ring_intr(struct qat_softc *sc, void *arg, void *msg) { struct arch_if_resp_hdr *resp; struct fw_init_resp *init_resp; struct qat_accel_init_cb *init_cb; int handled = 0; resp = (struct arch_if_resp_hdr *)msg; switch (resp->resp_type) { case ARCH_IF_REQ_QAT_FW_INIT: init_resp = (struct fw_init_resp *)msg; init_cb = (struct qat_accel_init_cb *) (uintptr_t)init_resp->comn_resp.opaque_data; init_cb->qaic_status = __SHIFTOUT(init_resp->comn_resp.comn_status, COMN_RESP_INIT_ADMIN_STATUS); wakeup(init_cb); break; default: device_printf(sc->sc_dev, "unknown resp type %d\n", resp->resp_type); break; } return handled; } static inline uint16_t qat_hw15_get_comn_req_flags(uint8_t ae) { if (ae == 0) { return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | COMN_REQ_AUTH0_SLICE_REQUIRED | COMN_REQ_CIPHER0_SLICE_REQUIRED; } else { return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | COMN_REQ_AUTH1_SLICE_REQUIRED | COMN_REQ_CIPHER1_SLICE_REQUIRED; } } static uint32_t qat_hw15_crypto_setup_cipher_desc(struct qat_crypto_desc *desc, struct qat_session *qs, struct fw_cipher_hdr *cipher_hdr, uint32_t hw_blk_offset, enum fw_slice next_slice) { desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ; cipher_hdr->state_padding_sz = 0; cipher_hdr->key_sz = qs->qs_cipher_klen / 8; cipher_hdr->state_sz = desc->qcd_cipher_blk_sz / 8; cipher_hdr->next_id = next_slice; cipher_hdr->curr_id = FW_SLICE_CIPHER; cipher_hdr->offset = hw_blk_offset / 8; cipher_hdr->resrvd = 0; return sizeof(struct hw_cipher_config) + qs->qs_cipher_klen; } static void qat_hw15_crypto_setup_cipher_config(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct cryptop *crp, struct hw_cipher_config *cipher_config) { const uint8_t *key; uint8_t *cipher_key; cipher_config->val = qat_crypto_load_cipher_session(desc, qs); cipher_config->reserved = 0; cipher_key = (uint8_t *)(cipher_config + 1); if (crp != NULL && crp->crp_cipher_key != NULL) key = crp->crp_cipher_key; else key = qs->qs_cipher_key; memcpy(cipher_key, key, qs->qs_cipher_klen); } static uint32_t qat_hw15_crypto_setup_auth_desc(struct qat_crypto_desc *desc, struct qat_session *qs, struct fw_auth_hdr *auth_hdr, uint32_t ctrl_blk_offset, uint32_t hw_blk_offset, enum fw_slice next_slice) { const struct qat_sym_hash_def *hash_def; (void)qat_crypto_load_auth_session(desc, qs, &hash_def); auth_hdr->next_id = next_slice; auth_hdr->curr_id = FW_SLICE_AUTH; auth_hdr->offset = hw_blk_offset / 8; auth_hdr->resrvd = 0; auth_hdr->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED; auth_hdr->u.inner_prefix_sz = 0; auth_hdr->outer_prefix_sz = 0; auth_hdr->final_sz = hash_def->qshd_alg->qshai_digest_len; auth_hdr->inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); auth_hdr->inner_res_sz = hash_def->qshd_alg->qshai_digest_len; auth_hdr->inner_state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8); auth_hdr->inner_state2_off = auth_hdr->offset + ((sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz) / 8); auth_hdr->outer_config_off = 0; auth_hdr->outer_state1_sz = 0; auth_hdr->outer_res_sz = 0; auth_hdr->outer_prefix_off = 0; desc->qcd_auth_sz = hash_def->qshd_alg->qshai_sah->hashsize; desc->qcd_state_storage_sz = (sizeof(struct hw_auth_counter) + roundup(hash_def->qshd_alg->qshai_state_size, 8)) / 8; desc->qcd_gcm_aad_sz_offset1 = desc->qcd_auth_offset + sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz + AES_BLOCK_LEN; desc->qcd_gcm_aad_sz_offset2 = ctrl_blk_offset + offsetof(struct fw_auth_hdr, u.aad_sz); return sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz + auth_hdr->inner_state2_sz; } static void qat_hw15_crypto_setup_auth_setup(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct cryptop *crp, struct hw_auth_setup *auth_setup) { const struct qat_sym_hash_def *hash_def; const uint8_t *key; uint8_t *state1, *state2; uint32_t state_sz, state1_sz, state2_sz, state1_pad_len, state2_pad_len; auth_setup->auth_config.config = qat_crypto_load_auth_session(desc, qs, &hash_def); auth_setup->auth_config.reserved = 0; auth_setup->auth_counter.counter = htobe32(hash_def->qshd_qat->qshqi_auth_counter); auth_setup->auth_counter.reserved = 0; state1 = (uint8_t *)(auth_setup + 1); state2 = state1 + roundup(hash_def->qshd_qat->qshqi_state1_len, 8); switch (qs->qs_auth_algo) { case HW_AUTH_ALGO_GALOIS_128: qat_crypto_gmac_precompute(desc, qs->qs_cipher_key, qs->qs_cipher_klen, hash_def, state2); break; case HW_AUTH_ALGO_SHA1: state_sz = hash_def->qshd_alg->qshai_state_size; state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8); if (qs->qs_auth_mode == HW_AUTH_MODE1) { state1_pad_len = state1_sz - state_sz; state2_pad_len = state2_sz - state_sz; if (state1_pad_len > 0) memset(state1 + state_sz, 0, state1_pad_len); if (state2_pad_len > 0) memset(state2 + state_sz, 0, state2_pad_len); } /* FALLTHROUGH */ case HW_AUTH_ALGO_SHA256: case HW_AUTH_ALGO_SHA384: case HW_AUTH_ALGO_SHA512: switch (qs->qs_auth_mode) { case HW_AUTH_MODE0: memcpy(state1, hash_def->qshd_alg->qshai_init_state, state1_sz); /* Override for mode 0 hashes. */ auth_setup->auth_counter.counter = 0; break; case HW_AUTH_MODE1: if (crp != NULL && crp->crp_auth_key != NULL) key = crp->crp_auth_key; else key = qs->qs_auth_key; if (key != NULL) { qat_crypto_hmac_precompute(desc, key, qs->qs_auth_klen, hash_def, state1, state2); } break; default: panic("%s: unhandled auth mode %d", __func__, qs->qs_auth_mode); } break; default: panic("%s: unhandled auth algorithm %d", __func__, qs->qs_auth_algo); } } void qat_hw15_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs, struct qat_crypto_desc *desc) { struct fw_cipher_hdr *cipher_hdr; struct fw_auth_hdr *auth_hdr; struct fw_la_bulk_req *req_cache; struct hw_auth_setup *auth_setup; struct hw_cipher_config *cipher_config; uint32_t ctrl_blk_sz, ctrl_blk_offset, hw_blk_offset; int i; uint16_t la_cmd_flags; uint8_t req_params_sz; uint8_t *ctrl_blk_ptr, *hw_blk_ptr; ctrl_blk_sz = 0; if (qs->qs_cipher_algo != HW_CIPHER_ALGO_NULL) ctrl_blk_sz += sizeof(struct fw_cipher_hdr); if (qs->qs_auth_algo != HW_AUTH_ALGO_NULL) ctrl_blk_sz += sizeof(struct fw_auth_hdr); ctrl_blk_ptr = desc->qcd_content_desc; ctrl_blk_offset = 0; hw_blk_ptr = ctrl_blk_ptr + ctrl_blk_sz; hw_blk_offset = 0; la_cmd_flags = 0; req_params_sz = 0; for (i = 0; i < MAX_FW_SLICE; i++) { switch (desc->qcd_slices[i]) { case FW_SLICE_CIPHER: cipher_hdr = (struct fw_cipher_hdr *)(ctrl_blk_ptr + ctrl_blk_offset); cipher_config = (struct hw_cipher_config *)(hw_blk_ptr + hw_blk_offset); desc->qcd_cipher_offset = ctrl_blk_sz + hw_blk_offset; hw_blk_offset += qat_hw15_crypto_setup_cipher_desc(desc, qs, cipher_hdr, hw_blk_offset, desc->qcd_slices[i + 1]); qat_hw15_crypto_setup_cipher_config(desc, qs, NULL, cipher_config); ctrl_blk_offset += sizeof(struct fw_cipher_hdr); req_params_sz += sizeof(struct fw_la_cipher_req_params); break; case FW_SLICE_AUTH: auth_hdr = (struct fw_auth_hdr *)(ctrl_blk_ptr + ctrl_blk_offset); auth_setup = (struct hw_auth_setup *)(hw_blk_ptr + hw_blk_offset); desc->qcd_auth_offset = ctrl_blk_sz + hw_blk_offset; hw_blk_offset += qat_hw15_crypto_setup_auth_desc(desc, qs, auth_hdr, ctrl_blk_offset, hw_blk_offset, desc->qcd_slices[i + 1]); qat_hw15_crypto_setup_auth_setup(desc, qs, NULL, auth_setup); ctrl_blk_offset += sizeof(struct fw_auth_hdr); req_params_sz += sizeof(struct fw_la_auth_req_params); la_cmd_flags |= LA_FLAGS_RET_AUTH_RES; /* no digest verify */ break; case FW_SLICE_DRAM_WR: i = MAX_FW_SLICE; /* end of chain */ break; default: MPASS(0); break; } } desc->qcd_hdr_sz = ctrl_blk_offset / 8; desc->qcd_hw_blk_sz = hw_blk_offset / 8; req_cache = (struct fw_la_bulk_req *)desc->qcd_req_cache; qat_msg_req_type_populate( &req_cache->comn_hdr.arch_if, ARCH_IF_REQ_QAT_FW_LA, 0); if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) la_cmd_flags |= LA_FLAGS_PROTO_GCM | LA_FLAGS_GCM_IV_LEN_FLAG; else la_cmd_flags |= LA_FLAGS_PROTO_NO; qat_msg_params_populate(req_cache, desc, req_params_sz, la_cmd_flags, 0); bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); } static void qat_hw15_crypto_req_setkey(const struct qat_crypto_desc *desc, const struct qat_session *qs, struct qat_sym_cookie *qsc, struct fw_la_bulk_req *bulk_req, struct cryptop *crp) { struct hw_auth_setup *auth_setup; struct hw_cipher_config *cipher_config; uint8_t *cdesc; int i; cdesc = qsc->qsc_content_desc; memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE); for (i = 0; i < MAX_FW_SLICE; i++) { switch (desc->qcd_slices[i]) { case FW_SLICE_CIPHER: cipher_config = (struct hw_cipher_config *) (cdesc + desc->qcd_cipher_offset); qat_hw15_crypto_setup_cipher_config(desc, qs, crp, cipher_config); break; case FW_SLICE_AUTH: auth_setup = (struct hw_auth_setup *) (cdesc + desc->qcd_auth_offset); qat_hw15_crypto_setup_auth_setup(desc, qs, crp, auth_setup); break; case FW_SLICE_DRAM_WR: i = MAX_FW_SLICE; /* end of chain */ break; default: MPASS(0); } } bulk_req->comn_hdr.content_desc_addr = qsc->qsc_content_desc_paddr; } void qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *qcb, struct qat_session *qs, struct qat_crypto_desc const *desc, struct qat_sym_cookie *qsc, struct cryptop *crp) { struct qat_sym_bulk_cookie *qsbc; struct fw_la_bulk_req *bulk_req; struct fw_la_cipher_req_params *cipher_req; struct fw_la_auth_req_params *auth_req; bus_addr_t digest_paddr; uint8_t *aad_szp2, *req_params_ptr; uint32_t aad_sz, *aad_szp1; enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id; enum fw_slice next_slice; - qsbc = &qsc->u.qsc_bulk_cookie; + qsbc = &qsc->qsc_bulk_cookie; bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg; memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE); bulk_req->comn_hdr.arch_if.resp_pipe_id = qcb->qcb_sym_rx->qr_ring_id; bulk_req->comn_hdr.comn_req_flags = qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2); bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr; - bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr; + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + bulk_req->comn_mid.dest_data_addr = + qsc->qsc_obuffer_list_desc_paddr; + } else { + bulk_req->comn_mid.dest_data_addr = + qsc->qsc_buffer_list_desc_paddr; + } bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr; bulk_req->comn_ftr.next_request_addr = 0; bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc; if (__predict_false(crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL)) { qat_hw15_crypto_req_setkey(desc, qs, qsc, bulk_req, crp); } digest_paddr = 0; if (desc->qcd_auth_sz != 0) digest_paddr = qsc->qsc_auth_res_paddr; req_params_ptr = qsbc->qsbc_req_params_buf; memset(req_params_ptr, 0, sizeof(qsbc->qsbc_req_params_buf)); /* * The SG list layout is a bit different for GCM and GMAC, it's simpler * to handle those cases separately. */ if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr; auth_req = (struct fw_la_auth_req_params *) (req_params_ptr + sizeof(struct fw_la_cipher_req_params)); cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8; cipher_req->curr_id = FW_SLICE_CIPHER; if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH) cipher_req->next_id = FW_SLICE_DRAM_WR; else cipher_req->next_id = FW_SLICE_AUTH; cipher_req->state_address = qsc->qsc_iv_buf_paddr; if (cmd_id != FW_LA_CMD_AUTH) { /* * Don't fill out the cipher block if we're doing GMAC * only. */ cipher_req->cipher_off = 0; cipher_req->cipher_len = crp->crp_payload_length; } auth_req->curr_id = FW_SLICE_AUTH; if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH) auth_req->next_id = FW_SLICE_CIPHER; else auth_req->next_id = FW_SLICE_DRAM_WR; auth_req->auth_res_address = digest_paddr; auth_req->auth_res_sz = desc->qcd_auth_sz; auth_req->auth_off = 0; auth_req->auth_len = crp->crp_payload_length; auth_req->hash_state_sz = roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) >> 3; auth_req->u1.aad_addr = crp->crp_aad_length > 0 ? qsc->qsc_gcm_aad_paddr : 0; /* * Update the hash state block if necessary. This only occurs * when the AAD length changes between requests in a session and * is synchronized by qat_process(). */ aad_sz = htobe32(crp->crp_aad_length); aad_szp1 = (uint32_t *)( __DECONST(uint8_t *, desc->qcd_content_desc) + desc->qcd_gcm_aad_sz_offset1); aad_szp2 = __DECONST(uint8_t *, desc->qcd_content_desc) + desc->qcd_gcm_aad_sz_offset2; if (__predict_false(*aad_szp1 != aad_sz)) { *aad_szp1 = aad_sz; *aad_szp2 = (uint8_t)roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN); bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); } } else { cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr; if (cmd_id != FW_LA_CMD_AUTH) { if (cmd_id == FW_LA_CMD_CIPHER || cmd_id == FW_LA_CMD_HASH_CIPHER) next_slice = FW_SLICE_DRAM_WR; else next_slice = FW_SLICE_AUTH; cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8; cipher_req->curr_id = FW_SLICE_CIPHER; cipher_req->next_id = next_slice; - cipher_req->cipher_off = crp->crp_aad_length == 0 ? 0 : - crp->crp_payload_start - crp->crp_aad_start; + if (crp->crp_aad_length == 0) { + cipher_req->cipher_off = 0; + } else if (crp->crp_aad == NULL) { + cipher_req->cipher_off = + crp->crp_payload_start - crp->crp_aad_start; + } else { + cipher_req->cipher_off = crp->crp_aad_length; + } cipher_req->cipher_len = crp->crp_payload_length; cipher_req->state_address = qsc->qsc_iv_buf_paddr; } if (cmd_id != FW_LA_CMD_CIPHER) { if (cmd_id == FW_LA_CMD_AUTH) auth_req = (struct fw_la_auth_req_params *) req_params_ptr; else auth_req = (struct fw_la_auth_req_params *) (cipher_req + 1); if (cmd_id == FW_LA_CMD_HASH_CIPHER) next_slice = FW_SLICE_CIPHER; else next_slice = FW_SLICE_DRAM_WR; auth_req->curr_id = FW_SLICE_AUTH; auth_req->next_id = next_slice; auth_req->auth_res_address = digest_paddr; auth_req->auth_res_sz = desc->qcd_auth_sz; auth_req->auth_len = crp->crp_payload_length + crp->crp_aad_length; auth_req->auth_off = 0; auth_req->hash_state_sz = 0; auth_req->u1.prefix_addr = desc->qcd_hash_state_paddr + desc->qcd_state_storage_sz; } } } diff --git a/sys/dev/qat/qat_hw17.c b/sys/dev/qat/qat_hw17.c index 643b624ba840..92f2c1ee251d 100644 --- a/sys/dev/qat/qat_hw17.c +++ b/sys/dev/qat/qat_hw17.c @@ -1,662 +1,674 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2014 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #if 0 __KERNEL_RCSID(0, "$NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); #endif #include #include #include #include #include #include #include #include #include "qatreg.h" #include "qat_hw17reg.h" #include "qatvar.h" #include "qat_hw17var.h" int qat_adm_mailbox_put_msg_sync(struct qat_softc *, uint32_t, void *, void *); int qat_adm_mailbox_send(struct qat_softc *, struct fw_init_admin_req *, struct fw_init_admin_resp *); int qat_adm_mailbox_send_init_me(struct qat_softc *); int qat_adm_mailbox_send_hb_timer(struct qat_softc *); int qat_adm_mailbox_send_fw_status(struct qat_softc *); int qat_adm_mailbox_send_constants(struct qat_softc *); int qat_adm_mailbox_init(struct qat_softc *sc) { uint64_t addr; int error; struct qat_dmamem *qdm; error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_dma, 1, PAGE_SIZE, PAGE_SIZE); if (error) return error; qdm = &sc->sc_admin_comms.qadc_const_tbl_dma; error = qat_alloc_dmamem(sc, qdm, 1, PAGE_SIZE, PAGE_SIZE); if (error) return error; memcpy(qdm->qdm_dma_vaddr, mailbox_const_tab, sizeof(mailbox_const_tab)); bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, BUS_DMASYNC_PREWRITE); error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_hb_dma, 1, PAGE_SIZE, PAGE_SIZE); if (error) return error; addr = (uint64_t)sc->sc_admin_comms.qadc_dma.qdm_dma_seg.ds_addr; qat_misc_write_4(sc, ADMINMSGUR, addr >> 32); qat_misc_write_4(sc, ADMINMSGLR, addr); return 0; } int qat_adm_mailbox_put_msg_sync(struct qat_softc *sc, uint32_t ae, void *in, void *out) { struct qat_dmamem *qdm; uint32_t mailbox; bus_size_t mb_offset = MAILBOX_BASE + (ae * MAILBOX_STRIDE); int offset = ae * ADMINMSG_LEN * 2; int times, received; uint8_t *buf = (uint8_t *)sc->sc_admin_comms.qadc_dma.qdm_dma_vaddr + offset; mailbox = qat_misc_read_4(sc, mb_offset); if (mailbox == 1) return EAGAIN; qdm = &sc->sc_admin_comms.qadc_dma; memcpy(buf, in, ADMINMSG_LEN); bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); qat_misc_write_4(sc, mb_offset, 1); received = 0; for (times = 0; times < 50; times++) { DELAY(20000); if (qat_misc_read_4(sc, mb_offset) == 0) { received = 1; break; } } if (received) { bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); memcpy(out, buf + ADMINMSG_LEN, ADMINMSG_LEN); } else { device_printf(sc->sc_dev, "Failed to send admin msg to accelerator\n"); } return received ? 0 : EFAULT; } int qat_adm_mailbox_send(struct qat_softc *sc, struct fw_init_admin_req *req, struct fw_init_admin_resp *resp) { int error; uint32_t mask; uint8_t ae; for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { if (!(mask & 1)) continue; error = qat_adm_mailbox_put_msg_sync(sc, ae, req, resp); if (error) return error; if (resp->init_resp_hdr.status) { device_printf(sc->sc_dev, "Failed to send admin msg: cmd %d\n", req->init_admin_cmd_id); return EFAULT; } } return 0; } int qat_adm_mailbox_send_init_me(struct qat_softc *sc) { struct fw_init_admin_req req; struct fw_init_admin_resp resp; memset(&req, 0, sizeof(req)); req.init_admin_cmd_id = FW_INIT_ME; return qat_adm_mailbox_send(sc, &req, &resp); } int qat_adm_mailbox_send_hb_timer(struct qat_softc *sc) { struct fw_init_admin_req req; struct fw_init_admin_resp resp; memset(&req, 0, sizeof(req)); req.init_admin_cmd_id = FW_HEARTBEAT_TIMER_SET; req.init_cfg_ptr = sc->sc_admin_comms.qadc_hb_dma.qdm_dma_seg.ds_addr; req.heartbeat_ticks = sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_HB_INTERVAL; return qat_adm_mailbox_send(sc, &req, &resp); } int qat_adm_mailbox_send_fw_status(struct qat_softc *sc) { int error; struct fw_init_admin_req req; struct fw_init_admin_resp resp; memset(&req, 0, sizeof(req)); req.init_admin_cmd_id = FW_STATUS_GET; error = qat_adm_mailbox_send(sc, &req, &resp); if (error) return error; return 0; } int qat_adm_mailbox_send_constants(struct qat_softc *sc) { struct fw_init_admin_req req; struct fw_init_admin_resp resp; memset(&req, 0, sizeof(req)); req.init_admin_cmd_id = FW_CONSTANTS_CFG; req.init_cfg_sz = 1024; req.init_cfg_ptr = sc->sc_admin_comms.qadc_const_tbl_dma.qdm_dma_seg.ds_addr; return qat_adm_mailbox_send(sc, &req, &resp); } int qat_adm_mailbox_send_init(struct qat_softc *sc) { int error; error = qat_adm_mailbox_send_init_me(sc); if (error) return error; error = qat_adm_mailbox_send_hb_timer(sc); if (error) return error; error = qat_adm_mailbox_send_fw_status(sc); if (error) return error; return qat_adm_mailbox_send_constants(sc); } int qat_arb_init(struct qat_softc *sc) { uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; uint32_t arb, i; const uint32_t *thd_2_arb_cfg; /* Service arb configured for 32 bytes responses and * ring flow control check enabled. */ for (arb = 0; arb < MAX_ARB; arb++) qat_arb_sarconfig_write_4(sc, arb, arb_cfg); /* Map worker threads to service arbiters */ sc->sc_hw.qhw_get_arb_mapping(sc, &thd_2_arb_cfg); if (!thd_2_arb_cfg) return EINVAL; for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) qat_arb_wrk_2_ser_map_write_4(sc, i, *(thd_2_arb_cfg + i)); return 0; } int qat_set_ssm_wdtimer(struct qat_softc *sc) { uint32_t timer; u_int mask; int i; timer = sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_SSM_WDT; for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { if (!(mask & 1)) continue; qat_misc_write_4(sc, SSMWDT(i), timer); qat_misc_write_4(sc, SSMWDTPKE(i), timer); } return 0; } int qat_check_slice_hang(struct qat_softc *sc) { int handled = 0; return handled; } static uint32_t qat_hw17_crypto_setup_cipher_ctrl(struct qat_crypto_desc *desc, struct qat_session *qs, uint32_t cd_blk_offset, struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice) { struct fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = (struct fw_cipher_cd_ctrl_hdr *)&req_tmpl->cd_ctrl; desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ; desc->qcd_cipher_offset = cd_blk_offset; cipher_cd_ctrl->cipher_state_sz = desc->qcd_cipher_blk_sz >> 3; cipher_cd_ctrl->cipher_key_sz = qs->qs_cipher_klen >> 3; cipher_cd_ctrl->cipher_cfg_offset = cd_blk_offset >> 3; FW_COMN_CURR_ID_SET(cipher_cd_ctrl, FW_SLICE_CIPHER); FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, next_slice); return roundup(sizeof(struct hw_cipher_config) + qs->qs_cipher_klen, 8); } static void qat_hw17_crypto_setup_cipher_cdesc(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct cryptop *crp, union hw_cipher_algo_blk *cipher) { const uint8_t *key; cipher->max.cipher_config.val = qat_crypto_load_cipher_session(desc, qs); if (crp != NULL && crp->crp_cipher_key != NULL) key = crp->crp_cipher_key; else key = qs->qs_cipher_key; memcpy(cipher->max.key, key, qs->qs_cipher_klen); } static uint32_t qat_hw17_crypto_setup_auth_ctrl(struct qat_crypto_desc *desc, struct qat_session *qs, uint32_t cd_blk_offset, struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice) { struct fw_auth_cd_ctrl_hdr *auth_cd_ctrl = (struct fw_auth_cd_ctrl_hdr *)&req_tmpl->cd_ctrl; struct qat_sym_hash_def const *hash_def; (void)qat_crypto_load_auth_session(desc, qs, &hash_def); auth_cd_ctrl->hash_cfg_offset = cd_blk_offset >> 3; auth_cd_ctrl->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED; auth_cd_ctrl->inner_res_sz = hash_def->qshd_alg->qshai_digest_len; auth_cd_ctrl->final_sz = hash_def->qshd_alg->qshai_sah->hashsize; auth_cd_ctrl->inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); auth_cd_ctrl->inner_state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8); auth_cd_ctrl->inner_state2_offset = auth_cd_ctrl->hash_cfg_offset + ((sizeof(struct hw_auth_setup) + auth_cd_ctrl->inner_state1_sz) >> 3); FW_COMN_CURR_ID_SET(auth_cd_ctrl, FW_SLICE_AUTH); FW_COMN_NEXT_ID_SET(auth_cd_ctrl, next_slice); desc->qcd_auth_sz = auth_cd_ctrl->final_sz; desc->qcd_auth_offset = cd_blk_offset; desc->qcd_gcm_aad_sz_offset1 = cd_blk_offset + offsetof(union hw_auth_algo_blk, max.state1) + auth_cd_ctrl->inner_state1_sz + AES_BLOCK_LEN; return roundup(auth_cd_ctrl->inner_state1_sz + auth_cd_ctrl->inner_state2_sz + sizeof(struct hw_auth_setup), 8); } static void qat_hw17_crypto_setup_auth_cdesc(const struct qat_crypto_desc *desc, const struct qat_session *qs, const struct cryptop *crp, union hw_auth_algo_blk *auth) { struct qat_sym_hash_def const *hash_def; uint8_t inner_state1_sz, *state1, *state2; const uint8_t *key; auth->max.inner_setup.auth_config.config = qat_crypto_load_auth_session(desc, qs, &hash_def); auth->max.inner_setup.auth_counter.counter = htobe32(hash_def->qshd_qat->qshqi_auth_counter); inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); state1 = auth->max.state1; state2 = auth->max.state1 + inner_state1_sz; switch (qs->qs_auth_algo) { case HW_AUTH_ALGO_GALOIS_128: key = NULL; if (crp != NULL && crp->crp_cipher_key != NULL) key = crp->crp_cipher_key; else if (qs->qs_cipher_key != NULL) key = qs->qs_cipher_key; if (key != NULL) { qat_crypto_gmac_precompute(desc, key, qs->qs_cipher_klen, hash_def, state2); } break; case HW_AUTH_ALGO_SHA1: case HW_AUTH_ALGO_SHA256: case HW_AUTH_ALGO_SHA384: case HW_AUTH_ALGO_SHA512: switch (qs->qs_auth_mode) { case HW_AUTH_MODE0: memcpy(state1, hash_def->qshd_alg->qshai_init_state, inner_state1_sz); /* Override for mode 0 hashes. */ auth->max.inner_setup.auth_counter.counter = 0; break; case HW_AUTH_MODE1: if (crp != NULL && crp->crp_auth_key != NULL) key = crp->crp_auth_key; else key = qs->qs_auth_key; if (key != NULL) { qat_crypto_hmac_precompute(desc, key, qs->qs_auth_klen, hash_def, state1, state2); } break; default: panic("%s: unhandled auth mode %d", __func__, qs->qs_auth_mode); } break; default: panic("%s: unhandled auth algorithm %d", __func__, qs->qs_auth_algo); } } static void qat_hw17_init_comn_req_hdr(struct qat_crypto_desc *desc, struct fw_la_bulk_req *req) { union fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; struct fw_comn_req_hdr *req_hdr = &req->comn_hdr; req_hdr->service_cmd_id = desc->qcd_cmd_id; req_hdr->hdr_flags = FW_COMN_VALID; req_hdr->service_type = FW_COMN_REQ_CPM_FW_LA; req_hdr->comn_req_flags = FW_COMN_FLAGS_BUILD( COMN_CD_FLD_TYPE_64BIT_ADR, COMN_PTR_TYPE_SGL); req_hdr->serv_specif_flags = 0; cd_pars->s.content_desc_addr = desc->qcd_desc_paddr; } void qat_hw17_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs, struct qat_crypto_desc *desc) { union hw_cipher_algo_blk *cipher; union hw_auth_algo_blk *auth; struct fw_la_bulk_req *req_tmpl; struct fw_comn_req_hdr *req_hdr; uint32_t cd_blk_offset = 0; int i; uint8_t *cd_blk_ptr; req_tmpl = (struct fw_la_bulk_req *)desc->qcd_req_cache; req_hdr = &req_tmpl->comn_hdr; cd_blk_ptr = desc->qcd_content_desc; memset(req_tmpl, 0, sizeof(struct fw_la_bulk_req)); qat_hw17_init_comn_req_hdr(desc, req_tmpl); for (i = 0; i < MAX_FW_SLICE; i++) { switch (desc->qcd_slices[i]) { case FW_SLICE_CIPHER: cipher = (union hw_cipher_algo_blk *)(cd_blk_ptr + cd_blk_offset); cd_blk_offset += qat_hw17_crypto_setup_cipher_ctrl(desc, qs, cd_blk_offset, req_tmpl, desc->qcd_slices[i + 1]); qat_hw17_crypto_setup_cipher_cdesc(desc, qs, NULL, cipher); break; case FW_SLICE_AUTH: auth = (union hw_auth_algo_blk *)(cd_blk_ptr + cd_blk_offset); cd_blk_offset += qat_hw17_crypto_setup_auth_ctrl(desc, qs, cd_blk_offset, req_tmpl, desc->qcd_slices[i + 1]); qat_hw17_crypto_setup_auth_cdesc(desc, qs, NULL, auth); req_hdr->serv_specif_flags |= FW_LA_RET_AUTH_RES; break; case FW_SLICE_DRAM_WR: i = MAX_FW_SLICE; /* end of chain */ break; default: MPASS(0); break; } } req_tmpl->cd_pars.s.content_desc_params_sz = roundup(cd_blk_offset, QAT_OPTIMAL_ALIGN) >> 3; if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) req_hdr->serv_specif_flags |= FW_LA_PROTO_GCM | FW_LA_GCM_IV_LEN_12_OCTETS; bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); } static void qat_hw17_crypto_req_setkey(const struct qat_crypto_desc *desc, const struct qat_session *qs, struct qat_sym_cookie *qsc, struct fw_la_bulk_req *bulk_req, const struct cryptop *crp) { union hw_auth_algo_blk *auth; union hw_cipher_algo_blk *cipher; uint8_t *cdesc; int i; cdesc = qsc->qsc_content_desc; memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE); for (i = 0; i < MAX_FW_SLICE; i++) { switch (desc->qcd_slices[i]) { case FW_SLICE_CIPHER: cipher = (union hw_cipher_algo_blk *) (cdesc + desc->qcd_cipher_offset); qat_hw17_crypto_setup_cipher_cdesc(desc, qs, crp, cipher); break; case FW_SLICE_AUTH: auth = (union hw_auth_algo_blk *) (cdesc + desc->qcd_auth_offset); qat_hw17_crypto_setup_auth_cdesc(desc, qs, crp, auth); break; case FW_SLICE_DRAM_WR: i = MAX_FW_SLICE; /* end of chain */ break; default: MPASS(0); } } bulk_req->cd_pars.s.content_desc_addr = qsc->qsc_content_desc_paddr; } void qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *qcb __unused, struct qat_session *qs, const struct qat_crypto_desc *desc, struct qat_sym_cookie *qsc, struct cryptop *crp) { struct qat_sym_bulk_cookie *qsbc; struct fw_la_bulk_req *bulk_req; struct fw_la_cipher_req_params *cipher_param; struct fw_la_auth_req_params *auth_param; bus_addr_t digest_paddr; uint32_t aad_sz, *aad_szp; uint8_t *req_params_ptr; enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id; - qsbc = &qsc->u.qsc_bulk_cookie; + qsbc = &qsc->qsc_bulk_cookie; bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg; memcpy(bulk_req, desc->qcd_req_cache, sizeof(struct fw_la_bulk_req)); bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc; bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr; - bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr; + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + bulk_req->comn_mid.dest_data_addr = + qsc->qsc_obuffer_list_desc_paddr; + } else { + bulk_req->comn_mid.dest_data_addr = + qsc->qsc_buffer_list_desc_paddr; + } if (__predict_false(crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL)) qat_hw17_crypto_req_setkey(desc, qs, qsc, bulk_req, crp); digest_paddr = 0; if (desc->qcd_auth_sz != 0) digest_paddr = qsc->qsc_auth_res_paddr; req_params_ptr = (uint8_t *)&bulk_req->serv_specif_rqpars; cipher_param = (struct fw_la_cipher_req_params *)req_params_ptr; auth_param = (struct fw_la_auth_req_params *) (req_params_ptr + sizeof(struct fw_la_cipher_req_params)); cipher_param->u.s.cipher_IV_ptr = qsc->qsc_iv_buf_paddr; /* * The SG list layout is a bit different for GCM and GMAC, it's simpler * to handle those cases separately. */ if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { if (cmd_id != FW_LA_CMD_AUTH) { /* * Don't fill out the cipher block if we're doing GMAC * only. */ cipher_param->cipher_offset = 0; cipher_param->cipher_length = crp->crp_payload_length; } auth_param->auth_off = 0; auth_param->auth_len = crp->crp_payload_length; auth_param->auth_res_addr = digest_paddr; auth_param->auth_res_sz = desc->qcd_auth_sz; auth_param->u1.aad_adr = crp->crp_aad_length > 0 ? qsc->qsc_gcm_aad_paddr : 0; auth_param->u2.aad_sz = roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN); auth_param->hash_state_sz = auth_param->u2.aad_sz >> 3; /* * Update the hash state block if necessary. This only occurs * when the AAD length changes between requests in a session and * is synchronized by qat_process(). */ aad_sz = htobe32(crp->crp_aad_length); aad_szp = (uint32_t *)( __DECONST(uint8_t *, desc->qcd_content_desc) + desc->qcd_gcm_aad_sz_offset1); if (__predict_false(*aad_szp != aad_sz)) { *aad_szp = aad_sz; bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); } } else { if (cmd_id != FW_LA_CMD_AUTH) { - cipher_param->cipher_offset = - crp->crp_aad_length == 0 ? 0 : - crp->crp_payload_start - crp->crp_aad_start; + if (crp->crp_aad_length == 0) { + cipher_param->cipher_offset = 0; + } else if (crp->crp_aad == NULL) { + cipher_param->cipher_offset = + crp->crp_payload_start - crp->crp_aad_start; + } else { + cipher_param->cipher_offset = + crp->crp_aad_length; + } cipher_param->cipher_length = crp->crp_payload_length; } if (cmd_id != FW_LA_CMD_CIPHER) { auth_param->auth_off = 0; auth_param->auth_len = crp->crp_payload_length + crp->crp_aad_length; auth_param->auth_res_addr = digest_paddr; auth_param->auth_res_sz = desc->qcd_auth_sz; auth_param->u1.aad_adr = 0; auth_param->u2.aad_sz = 0; auth_param->hash_state_sz = 0; } } } diff --git a/sys/dev/qat/qatreg.h b/sys/dev/qat/qatreg.h index 2a0be321b4f9..076354fc6108 100644 --- a/sys/dev/qat/qatreg.h +++ b/sys/dev/qat/qatreg.h @@ -1,1582 +1,1585 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qatreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef _DEV_PCI_QATREG_H_ #define _DEV_PCI_QATREG_H_ #define __BIT(__n) \ (((uintmax_t)(__n) >= NBBY * sizeof(uintmax_t)) ? 0 : \ ((uintmax_t)1 << (uintmax_t)((__n) & (NBBY * sizeof(uintmax_t) - 1)))) #define __BITS(__m, __n) \ ((__BIT(MAX((__m), (__n)) + 1) - 1) ^ (__BIT(MIN((__m), (__n))) - 1)) #define __LOWEST_SET_BIT(__mask) ((((__mask) - 1) & (__mask)) ^ (__mask)) #define __SHIFTOUT(__x, __mask) (((__x) & (__mask)) / __LOWEST_SET_BIT(__mask)) #define __SHIFTIN(__x, __mask) ((__x) * __LOWEST_SET_BIT(__mask)) /* Limits */ #define MAX_NUM_AE 0x10 #define MAX_NUM_ACCEL 6 #define MAX_AE 0x18 #define MAX_AE_CTX 8 #define MAX_ARB 4 #define MAX_USTORE_PER_SEG 0x8000 /* 16k * 2 */ #define MAX_USTORE MAX_USTORE_PER_SEG #define MAX_AE_PER_ACCEL 4 /* XXX */ #define MAX_BANK_PER_ACCEL 16 /* XXX */ #define MAX_RING_PER_BANK 16 #define MAX_XFER_REG 128 #define MAX_GPR_REG 128 #define MAX_NN_REG 128 #define MAX_LMEM_REG 1024 #define MAX_INP_STATE 16 #define MAX_CAM_REG 16 #define MAX_FIFO_QWADDR 160 #define MAX_EXEC_INST 100 #define UWORD_CPYBUF_SIZE 1024 /* micro-store copy buffer (bytes) */ #define INVLD_UWORD 0xffffffffffull /* invalid micro-instruction */ #define AEV2_PACKED_UWORD_BYTES 6 /* version 2 packed uword size */ #define UWORD_MASK 0xbffffffffffull /* micro-word mask without parity */ #define AE_ALL_CTX 0xff /* PCIe configuration space parameter */ #define NO_PCI_REG (-1) #define NO_REG_OFFSET 0 #define MAX_BARS 3 /* Fuse Control */ #define FUSECTL_REG 0x40 #define FUSECTL_MASK __BIT(31) #define LEGFUSE_REG 0x4c #define LEGFUSE_ACCEL_MASK_CIPHER_SLICE __BIT(0) #define LEGFUSE_ACCEL_MASK_AUTH_SLICE __BIT(1) #define LEGFUSE_ACCEL_MASK_PKE_SLICE __BIT(2) #define LEGFUSE_ACCEL_MASK_COMPRESS_SLICE __BIT(3) #define LEGFUSE_ACCEL_MASK_LZS_SLICE __BIT(4) #define LEGFUSE_ACCEL_MASK_EIA3_SLICE __BIT(5) #define LEGFUSE_ACCEL_MASK_SHA3_SLICE __BIT(6) /* -------------------------------------------------------------------------- */ /* PETRINGCSR region */ /* ETR parameters */ #define ETR_MAX_RINGS_PER_BANK 16 /* ETR registers */ #define ETR_RING_CONFIG 0x0000 #define ETR_RING_LBASE 0x0040 #define ETR_RING_UBASE 0x0080 #define ETR_RING_HEAD_OFFSET 0x00C0 #define ETR_RING_TAIL_OFFSET 0x0100 #define ETR_RING_STAT 0x0140 #define ETR_UO_STAT 0x0148 #define ETR_E_STAT 0x014C #define ETR_NE_STAT 0x0150 #define ETR_NF_STAT 0x0154 #define ETR_F_STAT 0x0158 #define ETR_C_STAT 0x015C #define ETR_INT_EN 0x016C #define ETR_INT_REG 0x0170 #define ETR_INT_SRCSEL 0x0174 #define ETR_INT_SRCSEL_2 0x0178 #define ETR_INT_COL_EN 0x017C #define ETR_INT_COL_CTL 0x0180 #define ETR_AP_NF_MASK 0x2000 #define ETR_AP_NF_DEST 0x2020 #define ETR_AP_NE_MASK 0x2040 #define ETR_AP_NE_DEST 0x2060 #define ETR_AP_DELAY 0x2080 /* ARB registers */ #define ARB_OFFSET 0x30000 #define ARB_REG_SIZE 0x4 #define ARB_WTR_SIZE 0x20 #define ARB_REG_SLOT 0x1000 #define ARB_WTR_OFFSET 0x010 #define ARB_RO_EN_OFFSET 0x090 #define ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ARB_RINGSRVARBEN_OFFSET 0x19c /* Ring Config */ #define ETR_RING_CONFIG_LATE_HEAD_POINTER_MODE __BIT(31) #define ETR_RING_CONFIG_NEAR_FULL_WM __BITS(14, 10) #define ETR_RING_CONFIG_NEAR_EMPTY_WM __BITS(9, 5) #define ETR_RING_CONFIG_RING_SIZE __BITS(4, 0) #define ETR_RING_CONFIG_NEAR_WM_0 0x00 #define ETR_RING_CONFIG_NEAR_WM_4 0x01 #define ETR_RING_CONFIG_NEAR_WM_8 0x02 #define ETR_RING_CONFIG_NEAR_WM_16 0x03 #define ETR_RING_CONFIG_NEAR_WM_32 0x04 #define ETR_RING_CONFIG_NEAR_WM_64 0x05 #define ETR_RING_CONFIG_NEAR_WM_128 0x06 #define ETR_RING_CONFIG_NEAR_WM_256 0x07 #define ETR_RING_CONFIG_NEAR_WM_512 0x08 #define ETR_RING_CONFIG_NEAR_WM_1K 0x09 #define ETR_RING_CONFIG_NEAR_WM_2K 0x0A #define ETR_RING_CONFIG_NEAR_WM_4K 0x0B #define ETR_RING_CONFIG_NEAR_WM_8K 0x0C #define ETR_RING_CONFIG_NEAR_WM_16K 0x0D #define ETR_RING_CONFIG_NEAR_WM_32K 0x0E #define ETR_RING_CONFIG_NEAR_WM_64K 0x0F #define ETR_RING_CONFIG_NEAR_WM_128K 0x10 #define ETR_RING_CONFIG_NEAR_WM_256K 0x11 #define ETR_RING_CONFIG_NEAR_WM_512K 0x12 #define ETR_RING_CONFIG_NEAR_WM_1M 0x13 #define ETR_RING_CONFIG_NEAR_WM_2M 0x14 #define ETR_RING_CONFIG_NEAR_WM_4M 0x15 #define ETR_RING_CONFIG_SIZE_64 0x00 #define ETR_RING_CONFIG_SIZE_128 0x01 #define ETR_RING_CONFIG_SIZE_256 0x02 #define ETR_RING_CONFIG_SIZE_512 0x03 #define ETR_RING_CONFIG_SIZE_1K 0x04 #define ETR_RING_CONFIG_SIZE_2K 0x05 #define ETR_RING_CONFIG_SIZE_4K 0x06 #define ETR_RING_CONFIG_SIZE_8K 0x07 #define ETR_RING_CONFIG_SIZE_16K 0x08 #define ETR_RING_CONFIG_SIZE_32K 0x09 #define ETR_RING_CONFIG_SIZE_64K 0x0A #define ETR_RING_CONFIG_SIZE_128K 0x0B #define ETR_RING_CONFIG_SIZE_256K 0x0C #define ETR_RING_CONFIG_SIZE_512K 0x0D #define ETR_RING_CONFIG_SIZE_1M 0x0E #define ETR_RING_CONFIG_SIZE_2M 0x0F #define ETR_RING_CONFIG_SIZE_4M 0x10 /* Default Ring Config is Nearly Full = Full and Nearly Empty = Empty */ #define ETR_RING_CONFIG_BUILD(size) \ (__SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0, \ ETR_RING_CONFIG_NEAR_FULL_WM) | \ __SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0, \ ETR_RING_CONFIG_NEAR_EMPTY_WM) | \ __SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE)) /* Response Ring Configuration */ #define ETR_RING_CONFIG_BUILD_RESP(size, wm_nf, wm_ne) \ (__SHIFTIN((wm_nf), ETR_RING_CONFIG_NEAR_FULL_WM) | \ __SHIFTIN((wm_ne), ETR_RING_CONFIG_NEAR_EMPTY_WM) | \ __SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE)) /* Ring Base */ #define ETR_RING_BASE_BUILD(addr, size) \ (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) #define ETR_INT_REG_CLEAR_MASK 0xffff /* Initial bank Interrupt Source mask */ #define ETR_INT_SRCSEL_MASK 0x44444444UL #define ETR_INT_SRCSEL_NEXT_OFFSET 4 #define ETR_RINGS_PER_INT_SRCSEL 8 #define ETR_INT_COL_CTL_ENABLE __BIT(31) #define ETR_AP_NF_MASK_INIT 0xAAAAAAAA #define ETR_AP_NE_MASK_INIT 0x55555555 /* Autopush destination AE bit */ #define ETR_AP_DEST_ENABLE __BIT(7) #define ETR_AP_DEST_AE __BITS(6, 2) #define ETR_AP_DEST_MAILBOX __BITS(1, 0) /* Autopush destination enable bit */ /* Autopush CSR Offset */ #define ETR_AP_BANK_OFFSET 4 /* Autopush maximum rings per bank */ #define ETR_MAX_RINGS_PER_AP_BANK 32 /* Maximum mailbox per acclerator */ #define ETR_MAX_MAILBOX_PER_ACCELERATOR 4 /* Maximum AEs per mailbox */ #define ETR_MAX_AE_PER_MAILBOX 4 /* Macro to get the ring's autopush bank number */ #define ETR_RING_AP_BANK_NUMBER(ring) ((ring) >> 5) /* Macro to get the ring's autopush mailbox number */ #define ETR_RING_AP_MAILBOX_NUMBER(ring) \ (ETR_RING_AP_BANK_NUMBER(ring) % ETR_MAX_MAILBOX_PER_ACCELERATOR) /* Macro to get the ring number in the autopush bank */ #define ETR_RING_NUMBER_IN_AP_BANK(ring) \ ((ring) % ETR_MAX_RINGS_PER_AP_BANK) #define ETR_RING_EMPTY_ENTRY_SIG (0x7F7F7F7F) /* -------------------------------------------------------------------------- */ /* CAP_GLOBAL_CTL region */ #define FCU_CTRL 0x8c0 #define FCU_CTRL_CMD_NOOP 0 #define FCU_CTRL_CMD_AUTH 1 #define FCU_CTRL_CMD_LOAD 2 #define FCU_CTRL_CMD_START 3 #define FCU_CTRL_AE __BITS(8, 31) #define FCU_STATUS 0x8c4 #define FCU_STATUS_STS __BITS(0, 2) #define FCU_STATUS_STS_NO 0 #define FCU_STATUS_STS_VERI_DONE 1 #define FCU_STATUS_STS_LOAD_DONE 2 #define FCU_STATUS_STS_VERI_FAIL 3 #define FCU_STATUS_STS_LOAD_FAIL 4 #define FCU_STATUS_STS_BUSY 5 #define FCU_STATUS_AUTHFWLD __BIT(8) #define FCU_STATUS_DONE __BIT(9) #define FCU_STATUS_LOADED_AE __BITS(22, 31) #define FCU_STATUS1 0x8c8 #define FCU_DRAM_ADDR_LO 0x8cc #define FCU_DRAM_ADDR_HI 0x8d0 #define FCU_RAMBASE_ADDR_HI 0x8d4 #define FCU_RAMBASE_ADDR_LO 0x8d8 #define FW_AUTH_WAIT_PERIOD 10 #define FW_AUTH_MAX_RETRY 300 #define CAP_GLOBAL_CTL_BASE 0xa00 #define CAP_GLOBAL_CTL_MISC CAP_GLOBAL_CTL_BASE + 0x04 #define CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN __BIT(7) #define CAP_GLOBAL_CTL_RESET CAP_GLOBAL_CTL_BASE + 0x0c #define CAP_GLOBAL_CTL_RESET_MASK __BITS(31, 26) #define CAP_GLOBAL_CTL_RESET_ACCEL_MASK __BITS(25, 20) #define CAP_GLOBAL_CTL_RESET_AE_MASK __BITS(19, 0) #define CAP_GLOBAL_CTL_CLK_EN CAP_GLOBAL_CTL_BASE + 0x50 #define CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK __BITS(25, 20) #define CAP_GLOBAL_CTL_CLK_EN_AE_MASK __BITS(19, 0) /* -------------------------------------------------------------------------- */ /* AE region */ #define UPC_MASK 0x1ffff #define USTORE_SIZE QAT_16K #define AE_LOCAL_AE_MASK __BITS(31, 12) #define AE_LOCAL_CSR_MASK __BITS(9, 0) /* AE_LOCAL registers */ /* Control Store Address Register */ #define USTORE_ADDRESS 0x000 #define USTORE_ADDRESS_ECS __BIT(31) #define USTORE_ECC_BIT_0 44 #define USTORE_ECC_BIT_1 45 #define USTORE_ECC_BIT_2 46 #define USTORE_ECC_BIT_3 47 #define USTORE_ECC_BIT_4 48 #define USTORE_ECC_BIT_5 49 #define USTORE_ECC_BIT_6 50 /* Control Store Data Lower Register */ #define USTORE_DATA_LOWER 0x004 /* Control Store Data Upper Register */ #define USTORE_DATA_UPPER 0x008 /* Control Store Error Status Register */ #define USTORE_ERROR_STATUS 0x00c /* Arithmetic Logic Unit Output Register */ #define ALU_OUT 0x010 /* Context Arbiter Control Register */ #define CTX_ARB_CNTL 0x014 #define CTX_ARB_CNTL_INIT 0x00000000 /* Context Enables Register */ #define CTX_ENABLES 0x018 #define CTX_ENABLES_INIT 0 #define CTX_ENABLES_INUSE_CONTEXTS __BIT(31) #define CTX_ENABLES_CNTL_STORE_PARITY_ERROR __BIT(29) #define CTX_ENABLES_CNTL_STORE_PARITY_ENABLE __BIT(28) #define CTX_ENABLES_BREAKPOINT __BIT(27) #define CTX_ENABLES_PAR_ERR __BIT(25) #define CTX_ENABLES_NN_MODE __BIT(20) #define CTX_ENABLES_NN_RING_EMPTY __BIT(18) #define CTX_ENABLES_LMADDR_1_GLOBAL __BIT(17) #define CTX_ENABLES_LMADDR_0_GLOBAL __BIT(16) #define CTX_ENABLES_ENABLE __BITS(15,8) #define CTX_ENABLES_IGNORE_W1C_MASK \ (~(CTX_ENABLES_PAR_ERR | \ CTX_ENABLES_BREAKPOINT | \ CTX_ENABLES_CNTL_STORE_PARITY_ERROR)) /* cycles from CTX_ENABLE high to CTX entering executing state */ #define CYCLES_FROM_READY2EXE 8 /* Condition Code Enable Register */ #define CC_ENABLE 0x01c #define CC_ENABLE_INIT 0x2000 /* CSR Context Pointer Register */ #define CSR_CTX_POINTER 0x020 #define CSR_CTX_POINTER_CONTEXT __BITS(2,0) /* Register Error Status Register */ #define REG_ERROR_STATUS 0x030 /* Indirect Context Status Register */ #define CTX_STS_INDIRECT 0x040 #define CTX_STS_INDIRECT_UPC_INIT 0x00000000 /* Active Context Status Register */ #define ACTIVE_CTX_STATUS 0x044 #define ACTIVE_CTX_STATUS_ABO __BIT(31) #define ACTIVE_CTX_STATUS_ACNO __BITS(0, 2) /* Indirect Context Signal Events Register */ #define CTX_SIG_EVENTS_INDIRECT 0x048 #define CTX_SIG_EVENTS_INDIRECT_INIT 0x00000001 /* Active Context Signal Events Register */ #define CTX_SIG_EVENTS_ACTIVE 0x04c /* Indirect Context Wakeup Events Register */ #define CTX_WAKEUP_EVENTS_INDIRECT 0x050 #define CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY 0x00000001 #define CTX_WAKEUP_EVENTS_INDIRECT_SLEEP 0x00010000 #define CTX_WAKEUP_EVENTS_INDIRECT_INIT 0x00000001 /* Active Context Wakeup Events Register */ #define CTX_WAKEUP_EVENTS_ACTIVE 0x054 /* Indirect Context Future Count Register */ #define CTX_FUTURE_COUNT_INDIRECT 0x058 /* Active Context Future Count Register */ #define CTX_FUTURE_COUNT_ACTIVE 0x05c /* Indirect Local Memory Address 0 Register */ #define LM_ADDR_0_INDIRECT 0x060 /* Active Local Memory Address 0 Register */ #define LM_ADDR_0_ACTIVE 0x064 /* Indirect Local Memory Address 1 Register */ #define LM_ADDR_1_INDIRECT 0x068 /* Active Local Memory Address 1 Register */ #define LM_ADDR_1_ACTIVE 0x06c /* Byte Index Register */ #define BYTE_INDEX 0x070 /* Indirect Local Memory Address 0 Byte Index Register */ #define INDIRECT_LM_ADDR_0_BYTE_INDEX 0x0e0 /* Active Local Memory Address 0 Byte Index Register */ #define ACTIVE_LM_ADDR_0_BYTE_INDEX 0x0e4 /* Indirect Local Memory Address 1 Byte Index Register */ #define INDIRECT_LM_ADDR_1_BYTE_INDEX 0x0e8 /* Active Local Memory Address 1 Byte Index Register */ #define ACTIVE_LM_ADDR_1_BYTE_INDEX 0x0ec /* Transfer Index Concatenated with Byte Index Register */ #define T_INDEX_BYTE_INDEX 0x0f4 /* Transfer Index Register */ #define T_INDEX 0x074 /* Indirect Future Count Signal Signal Register */ #define FUTURE_COUNT_SIGNAL_INDIRECT 0x078 /* Active Context Future Count Register */ #define FUTURE_COUNT_SIGNAL_ACTIVE 0x07c /* Next Neighbor Put Register */ #define NN_PUT 0x080 /* Next Neighbor Get Register */ #define NN_GET 0x084 /* Timestamp Low Register */ #define TIMESTAMP_LOW 0x0c0 /* Timestamp High Register */ #define TIMESTAMP_HIGH 0x0c4 /* Next Neighbor Signal Register */ #define NEXT_NEIGHBOR_SIGNAL 0x100 /* Previous Neighbor Signal Register */ #define PREV_NEIGHBOR_SIGNAL 0x104 /* Same AccelEngine Signal Register */ #define SAME_AE_SIGNAL 0x108 /* Cyclic Redundancy Check Remainder Register */ #define CRC_REMAINDER 0x140 /* Profile Count Register */ #define PROFILE_COUNT 0x144 /* Pseudorandom Number Register */ #define PSEUDO_RANDOM_NUMBER 0x148 /* Signature Enable Register */ #define SIGNATURE_ENABLE 0x150 /* Miscellaneous Control Register */ #define AE_MISC_CONTROL 0x160 #define AE_MISC_CONTROL_PARITY_ENABLE __BIT(24) #define AE_MISC_CONTROL_FORCE_BAD_PARITY __BIT(23) #define AE_MISC_CONTROL_ONE_CTX_RELOAD __BIT(22) #define AE_MISC_CONTROL_CS_RELOAD __BITS(21, 20) #define AE_MISC_CONTROL_SHARE_CS __BIT(2) /* Control Store Address 1 Register */ #define USTORE_ADDRESS1 0x158 /* Local CSR Status Register */ #define LOCAL_CSR_STATUS 0x180 #define LOCAL_CSR_STATUS_STATUS 0x1 /* NULL Register */ #define NULL_CSR 0x3fc /* AE_XFER macros */ #define AE_XFER_AE_MASK __BITS(31, 12) #define AE_XFER_CSR_MASK __BITS(9, 2) #define AEREG_BAD_REGADDR 0xffff /* bad register address */ /* -------------------------------------------------------------------------- */ #define SSMWDT(i) ((i) * 0x4000 + 0x54) #define SSMWDTPKE(i) ((i) * 0x4000 + 0x58) #define INTSTATSSM(i) ((i) * 0x4000 + 0x04) #define INTSTATSSM_SHANGERR __BIT(13) #define PPERR(i) ((i) * 0x4000 + 0x08) #define PPERRID(i) ((i) * 0x4000 + 0x0C) #define CERRSSMSH(i) ((i) * 0x4000 + 0x10) #define UERRSSMSH(i) ((i) * 0x4000 + 0x18) #define UERRSSMSHAD(i) ((i) * 0x4000 + 0x1C) #define SLICEHANGSTATUS(i) ((i) * 0x4000 + 0x4C) #define SLICE_HANG_AUTH0_MASK __BIT(0) #define SLICE_HANG_AUTH1_MASK __BIT(1) #define SLICE_HANG_CPHR0_MASK __BIT(4) #define SLICE_HANG_CPHR1_MASK __BIT(5) #define SLICE_HANG_CMP0_MASK __BIT(8) #define SLICE_HANG_CMP1_MASK __BIT(9) #define SLICE_HANG_XLT0_MASK __BIT(12) #define SLICE_HANG_XLT1_MASK __BIT(13) #define SLICE_HANG_MMP0_MASK __BIT(16) #define SLICE_HANG_MMP1_MASK __BIT(17) #define SLICE_HANG_MMP2_MASK __BIT(18) #define SLICE_HANG_MMP3_MASK __BIT(19) #define SLICE_HANG_MMP4_MASK __BIT(20) #define SHINTMASKSSM(i) ((i) * 0x4000 + 0x1018) #define ENABLE_SLICE_HANG 0x000000 #define MAX_MMP (5) #define MMP_BASE(i) ((i) * 0x1000 % 0x3800) #define CERRSSMMMP(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x380) #define UERRSSMMMP(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x388) #define UERRSSMMMPAD(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x38C) #define CPP_CFC_ERR_STATUS (0x30000 + 0xC04) #define CPP_CFC_ERR_PPID (0x30000 + 0xC08) #define ERRSOU0 (0x3A000 + 0x00) #define ERRSOU1 (0x3A000 + 0x04) #define ERRSOU2 (0x3A000 + 0x08) #define ERRSOU3 (0x3A000 + 0x0C) #define ERRSOU4 (0x3A000 + 0xD0) #define ERRSOU5 (0x3A000 + 0xD8) #define ERRMSK0 (0x3A000 + 0x10) #define ERRMSK1 (0x3A000 + 0x14) #define ERRMSK2 (0x3A000 + 0x18) #define ERRMSK3 (0x3A000 + 0x1C) #define ERRMSK4 (0x3A000 + 0xD4) #define ERRMSK5 (0x3A000 + 0xDC) #define EMSK3_CPM0_MASK __BIT(2) #define EMSK3_CPM1_MASK __BIT(3) #define EMSK5_CPM2_MASK __BIT(16) #define EMSK5_CPM3_MASK __BIT(17) #define EMSK5_CPM4_MASK __BIT(18) #define RICPPINTSTS (0x3A000 + 0x114) #define RIERRPUSHID (0x3A000 + 0x118) #define RIERRPULLID (0x3A000 + 0x11C) #define TICPPINTSTS (0x3A400 + 0x13C) #define TIERRPUSHID (0x3A400 + 0x140) #define TIERRPULLID (0x3A400 + 0x144) #define SECRAMUERR (0x3AC00 + 0x04) #define SECRAMUERRAD (0x3AC00 + 0x0C) #define CPPMEMTGTERR (0x3AC00 + 0x10) #define ERRPPID (0x3AC00 + 0x14) #define ADMINMSGUR 0x3a574 #define ADMINMSGLR 0x3a578 #define MAILBOX_BASE 0x20970 #define MAILBOX_STRIDE 0x1000 #define ADMINMSG_LEN 32 /* -------------------------------------------------------------------------- */ static const uint8_t mailbox_const_tab[1024] __aligned(1024) = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, 0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; /* -------------------------------------------------------------------------- */ /* Microcode */ /* Clear GPR of AE */ static const uint64_t ae_clear_gprs_inst[] = { 0x0F0000C0000ull, /* .0 l0000!val = 0 ; immed[l0000!val, 0x0] */ 0x0F000000380ull, /* .1 l0000!count = 128 ; immed[l0000!count, 0x80] */ 0x0D805000011ull, /* .2 br!=ctx[0, ctx_init#] */ 0x0FC082C0300ull, /* .3 local_csr_wr[nn_put, 0] */ 0x0F0000C0300ull, /* .4 nop */ 0x0F0000C0300ull, /* .5 nop */ 0x0F0000C0300ull, /* .6 nop */ 0x0F0000C0300ull, /* .7 nop */ 0x0A0643C0000ull, /* .8 init_nn#:alu[*n$index++, --, b, l0000!val] */ 0x0BAC0000301ull, /* .9 alu[l0000!count, l0000!count, -, 1] */ 0x0D802000101ull, /* .10 bne[init_nn#] */ 0x0F0000C0001ull, /* .11 l0000!indx = 0 ; immed[l0000!indx, 0x0] */ 0x0FC066C0001ull, /* .12 local_csr_wr[active_lm_addr_0, l0000!indx]; * put indx to lm_addr */ 0x0F0000C0300ull, /* .13 nop */ 0x0F0000C0300ull, /* .14 nop */ 0x0F0000C0300ull, /* .15 nop */ 0x0F000400300ull, /* .16 l0000!count = 1024 ; immed[l0000!count, 0x400] */ 0x0A0610C0000ull, /* .17 init_lm#:alu[*l$index0++, --, b, l0000!val] */ 0x0BAC0000301ull, /* .18 alu[l0000!count, l0000!count, -, 1] */ 0x0D804400101ull, /* .19 bne[init_lm#] */ 0x0A0580C0000ull, /* .20 ctx_init#:alu[$l0000!xfers[0], --, b, l0000!val] */ 0x0A0581C0000ull, /* .21 alu[$l0000!xfers[1], --, b, l0000!val] */ 0x0A0582C0000ull, /* .22 alu[$l0000!xfers[2], --, b, l0000!val] */ 0x0A0583C0000ull, /* .23 alu[$l0000!xfers[3], --, b, l0000!val] */ 0x0A0584C0000ull, /* .24 alu[$l0000!xfers[4], --, b, l0000!val] */ 0x0A0585C0000ull, /* .25 alu[$l0000!xfers[5], --, b, l0000!val] */ 0x0A0586C0000ull, /* .26 alu[$l0000!xfers[6], --, b, l0000!val] */ 0x0A0587C0000ull, /* .27 alu[$l0000!xfers[7], --, b, l0000!val] */ 0x0A0588C0000ull, /* .28 alu[$l0000!xfers[8], --, b, l0000!val] */ 0x0A0589C0000ull, /* .29 alu[$l0000!xfers[9], --, b, l0000!val] */ 0x0A058AC0000ull, /* .30 alu[$l0000!xfers[10], --, b, l0000!val] */ 0x0A058BC0000ull, /* .31 alu[$l0000!xfers[11], --, b, l0000!val] */ 0x0A058CC0000ull, /* .32 alu[$l0000!xfers[12], --, b, l0000!val] */ 0x0A058DC0000ull, /* .33 alu[$l0000!xfers[13], --, b, l0000!val] */ 0x0A058EC0000ull, /* .34 alu[$l0000!xfers[14], --, b, l0000!val] */ 0x0A058FC0000ull, /* .35 alu[$l0000!xfers[15], --, b, l0000!val] */ 0x0A05C0C0000ull, /* .36 alu[$l0000!xfers[16], --, b, l0000!val] */ 0x0A05C1C0000ull, /* .37 alu[$l0000!xfers[17], --, b, l0000!val] */ 0x0A05C2C0000ull, /* .38 alu[$l0000!xfers[18], --, b, l0000!val] */ 0x0A05C3C0000ull, /* .39 alu[$l0000!xfers[19], --, b, l0000!val] */ 0x0A05C4C0000ull, /* .40 alu[$l0000!xfers[20], --, b, l0000!val] */ 0x0A05C5C0000ull, /* .41 alu[$l0000!xfers[21], --, b, l0000!val] */ 0x0A05C6C0000ull, /* .42 alu[$l0000!xfers[22], --, b, l0000!val] */ 0x0A05C7C0000ull, /* .43 alu[$l0000!xfers[23], --, b, l0000!val] */ 0x0A05C8C0000ull, /* .44 alu[$l0000!xfers[24], --, b, l0000!val] */ 0x0A05C9C0000ull, /* .45 alu[$l0000!xfers[25], --, b, l0000!val] */ 0x0A05CAC0000ull, /* .46 alu[$l0000!xfers[26], --, b, l0000!val] */ 0x0A05CBC0000ull, /* .47 alu[$l0000!xfers[27], --, b, l0000!val] */ 0x0A05CCC0000ull, /* .48 alu[$l0000!xfers[28], --, b, l0000!val] */ 0x0A05CDC0000ull, /* .49 alu[$l0000!xfers[29], --, b, l0000!val] */ 0x0A05CEC0000ull, /* .50 alu[$l0000!xfers[30], --, b, l0000!val] */ 0x0A05CFC0000ull, /* .51 alu[$l0000!xfers[31], --, b, l0000!val] */ 0x0A0400C0000ull, /* .52 alu[l0000!gprega[0], --, b, l0000!val] */ 0x0B0400C0000ull, /* .53 alu[l0000!gpregb[0], --, b, l0000!val] */ 0x0A0401C0000ull, /* .54 alu[l0000!gprega[1], --, b, l0000!val] */ 0x0B0401C0000ull, /* .55 alu[l0000!gpregb[1], --, b, l0000!val] */ 0x0A0402C0000ull, /* .56 alu[l0000!gprega[2], --, b, l0000!val] */ 0x0B0402C0000ull, /* .57 alu[l0000!gpregb[2], --, b, l0000!val] */ 0x0A0403C0000ull, /* .58 alu[l0000!gprega[3], --, b, l0000!val] */ 0x0B0403C0000ull, /* .59 alu[l0000!gpregb[3], --, b, l0000!val] */ 0x0A0404C0000ull, /* .60 alu[l0000!gprega[4], --, b, l0000!val] */ 0x0B0404C0000ull, /* .61 alu[l0000!gpregb[4], --, b, l0000!val] */ 0x0A0405C0000ull, /* .62 alu[l0000!gprega[5], --, b, l0000!val] */ 0x0B0405C0000ull, /* .63 alu[l0000!gpregb[5], --, b, l0000!val] */ 0x0A0406C0000ull, /* .64 alu[l0000!gprega[6], --, b, l0000!val] */ 0x0B0406C0000ull, /* .65 alu[l0000!gpregb[6], --, b, l0000!val] */ 0x0A0407C0000ull, /* .66 alu[l0000!gprega[7], --, b, l0000!val] */ 0x0B0407C0000ull, /* .67 alu[l0000!gpregb[7], --, b, l0000!val] */ 0x0A0408C0000ull, /* .68 alu[l0000!gprega[8], --, b, l0000!val] */ 0x0B0408C0000ull, /* .69 alu[l0000!gpregb[8], --, b, l0000!val] */ 0x0A0409C0000ull, /* .70 alu[l0000!gprega[9], --, b, l0000!val] */ 0x0B0409C0000ull, /* .71 alu[l0000!gpregb[9], --, b, l0000!val] */ 0x0A040AC0000ull, /* .72 alu[l0000!gprega[10], --, b, l0000!val] */ 0x0B040AC0000ull, /* .73 alu[l0000!gpregb[10], --, b, l0000!val] */ 0x0A040BC0000ull, /* .74 alu[l0000!gprega[11], --, b, l0000!val] */ 0x0B040BC0000ull, /* .75 alu[l0000!gpregb[11], --, b, l0000!val] */ 0x0A040CC0000ull, /* .76 alu[l0000!gprega[12], --, b, l0000!val] */ 0x0B040CC0000ull, /* .77 alu[l0000!gpregb[12], --, b, l0000!val] */ 0x0A040DC0000ull, /* .78 alu[l0000!gprega[13], --, b, l0000!val] */ 0x0B040DC0000ull, /* .79 alu[l0000!gpregb[13], --, b, l0000!val] */ 0x0A040EC0000ull, /* .80 alu[l0000!gprega[14], --, b, l0000!val] */ 0x0B040EC0000ull, /* .81 alu[l0000!gpregb[14], --, b, l0000!val] */ 0x0A040FC0000ull, /* .82 alu[l0000!gprega[15], --, b, l0000!val] */ 0x0B040FC0000ull, /* .83 alu[l0000!gpregb[15], --, b, l0000!val] */ 0x0D81581C010ull, /* .84 br=ctx[7, exit#] */ 0x0E000010000ull, /* .85 ctx_arb[kill], any */ 0x0E000010000ull, /* .86 exit#:ctx_arb[kill], any */ }; static const uint64_t ae_inst_4b[] = { 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, l0000!indx]; put indx to lm_addr */ 0x0F0000C0300ull, /* .5 nop */ 0x0F0000C0300ull, /* .6 nop */ 0x0F0000C0300ull, /* .7 nop */ 0x0A021000000ull, /* .8 alu[*l$index0++, --, b, l0000!myvalue] */ }; static const uint64_t ae_inst_1b[] = { 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, l0000!indx]; put indx to lm_addr */ 0x0F0000C0300ull, /* .5 nop */ 0x0F0000C0300ull, /* .6 nop */ 0x0F0000C0300ull, /* .7 nop */ 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ 0x09080000200ull, /* .9 alu_shf[l0000!myvalue, --, b, l0000!myvalue, <<24 ] */ 0x08180280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<8 ] */ 0x08080280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>8 ] */ 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ }; static const uint64_t ae_inst_2b[] = { 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, l0000!indx]; put indx to lm_addr */ 0x0F0000C0300ull, /* .5 nop */ 0x0F0000C0300ull, /* .6 nop */ 0x0F0000C0300ull, /* .7 nop */ 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ 0x09100000200ull, /* .9 alu_shf[l0000!myvalue, --, b, l0000!myvalue, <<16 ] */ 0x08100280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<16 ] */ 0x08100280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>16 ] */ 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ }; static const uint64_t ae_inst_3b[] = { 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, l0000!indx]; put indx to lm_addr */ 0x0F0000C0300ull, /* .5 nop */ 0x0F0000C0300ull, /* .6 nop */ 0x0F0000C0300ull, /* .7 nop */ 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ 0x09180000200ull, /* .9 alu_shf[l0000!myvalue, --, b, l0000!myvalue, <<8 ] */ 0x08080280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<24 ] */ 0x08180280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>24 ] */ 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ }; /* micro-instr fixup */ #define INSERT_IMMED_GPRA_CONST(inst, const_val) \ inst = (inst & 0xFFFF00C03FFull) | \ ((((const_val) << 12) & 0x0FF00000ull) | \ (((const_val) << 10) & 0x0003FC00ull)) #define INSERT_IMMED_GPRB_CONST(inst, const_val) \ inst = (inst & 0xFFFF00FFF00ull) | \ ((((const_val) << 12) & 0x0FF00000ull) | \ (((const_val) << 0) & 0x000000FFull)) enum aereg_type { AEREG_NO_DEST, /* no destination */ AEREG_GPA_REL, /* general-purpose A register under relative mode */ AEREG_GPA_ABS, /* general-purpose A register under absolute mode */ AEREG_GPB_REL, /* general-purpose B register under relative mode */ AEREG_GPB_ABS, /* general-purpose B register under absolute mode */ AEREG_SR_REL, /* sram register under relative mode */ AEREG_SR_RD_REL, /* sram read register under relative mode */ AEREG_SR_WR_REL, /* sram write register under relative mode */ AEREG_SR_ABS, /* sram register under absolute mode */ AEREG_SR_RD_ABS, /* sram read register under absolute mode */ AEREG_SR_WR_ABS, /* sram write register under absolute mode */ AEREG_SR0_SPILL, /* sram0 spill register */ AEREG_SR1_SPILL, /* sram1 spill register */ AEREG_SR2_SPILL, /* sram2 spill register */ AEREG_SR3_SPILL, /* sram3 spill register */ AEREG_SR0_MEM_ADDR, /* sram0 memory address register */ AEREG_SR1_MEM_ADDR, /* sram1 memory address register */ AEREG_SR2_MEM_ADDR, /* sram2 memory address register */ AEREG_SR3_MEM_ADDR, /* sram3 memory address register */ AEREG_DR_REL, /* dram register under relative mode */ AEREG_DR_RD_REL, /* dram read register under relative mode */ AEREG_DR_WR_REL, /* dram write register under relative mode */ AEREG_DR_ABS, /* dram register under absolute mode */ AEREG_DR_RD_ABS, /* dram read register under absolute mode */ AEREG_DR_WR_ABS, /* dram write register under absolute mode */ AEREG_DR_MEM_ADDR, /* dram memory address register */ AEREG_LMEM, /* local memory */ AEREG_LMEM0, /* local memory bank0 */ AEREG_LMEM1, /* local memory bank1 */ AEREG_LMEM_SPILL, /* local memory spill */ AEREG_LMEM_ADDR, /* local memory address */ AEREG_NEIGH_REL, /* next neighbour register under relative mode */ AEREG_NEIGH_INDX, /* next neighbour register under index mode */ AEREG_SIG_REL, /* signal register under relative mode */ AEREG_SIG_INDX, /* signal register under index mode */ AEREG_SIG_DOUBLE, /* signal register */ AEREG_SIG_SINGLE, /* signal register */ AEREG_SCRATCH_MEM_ADDR, /* scratch memory address */ AEREG_UMEM0, /* ustore memory bank0 */ AEREG_UMEM1, /* ustore memory bank1 */ AEREG_UMEM_SPILL, /* ustore memory spill */ AEREG_UMEM_ADDR, /* ustore memory address */ AEREG_DR1_MEM_ADDR, /* dram segment1 address */ AEREG_SR0_IMPORTED, /* sram segment0 imported data */ AEREG_SR1_IMPORTED, /* sram segment1 imported data */ AEREG_SR2_IMPORTED, /* sram segment2 imported data */ AEREG_SR3_IMPORTED, /* sram segment3 imported data */ AEREG_DR_IMPORTED, /* dram segment0 imported data */ AEREG_DR1_IMPORTED, /* dram segment1 imported data */ AEREG_SCRATCH_IMPORTED, /* scratch imported data */ AEREG_XFER_RD_ABS, /* transfer read register under absolute mode */ AEREG_XFER_WR_ABS, /* transfer write register under absolute mode */ AEREG_CONST_VALUE, /* const alue */ AEREG_ADDR_TAKEN, /* address taken */ AEREG_OPTIMIZED_AWAY, /* optimized away */ AEREG_SHRAM_ADDR, /* shared ram0 address */ AEREG_SHRAM1_ADDR, /* shared ram1 address */ AEREG_SHRAM2_ADDR, /* shared ram2 address */ AEREG_SHRAM3_ADDR, /* shared ram3 address */ AEREG_SHRAM4_ADDR, /* shared ram4 address */ AEREG_SHRAM5_ADDR, /* shared ram5 address */ AEREG_ANY = 0xffff /* any register */ }; #define AEREG_SR_INDX AEREG_SR_ABS /* sram transfer register under index mode */ #define AEREG_DR_INDX AEREG_DR_ABS /* dram transfer register under index mode */ #define AEREG_NEIGH_ABS AEREG_NEIGH_INDX /* next neighbor register under absolute mode */ #define QAT_2K 0x0800 #define QAT_4K 0x1000 #define QAT_6K 0x1800 #define QAT_8K 0x2000 #define QAT_16K 0x4000 #define MOF_OBJ_ID_LEN 8 #define MOF_FID 0x00666f6d #define MOF_MIN_VER 0x1 #define MOF_MAJ_VER 0x0 #define SYM_OBJS "SYM_OBJS" /* symbol object string */ #define UOF_OBJS "UOF_OBJS" /* uof object string */ #define SUOF_OBJS "SUF_OBJS" /* suof object string */ #define SUOF_IMAG "SUF_IMAG" /* suof chunk ID string */ #define UOF_STRT "UOF_STRT" /* string table section ID */ #define UOF_GTID "UOF_GTID" /* GTID section ID */ #define UOF_IMAG "UOF_IMAG" /* image section ID */ #define UOF_IMEM "UOF_IMEM" /* import section ID */ #define UOF_MSEG "UOF_MSEG" /* memory section ID */ #define CRC_POLY 0x1021 #define CRC_WIDTH 16 #define CRC_BITMASK(x) (1L << (x)) #define CRC_WIDTHMASK(width) ((((1L<<(width-1))-1L)<<1)|1L) struct mof_file_hdr { u_int mfh_fid; u_int mfh_csum; char mfh_min_ver; char mfh_maj_ver; u_short mfh_reserved; u_short mfh_max_chunks; u_short mfh_num_chunks; }; struct mof_file_chunk_hdr { char mfch_id[MOF_OBJ_ID_LEN]; uint64_t mfch_offset; uint64_t mfch_size; }; struct mof_uof_hdr { u_short muh_max_chunks; u_short muh_num_chunks; u_int muh_reserved; }; struct mof_uof_chunk_hdr { char much_id[MOF_OBJ_ID_LEN]; /* should be UOF_IMAG */ uint64_t much_offset; /* uof image */ uint64_t much_size; /* uof image size */ u_int much_name; /* uof name string-table offset */ u_int much_reserved; }; #define UOF_MAX_NUM_OF_AE 16 /* maximum number of AE */ #define UOF_OBJ_ID_LEN 8 /* length of object ID */ #define UOF_FIELD_POS_SIZE 12 /* field postion size */ #define MIN_UOF_SIZE 24 /* minimum .uof file size */ #define UOF_FID 0xc6c2 /* uof magic number */ #define UOF_MIN_VER 0x11 #define UOF_MAJ_VER 0x4 struct uof_file_hdr { u_short ufh_id; /* file id and endian indicator */ u_short ufh_reserved1; /* reserved for future use */ char ufh_min_ver; /* file format minor version */ char ufh_maj_ver; /* file format major version */ u_short ufh_reserved2; /* reserved for future use */ u_short ufh_max_chunks; /* max chunks in file */ u_short ufh_num_chunks; /* num of actual chunks */ }; struct uof_file_chunk_hdr { char ufch_id[UOF_OBJ_ID_LEN]; /* chunk identifier */ u_int ufch_csum; /* chunk checksum */ u_int ufch_offset; /* offset of the chunk in the file */ u_int ufch_size; /* size of the chunk */ }; struct uof_obj_hdr { u_int uoh_cpu_type; /* CPU type */ u_short uoh_min_cpu_ver; /* starting CPU version */ u_short uoh_max_cpu_ver; /* ending CPU version */ short uoh_max_chunks; /* max chunks in chunk obj */ short uoh_num_chunks; /* num of actual chunks */ u_int uoh_reserved1; u_int uoh_reserved2; }; struct uof_chunk_hdr { char uch_id[UOF_OBJ_ID_LEN]; u_int uch_offset; u_int uch_size; }; struct uof_str_tab { u_int ust_table_len; /* length of table */ u_int ust_reserved; /* reserved for future use */ uint64_t ust_strings; /* pointer to string table. * NULL terminated strings */ }; #define AE_MODE_RELOAD_CTX_SHARED __BIT(12) #define AE_MODE_SHARED_USTORE __BIT(11) #define AE_MODE_LMEM1 __BIT(9) #define AE_MODE_LMEM0 __BIT(8) #define AE_MODE_NN_MODE __BITS(7, 4) #define AE_MODE_CTX_MODE __BITS(3, 0) #define AE_MODE_NN_MODE_NEIGH 0 #define AE_MODE_NN_MODE_SELF 1 #define AE_MODE_NN_MODE_DONTCARE 0xff struct uof_image { u_int ui_name; /* image name */ u_int ui_ae_assigned; /* AccelEngines assigned */ u_int ui_ctx_assigned; /* AccelEngine contexts assigned */ u_int ui_cpu_type; /* cpu type */ u_int ui_entry_address; /* entry uaddress */ u_int ui_fill_pattern[2]; /* uword fill value */ u_int ui_reloadable_size; /* size of reloadable ustore section */ u_char ui_sensitivity; /* * case sensitivity: 0 = insensitive, * 1 = sensitive */ u_char ui_reserved; /* reserved for future use */ u_short ui_ae_mode; /* * unused<15:14>, legacyMode<13>, * reloadCtxShared<12>, sharedUstore<11>, * ecc<10>, locMem1<9>, locMem0<8>, * nnMode<7:4>, ctx<3:0> */ u_short ui_max_ver; /* max cpu ver on which the image can run */ u_short ui_min_ver; /* min cpu ver on which the image can run */ u_short ui_image_attrib; /* image attributes */ u_short ui_reserved2; /* reserved for future use */ u_short ui_num_page_regions; /* number of page regions */ u_short ui_num_pages; /* number of pages */ u_int ui_reg_tab; /* offset to register table */ u_int ui_init_reg_sym_tab; /* reg/sym init table */ u_int ui_sbreak_tab; /* offset to sbreak table */ u_int ui_app_metadata; /* application meta-data */ /* ui_npages of code page follows this header */ }; struct uof_obj_table { u_int uot_nentries; /* number of table entries */ /* uot_nentries of object follows */ }; struct uof_ae_reg { u_int uar_name; /* reg name string-table offset */ u_int uar_vis_name; /* reg visible name string-table offset */ u_short uar_type; /* reg type */ u_short uar_addr; /* reg address */ u_short uar_access_mode; /* uof_RegAccessMode_T: read/write/both/undef */ u_char uar_visible; /* register visibility */ u_char uar_reserved1; /* reserved for future use */ u_short uar_ref_count; /* number of contiguous registers allocated */ u_short uar_reserved2; /* reserved for future use */ u_int uar_xoid; /* xfer order ID */ }; enum uof_value_kind { UNDEF_VAL, /* undefined value */ CHAR_VAL, /* character value */ SHORT_VAL, /* short value */ INT_VAL, /* integer value */ STR_VAL, /* string value */ STRTAB_VAL, /* string table value */ NUM_VAL, /* number value */ EXPR_VAL /* expression value */ }; enum uof_init_type { INIT_EXPR, INIT_REG, INIT_REG_CTX, INIT_EXPR_ENDIAN_SWAP }; struct uof_init_reg_sym { u_int uirs_name; /* symbol name */ char uirs_init_type; /* 0=expr, 1=register, 2=ctxReg, * 3=expr_endian_swap */ char uirs_value_type; /* EXPR_VAL, STRTAB_VAL */ char uirs_reg_type; /* register type: ae_reg_type */ u_char uirs_ctx; /* AE context when initType=2 */ u_int uirs_addr_offset; /* reg address, or sym-value offset */ u_int uirs_value; /* integer value, or expression */ }; struct uof_sbreak { u_int us_page_num; /* page number */ u_int us_virt_uaddr; /* virt uaddress */ u_char us_sbreak_type; /* sbreak type */ u_char us_reg_type; /* register type: ae_reg_type */ u_short us_reserved1; /* reserved for future use */ u_int us_addr_offset; /* branch target address or offset * to be used with the reg value to * calculate the target address */ u_int us_reg_rddr; /* register address */ }; struct uof_code_page { u_int ucp_page_region; /* page associated region */ u_int ucp_page_num; /* code-page number */ u_char ucp_def_page; /* default page indicator */ u_char ucp_reserved2; /* reserved for future use */ u_short ucp_reserved1; /* reserved for future use */ u_int ucp_beg_vaddr; /* starting virtual uaddr */ u_int ucp_beg_paddr; /* starting physical uaddr */ u_int ucp_neigh_reg_tab; /* offset to neighbour-reg table */ u_int ucp_uc_var_tab; /* offset to uC var table */ u_int ucp_imp_var_tab; /* offset to import var table */ u_int ucp_imp_expr_tab; /* offset to import expression table */ u_int ucp_code_area; /* offset to code area */ }; struct uof_code_area { u_int uca_num_micro_words; /* number of micro words */ u_int uca_uword_block_tab; /* offset to ublock table */ }; struct uof_uword_block { u_int uub_start_addr; /* start address */ u_int uub_num_words; /* number of microwords */ u_int uub_uword_offset; /* offset to the uwords */ u_int uub_reserved; /* reserved for future use */ }; struct uof_uword_fixup { u_int uuf_name; /* offset to string table */ u_int uuf_uword_address; /* micro word address */ u_int uuf_expr_value; /* string table offset of expr string, or value */ u_char uuf_val_type; /* VALUE_UNDEF, VALUE_NUM, VALUE_EXPR */ u_char uuf_value_attrs; /* bit<0> (Scope: 0=global, 1=local), * bit<1> (init: 0=no, 1=yes) */ u_short uuf_reserved1; /* reserved for future use */ char uuf_field_attrs[UOF_FIELD_POS_SIZE]; /* field pos, size, and right shift value */ }; struct uof_import_var { u_int uiv_name; /* import var name string-table offset */ u_char uiv_value_attrs; /* bit<0> (Scope: 0=global), * bit<1> (init: 0=no, 1=yes) */ u_char uiv_reserved1; /* reserved for future use */ u_short uiv_reserved2; /* reserved for future use */ uint64_t uiv_value; /* 64-bit imported value */ }; struct uof_mem_val_attr { u_int umva_byte_offset; /* byte-offset from the allocated memory */ u_int umva_value; /* memory value */ }; enum uof_mem_region { SRAM_REGION, /* SRAM region */ DRAM_REGION, /* DRAM0 region */ DRAM1_REGION, /* DRAM1 region */ LMEM_REGION, /* local memory region */ SCRATCH_REGION, /* SCRATCH region */ UMEM_REGION, /* micro-store region */ RAM_REGION, /* RAM region */ SHRAM_REGION, /* shared memory-0 region */ SHRAM1_REGION, /* shared memory-1 region */ SHRAM2_REGION, /* shared memory-2 region */ SHRAM3_REGION, /* shared memory-3 region */ SHRAM4_REGION, /* shared memory-4 region */ SHRAM5_REGION /* shared memory-5 region */ }; #define UOF_SCOPE_GLOBAL 0 #define UOF_SCOPE_LOCAL 1 struct uof_init_mem { u_int uim_sym_name; /* symbol name */ char uim_region; /* memory region -- uof_mem_region */ char uim_scope; /* visibility scope */ u_short uim_reserved1; /* reserved for future use */ u_int uim_addr; /* memory address */ u_int uim_num_bytes; /* number of bytes */ u_int uim_num_val_attr; /* number of values attributes */ /* uim_num_val_attr of uof_mem_val_attr follows this header */ }; struct uof_var_mem_seg { u_int uvms_sram_base; /* SRAM memory segment base addr */ u_int uvms_sram_size; /* SRAM segment size bytes */ u_int uvms_sram_alignment; /* SRAM segment alignment bytes */ u_int uvms_sdram_base; /* DRAM0 memory segment base addr */ u_int uvms_sdram_size; /* DRAM0 segment size bytes */ u_int uvms_sdram_alignment; /* DRAM0 segment alignment bytes */ u_int uvms_sdram1_base; /* DRAM1 memory segment base addr */ u_int uvms_sdram1_size; /* DRAM1 segment size bytes */ u_int uvms_sdram1_alignment; /* DRAM1 segment alignment bytes */ u_int uvms_scratch_base; /* SCRATCH memory segment base addr */ u_int uvms_scratch_size; /* SCRATCH segment size bytes */ u_int uvms_scratch_alignment; /* SCRATCH segment alignment bytes */ }; #define SUOF_OBJ_ID_LEN 8 #define SUOF_FID 0x53554f46 #define SUOF_MAJ_VER 0x0 #define SUOF_MIN_VER 0x1 #define SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) #define CSS_FWSK_MODULUS_LEN 256 #define CSS_FWSK_EXPONENT_LEN 4 #define CSS_FWSK_PAD_LEN 252 #define CSS_FWSK_PUB_LEN (CSS_FWSK_MODULUS_LEN + \ CSS_FWSK_EXPONENT_LEN + \ CSS_FWSK_PAD_LEN) #define CSS_SIGNATURE_LEN 256 #define CSS_AE_IMG_LEN (sizeof(struct simg_ae_mode) + \ SIMG_AE_INIT_SEQ_LEN + \ SIMG_AE_INSTS_LEN) #define CSS_AE_SIMG_LEN (sizeof(struct css_hdr) + \ CSS_FWSK_PUB_LEN + \ CSS_SIGNATURE_LEN + \ CSS_AE_IMG_LEN) #define AE_IMG_OFFSET (sizeof(struct css_hdr) + \ CSS_FWSK_MODULUS_LEN + \ CSS_FWSK_EXPONENT_LEN + \ CSS_SIGNATURE_LEN) #define CSS_MAX_IMAGE_LEN 0x40000 struct fw_auth_desc { u_int fad_img_len; u_int fad_reserved; u_int fad_css_hdr_high; u_int fad_css_hdr_low; u_int fad_img_high; u_int fad_img_low; u_int fad_signature_high; u_int fad_signature_low; u_int fad_fwsk_pub_high; u_int fad_fwsk_pub_low; u_int fad_img_ae_mode_data_high; u_int fad_img_ae_mode_data_low; u_int fad_img_ae_init_data_high; u_int fad_img_ae_init_data_low; u_int fad_img_ae_insts_high; u_int fad_img_ae_insts_low; }; struct auth_chunk { struct fw_auth_desc ac_fw_auth_desc; uint64_t ac_chunk_size; uint64_t ac_chunk_bus_addr; }; enum css_fwtype { CSS_AE_FIRMWARE = 0, CSS_MMP_FIRMWARE = 1 }; struct css_hdr { u_int css_module_type; u_int css_header_len; u_int css_header_ver; u_int css_module_id; u_int css_module_vendor; u_int css_date; u_int css_size; u_int css_key_size; u_int css_module_size; u_int css_exponent_size; u_int css_fw_type; u_int css_reserved[21]; }; struct simg_ae_mode { u_int sam_file_id; u_short sam_maj_ver; u_short sam_min_ver; u_int sam_dev_type; u_short sam_devmax_ver; u_short sam_devmin_ver; u_int sam_ae_mask; u_int sam_ctx_enables; char sam_fw_type; char sam_ctx_mode; char sam_nn_mode; char sam_lm0_mode; char sam_lm1_mode; char sam_scs_mode; char sam_lm2_mode; char sam_lm3_mode; char sam_tindex_mode; u_char sam_reserved[7]; char sam_simg_name[256]; char sam_appmeta_data[256]; }; struct suof_file_hdr { u_int sfh_file_id; u_int sfh_check_sum; char sfh_min_ver; char sfh_maj_ver; char sfh_fw_type; char sfh_reserved; u_short sfh_max_chunks; u_short sfh_num_chunks; }; struct suof_chunk_hdr { char sch_chunk_id[SUOF_OBJ_ID_LEN]; uint64_t sch_offset; uint64_t sch_size; }; struct suof_str_tab { u_int sst_tab_length; u_int sst_strings; }; struct suof_obj_hdr { u_int soh_img_length; u_int soh_reserved; }; /* -------------------------------------------------------------------------- */ /* accel */ enum fw_slice { FW_SLICE_NULL = 0, /* NULL slice type */ FW_SLICE_CIPHER = 1, /* CIPHER slice type */ FW_SLICE_AUTH = 2, /* AUTH slice type */ FW_SLICE_DRAM_RD = 3, /* DRAM_RD Logical slice type */ FW_SLICE_DRAM_WR = 4, /* DRAM_WR Logical slice type */ FW_SLICE_COMP = 5, /* Compression slice type */ FW_SLICE_XLAT = 6, /* Translator slice type */ FW_SLICE_DELIMITER /* End delimiter */ }; #define MAX_FW_SLICE FW_SLICE_DELIMITER #define QAT_OPTIMAL_ALIGN_SHIFT 6 #define QAT_OPTIMAL_ALIGN (1 << QAT_OPTIMAL_ALIGN_SHIFT) enum hw_auth_algo { HW_AUTH_ALGO_NULL = 0, /* Null hashing */ HW_AUTH_ALGO_SHA1 = 1, /* SHA1 hashing */ HW_AUTH_ALGO_MD5 = 2, /* MD5 hashing */ HW_AUTH_ALGO_SHA224 = 3, /* SHA-224 hashing */ HW_AUTH_ALGO_SHA256 = 4, /* SHA-256 hashing */ HW_AUTH_ALGO_SHA384 = 5, /* SHA-384 hashing */ HW_AUTH_ALGO_SHA512 = 6, /* SHA-512 hashing */ HW_AUTH_ALGO_AES_XCBC_MAC = 7, /* AES-XCBC-MAC hashing */ HW_AUTH_ALGO_AES_CBC_MAC = 8, /* AES-CBC-MAC hashing */ HW_AUTH_ALGO_AES_F9 = 9, /* AES F9 hashing */ HW_AUTH_ALGO_GALOIS_128 = 10, /* Galois 128 bit hashing */ HW_AUTH_ALGO_GALOIS_64 = 11, /* Galois 64 hashing */ HW_AUTH_ALGO_KASUMI_F9 = 12, /* Kasumi F9 hashing */ HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, /* UIA2/SNOW_3H F9 hashing */ HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14, HW_AUTH_RESERVED_1 = 15, HW_AUTH_RESERVED_2 = 16, HW_AUTH_ALGO_SHA3_256 = 17, HW_AUTH_RESERVED_3 = 18, HW_AUTH_ALGO_SHA3_512 = 19, HW_AUTH_ALGO_DELIMITER = 20 }; enum hw_auth_mode { HW_AUTH_MODE0, HW_AUTH_MODE1, HW_AUTH_MODE2, HW_AUTH_MODE_DELIMITER }; struct hw_auth_config { uint32_t config; /* Configuration used for setting up the slice */ uint32_t reserved; /* Reserved */ }; #define HW_AUTH_CONFIG_SHA3_ALGO __BITS(22, 23) #define HW_AUTH_CONFIG_SHA3_PADDING __BIT(16) #define HW_AUTH_CONFIG_CMPLEN __BITS(14, 8) /* The length of the digest if the QAT is to the check*/ #define HW_AUTH_CONFIG_MODE __BITS(7, 4) #define HW_AUTH_CONFIG_ALGO __BITS(3, 0) #define HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \ __SHIFTIN(mode, HW_AUTH_CONFIG_MODE) | \ __SHIFTIN(algo, HW_AUTH_CONFIG_ALGO) | \ __SHIFTIN(cmp_len, HW_AUTH_CONFIG_CMPLEN) struct hw_auth_counter { uint32_t counter; /* Counter value */ uint32_t reserved; /* Reserved */ }; struct hw_auth_setup { struct hw_auth_config auth_config; /* Configuration word for the auth slice */ struct hw_auth_counter auth_counter; /* Auth counter value for this request */ }; #define HW_NULL_STATE1_SZ 32 #define HW_MD5_STATE1_SZ 16 #define HW_SHA1_STATE1_SZ 20 #define HW_SHA224_STATE1_SZ 32 #define HW_SHA256_STATE1_SZ 32 #define HW_SHA3_256_STATE1_SZ 32 #define HW_SHA384_STATE1_SZ 64 #define HW_SHA512_STATE1_SZ 64 #define HW_SHA3_512_STATE1_SZ 64 #define HW_SHA3_224_STATE1_SZ 28 #define HW_SHA3_384_STATE1_SZ 48 #define HW_AES_XCBC_MAC_STATE1_SZ 16 #define HW_AES_CBC_MAC_STATE1_SZ 16 #define HW_AES_F9_STATE1_SZ 32 #define HW_KASUMI_F9_STATE1_SZ 16 #define HW_GALOIS_128_STATE1_SZ 16 #define HW_SNOW_3G_UIA2_STATE1_SZ 8 #define HW_ZUC_3G_EIA3_STATE1_SZ 8 #define HW_NULL_STATE2_SZ 32 #define HW_MD5_STATE2_SZ 16 #define HW_SHA1_STATE2_SZ 20 #define HW_SHA224_STATE2_SZ 32 #define HW_SHA256_STATE2_SZ 32 #define HW_SHA3_256_STATE2_SZ 0 #define HW_SHA384_STATE2_SZ 64 #define HW_SHA512_STATE2_SZ 64 #define HW_SHA3_512_STATE2_SZ 0 #define HW_SHA3_224_STATE2_SZ 0 #define HW_SHA3_384_STATE2_SZ 0 #define HW_AES_XCBC_MAC_KEY_SZ 16 #define HW_AES_CBC_MAC_KEY_SZ 16 #define HW_AES_CCM_CBC_E_CTR0_SZ 16 #define HW_F9_IK_SZ 16 #define HW_F9_FK_SZ 16 #define HW_KASUMI_F9_STATE2_SZ (HW_F9_IK_SZ + HW_F9_FK_SZ) #define HW_AES_F9_STATE2_SZ HW_KASUMI_F9_STATE2_SZ #define HW_SNOW_3G_UIA2_STATE2_SZ 24 #define HW_ZUC_3G_EIA3_STATE2_SZ 32 #define HW_GALOIS_H_SZ 16 #define HW_GALOIS_LEN_A_SZ 8 #define HW_GALOIS_E_CTR0_SZ 16 struct hw_auth_sha512 { struct hw_auth_setup inner_setup; /* Inner loop configuration word for the slice */ uint8_t state1[HW_SHA512_STATE1_SZ]; /* Slice state1 variable */ struct hw_auth_setup outer_setup; /* Outer configuration word for the slice */ uint8_t state2[HW_SHA512_STATE2_SZ]; /* Slice state2 variable */ }; union hw_auth_algo_blk { struct hw_auth_sha512 max; /* This is the largest possible auth setup block size */ }; enum hw_cipher_algo { HW_CIPHER_ALGO_NULL = 0, /* Null ciphering */ HW_CIPHER_ALGO_DES = 1, /* DES ciphering */ HW_CIPHER_ALGO_3DES = 2, /* 3DES ciphering */ HW_CIPHER_ALGO_AES128 = 3, /* AES-128 ciphering */ HW_CIPHER_ALGO_AES192 = 4, /* AES-192 ciphering */ HW_CIPHER_ALGO_AES256 = 5, /* AES-256 ciphering */ HW_CIPHER_ALGO_ARC4 = 6, /* ARC4 ciphering */ HW_CIPHER_ALGO_KASUMI = 7, /* Kasumi */ HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, /* Snow_3G */ HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, HW_CIPHER_DELIMITER = 10 /* Delimiter type */ }; enum hw_cipher_mode { HW_CIPHER_ECB_MODE = 0, /* ECB mode */ HW_CIPHER_CBC_MODE = 1, /* CBC mode */ HW_CIPHER_CTR_MODE = 2, /* CTR mode */ HW_CIPHER_F8_MODE = 3, /* F8 mode */ HW_CIPHER_XTS_MODE = 6, HW_CIPHER_MODE_DELIMITER = 7 /* Delimiter type */ }; struct hw_cipher_config { uint32_t val; /* Cipher slice configuration */ uint32_t reserved; /* Reserved */ }; #define CIPHER_CONFIG_CONVERT __BIT(9) #define CIPHER_CONFIG_DIR __BIT(8) #define CIPHER_CONFIG_MODE __BITS(7, 4) #define CIPHER_CONFIG_ALGO __BITS(3, 0) #define HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir) \ __SHIFTIN(mode, CIPHER_CONFIG_MODE) | \ __SHIFTIN(algo, CIPHER_CONFIG_ALGO) | \ __SHIFTIN(convert, CIPHER_CONFIG_CONVERT) | \ __SHIFTIN(dir, CIPHER_CONFIG_DIR) enum hw_cipher_dir { HW_CIPHER_ENCRYPT = 0, /* encryption is required */ HW_CIPHER_DECRYPT = 1, /* decryption is required */ }; enum hw_cipher_convert { HW_CIPHER_NO_CONVERT = 0, /* no key convert is required*/ HW_CIPHER_KEY_CONVERT = 1, /* key conversion is required*/ }; #define CIPHER_MODE_F8_KEY_SZ_MULT 2 #define CIPHER_MODE_XTS_KEY_SZ_MULT 2 #define HW_DES_BLK_SZ 8 #define HW_3DES_BLK_SZ 8 #define HW_NULL_BLK_SZ 8 #define HW_AES_BLK_SZ 16 #define HW_KASUMI_BLK_SZ 8 #define HW_SNOW_3G_BLK_SZ 8 #define HW_ZUC_3G_BLK_SZ 8 #define HW_NULL_KEY_SZ 256 #define HW_DES_KEY_SZ 8 #define HW_3DES_KEY_SZ 24 #define HW_AES_128_KEY_SZ 16 #define HW_AES_192_KEY_SZ 24 #define HW_AES_256_KEY_SZ 32 #define HW_AES_128_F8_KEY_SZ (HW_AES_128_KEY_SZ * \ CIPHER_MODE_F8_KEY_SZ_MULT) #define HW_AES_192_F8_KEY_SZ (HW_AES_192_KEY_SZ * \ CIPHER_MODE_F8_KEY_SZ_MULT) #define HW_AES_256_F8_KEY_SZ (HW_AES_256_KEY_SZ * \ CIPHER_MODE_F8_KEY_SZ_MULT) #define HW_AES_128_XTS_KEY_SZ (HW_AES_128_KEY_SZ * \ CIPHER_MODE_XTS_KEY_SZ_MULT) #define HW_AES_256_XTS_KEY_SZ (HW_AES_256_KEY_SZ * \ CIPHER_MODE_XTS_KEY_SZ_MULT) #define HW_KASUMI_KEY_SZ 16 #define HW_KASUMI_F8_KEY_SZ (HW_KASUMI_KEY_SZ * \ CIPHER_MODE_F8_KEY_SZ_MULT) #define HW_AES_128_XTS_KEY_SZ (HW_AES_128_KEY_SZ * \ CIPHER_MODE_XTS_KEY_SZ_MULT) #define HW_AES_256_XTS_KEY_SZ (HW_AES_256_KEY_SZ * \ CIPHER_MODE_XTS_KEY_SZ_MULT) #define HW_ARC4_KEY_SZ 256 #define HW_SNOW_3G_UEA2_KEY_SZ 16 #define HW_SNOW_3G_UEA2_IV_SZ 16 #define HW_ZUC_3G_EEA3_KEY_SZ 16 #define HW_ZUC_3G_EEA3_IV_SZ 16 #define HW_MODE_F8_NUM_REG_TO_CLEAR 2 struct hw_cipher_aes256_f8 { struct hw_cipher_config cipher_config; /* Cipher configuration word for the slice set to * AES-256 and the F8 mode */ uint8_t key[HW_AES_256_F8_KEY_SZ]; /* Cipher key */ }; union hw_cipher_algo_blk { struct hw_cipher_aes256_f8 max; /* AES-256 F8 Cipher */ /* This is the largest possible cipher setup block size */ }; struct flat_buffer_desc { uint32_t data_len_in_bytes; uint32_t reserved; uint64_t phy_buffer; }; +#define HW_MAXSEG 32 + struct buffer_list_desc { uint64_t resrvd; uint32_t num_buffers; uint32_t reserved; + struct flat_buffer_desc flat_bufs[HW_MAXSEG]; }; /* -------------------------------------------------------------------------- */ /* look aside */ enum fw_la_cmd_id { FW_LA_CMD_CIPHER, /* Cipher Request */ FW_LA_CMD_AUTH, /* Auth Request */ FW_LA_CMD_CIPHER_HASH, /* Cipher-Hash Request */ FW_LA_CMD_HASH_CIPHER, /* Hash-Cipher Request */ FW_LA_CMD_TRNG_GET_RANDOM, /* TRNG Get Random Request */ FW_LA_CMD_TRNG_TEST, /* TRNG Test Request */ FW_LA_CMD_SSL3_KEY_DERIVE, /* SSL3 Key Derivation Request */ FW_LA_CMD_TLS_V1_1_KEY_DERIVE, /* TLS Key Derivation Request */ FW_LA_CMD_TLS_V1_2_KEY_DERIVE, /* TLS Key Derivation Request */ FW_LA_CMD_MGF1, /* MGF1 Request */ FW_LA_CMD_AUTH_PRE_COMP, /* Auth Pre-Compute Request */ #if 0 /* incompatible between qat 1.5 and 1.7 */ FW_LA_CMD_CIPHER_CIPHER, /* Cipher-Cipher Request */ FW_LA_CMD_HASH_HASH, /* Hash-Hash Request */ FW_LA_CMD_CIPHER_PRE_COMP, /* Auth Pre-Compute Request */ #endif FW_LA_CMD_DELIMITER, /* Delimiter type */ }; #endif diff --git a/sys/dev/qat/qatvar.h b/sys/dev/qat/qatvar.h index 3326dc268fc1..3f0ecd411874 100644 --- a/sys/dev/qat/qatvar.h +++ b/sys/dev/qat/qatvar.h @@ -1,1074 +1,1071 @@ /* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ /* $NetBSD: qatvar.h,v 1.2 2020/03/14 18:08:39 ad Exp $ */ /* * Copyright (c) 2019 Internet Initiative Japan, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef _DEV_PCI_QATVAR_H_ #define _DEV_PCI_QATVAR_H_ #include #include #include #define QAT_NSYMREQ 256 #define QAT_NSYMCOOKIE ((QAT_NSYMREQ * 2 + 1) * 2) -#define QAT_NASYMREQ 64 -#define QAT_BATCH_SUBMIT_FREE_SPACE 2 #define QAT_EV_NAME_SIZE 32 #define QAT_RING_NAME_SIZE 32 -#define QAT_MAXSEG 32 /* max segments for sg dma */ +#define QAT_MAXSEG HW_MAXSEG /* max segments for sg dma */ #define QAT_MAXLEN 65535 /* IP_MAXPACKET */ #define QAT_HB_INTERVAL 500 /* heartbeat msec */ #define QAT_SSM_WDT 100 enum qat_chip_type { QAT_CHIP_C2XXX = 0, /* NanoQAT: Atom C2000 */ QAT_CHIP_C2XXX_IOV, QAT_CHIP_C3XXX, /* Atom C3000 */ QAT_CHIP_C3XXX_IOV, QAT_CHIP_C62X, QAT_CHIP_C62X_IOV, QAT_CHIP_D15XX, QAT_CHIP_D15XX_IOV, QAT_CHIP_DH895XCC, QAT_CHIP_DH895XCC_IOV, }; enum qat_sku { QAT_SKU_UNKNOWN = 0, QAT_SKU_1, QAT_SKU_2, QAT_SKU_3, QAT_SKU_4, QAT_SKU_VF, }; enum qat_ae_status { QAT_AE_ENABLED = 1, QAT_AE_ACTIVE, QAT_AE_DISABLED }; #define TIMEOUT_AE_RESET 100 #define TIMEOUT_AE_CHECK 10000 #define TIMEOUT_AE_CSR 500 #define AE_EXEC_CYCLE 20 #define QAT_UOF_MAX_PAGE 1 #define QAT_UOF_MAX_PAGE_REGION 1 struct qat_dmamem { bus_dma_tag_t qdm_dma_tag; bus_dmamap_t qdm_dma_map; bus_size_t qdm_dma_size; bus_dma_segment_t qdm_dma_seg; void *qdm_dma_vaddr; }; /* Valid internal ring size values */ #define QAT_RING_SIZE_128 0x01 #define QAT_RING_SIZE_256 0x02 #define QAT_RING_SIZE_512 0x03 #define QAT_RING_SIZE_4K 0x06 #define QAT_RING_SIZE_16K 0x08 #define QAT_RING_SIZE_4M 0x10 #define QAT_MIN_RING_SIZE QAT_RING_SIZE_128 #define QAT_MAX_RING_SIZE QAT_RING_SIZE_4M #define QAT_DEFAULT_RING_SIZE QAT_RING_SIZE_16K /* Valid internal msg size values */ #define QAT_MSG_SIZE_32 0x01 #define QAT_MSG_SIZE_64 0x02 #define QAT_MSG_SIZE_128 0x04 #define QAT_MIN_MSG_SIZE QAT_MSG_SIZE_32 #define QAT_MAX_MSG_SIZE QAT_MSG_SIZE_128 /* Size to bytes conversion macros for ring and msg size values */ #define QAT_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5) #define QAT_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5) #define QAT_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7) #define QAT_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) /* Minimum ring bufer size for memory allocation */ #define QAT_RING_SIZE_BYTES_MIN(SIZE) \ ((SIZE < QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K)) ? \ QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K) : SIZE) #define QAT_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) #define QAT_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ SIZE) & ~0x4) /* Max outstanding requests */ #define QAT_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \ ((((1 << (RING_SIZE - 1)) << 3) >> QAT_SIZE_TO_POW(MSG_SIZE)) - 1) #define QAT_RING_PATTERN 0x7f struct qat_softc; typedef int (*qat_cb_t)(struct qat_softc *, void *, void *); struct qat_ring { struct mtx qr_ring_mtx; /* Lock per ring */ bool qr_need_wakeup; void *qr_ring_vaddr; uint32_t * volatile qr_inflight; /* tx/rx shared */ uint32_t qr_head; uint32_t qr_tail; uint8_t qr_msg_size; uint8_t qr_ring_size; uint32_t qr_ring; /* ring number in bank */ uint32_t qr_bank; /* bank number in device */ uint32_t qr_ring_id; uint32_t qr_ring_mask; qat_cb_t qr_cb; void *qr_cb_arg; struct qat_dmamem qr_dma; bus_addr_t qr_ring_paddr; const char *qr_name; }; struct qat_bank { struct qat_softc *qb_sc; /* back pointer to softc */ uint32_t qb_intr_mask; /* current interrupt mask */ uint32_t qb_allocated_rings; /* current allocated ring bitfiled */ uint32_t qb_coalescing_time; /* timer in nano sec, 0: disabled */ #define COALESCING_TIME_INTERVAL_DEFAULT 10000 #define COALESCING_TIME_INTERVAL_MIN 500 #define COALESCING_TIME_INTERVAL_MAX 0xfffff uint32_t qb_bank; /* bank index */ struct mtx qb_bank_mtx; struct resource *qb_ih; void *qb_ih_cookie; struct qat_ring qb_et_rings[MAX_RING_PER_BANK]; }; struct qat_ap_bank { uint32_t qab_nf_mask; uint32_t qab_nf_dest; uint32_t qab_ne_mask; uint32_t qab_ne_dest; }; struct qat_ae_page { struct qat_ae_page *qap_next; struct qat_uof_page *qap_page; struct qat_ae_region *qap_region; u_int qap_flags; }; #define QAT_AE_PAGA_FLAG_WAITING (1 << 0) struct qat_ae_region { struct qat_ae_page *qar_loaded_page; STAILQ_HEAD(, qat_ae_page) qar_waiting_pages; }; struct qat_ae_slice { u_int qas_assigned_ctx_mask; struct qat_ae_region qas_regions[QAT_UOF_MAX_PAGE_REGION]; struct qat_ae_page qas_pages[QAT_UOF_MAX_PAGE]; struct qat_ae_page *qas_cur_pages[MAX_AE_CTX]; struct qat_uof_image *qas_image; }; #define QAT_AE(sc, ae) \ ((sc)->sc_ae[ae]) struct qat_ae { u_int qae_state; /* AE state */ u_int qae_ustore_size; /* free micro-store address */ u_int qae_free_addr; /* free micro-store address */ u_int qae_free_size; /* free micro-store size */ u_int qae_live_ctx_mask; /* live context mask */ u_int qae_ustore_dram_addr; /* mirco-store DRAM address */ u_int qae_reload_size; /* reloadable code size */ /* aefw */ u_int qae_num_slices; struct qat_ae_slice qae_slices[MAX_AE_CTX]; u_int qae_reloc_ustore_dram; /* reloadable ustore-dram address */ u_int qae_effect_ustore_size; /* effective AE ustore size */ u_int qae_shareable_ustore; }; struct qat_mof { void *qmf_sym; /* SYM_OBJS in sc_fw_mof */ size_t qmf_sym_size; void *qmf_uof_objs; /* UOF_OBJS in sc_fw_mof */ size_t qmf_uof_objs_size; void *qmf_suof_objs; /* SUOF_OBJS in sc_fw_mof */ size_t qmf_suof_objs_size; }; struct qat_ae_batch_init { u_int qabi_ae; u_int qabi_addr; u_int *qabi_value; u_int qabi_size; STAILQ_ENTRY(qat_ae_batch_init) qabi_next; }; STAILQ_HEAD(qat_ae_batch_init_list, qat_ae_batch_init); /* overwritten struct uof_uword_block */ struct qat_uof_uword_block { u_int quub_start_addr; /* start address */ u_int quub_num_words; /* number of microwords */ uint64_t quub_micro_words; /* pointer to the uwords */ }; struct qat_uof_page { u_int qup_page_num; /* page number */ u_int qup_def_page; /* default page */ u_int qup_page_region; /* region of page */ u_int qup_beg_vaddr; /* begin virtual address */ u_int qup_beg_paddr; /* begin physical address */ u_int qup_num_uc_var; /* num of uC var in array */ struct uof_uword_fixup *qup_uc_var; /* array of import variables */ u_int qup_num_imp_var; /* num of import var in array */ struct uof_import_var *qup_imp_var; /* array of import variables */ u_int qup_num_imp_expr; /* num of import expr in array */ struct uof_uword_fixup *qup_imp_expr; /* array of import expressions */ u_int qup_num_neigh_reg; /* num of neigh-reg in array */ struct uof_uword_fixup *qup_neigh_reg; /* array of neigh-reg assignments */ u_int qup_num_micro_words; /* number of microwords in the seg */ u_int qup_num_uw_blocks; /* number of uword blocks */ struct qat_uof_uword_block *qup_uw_blocks; /* array of uword blocks */ }; struct qat_uof_image { struct uof_image *qui_image; /* image pointer */ struct qat_uof_page qui_pages[QAT_UOF_MAX_PAGE]; /* array of pages */ u_int qui_num_ae_reg; /* num of registers */ struct uof_ae_reg *qui_ae_reg; /* array of registers */ u_int qui_num_init_reg_sym; /* num of reg/sym init values */ struct uof_init_reg_sym *qui_init_reg_sym; /* array of reg/sym init values */ u_int qui_num_sbreak; /* num of sbreak values */ struct qui_sbreak *qui_sbreak; /* array of sbreak values */ u_int qui_num_uwords_used; /* highest uword addressreferenced + 1 */ }; struct qat_aefw_uof { size_t qafu_size; /* uof size */ struct uof_obj_hdr *qafu_obj_hdr; /* UOF_OBJS */ void *qafu_str_tab; size_t qafu_str_tab_size; u_int qafu_num_init_mem; struct uof_init_mem *qafu_init_mem; size_t qafu_init_mem_size; struct uof_var_mem_seg *qafu_var_mem_seg; struct qat_ae_batch_init_list qafu_lm_init[MAX_AE]; size_t qafu_num_lm_init[MAX_AE]; size_t qafu_num_lm_init_inst[MAX_AE]; u_int qafu_num_imgs; /* number of uof image */ struct qat_uof_image qafu_imgs[MAX_NUM_AE * MAX_AE_CTX]; /* uof images */ }; #define QAT_SERVICE_CRYPTO_A (1 << 0) #define QAT_SERVICE_CRYPTO_B (1 << 1) struct qat_admin_rings { uint32_t qadr_active_aes_per_accel; uint8_t qadr_srv_mask[MAX_AE_PER_ACCEL]; struct qat_dmamem qadr_dma; struct fw_init_ring_table *qadr_master_ring_tbl; struct fw_init_ring_table *qadr_cya_ring_tbl; struct fw_init_ring_table *qadr_cyb_ring_tbl; struct qat_ring *qadr_admin_tx; struct qat_ring *qadr_admin_rx; }; struct qat_accel_init_cb { int qaic_status; }; struct qat_admin_comms { struct qat_dmamem qadc_dma; struct qat_dmamem qadc_const_tbl_dma; struct qat_dmamem qadc_hb_dma; }; #define QAT_PID_MINOR_REV 0xf #define QAT_PID_MAJOR_REV (0xf << 4) struct qat_suof_image { char *qsi_simg_buf; u_long qsi_simg_len; char *qsi_css_header; char *qsi_css_key; char *qsi_css_signature; char *qsi_css_simg; u_long qsi_simg_size; u_int qsi_ae_num; u_int qsi_ae_mask; u_int qsi_fw_type; u_long qsi_simg_name; u_long qsi_appmeta_data; struct qat_dmamem qsi_dma; }; struct qat_aefw_suof { u_int qafs_file_id; u_int qafs_check_sum; char qafs_min_ver; char qafs_maj_ver; char qafs_fw_type; char *qafs_suof_buf; u_int qafs_suof_size; char *qafs_sym_str; u_int qafs_sym_size; u_int qafs_num_simgs; struct qat_suof_image *qafs_simg; }; enum qat_sym_hash_algorithm { QAT_SYM_HASH_NONE = 0, QAT_SYM_HASH_MD5 = 1, QAT_SYM_HASH_SHA1 = 2, QAT_SYM_HASH_SHA224 = 3, QAT_SYM_HASH_SHA256 = 4, QAT_SYM_HASH_SHA384 = 5, QAT_SYM_HASH_SHA512 = 6, QAT_SYM_HASH_AES_XCBC = 7, QAT_SYM_HASH_AES_CCM = 8, QAT_SYM_HASH_AES_GCM = 9, QAT_SYM_HASH_KASUMI_F9 = 10, QAT_SYM_HASH_SNOW3G_UIA2 = 11, QAT_SYM_HASH_AES_CMAC = 12, QAT_SYM_HASH_AES_GMAC = 13, QAT_SYM_HASH_AES_CBC_MAC = 14, }; #define QAT_HASH_MD5_BLOCK_SIZE 64 #define QAT_HASH_MD5_DIGEST_SIZE 16 #define QAT_HASH_MD5_STATE_SIZE 16 #define QAT_HASH_SHA1_BLOCK_SIZE 64 #define QAT_HASH_SHA1_DIGEST_SIZE 20 #define QAT_HASH_SHA1_STATE_SIZE 20 #define QAT_HASH_SHA224_BLOCK_SIZE 64 #define QAT_HASH_SHA224_DIGEST_SIZE 28 #define QAT_HASH_SHA224_STATE_SIZE 32 #define QAT_HASH_SHA256_BLOCK_SIZE 64 #define QAT_HASH_SHA256_DIGEST_SIZE 32 #define QAT_HASH_SHA256_STATE_SIZE 32 #define QAT_HASH_SHA384_BLOCK_SIZE 128 #define QAT_HASH_SHA384_DIGEST_SIZE 48 #define QAT_HASH_SHA384_STATE_SIZE 64 #define QAT_HASH_SHA512_BLOCK_SIZE 128 #define QAT_HASH_SHA512_DIGEST_SIZE 64 #define QAT_HASH_SHA512_STATE_SIZE 64 #define QAT_HASH_XCBC_PRECOMP_KEY_NUM 3 #define QAT_HASH_XCBC_MAC_BLOCK_SIZE 16 #define QAT_HASH_XCBC_MAC_128_DIGEST_SIZE 16 #define QAT_HASH_CMAC_BLOCK_SIZE 16 #define QAT_HASH_CMAC_128_DIGEST_SIZE 16 #define QAT_HASH_AES_CCM_BLOCK_SIZE 16 #define QAT_HASH_AES_CCM_DIGEST_SIZE 16 #define QAT_HASH_AES_GCM_BLOCK_SIZE 16 #define QAT_HASH_AES_GCM_DIGEST_SIZE 16 #define QAT_HASH_AES_GCM_STATE_SIZE 16 #define QAT_HASH_KASUMI_F9_BLOCK_SIZE 8 #define QAT_HASH_KASUMI_F9_DIGEST_SIZE 4 #define QAT_HASH_SNOW3G_UIA2_BLOCK_SIZE 8 #define QAT_HASH_SNOW3G_UIA2_DIGEST_SIZE 4 #define QAT_HASH_AES_CBC_MAC_BLOCK_SIZE 16 #define QAT_HASH_AES_CBC_MAC_DIGEST_SIZE 16 #define QAT_HASH_AES_GCM_ICV_SIZE_8 8 #define QAT_HASH_AES_GCM_ICV_SIZE_12 12 #define QAT_HASH_AES_GCM_ICV_SIZE_16 16 #define QAT_HASH_AES_CCM_ICV_SIZE_MIN 4 #define QAT_HASH_AES_CCM_ICV_SIZE_MAX 16 #define QAT_HASH_IPAD_BYTE 0x36 #define QAT_HASH_OPAD_BYTE 0x5c #define QAT_HASH_IPAD_4_BYTES 0x36363636 #define QAT_HASH_OPAD_4_BYTES 0x5c5c5c5c #define QAT_HASH_KASUMI_F9_KEY_MODIFIER_4_BYTES 0xAAAAAAAA #define QAT_SYM_XCBC_STATE_SIZE ((QAT_HASH_XCBC_MAC_BLOCK_SIZE) * 3) #define QAT_SYM_CMAC_STATE_SIZE ((QAT_HASH_CMAC_BLOCK_SIZE) * 3) struct qat_sym_hash_alg_info { uint32_t qshai_digest_len; /* Digest length in bytes */ uint32_t qshai_block_len; /* Block length in bytes */ uint32_t qshai_state_size; /* size of above state in bytes */ const uint8_t *qshai_init_state; /* Initial state */ const struct auth_hash *qshai_sah; /* software auth hash */ uint32_t qshai_state_offset; /* offset to state in *_CTX */ uint32_t qshai_state_word; }; struct qat_sym_hash_qat_info { uint32_t qshqi_algo_enc; /* QAT Algorithm encoding */ uint32_t qshqi_auth_counter; /* Counter value for Auth */ uint32_t qshqi_state1_len; /* QAT state1 length in bytes */ uint32_t qshqi_state2_len; /* QAT state2 length in bytes */ }; struct qat_sym_hash_def { const struct qat_sym_hash_alg_info *qshd_alg; const struct qat_sym_hash_qat_info *qshd_qat; }; #define QAT_SYM_REQ_PARAMS_SIZE_MAX (24 + 32) /* Reserve enough space for cipher and authentication request params */ /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */ #define QAT_SYM_REQ_PARAMS_SIZE_PADDED \ roundup(QAT_SYM_REQ_PARAMS_SIZE_MAX, QAT_OPTIMAL_ALIGN) /* Pad out to 64-byte multiple to ensure optimal alignment of next field */ #define QAT_SYM_KEY_TLS_PREFIX_SIZE (128) /* Hash Prefix size in bytes for TLS (128 = MAX = SHA2 (384, 512)*/ #define QAT_SYM_KEY_MAX_HASH_STATE_BUFFER \ (QAT_SYM_KEY_TLS_PREFIX_SIZE * 2) /* hash state prefix buffer structure that holds the maximum sized secret */ #define QAT_SYM_HASH_BUFFER_LEN QAT_HASH_SHA512_STATE_SIZE /* Buffer length to hold 16 byte MD5 key and 20 byte SHA1 key */ #define QAT_GCM_AAD_SIZE_MAX 240 /* Maximum AAD size */ #define QAT_AES_GCM_AAD_ALIGN 16 struct qat_sym_bulk_cookie { uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED]; - /* memory block reserved for request params + /* memory block reserved for request params, QAT 1.5 only * NOTE: Field must be correctly aligned in memory for access by QAT * engine */ struct qat_crypto *qsbc_crypto; struct qat_session *qsbc_session; /* Session context */ void *qsbc_cb_tag; /* correlator supplied by the client */ uint8_t qsbc_msg[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)]; /* QAT request message */ } __aligned(QAT_OPTIMAL_ALIGN); /* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */ #define HASH_CONTENT_DESC_SIZE 176 #define CIPHER_CONTENT_DESC_SIZE 64 #define CONTENT_DESC_MAX_SIZE roundup( \ HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE, \ QAT_OPTIMAL_ALIGN) +enum qat_sym_dma { + QAT_SYM_DMA_AADBUF = 0, + QAT_SYM_DMA_BUF, + QAT_SYM_DMA_OBUF, + QAT_SYM_DMA_COUNT, +}; + +struct qat_sym_dmamap { + bus_dmamap_t qsd_dmamap; + bus_dma_tag_t qsd_dma_tag; +}; + struct qat_sym_cookie { - union qat_sym_cookie_u { - /* should be 64byte aligned */ - struct qat_sym_bulk_cookie qsc_bulk_cookie; - /* symmetric bulk cookie */ -#ifdef notyet - struct qat_sym_key_cookie qsc_key_cookie; - /* symmetric key cookie */ - struct qat_sym_nrbg_cookie qsc_nrbg_cookie; - /* symmetric NRBG cookie */ -#endif - } u; + struct qat_sym_bulk_cookie qsc_bulk_cookie; /* should be 64-byte aligned */ struct buffer_list_desc qsc_buf_list; - struct flat_buffer_desc qsc_flat_bufs[QAT_MAXSEG]; /* should be here */ + struct buffer_list_desc qsc_obuf_list; - bus_dmamap_t qsc_self_dmamap; /* self DMA mapping and - end of DMA region */ + bus_dmamap_t qsc_self_dmamap; bus_dma_tag_t qsc_self_dma_tag; uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN]; uint8_t qsc_auth_res[QAT_SYM_HASH_BUFFER_LEN]; uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX]; uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE]; - bus_dmamap_t qsc_buf_dmamap; /* qsc_flat_bufs DMA mapping */ - bus_dma_tag_t qsc_buf_dma_tag; - void *qsc_buf; + struct qat_sym_dmamap qsc_dma[QAT_SYM_DMA_COUNT]; bus_addr_t qsc_bulk_req_params_buf_paddr; bus_addr_t qsc_buffer_list_desc_paddr; + bus_addr_t qsc_obuffer_list_desc_paddr; bus_addr_t qsc_iv_buf_paddr; bus_addr_t qsc_auth_res_paddr; bus_addr_t qsc_gcm_aad_paddr; bus_addr_t qsc_content_desc_paddr; }; CTASSERT(offsetof(struct qat_sym_cookie, - u.qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0); + qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0); CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN == 0); #define MAX_CIPHER_SETUP_BLK_SZ \ (sizeof(struct hw_cipher_config) + \ 2 * HW_KASUMI_KEY_SZ + 2 * HW_KASUMI_BLK_SZ) #define MAX_HASH_SETUP_BLK_SZ sizeof(union hw_auth_algo_blk) struct qat_crypto_desc { uint8_t qcd_content_desc[CONTENT_DESC_MAX_SIZE]; /* must be first */ /* using only for qat 1.5 */ uint8_t qcd_hash_state_prefix_buf[QAT_GCM_AAD_SIZE_MAX]; bus_addr_t qcd_desc_paddr; bus_addr_t qcd_hash_state_paddr; enum fw_slice qcd_slices[MAX_FW_SLICE + 1]; enum fw_la_cmd_id qcd_cmd_id; enum hw_cipher_dir qcd_cipher_dir; /* content desc info */ uint8_t qcd_hdr_sz; /* in quad words */ uint8_t qcd_hw_blk_sz; /* in quad words */ uint32_t qcd_cipher_offset; uint32_t qcd_auth_offset; /* hash info */ uint8_t qcd_state_storage_sz; /* in quad words */ uint32_t qcd_gcm_aad_sz_offset1; uint32_t qcd_gcm_aad_sz_offset2; /* cipher info */ uint16_t qcd_cipher_blk_sz; /* in bytes */ uint16_t qcd_auth_sz; /* in bytes */ uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)]; } __aligned(QAT_OPTIMAL_ALIGN); -/* should be aligned to 64bytes */ struct qat_session { struct qat_crypto_desc *qs_dec_desc; /* should be at top of struct*/ /* decrypt or auth then decrypt or auth */ struct qat_crypto_desc *qs_enc_desc; /* encrypt or encrypt then auth */ struct qat_dmamem qs_desc_mem; enum hw_cipher_algo qs_cipher_algo; enum hw_cipher_mode qs_cipher_mode; enum hw_auth_algo qs_auth_algo; enum hw_auth_mode qs_auth_mode; const uint8_t *qs_cipher_key; int qs_cipher_klen; const uint8_t *qs_auth_key; int qs_auth_klen; int qs_auth_mlen; uint32_t qs_status; #define QAT_SESSION_STATUS_ACTIVE (1 << 0) #define QAT_SESSION_STATUS_FREEING (1 << 1) uint32_t qs_inflight; int qs_aad_length; bool qs_need_wakeup; struct mtx qs_session_mtx; }; struct qat_crypto_bank { uint16_t qcb_bank; struct qat_ring *qcb_sym_tx; struct qat_ring *qcb_sym_rx; struct qat_dmamem qcb_symck_dmamems[QAT_NSYMCOOKIE]; struct qat_sym_cookie *qcb_symck_free[QAT_NSYMCOOKIE]; uint32_t qcb_symck_free_count; struct mtx qcb_bank_mtx; char qcb_ring_names[2][QAT_RING_NAME_SIZE]; /* sym tx,rx */ }; struct qat_crypto { struct qat_softc *qcy_sc; uint32_t qcy_bank_mask; uint16_t qcy_num_banks; int32_t qcy_cid; /* OpenCrypto driver ID */ struct qat_crypto_bank *qcy_banks; /* array of qat_crypto_bank */ uint32_t qcy_session_free_count; struct mtx qcy_crypto_mtx; }; struct qat_hw { int8_t qhw_sram_bar_id; int8_t qhw_misc_bar_id; int8_t qhw_etr_bar_id; bus_size_t qhw_cap_global_offset; bus_size_t qhw_ae_offset; bus_size_t qhw_ae_local_offset; bus_size_t qhw_etr_bundle_size; /* crypto processing callbacks */ size_t qhw_crypto_opaque_offset; void (*qhw_crypto_setup_req_params)(struct qat_crypto_bank *, struct qat_session *, struct qat_crypto_desc const *, struct qat_sym_cookie *, struct cryptop *); void (*qhw_crypto_setup_desc)(struct qat_crypto *, struct qat_session *, struct qat_crypto_desc *); uint8_t qhw_num_banks; /* max number of banks */ uint8_t qhw_num_ap_banks; /* max number of AutoPush banks */ uint8_t qhw_num_rings_per_bank; /* rings per bank */ uint8_t qhw_num_accel; /* max number of accelerators */ uint8_t qhw_num_engines; /* max number of accelerator engines */ uint8_t qhw_tx_rx_gap; uint32_t qhw_tx_rings_mask; uint32_t qhw_clock_per_sec; bool qhw_fw_auth; uint32_t qhw_fw_req_size; uint32_t qhw_fw_resp_size; uint8_t qhw_ring_sym_tx; uint8_t qhw_ring_sym_rx; uint8_t qhw_ring_asym_tx; uint8_t qhw_ring_asym_rx; /* MSIx */ uint32_t qhw_msix_ae_vec_gap; /* gap to ae vec from bank */ const char *qhw_mof_fwname; const char *qhw_mmp_fwname; uint32_t qhw_prod_type; /* cpu type */ /* setup callbacks */ uint32_t (*qhw_get_accel_mask)(struct qat_softc *); uint32_t (*qhw_get_ae_mask)(struct qat_softc *); enum qat_sku (*qhw_get_sku)(struct qat_softc *); uint32_t (*qhw_get_accel_cap)(struct qat_softc *); const char *(*qhw_get_fw_uof_name)(struct qat_softc *); void (*qhw_enable_intr)(struct qat_softc *); void (*qhw_init_etr_intr)(struct qat_softc *, int); int (*qhw_init_admin_comms)(struct qat_softc *); int (*qhw_send_admin_init)(struct qat_softc *); int (*qhw_init_arb)(struct qat_softc *); void (*qhw_get_arb_mapping)(struct qat_softc *, const uint32_t **); void (*qhw_enable_error_correction)(struct qat_softc *); int (*qhw_check_uncorrectable_error)(struct qat_softc *); void (*qhw_print_err_registers)(struct qat_softc *); void (*qhw_disable_error_interrupts)(struct qat_softc *); int (*qhw_check_slice_hang)(struct qat_softc *); int (*qhw_set_ssm_wdtimer)(struct qat_softc *); }; /* sc_flags */ #define QAT_FLAG_ESRAM_ENABLE_AUTO_INIT (1 << 0) #define QAT_FLAG_SHRAM_WAIT_READY (1 << 1) /* sc_accel_cap */ #define QAT_ACCEL_CAP_CRYPTO_SYMMETRIC (1 << 0) #define QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC (1 << 1) #define QAT_ACCEL_CAP_CIPHER (1 << 2) #define QAT_ACCEL_CAP_AUTHENTICATION (1 << 3) #define QAT_ACCEL_CAP_REGEX (1 << 4) #define QAT_ACCEL_CAP_COMPRESSION (1 << 5) #define QAT_ACCEL_CAP_LZS_COMPRESSION (1 << 6) #define QAT_ACCEL_CAP_RANDOM_NUMBER (1 << 7) #define QAT_ACCEL_CAP_ZUC (1 << 8) #define QAT_ACCEL_CAP_SHA3 (1 << 9) #define QAT_ACCEL_CAP_KPT (1 << 10) #define QAT_ACCEL_CAP_BITS \ "\177\020" \ "b\x0a" "KPT\0" \ "b\x09" "SHA3\0" \ "b\x08" "ZUC\0" \ "b\x07" "RANDOM_NUMBER\0" \ "b\x06" "LZS_COMPRESSION\0" \ "b\x05" "COMPRESSION\0" \ "b\x04" "REGEX\0" \ "b\x03" "AUTHENTICATION\0" \ "b\x02" "CIPHER\0" \ "b\x01" "CRYPTO_ASYMMETRIC\0" \ "b\x00" "CRYPTO_SYMMETRIC\0" #define QAT_HI_PRIO_RING_WEIGHT 0xfc #define QAT_LO_PRIO_RING_WEIGHT 0xfe #define QAT_DEFAULT_RING_WEIGHT 0xff #define QAT_DEFAULT_PVL 0 struct firmware; struct resource; struct qat_softc { device_t sc_dev; struct resource *sc_res[MAX_BARS]; int sc_rid[MAX_BARS]; bus_space_tag_t sc_csrt[MAX_BARS]; bus_space_handle_t sc_csrh[MAX_BARS]; uint32_t sc_ae_num; uint32_t sc_ae_mask; struct qat_crypto sc_crypto; /* crypto services */ struct qat_hw sc_hw; uint8_t sc_rev; enum qat_sku sc_sku; uint32_t sc_flags; uint32_t sc_accel_num; uint32_t sc_accel_mask; uint32_t sc_accel_cap; struct qat_admin_rings sc_admin_rings; /* use only for qat 1.5 */ struct qat_admin_comms sc_admin_comms; /* use only for qat 1.7 */ /* ETR */ struct qat_bank *sc_etr_banks; /* array of etr banks */ struct qat_ap_bank *sc_etr_ap_banks; /* array of etr auto push banks */ /* AE */ struct qat_ae sc_ae[MAX_NUM_AE]; /* Interrupt */ struct resource *sc_ih; /* ae cluster ih */ void *sc_ih_cookie; /* ae cluster ih cookie */ /* Counters */ counter_u64_t sc_gcm_aad_restarts; counter_u64_t sc_gcm_aad_updates; counter_u64_t sc_ring_full_restarts; counter_u64_t sc_sym_alloc_failures; /* Firmware */ void *sc_fw_mof; /* mof data */ size_t sc_fw_mof_size; /* mof size */ struct qat_mof sc_mof; /* mof sections */ const char *sc_fw_uof_name; /* uof/suof name in mof */ void *sc_fw_uof; /* uof head */ size_t sc_fw_uof_size; /* uof size */ struct qat_aefw_uof sc_aefw_uof; /* UOF_OBJS in uof */ void *sc_fw_suof; /* suof head */ size_t sc_fw_suof_size; /* suof size */ struct qat_aefw_suof sc_aefw_suof; /* suof context */ void *sc_fw_mmp; /* mmp data */ size_t sc_fw_mmp_size; /* mmp size */ }; static inline void qat_bar_write_4(struct qat_softc *sc, int baroff, bus_size_t offset, uint32_t value) { MPASS(baroff >= 0 && baroff < MAX_BARS); bus_space_write_4(sc->sc_csrt[baroff], sc->sc_csrh[baroff], offset, value); } static inline uint32_t qat_bar_read_4(struct qat_softc *sc, int baroff, bus_size_t offset) { MPASS(baroff >= 0 && baroff < MAX_BARS); return bus_space_read_4(sc->sc_csrt[baroff], sc->sc_csrh[baroff], offset); } static inline void qat_misc_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) { qat_bar_write_4(sc, sc->sc_hw.qhw_misc_bar_id, offset, value); } static inline uint32_t qat_misc_read_4(struct qat_softc *sc, bus_size_t offset) { return qat_bar_read_4(sc, sc->sc_hw.qhw_misc_bar_id, offset); } static inline void qat_misc_read_write_or_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) { uint32_t reg; reg = qat_misc_read_4(sc, offset); reg |= value; qat_misc_write_4(sc, offset, reg); } static inline void qat_misc_read_write_and_4(struct qat_softc *sc, bus_size_t offset, uint32_t mask) { uint32_t reg; reg = qat_misc_read_4(sc, offset); reg &= mask; qat_misc_write_4(sc, offset, reg); } static inline void qat_etr_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) { qat_bar_write_4(sc, sc->sc_hw.qhw_etr_bar_id, offset, value); } static inline uint32_t qat_etr_read_4(struct qat_softc *sc, bus_size_t offset) { return qat_bar_read_4(sc, sc->sc_hw.qhw_etr_bar_id, offset); } static inline void qat_ae_local_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, uint32_t value) { offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) | (offset & AE_LOCAL_CSR_MASK); qat_misc_write_4(sc, sc->sc_hw.qhw_ae_local_offset + offset, value); } static inline uint32_t qat_ae_local_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset) { offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) | (offset & AE_LOCAL_CSR_MASK); return qat_misc_read_4(sc, sc->sc_hw.qhw_ae_local_offset + offset); } static inline void qat_ae_xfer_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, uint32_t value) { offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_XFER_AE_MASK) | __SHIFTIN(offset, AE_XFER_CSR_MASK); qat_misc_write_4(sc, sc->sc_hw.qhw_ae_offset + offset, value); } static inline void qat_cap_global_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) { qat_misc_write_4(sc, sc->sc_hw.qhw_cap_global_offset + offset, value); } static inline uint32_t qat_cap_global_read_4(struct qat_softc *sc, bus_size_t offset) { return qat_misc_read_4(sc, sc->sc_hw.qhw_cap_global_offset + offset); } static inline void qat_etr_bank_write_4(struct qat_softc *sc, int bank, bus_size_t offset, uint32_t value) { qat_etr_write_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset, value); } static inline uint32_t qat_etr_bank_read_4(struct qat_softc *sc, int bank, bus_size_t offset) { return qat_etr_read_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset); } static inline void qat_etr_ap_bank_write_4(struct qat_softc *sc, int ap_bank, bus_size_t offset, uint32_t value) { qat_etr_write_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset, value); } static inline uint32_t qat_etr_ap_bank_read_4(struct qat_softc *sc, int ap_bank, bus_size_t offset) { return qat_etr_read_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset); } static inline void qat_etr_bank_ring_write_4(struct qat_softc *sc, int bank, int ring, bus_size_t offset, uint32_t value) { qat_etr_bank_write_4(sc, bank, (ring << 2) + offset, value); } static inline uint32_t qat_etr_bank_ring_read_4(struct qat_softc *sc, int bank, int ring, bus_size_t offset) { return qat_etr_bank_read_4(sc, bank, (ring << 2) * offset); } static inline void qat_etr_bank_ring_base_write_8(struct qat_softc *sc, int bank, int ring, uint64_t value) { uint32_t lo, hi; lo = (uint32_t)(value & 0xffffffff); hi = (uint32_t)((value & 0xffffffff00000000ULL) >> 32); qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_LBASE, lo); qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_UBASE, hi); } static inline void qat_arb_ringsrvarben_write_4(struct qat_softc *sc, int index, uint32_t value) { qat_etr_write_4(sc, ARB_RINGSRVARBEN_OFFSET + (ARB_REG_SLOT * index), value); } static inline void qat_arb_sarconfig_write_4(struct qat_softc *sc, int index, uint32_t value) { qat_etr_write_4(sc, ARB_OFFSET + (ARB_REG_SIZE * index), value); } static inline void qat_arb_wrk_2_ser_map_write_4(struct qat_softc *sc, int index, uint32_t value) { qat_etr_write_4(sc, ARB_OFFSET + ARB_WRK_2_SER_MAP_OFFSET + (ARB_REG_SIZE * index), value); } void * qat_alloc_mem(size_t); void qat_free_mem(void *); void qat_free_dmamem(struct qat_softc *, struct qat_dmamem *); int qat_alloc_dmamem(struct qat_softc *, struct qat_dmamem *, int, bus_size_t, bus_size_t); int qat_etr_setup_ring(struct qat_softc *, int, uint32_t, uint32_t, uint32_t, qat_cb_t, void *, const char *, struct qat_ring **); int qat_etr_put_msg(struct qat_softc *, struct qat_ring *, uint32_t *); void qat_memcpy_htobe64(void *, const void *, size_t); void qat_memcpy_htobe32(void *, const void *, size_t); void qat_memcpy_htobe(void *, const void *, size_t, uint32_t); void qat_crypto_gmac_precompute(const struct qat_crypto_desc *, const uint8_t *key, int klen, const struct qat_sym_hash_def *, uint8_t *); void qat_crypto_hmac_precompute(const struct qat_crypto_desc *, const uint8_t *, int, const struct qat_sym_hash_def *, uint8_t *, uint8_t *); uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *, const struct qat_session *); uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *, const struct qat_session *, struct qat_sym_hash_def const **); #endif