Index: head/share/man/man4/Makefile =================================================================== --- head/share/man/man4/Makefile +++ head/share/man/man4/Makefile @@ -431,6 +431,7 @@ pty.4 \ puc.4 \ pwmc.4 \ + ${_qat.4} \ ${_qlxge.4} \ ${_qlxgb.4} \ ${_qlxgbe.4} \ @@ -823,6 +824,7 @@ _ossl.4= ossl.4 _padlock.4= padlock.4 _pchtherm.4= pchtherm.4 +_qat.4= qat.4 _rr232x.4= rr232x.4 _speaker.4= speaker.4 _spkr.4= spkr.4 Index: head/share/man/man4/qat.4 =================================================================== --- head/share/man/man4/qat.4 +++ head/share/man/man4/qat.4 @@ -0,0 +1,99 @@ +.\"- +.\" Copyright (c) 2020 Rubicon Communications, LLC (Netgate) +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd November 5, 2020 +.Dt QAT 4 +.Os +.Sh NAME +.Nm qat +.Nd Intel QuickAssist Technology (QAT) driver +.Sh SYNOPSIS +To compile this driver into the kernel, +place the following lines in your +kernel configuration file: +.Bd -ragged -offset indent +.Cd "device crypto" +.Cd "device cryptodev" +.Cd "device qat" +.Ed +.Pp +Alternatively, to load the driver as a +module at boot time, place the following lines in +.Xr loader.conf 5 : +.Bd -literal -offset indent +qat_load="YES" +qat_c2xxxfw_load="YES" +qat_c3xxxfw_load="YES" +qat_c63xfw_load="YES" +qat_d15xxfw_load="YES" +qat_dh895xcc_load="YES" +.Ed +.Sh DESCRIPTION +The +.Nm +driver implements +.Xr crypto 4 +support for some of the cryptographic acceleration functions of the Intel +QuickAssist device. +The +.Nm +driver supports the QAT devices integrated with Atom C2000 and C3000 and Xeon +C620 and D-1500 chipsets, and the Intel QAT Adapter 8950. +It can accelerate AES in CBC, CTR, XTS (except for the C2000) and GCM modes, +and can perform authenticated encryption combining the CBC, CTR and XTS modes +with SHA1-HMAC and SHA2-HMAC. +The +.Nm +driver can also compute SHA1 and SHA2 digests. +.Sh SEE ALSO +.Xr crypto 4 , +.Xr ipsec 4 , +.Xr pci 4 , +.Xr random 4 , +.Xr crypto 7 , +.Xr crypto 9 +.Sh HISTORY +The +.Nm +driver first appeared in +.Fx 13.0 . +.Sh AUTHORS +The +.Nm +driver was written for +.Nx +by +.An Hikaru Abe Aq Mt hikaru@iij.ad.jp +and ported to +.Fx +by +.An Mark Johnston Aq Mt markj@FreeBSD.org . +.Sh BUGS +Some Atom C2000 QAT devices have two acceleration engines instead of one. +The +.Nm +driver currently misbehaves when both are enabled and thus does not enable +the second acceleration engine if one is present. Index: head/sys/amd64/conf/NOTES =================================================================== --- head/sys/amd64/conf/NOTES +++ head/sys/amd64/conf/NOTES @@ -471,6 +471,10 @@ device pmspcv # +# Intel QuickAssist +device qat + +# # SafeNet crypto driver: can be moved to the MI NOTES as soon as # it's tested on a big-endian machine # Index: head/sys/conf/files.x86 =================================================================== --- head/sys/conf/files.x86 +++ head/sys/conf/files.x86 @@ -291,6 +291,15 @@ dev/isci/scil/scif_sas_task_request_states.c optional isci dev/isci/scil/scif_sas_timer.c optional isci dev/itwd/itwd.c optional itwd +dev/qat/qat.c optional qat +dev/qat/qat_ae.c optional qat +dev/qat/qat_c2xxx.c optional qat +dev/qat/qat_c3xxx.c optional qat +dev/qat/qat_c62x.c optional qat +dev/qat/qat_d15xx.c optional qat +dev/qat/qat_dh895xcc.c optional qat +dev/qat/qat_hw15.c optional qat +dev/qat/qat_hw17.c optional qat libkern/x86/crc32_sse42.c standard # # x86 shared code between IA32 and AMD64 architectures Index: head/sys/dev/qat/qat.c =================================================================== --- head/sys/dev/qat/qat.c +++ head/sys/dev/qat/qat.c @@ -0,0 +1,2140 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat.c,v 1.6 2020/06/14 23:23:12 riastradh Exp $"); +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "cryptodev_if.h" + +#include +#include + +#include "qatreg.h" +#include "qatvar.h" +#include "qat_aevar.h" + +extern struct qat_hw qat_hw_c2xxx; +extern struct qat_hw qat_hw_c3xxx; +extern struct qat_hw qat_hw_c62x; +extern struct qat_hw qat_hw_d15xx; +extern struct qat_hw qat_hw_dh895xcc; + +#define PCI_VENDOR_INTEL 0x8086 +#define PCI_PRODUCT_INTEL_C2000_IQIA_PHYS 0x1f18 +#define PCI_PRODUCT_INTEL_C3K_QAT 0x19e2 +#define PCI_PRODUCT_INTEL_C3K_QAT_VF 0x19e3 +#define PCI_PRODUCT_INTEL_C620_QAT 0x37c8 +#define PCI_PRODUCT_INTEL_C620_QAT_VF 0x37c9 +#define PCI_PRODUCT_INTEL_XEOND_QAT 0x6f54 +#define PCI_PRODUCT_INTEL_XEOND_QAT_VF 0x6f55 +#define PCI_PRODUCT_INTEL_DH895XCC_QAT 0x0435 +#define PCI_PRODUCT_INTEL_DH895XCC_QAT_VF 0x0443 + +static const struct qat_product { + uint16_t qatp_vendor; + uint16_t qatp_product; + const char *qatp_name; + enum qat_chip_type qatp_chip; + const struct qat_hw *qatp_hw; +} qat_products[] = { + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C2000_IQIA_PHYS, + "Intel C2000 QuickAssist PF", + QAT_CHIP_C2XXX, &qat_hw_c2xxx }, + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C3K_QAT, + "Intel C3000 QuickAssist PF", + QAT_CHIP_C3XXX, &qat_hw_c3xxx }, + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_C620_QAT, + "Intel C620/Xeon D-2100 QuickAssist PF", + QAT_CHIP_C62X, &qat_hw_c62x }, + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XEOND_QAT, + "Intel Xeon D-1500 QuickAssist PF", + QAT_CHIP_D15XX, &qat_hw_d15xx }, + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH895XCC_QAT, + "Intel 8950 QuickAssist PCIe Adapter PF", + QAT_CHIP_DH895XCC, &qat_hw_dh895xcc }, + { 0, 0, NULL, 0, NULL }, +}; + +/* Hash Algorithm specific structure */ + +/* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha1_initial_state[QAT_HASH_SHA1_STATE_SIZE] = { + 0x67, 0x45, 0x23, 0x01, + 0xef, 0xcd, 0xab, 0x89, + 0x98, 0xba, 0xdc, 0xfe, + 0x10, 0x32, 0x54, 0x76, + 0xc3, 0xd2, 0xe1, 0xf0 +}; + +/* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha256_initial_state[QAT_HASH_SHA256_STATE_SIZE] = { + 0x6a, 0x09, 0xe6, 0x67, + 0xbb, 0x67, 0xae, 0x85, + 0x3c, 0x6e, 0xf3, 0x72, + 0xa5, 0x4f, 0xf5, 0x3a, + 0x51, 0x0e, 0x52, 0x7f, + 0x9b, 0x05, 0x68, 0x8c, + 0x1f, 0x83, 0xd9, 0xab, + 0x5b, 0xe0, 0xcd, 0x19 +}; + +/* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha384_initial_state[QAT_HASH_SHA384_STATE_SIZE] = { + 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, + 0x62, 0x9a, 0x29, 0x2a, 0x36, 0x7c, 0xd5, 0x07, + 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, + 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, + 0x67, 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, + 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, + 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, + 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4 +}; + +/* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha512_initial_state[QAT_HASH_SHA512_STATE_SIZE] = { + 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, + 0xbb, 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, + 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 0xf8, 0x2b, + 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, + 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, + 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, + 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, + 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79 +}; + +static const struct qat_sym_hash_alg_info sha1_info = { + .qshai_digest_len = QAT_HASH_SHA1_DIGEST_SIZE, + .qshai_block_len = QAT_HASH_SHA1_BLOCK_SIZE, + .qshai_state_size = QAT_HASH_SHA1_STATE_SIZE, + .qshai_init_state = sha1_initial_state, + .qshai_sah = &auth_hash_hmac_sha1, + .qshai_state_offset = 0, + .qshai_state_word = 4, +}; + +static const struct qat_sym_hash_alg_info sha256_info = { + .qshai_digest_len = QAT_HASH_SHA256_DIGEST_SIZE, + .qshai_block_len = QAT_HASH_SHA256_BLOCK_SIZE, + .qshai_state_size = QAT_HASH_SHA256_STATE_SIZE, + .qshai_init_state = sha256_initial_state, + .qshai_sah = &auth_hash_hmac_sha2_256, + .qshai_state_offset = offsetof(SHA256_CTX, state), + .qshai_state_word = 4, +}; + +static const struct qat_sym_hash_alg_info sha384_info = { + .qshai_digest_len = QAT_HASH_SHA384_DIGEST_SIZE, + .qshai_block_len = QAT_HASH_SHA384_BLOCK_SIZE, + .qshai_state_size = QAT_HASH_SHA384_STATE_SIZE, + .qshai_init_state = sha384_initial_state, + .qshai_sah = &auth_hash_hmac_sha2_384, + .qshai_state_offset = offsetof(SHA384_CTX, state), + .qshai_state_word = 8, +}; + +static const struct qat_sym_hash_alg_info sha512_info = { + .qshai_digest_len = QAT_HASH_SHA512_DIGEST_SIZE, + .qshai_block_len = QAT_HASH_SHA512_BLOCK_SIZE, + .qshai_state_size = QAT_HASH_SHA512_STATE_SIZE, + .qshai_init_state = sha512_initial_state, + .qshai_sah = &auth_hash_hmac_sha2_512, + .qshai_state_offset = offsetof(SHA512_CTX, state), + .qshai_state_word = 8, +}; + +static const struct qat_sym_hash_alg_info aes_gcm_info = { + .qshai_digest_len = QAT_HASH_AES_GCM_DIGEST_SIZE, + .qshai_block_len = QAT_HASH_AES_GCM_BLOCK_SIZE, + .qshai_state_size = QAT_HASH_AES_GCM_STATE_SIZE, + .qshai_sah = &auth_hash_nist_gmac_aes_128, +}; + +/* Hash QAT specific structures */ + +static const struct qat_sym_hash_qat_info sha1_config = { + .qshqi_algo_enc = HW_AUTH_ALGO_SHA1, + .qshqi_auth_counter = QAT_HASH_SHA1_BLOCK_SIZE, + .qshqi_state1_len = HW_SHA1_STATE1_SZ, + .qshqi_state2_len = HW_SHA1_STATE2_SZ, +}; + +static const struct qat_sym_hash_qat_info sha256_config = { + .qshqi_algo_enc = HW_AUTH_ALGO_SHA256, + .qshqi_auth_counter = QAT_HASH_SHA256_BLOCK_SIZE, + .qshqi_state1_len = HW_SHA256_STATE1_SZ, + .qshqi_state2_len = HW_SHA256_STATE2_SZ +}; + +static const struct qat_sym_hash_qat_info sha384_config = { + .qshqi_algo_enc = HW_AUTH_ALGO_SHA384, + .qshqi_auth_counter = QAT_HASH_SHA384_BLOCK_SIZE, + .qshqi_state1_len = HW_SHA384_STATE1_SZ, + .qshqi_state2_len = HW_SHA384_STATE2_SZ +}; + +static const struct qat_sym_hash_qat_info sha512_config = { + .qshqi_algo_enc = HW_AUTH_ALGO_SHA512, + .qshqi_auth_counter = QAT_HASH_SHA512_BLOCK_SIZE, + .qshqi_state1_len = HW_SHA512_STATE1_SZ, + .qshqi_state2_len = HW_SHA512_STATE2_SZ +}; + +static const struct qat_sym_hash_qat_info aes_gcm_config = { + .qshqi_algo_enc = HW_AUTH_ALGO_GALOIS_128, + .qshqi_auth_counter = QAT_HASH_AES_GCM_BLOCK_SIZE, + .qshqi_state1_len = HW_GALOIS_128_STATE1_SZ, + .qshqi_state2_len = + HW_GALOIS_H_SZ + HW_GALOIS_LEN_A_SZ + HW_GALOIS_E_CTR0_SZ, +}; + +static const struct qat_sym_hash_def qat_sym_hash_defs[] = { + [QAT_SYM_HASH_SHA1] = { &sha1_info, &sha1_config }, + [QAT_SYM_HASH_SHA256] = { &sha256_info, &sha256_config }, + [QAT_SYM_HASH_SHA384] = { &sha384_info, &sha384_config }, + [QAT_SYM_HASH_SHA512] = { &sha512_info, &sha512_config }, + [QAT_SYM_HASH_AES_GCM] = { &aes_gcm_info, &aes_gcm_config }, +}; + +static const struct qat_product *qat_lookup(device_t); +static int qat_probe(device_t); +static int qat_attach(device_t); +static int qat_init(struct device *); +static int qat_start(struct device *); +static int qat_detach(device_t); + +static int qat_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp); +static void qat_freesession(device_t dev, crypto_session_t cses); + +static int qat_setup_msix_intr(struct qat_softc *); + +static void qat_etr_init(struct qat_softc *); +static void qat_etr_deinit(struct qat_softc *); +static void qat_etr_bank_init(struct qat_softc *, int); +static void qat_etr_bank_deinit(struct qat_softc *sc, int); + +static void qat_etr_ap_bank_init(struct qat_softc *); +static void qat_etr_ap_bank_set_ring_mask(uint32_t *, uint32_t, int); +static void qat_etr_ap_bank_set_ring_dest(struct qat_softc *, uint32_t *, + uint32_t, int); +static void qat_etr_ap_bank_setup_ring(struct qat_softc *, + struct qat_ring *); +static int qat_etr_verify_ring_size(uint32_t, uint32_t); + +static int qat_etr_ring_intr(struct qat_softc *, struct qat_bank *, + struct qat_ring *); +static void qat_etr_bank_intr(void *); + +static void qat_arb_update(struct qat_softc *, struct qat_bank *); + +static struct qat_sym_cookie *qat_crypto_alloc_sym_cookie( + struct qat_crypto_bank *); +static void qat_crypto_free_sym_cookie(struct qat_crypto_bank *, + struct qat_sym_cookie *); +static int qat_crypto_setup_ring(struct qat_softc *, + struct qat_crypto_bank *); +static int qat_crypto_bank_init(struct qat_softc *, + struct qat_crypto_bank *); +static int qat_crypto_init(struct qat_softc *); +static void qat_crypto_deinit(struct qat_softc *); +static int qat_crypto_start(struct qat_softc *); +static void qat_crypto_stop(struct qat_softc *); +static int qat_crypto_sym_rxintr(struct qat_softc *, void *, void *); + +static MALLOC_DEFINE(M_QAT, "qat", "Intel QAT driver"); + +static const struct qat_product * +qat_lookup(device_t dev) +{ + const struct qat_product *qatp; + + for (qatp = qat_products; qatp->qatp_name != NULL; qatp++) { + if (pci_get_vendor(dev) == qatp->qatp_vendor && + pci_get_device(dev) == qatp->qatp_product) + return qatp; + } + return NULL; +} + +static int +qat_probe(device_t dev) +{ + const struct qat_product *prod; + + prod = qat_lookup(dev); + if (prod != NULL) { + device_set_desc(dev, prod->qatp_name); + return BUS_PROBE_DEFAULT; + } + return ENXIO; +} + +static int +qat_attach(device_t dev) +{ + struct qat_softc *sc = device_get_softc(dev); + const struct qat_product *qatp; + bus_size_t msixtbl_offset; + int bar, count, error, i, msixoff, msixtbl_bar; + + sc->sc_dev = dev; + sc->sc_rev = pci_get_revid(dev); + + qatp = qat_lookup(dev); + memcpy(&sc->sc_hw, qatp->qatp_hw, sizeof(struct qat_hw)); + + /* Determine active accelerators and engines */ + sc->sc_accel_mask = sc->sc_hw.qhw_get_accel_mask(sc); + sc->sc_ae_mask = sc->sc_hw.qhw_get_ae_mask(sc); + + sc->sc_accel_num = 0; + for (i = 0; i < sc->sc_hw.qhw_num_accel; i++) { + if (sc->sc_accel_mask & (1 << i)) + sc->sc_accel_num++; + } + sc->sc_ae_num = 0; + for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) { + if (sc->sc_ae_mask & (1 << i)) + sc->sc_ae_num++; + } + + if (!sc->sc_accel_mask || (sc->sc_ae_mask & 0x01) == 0) { + device_printf(sc->sc_dev, "couldn't find acceleration"); + goto fail; + } + + MPASS(sc->sc_accel_num <= MAX_NUM_ACCEL); + MPASS(sc->sc_ae_num <= MAX_NUM_AE); + + /* Determine SKU and capabilities */ + sc->sc_sku = sc->sc_hw.qhw_get_sku(sc); + sc->sc_accel_cap = sc->sc_hw.qhw_get_accel_cap(sc); + sc->sc_fw_uof_name = sc->sc_hw.qhw_get_fw_uof_name(sc); + + /* Map BARs */ + msixtbl_bar = 0; + msixtbl_offset = 0; + if (pci_find_cap(dev, PCIY_MSIX, &msixoff) == 0) { + uint32_t msixtbl; + msixtbl = pci_read_config(dev, msixoff + PCIR_MSIX_TABLE, 4); + msixtbl_offset = msixtbl & ~PCIM_MSIX_BIR_MASK; + msixtbl_bar = PCIR_BAR(msixtbl & PCIM_MSIX_BIR_MASK); + } + + i = 0; + if (sc->sc_hw.qhw_sram_bar_id != NO_PCI_REG) { + MPASS(sc->sc_hw.qhw_sram_bar_id == 0); + uint32_t fusectl = pci_read_config(dev, FUSECTL_REG, 4); + /* Skip SRAM BAR */ + i = (fusectl & FUSECTL_MASK) ? 1 : 0; + } + for (bar = 0; bar < PCIR_MAX_BAR_0; bar++) { + uint32_t val = pci_read_config(dev, PCIR_BAR(bar), 4); + if (val == 0 || !PCI_BAR_MEM(val)) + continue; + + sc->sc_rid[i] = PCIR_BAR(bar); + sc->sc_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->sc_rid[i], RF_ACTIVE); + if (sc->sc_res[i] == NULL) { + device_printf(dev, "couldn't map BAR %d\n", bar); + goto fail; + } + + sc->sc_csrt[i] = rman_get_bustag(sc->sc_res[i]); + sc->sc_csrh[i] = rman_get_bushandle(sc->sc_res[i]); + + i++; + if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) + bar++; + } + + pci_enable_busmaster(dev); + + count = sc->sc_hw.qhw_num_banks + 1; + if (pci_msix_count(dev) < count) { + device_printf(dev, "insufficient MSI-X vectors (%d vs. %d)\n", + pci_msix_count(dev), count); + goto fail; + } + error = pci_alloc_msix(dev, &count); + if (error != 0) { + device_printf(dev, "failed to allocate MSI-X vectors\n"); + goto fail; + } + + error = qat_init(dev); + if (error == 0) + return 0; + +fail: + qat_detach(dev); + return ENXIO; +} + +static int +qat_init(device_t dev) +{ + struct qat_softc *sc = device_get_softc(dev); + int error; + + qat_etr_init(sc); + + if (sc->sc_hw.qhw_init_admin_comms != NULL && + (error = sc->sc_hw.qhw_init_admin_comms(sc)) != 0) { + device_printf(sc->sc_dev, + "Could not initialize admin comms: %d\n", error); + return error; + } + + if (sc->sc_hw.qhw_init_arb != NULL && + (error = sc->sc_hw.qhw_init_arb(sc)) != 0) { + device_printf(sc->sc_dev, + "Could not initialize hw arbiter: %d\n", error); + return error; + } + + error = qat_ae_init(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not initialize Acceleration Engine: %d\n", error); + return error; + } + + error = qat_aefw_load(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not load firmware: %d\n", error); + return error; + } + + error = qat_setup_msix_intr(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not setup interrupts: %d\n", error); + return error; + } + + sc->sc_hw.qhw_enable_intr(sc); + + error = qat_crypto_init(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not initialize service: %d\n", error); + return error; + } + + if (sc->sc_hw.qhw_enable_error_correction != NULL) + sc->sc_hw.qhw_enable_error_correction(sc); + + if (sc->sc_hw.qhw_set_ssm_wdtimer != NULL && + (error = sc->sc_hw.qhw_set_ssm_wdtimer(sc)) != 0) { + device_printf(sc->sc_dev, + "Could not initialize watchdog timer: %d\n", error); + return error; + } + + error = qat_start(dev); + if (error) { + device_printf(sc->sc_dev, + "Could not start: %d\n", error); + return error; + } + + return 0; +} + +static int +qat_start(device_t dev) +{ + struct qat_softc *sc = device_get_softc(dev); + int error; + + error = qat_ae_start(sc); + if (error) + return error; + + if (sc->sc_hw.qhw_send_admin_init != NULL && + (error = sc->sc_hw.qhw_send_admin_init(sc)) != 0) { + return error; + } + + error = qat_crypto_start(sc); + if (error) + return error; + + return 0; +} + +static int +qat_detach(device_t dev) +{ + struct qat_softc *sc; + int bar, i; + + sc = device_get_softc(dev); + + qat_crypto_stop(sc); + qat_crypto_deinit(sc); + qat_aefw_unload(sc); + + if (sc->sc_etr_banks != NULL) { + for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) { + struct qat_bank *qb = &sc->sc_etr_banks[i]; + + if (qb->qb_ih_cookie != NULL) + (void)bus_teardown_intr(dev, qb->qb_ih, + qb->qb_ih_cookie); + if (qb->qb_ih != NULL) + (void)bus_release_resource(dev, SYS_RES_IRQ, + i + 1, qb->qb_ih); + } + } + if (sc->sc_ih_cookie != NULL) { + (void)bus_teardown_intr(dev, sc->sc_ih, sc->sc_ih_cookie); + sc->sc_ih_cookie = NULL; + } + if (sc->sc_ih != NULL) { + (void)bus_release_resource(dev, SYS_RES_IRQ, i + 1, sc->sc_ih); + sc->sc_ih = NULL; + } + pci_release_msi(dev); + + qat_etr_deinit(sc); + + for (bar = 0; bar < MAX_BARS; bar++) { + if (sc->sc_res[bar] != NULL) { + (void)bus_release_resource(dev, SYS_RES_MEMORY, + sc->sc_rid[bar], sc->sc_res[bar]); + sc->sc_res[bar] = NULL; + } + } + + return 0; +} + +void * +qat_alloc_mem(size_t size) +{ + return (malloc(size, M_QAT, M_WAITOK | M_ZERO)); +} + +void +qat_free_mem(void *ptr) +{ + free(ptr, M_QAT); +} + +static void +qat_alloc_dmamem_cb(void *arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct qat_dmamem *qdm; + + if (error != 0) + return; + + KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg)); + qdm = arg; + qdm->qdm_dma_seg = segs[0]; +} + +int +qat_alloc_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm, + int nseg, bus_size_t size, bus_size_t alignment) +{ + int error; + + KASSERT(qdm->qdm_dma_vaddr == NULL, + ("%s: DMA memory descriptor in use", __func__)); + + error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), + alignment, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + size, /* maxsize */ + nseg, /* nsegments */ + size, /* maxsegsize */ + BUS_DMA_COHERENT, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &qdm->qdm_dma_tag); + if (error != 0) + return error; + + error = bus_dmamem_alloc(qdm->qdm_dma_tag, &qdm->qdm_dma_vaddr, + BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, + &qdm->qdm_dma_map); + if (error != 0) { + device_printf(sc->sc_dev, + "couldn't allocate dmamem, error = %d\n", error); + goto fail_0; + } + + error = bus_dmamap_load(qdm->qdm_dma_tag, qdm->qdm_dma_map, + qdm->qdm_dma_vaddr, size, qat_alloc_dmamem_cb, qdm, + BUS_DMA_NOWAIT); + if (error) { + device_printf(sc->sc_dev, + "couldn't load dmamem map, error = %d\n", error); + goto fail_1; + } + + return 0; +fail_1: + bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, qdm->qdm_dma_map); +fail_0: + bus_dma_tag_destroy(qdm->qdm_dma_tag); + return error; +} + +void +qat_free_dmamem(struct qat_softc *sc, struct qat_dmamem *qdm) +{ + if (qdm->qdm_dma_tag != NULL) { + bus_dmamap_unload(qdm->qdm_dma_tag, qdm->qdm_dma_map); + bus_dmamem_free(qdm->qdm_dma_tag, qdm->qdm_dma_vaddr, + qdm->qdm_dma_map); + bus_dma_tag_destroy(qdm->qdm_dma_tag); + explicit_bzero(qdm, sizeof(*qdm)); + } +} + +static int +qat_setup_msix_intr(struct qat_softc *sc) +{ + device_t dev; + int error, i, rid; + + dev = sc->sc_dev; + + for (i = 1; i <= sc->sc_hw.qhw_num_banks; i++) { + struct qat_bank *qb = &sc->sc_etr_banks[i - 1]; + + rid = i; + qb->qb_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (qb->qb_ih == NULL) { + device_printf(dev, + "failed to allocate bank intr resource\n"); + return ENXIO; + } + error = bus_setup_intr(dev, qb->qb_ih, + INTR_TYPE_NET | INTR_MPSAFE, NULL, qat_etr_bank_intr, qb, + &qb->qb_ih_cookie); + if (error != 0) { + device_printf(dev, "failed to set up bank intr\n"); + return error; + } + error = bus_bind_intr(dev, qb->qb_ih, (i - 1) % mp_ncpus); + if (error != 0) + device_printf(dev, "failed to bind intr %d\n", i); + } + + rid = i; + sc->sc_ih = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->sc_ih == NULL) + return ENXIO; + error = bus_setup_intr(dev, sc->sc_ih, INTR_TYPE_NET | INTR_MPSAFE, + NULL, qat_ae_cluster_intr, sc, &sc->sc_ih_cookie); + + return error; +} + +static void +qat_etr_init(struct qat_softc *sc) +{ + int i; + + sc->sc_etr_banks = qat_alloc_mem( + sizeof(struct qat_bank) * sc->sc_hw.qhw_num_banks); + + for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) + qat_etr_bank_init(sc, i); + + if (sc->sc_hw.qhw_num_ap_banks) { + sc->sc_etr_ap_banks = qat_alloc_mem( + sizeof(struct qat_ap_bank) * sc->sc_hw.qhw_num_ap_banks); + qat_etr_ap_bank_init(sc); + } +} + +static void +qat_etr_deinit(struct qat_softc *sc) +{ + int i; + + if (sc->sc_etr_banks != NULL) { + for (i = 0; i < sc->sc_hw.qhw_num_banks; i++) + qat_etr_bank_deinit(sc, i); + qat_free_mem(sc->sc_etr_banks); + sc->sc_etr_banks = NULL; + } + if (sc->sc_etr_ap_banks != NULL) { + qat_free_mem(sc->sc_etr_ap_banks); + sc->sc_etr_ap_banks = NULL; + } +} + +static void +qat_etr_bank_init(struct qat_softc *sc, int bank) +{ + struct qat_bank *qb = &sc->sc_etr_banks[bank]; + int i, tx_rx_gap = sc->sc_hw.qhw_tx_rx_gap; + + MPASS(bank < sc->sc_hw.qhw_num_banks); + + mtx_init(&qb->qb_bank_mtx, "qb bank", NULL, MTX_DEF); + + qb->qb_sc = sc; + qb->qb_bank = bank; + qb->qb_coalescing_time = COALESCING_TIME_INTERVAL_DEFAULT; + + /* Clean CSRs for all rings within the bank */ + for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { + struct qat_ring *qr = &qb->qb_et_rings[i]; + + qat_etr_bank_ring_write_4(sc, bank, i, + ETR_RING_CONFIG, 0); + qat_etr_bank_ring_base_write_8(sc, bank, i, 0); + + if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { + qr->qr_inflight = qat_alloc_mem(sizeof(uint32_t)); + } else if (sc->sc_hw.qhw_tx_rings_mask & + (1 << (i - tx_rx_gap))) { + /* Share inflight counter with rx and tx */ + qr->qr_inflight = + qb->qb_et_rings[i - tx_rx_gap].qr_inflight; + } + } + + if (sc->sc_hw.qhw_init_etr_intr != NULL) { + sc->sc_hw.qhw_init_etr_intr(sc, bank); + } else { + /* common code in qat 1.7 */ + qat_etr_bank_write_4(sc, bank, ETR_INT_REG, + ETR_INT_REG_CLEAR_MASK); + for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank / + ETR_RINGS_PER_INT_SRCSEL; i++) { + qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL + + (i * ETR_INT_SRCSEL_NEXT_OFFSET), + ETR_INT_SRCSEL_MASK); + } + } +} + +static void +qat_etr_bank_deinit(struct qat_softc *sc, int bank) +{ + struct qat_bank *qb; + struct qat_ring *qr; + int i; + + qb = &sc->sc_etr_banks[bank]; + for (i = 0; i < sc->sc_hw.qhw_num_rings_per_bank; i++) { + if (sc->sc_hw.qhw_tx_rings_mask & (1 << i)) { + qr = &qb->qb_et_rings[i]; + qat_free_mem(qr->qr_inflight); + } + } +} + +static void +qat_etr_ap_bank_init(struct qat_softc *sc) +{ + int ap_bank; + + for (ap_bank = 0; ap_bank < sc->sc_hw.qhw_num_ap_banks; ap_bank++) { + struct qat_ap_bank *qab = &sc->sc_etr_ap_banks[ap_bank]; + + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_MASK, + ETR_AP_NF_MASK_INIT); + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, 0); + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_MASK, + ETR_AP_NE_MASK_INIT); + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, 0); + + memset(qab, 0, sizeof(*qab)); + } +} + +static void +qat_etr_ap_bank_set_ring_mask(uint32_t *ap_mask, uint32_t ring, int set_mask) +{ + if (set_mask) + *ap_mask |= (1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); + else + *ap_mask &= ~(1 << ETR_RING_NUMBER_IN_AP_BANK(ring)); +} + +static void +qat_etr_ap_bank_set_ring_dest(struct qat_softc *sc, uint32_t *ap_dest, + uint32_t ring, int set_dest) +{ + uint32_t ae_mask; + uint8_t mailbox, ae, nae; + uint8_t *dest = (uint8_t *)ap_dest; + + mailbox = ETR_RING_AP_MAILBOX_NUMBER(ring); + + nae = 0; + ae_mask = sc->sc_ae_mask; + for (ae = 0; ae < sc->sc_hw.qhw_num_engines; ae++) { + if ((ae_mask & (1 << ae)) == 0) + continue; + + if (set_dest) { + dest[nae] = __SHIFTIN(ae, ETR_AP_DEST_AE) | + __SHIFTIN(mailbox, ETR_AP_DEST_MAILBOX) | + ETR_AP_DEST_ENABLE; + } else { + dest[nae] = 0; + } + nae++; + if (nae == ETR_MAX_AE_PER_MAILBOX) + break; + } +} + +static void +qat_etr_ap_bank_setup_ring(struct qat_softc *sc, struct qat_ring *qr) +{ + struct qat_ap_bank *qab; + int ap_bank; + + if (sc->sc_hw.qhw_num_ap_banks == 0) + return; + + ap_bank = ETR_RING_AP_BANK_NUMBER(qr->qr_ring); + MPASS(ap_bank < sc->sc_hw.qhw_num_ap_banks); + qab = &sc->sc_etr_ap_banks[ap_bank]; + + if (qr->qr_cb == NULL) { + qat_etr_ap_bank_set_ring_mask(&qab->qab_ne_mask, qr->qr_ring, 1); + if (!qab->qab_ne_dest) { + qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_ne_dest, + qr->qr_ring, 1); + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NE_DEST, + qab->qab_ne_dest); + } + } else { + qat_etr_ap_bank_set_ring_mask(&qab->qab_nf_mask, qr->qr_ring, 1); + if (!qab->qab_nf_dest) { + qat_etr_ap_bank_set_ring_dest(sc, &qab->qab_nf_dest, + qr->qr_ring, 1); + qat_etr_ap_bank_write_4(sc, ap_bank, ETR_AP_NF_DEST, + qab->qab_nf_dest); + } + } +} + +static int +qat_etr_verify_ring_size(uint32_t msg_size, uint32_t num_msgs) +{ + int i = QAT_MIN_RING_SIZE; + + for (; i <= QAT_MAX_RING_SIZE; i++) + if ((msg_size * num_msgs) == QAT_SIZE_TO_RING_SIZE_IN_BYTES(i)) + return i; + + return QAT_DEFAULT_RING_SIZE; +} + +int +qat_etr_setup_ring(struct qat_softc *sc, int bank, uint32_t ring, + uint32_t num_msgs, uint32_t msg_size, qat_cb_t cb, void *cb_arg, + const char *name, struct qat_ring **rqr) +{ + struct qat_bank *qb; + struct qat_ring *qr = NULL; + int error; + uint32_t ring_size_bytes, ring_config; + uint64_t ring_base; + uint32_t wm_nf = ETR_RING_CONFIG_NEAR_WM_512; + uint32_t wm_ne = ETR_RING_CONFIG_NEAR_WM_0; + + MPASS(bank < sc->sc_hw.qhw_num_banks); + + /* Allocate a ring from specified bank */ + qb = &sc->sc_etr_banks[bank]; + + if (ring >= sc->sc_hw.qhw_num_rings_per_bank) + return EINVAL; + if (qb->qb_allocated_rings & (1 << ring)) + return ENOENT; + qr = &qb->qb_et_rings[ring]; + qb->qb_allocated_rings |= 1 << ring; + + /* Initialize allocated ring */ + qr->qr_ring = ring; + qr->qr_bank = bank; + qr->qr_name = name; + qr->qr_ring_id = qr->qr_bank * sc->sc_hw.qhw_num_rings_per_bank + ring; + qr->qr_ring_mask = (1 << ring); + qr->qr_cb = cb; + qr->qr_cb_arg = cb_arg; + + /* Setup the shadow variables */ + qr->qr_head = 0; + qr->qr_tail = 0; + qr->qr_msg_size = QAT_BYTES_TO_MSG_SIZE(msg_size); + qr->qr_ring_size = qat_etr_verify_ring_size(msg_size, num_msgs); + + /* + * To make sure that ring is alligned to ring size allocate + * at least 4k and then tell the user it is smaller. + */ + ring_size_bytes = QAT_SIZE_TO_RING_SIZE_IN_BYTES(qr->qr_ring_size); + ring_size_bytes = QAT_RING_SIZE_BYTES_MIN(ring_size_bytes); + error = qat_alloc_dmamem(sc, &qr->qr_dma, 1, ring_size_bytes, + ring_size_bytes); + if (error) + return error; + + qr->qr_ring_vaddr = qr->qr_dma.qdm_dma_vaddr; + qr->qr_ring_paddr = qr->qr_dma.qdm_dma_seg.ds_addr; + + memset(qr->qr_ring_vaddr, QAT_RING_PATTERN, + qr->qr_dma.qdm_dma_seg.ds_len); + + bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + + if (cb == NULL) { + ring_config = ETR_RING_CONFIG_BUILD(qr->qr_ring_size); + } else { + ring_config = + ETR_RING_CONFIG_BUILD_RESP(qr->qr_ring_size, wm_nf, wm_ne); + } + qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_CONFIG, ring_config); + + ring_base = ETR_RING_BASE_BUILD(qr->qr_ring_paddr, qr->qr_ring_size); + qat_etr_bank_ring_base_write_8(sc, bank, ring, ring_base); + + if (sc->sc_hw.qhw_init_arb != NULL) + qat_arb_update(sc, qb); + + mtx_init(&qr->qr_ring_mtx, "qr ring", NULL, MTX_DEF); + + qat_etr_ap_bank_setup_ring(sc, qr); + + if (cb != NULL) { + uint32_t intr_mask; + + qb->qb_intr_mask |= qr->qr_ring_mask; + intr_mask = qb->qb_intr_mask; + + qat_etr_bank_write_4(sc, bank, ETR_INT_COL_EN, intr_mask); + qat_etr_bank_write_4(sc, bank, ETR_INT_COL_CTL, + ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); + } + + *rqr = qr; + + return 0; +} + +static inline u_int +qat_modulo(u_int data, u_int shift) +{ + u_int div = data >> shift; + u_int mult = div << shift; + return data - mult; +} + +int +qat_etr_put_msg(struct qat_softc *sc, struct qat_ring *qr, uint32_t *msg) +{ + uint32_t inflight; + uint32_t *addr; + + mtx_lock(&qr->qr_ring_mtx); + + inflight = atomic_fetchadd_32(qr->qr_inflight, 1) + 1; + if (inflight > QAT_MAX_INFLIGHTS(qr->qr_ring_size, qr->qr_msg_size)) { + atomic_subtract_32(qr->qr_inflight, 1); + qr->qr_need_wakeup = true; + mtx_unlock(&qr->qr_ring_mtx); + counter_u64_add(sc->sc_ring_full_restarts, 1); + return ERESTART; + } + + addr = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_tail); + + memcpy(addr, msg, QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size)); + + bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, + BUS_DMASYNC_PREWRITE); + + qr->qr_tail = qat_modulo(qr->qr_tail + + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), + QAT_RING_SIZE_MODULO(qr->qr_ring_size)); + + qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, + ETR_RING_TAIL_OFFSET, qr->qr_tail); + + mtx_unlock(&qr->qr_ring_mtx); + + return 0; +} + +static int +qat_etr_ring_intr(struct qat_softc *sc, struct qat_bank *qb, + struct qat_ring *qr) +{ + uint32_t *msg, nmsg = 0; + int handled = 0; + bool blocked = false; + + mtx_lock(&qr->qr_ring_mtx); + + msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); + + bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + while (atomic_load_32(msg) != ETR_RING_EMPTY_ENTRY_SIG) { + atomic_subtract_32(qr->qr_inflight, 1); + + if (qr->qr_cb != NULL) { + mtx_unlock(&qr->qr_ring_mtx); + handled |= qr->qr_cb(sc, qr->qr_cb_arg, msg); + mtx_lock(&qr->qr_ring_mtx); + } + + atomic_store_32(msg, ETR_RING_EMPTY_ENTRY_SIG); + + qr->qr_head = qat_modulo(qr->qr_head + + QAT_MSG_SIZE_TO_BYTES(qr->qr_msg_size), + QAT_RING_SIZE_MODULO(qr->qr_ring_size)); + nmsg++; + + msg = (uint32_t *)((uintptr_t)qr->qr_ring_vaddr + qr->qr_head); + } + + bus_dmamap_sync(qr->qr_dma.qdm_dma_tag, qr->qr_dma.qdm_dma_map, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + + if (nmsg > 0) { + qat_etr_bank_ring_write_4(sc, qr->qr_bank, qr->qr_ring, + ETR_RING_HEAD_OFFSET, qr->qr_head); + if (qr->qr_need_wakeup) { + blocked = true; + qr->qr_need_wakeup = false; + } + } + + mtx_unlock(&qr->qr_ring_mtx); + + if (blocked) + crypto_unblock(sc->sc_crypto.qcy_cid, CRYPTO_SYMQ); + + return handled; +} + +static void +qat_etr_bank_intr(void *arg) +{ + struct qat_bank *qb = arg; + struct qat_softc *sc = qb->qb_sc; + uint32_t estat; + int i, handled = 0; + + mtx_lock(&qb->qb_bank_mtx); + + qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, 0); + + /* Now handle all the responses */ + estat = ~qat_etr_bank_read_4(sc, qb->qb_bank, ETR_E_STAT); + estat &= qb->qb_intr_mask; + + qat_etr_bank_write_4(sc, qb->qb_bank, ETR_INT_COL_CTL, + ETR_INT_COL_CTL_ENABLE | qb->qb_coalescing_time); + + mtx_unlock(&qb->qb_bank_mtx); + + while ((i = ffs(estat)) != 0) { + struct qat_ring *qr = &qb->qb_et_rings[--i]; + estat &= ~(1 << i); + handled |= qat_etr_ring_intr(sc, qb, qr); + } +} + +void +qat_arb_update(struct qat_softc *sc, struct qat_bank *qb) +{ + + qat_arb_ringsrvarben_write_4(sc, qb->qb_bank, + qb->qb_allocated_rings & 0xff); +} + +static struct qat_sym_cookie * +qat_crypto_alloc_sym_cookie(struct qat_crypto_bank *qcb) +{ + struct qat_sym_cookie *qsc; + + mtx_lock(&qcb->qcb_bank_mtx); + + if (qcb->qcb_symck_free_count == 0) { + mtx_unlock(&qcb->qcb_bank_mtx); + return NULL; + } + + qsc = qcb->qcb_symck_free[--qcb->qcb_symck_free_count]; + + mtx_unlock(&qcb->qcb_bank_mtx); + + return qsc; +} + +static void +qat_crypto_free_sym_cookie(struct qat_crypto_bank *qcb, + struct qat_sym_cookie *qsc) +{ + + explicit_bzero(qsc->qsc_iv_buf, sizeof(qsc->qsc_iv_buf)); + explicit_bzero(qsc->qsc_auth_res, sizeof(qsc->qsc_auth_res)); + + mtx_lock(&qcb->qcb_bank_mtx); + qcb->qcb_symck_free[qcb->qcb_symck_free_count++] = qsc; + mtx_unlock(&qcb->qcb_bank_mtx); +} + +void +qat_memcpy_htobe64(void *dst, const void *src, size_t len) +{ + uint64_t *dst0 = dst; + const uint64_t *src0 = src; + size_t i; + + MPASS(len % sizeof(*dst0) == 0); + + for (i = 0; i < len / sizeof(*dst0); i++) + *(dst0 + i) = htobe64(*(src0 + i)); +} + +void +qat_memcpy_htobe32(void *dst, const void *src, size_t len) +{ + uint32_t *dst0 = dst; + const uint32_t *src0 = src; + size_t i; + + MPASS(len % sizeof(*dst0) == 0); + + for (i = 0; i < len / sizeof(*dst0); i++) + *(dst0 + i) = htobe32(*(src0 + i)); +} + +void +qat_memcpy_htobe(void *dst, const void *src, size_t len, uint32_t wordbyte) +{ + switch (wordbyte) { + case 4: + qat_memcpy_htobe32(dst, src, len); + break; + case 8: + qat_memcpy_htobe64(dst, src, len); + break; + default: + panic("invalid word size %u", wordbyte); + } +} + +void +qat_crypto_gmac_precompute(const struct qat_crypto_desc *desc, + const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, + uint8_t *state) +{ + uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)]; + char zeros[AES_BLOCK_LEN]; + int rounds; + + memset(zeros, 0, sizeof(zeros)); + rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY); + rijndaelEncrypt(ks, rounds, zeros, state); + explicit_bzero(ks, sizeof(ks)); +} + +void +qat_crypto_hmac_precompute(const struct qat_crypto_desc *desc, + const uint8_t *key, int klen, const struct qat_sym_hash_def *hash_def, + uint8_t *state1, uint8_t *state2) +{ + union authctx ctx; + const struct auth_hash *sah = hash_def->qshd_alg->qshai_sah; + uint32_t state_offset = hash_def->qshd_alg->qshai_state_offset; + uint32_t state_size = hash_def->qshd_alg->qshai_state_size; + uint32_t state_word = hash_def->qshd_alg->qshai_state_word; + + hmac_init_ipad(sah, key, klen, &ctx); + qat_memcpy_htobe(state1, (uint8_t *)&ctx + state_offset, state_size, + state_word); + hmac_init_opad(sah, key, klen, &ctx); + qat_memcpy_htobe(state2, (uint8_t *)&ctx + state_offset, state_size, + state_word); + explicit_bzero(&ctx, sizeof(ctx)); +} + +static enum hw_cipher_algo +qat_aes_cipher_algo(int klen) +{ + switch (klen) { + case HW_AES_128_KEY_SZ: + return HW_CIPHER_ALGO_AES128; + case HW_AES_192_KEY_SZ: + return HW_CIPHER_ALGO_AES192; + case HW_AES_256_KEY_SZ: + return HW_CIPHER_ALGO_AES256; + default: + panic("invalid key length %d", klen); + } +} + +uint16_t +qat_crypto_load_cipher_session(const struct qat_crypto_desc *desc, + const struct qat_session *qs) +{ + enum hw_cipher_algo algo; + enum hw_cipher_dir dir; + enum hw_cipher_convert key_convert; + enum hw_cipher_mode mode; + + dir = desc->qcd_cipher_dir; + key_convert = HW_CIPHER_NO_CONVERT; + mode = qs->qs_cipher_mode; + switch (mode) { + case HW_CIPHER_CBC_MODE: + case HW_CIPHER_XTS_MODE: + algo = qs->qs_cipher_algo; + + /* + * AES decrypt key needs to be reversed. + * Instead of reversing the key at session registration, + * it is instead reversed on-the-fly by setting the KEY_CONVERT + * bit here. + */ + if (desc->qcd_cipher_dir == HW_CIPHER_DECRYPT) + key_convert = HW_CIPHER_KEY_CONVERT; + break; + case HW_CIPHER_CTR_MODE: + algo = qs->qs_cipher_algo; + dir = HW_CIPHER_ENCRYPT; + break; + default: + panic("unhandled cipher mode %d", mode); + break; + } + + return HW_CIPHER_CONFIG_BUILD(mode, algo, key_convert, dir); +} + +uint16_t +qat_crypto_load_auth_session(const struct qat_crypto_desc *desc, + const struct qat_session *qs, const struct qat_sym_hash_def **hash_def) +{ + enum qat_sym_hash_algorithm algo; + + switch (qs->qs_auth_algo) { + case HW_AUTH_ALGO_SHA1: + algo = QAT_SYM_HASH_SHA1; + break; + case HW_AUTH_ALGO_SHA256: + algo = QAT_SYM_HASH_SHA256; + break; + case HW_AUTH_ALGO_SHA384: + algo = QAT_SYM_HASH_SHA384; + break; + case HW_AUTH_ALGO_SHA512: + algo = QAT_SYM_HASH_SHA512; + break; + case HW_AUTH_ALGO_GALOIS_128: + algo = QAT_SYM_HASH_AES_GCM; + break; + default: + panic("unhandled auth algorithm %d", qs->qs_auth_algo); + break; + } + *hash_def = &qat_sym_hash_defs[algo]; + + return HW_AUTH_CONFIG_BUILD(qs->qs_auth_mode, + (*hash_def)->qshd_qat->qshqi_algo_enc, + (*hash_def)->qshd_alg->qshai_digest_len); +} + +struct qat_crypto_load_cb_arg { + struct qat_session *qs; + struct qat_sym_cookie *qsc; + struct cryptop *crp; + int error; +}; + +static void +qat_crypto_load_cb(void *_arg, bus_dma_segment_t *segs, int nseg, + int error) +{ + struct cryptop *crp; + struct flat_buffer_desc *flatbuf; + struct qat_crypto_load_cb_arg *arg; + struct qat_session *qs; + struct qat_sym_cookie *qsc; + bus_addr_t addr; + bus_size_t len; + int iseg, oseg, skip; + + arg = _arg; + if (error != 0) { + arg->error = error; + return; + } + + crp = arg->crp; + qs = arg->qs; + qsc = arg->qsc; + + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { + /* + * The firmware expects AAD to be in a contiguous buffer and + * padded to a multiple of 16 bytes. To satisfy these + * constraints we bounce the AAD into a per-request buffer. + */ + crypto_copydata(crp, crp->crp_aad_start, crp->crp_aad_length, + qsc->qsc_gcm_aad); + memset(qsc->qsc_gcm_aad + crp->crp_aad_length, 0, + roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) - + crp->crp_aad_length); + skip = crp->crp_payload_start; + } else if (crp->crp_aad_length > 0) { + skip = crp->crp_aad_start; + } else { + skip = crp->crp_payload_start; + } + + for (iseg = oseg = 0; iseg < nseg; iseg++) { + addr = segs[iseg].ds_addr; + len = segs[iseg].ds_len; + + if (skip > 0) { + if (skip < len) { + addr += skip; + len -= skip; + skip = 0; + } else { + skip -= len; + continue; + } + } + + flatbuf = &qsc->qsc_flat_bufs[oseg++]; + flatbuf->data_len_in_bytes = (uint32_t)len; + flatbuf->phy_buffer = (uint64_t)addr; + } + qsc->qsc_buf_list.num_buffers = oseg; +} + +static int +qat_crypto_load(struct qat_session *qs, struct qat_sym_cookie *qsc, + struct qat_crypto_desc const *desc, struct cryptop *crp) +{ + struct qat_crypto_load_cb_arg arg; + int error; + + crypto_read_iv(crp, qsc->qsc_iv_buf); + + arg.crp = crp; + arg.qs = qs; + arg.qsc = qsc; + arg.error = 0; + error = bus_dmamap_load_crp(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, + crp, qat_crypto_load_cb, &arg, BUS_DMA_NOWAIT); + if (error == 0) + error = arg.error; + return error; +} + +static inline struct qat_crypto_bank * +qat_crypto_select_bank(struct qat_crypto *qcy) +{ + u_int cpuid = PCPU_GET(cpuid); + + return &qcy->qcy_banks[cpuid % qcy->qcy_num_banks]; +} + +static int +qat_crypto_setup_ring(struct qat_softc *sc, struct qat_crypto_bank *qcb) +{ + int error, i, bank; + int curname = 0; + char *name; + + bank = qcb->qcb_bank; + + name = qcb->qcb_ring_names[curname++]; + snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_tx", bank); + error = qat_etr_setup_ring(sc, qcb->qcb_bank, + sc->sc_hw.qhw_ring_sym_tx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_req_size, + NULL, NULL, name, &qcb->qcb_sym_tx); + if (error) + return error; + + name = qcb->qcb_ring_names[curname++]; + snprintf(name, QAT_RING_NAME_SIZE, "bank%d sym_rx", bank); + error = qat_etr_setup_ring(sc, qcb->qcb_bank, + sc->sc_hw.qhw_ring_sym_rx, QAT_NSYMREQ, sc->sc_hw.qhw_fw_resp_size, + qat_crypto_sym_rxintr, qcb, name, &qcb->qcb_sym_rx); + if (error) + return error; + + for (i = 0; i < QAT_NSYMCOOKIE; i++) { + struct qat_dmamem *qdm = &qcb->qcb_symck_dmamems[i]; + struct qat_sym_cookie *qsc; + + error = qat_alloc_dmamem(sc, qdm, 1, + sizeof(struct qat_sym_cookie), QAT_OPTIMAL_ALIGN); + if (error) + return error; + + qsc = qdm->qdm_dma_vaddr; + qsc->qsc_self_dmamap = qdm->qdm_dma_map; + qsc->qsc_self_dma_tag = qdm->qdm_dma_tag; + qsc->qsc_bulk_req_params_buf_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + u.qsc_bulk_cookie.qsbc_req_params_buf); + qsc->qsc_buffer_list_desc_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_buf_list); + qsc->qsc_iv_buf_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_iv_buf); + qsc->qsc_auth_res_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_auth_res); + qsc->qsc_gcm_aad_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_gcm_aad); + qsc->qsc_content_desc_paddr = + qdm->qdm_dma_seg.ds_addr + offsetof(struct qat_sym_cookie, + qsc_content_desc); + qcb->qcb_symck_free[i] = qsc; + qcb->qcb_symck_free_count++; + + error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + QAT_MAXLEN, /* maxsize */ + QAT_MAXSEG, /* nsegments */ + QAT_MAXLEN, /* maxsegsize */ + BUS_DMA_COHERENT, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &qsc->qsc_buf_dma_tag); + if (error != 0) + return error; + + error = bus_dmamap_create(qsc->qsc_buf_dma_tag, + BUS_DMA_COHERENT, &qsc->qsc_buf_dmamap); + if (error) + return error; + } + + return 0; +} + +static int +qat_crypto_bank_init(struct qat_softc *sc, struct qat_crypto_bank *qcb) +{ + mtx_init(&qcb->qcb_bank_mtx, "qcb bank", NULL, MTX_DEF); + + return qat_crypto_setup_ring(sc, qcb); +} + +static void +qat_crypto_bank_deinit(struct qat_softc *sc, struct qat_crypto_bank *qcb) +{ + struct qat_dmamem *qdm; + int i; + + for (i = 0; i < QAT_NSYMCOOKIE; i++) { + qdm = &qcb->qcb_symck_dmamems[i]; + qat_free_dmamem(sc, qdm); + } + qat_free_dmamem(sc, &qcb->qcb_sym_tx->qr_dma); + qat_free_dmamem(sc, &qcb->qcb_sym_rx->qr_dma); + + mtx_destroy(&qcb->qcb_bank_mtx); +} + +static int +qat_crypto_init(struct qat_softc *sc) +{ + struct qat_crypto *qcy = &sc->sc_crypto; + struct sysctl_ctx_list *ctx; + struct sysctl_oid *oid; + struct sysctl_oid_list *children; + int bank, error, num_banks; + + qcy->qcy_sc = sc; + + if (sc->sc_hw.qhw_init_arb != NULL) + num_banks = imin(mp_ncpus, sc->sc_hw.qhw_num_banks); + else + num_banks = sc->sc_ae_num; + + qcy->qcy_num_banks = num_banks; + + qcy->qcy_banks = + qat_alloc_mem(sizeof(struct qat_crypto_bank) * num_banks); + + for (bank = 0; bank < num_banks; bank++) { + struct qat_crypto_bank *qcb = &qcy->qcy_banks[bank]; + qcb->qcb_bank = bank; + error = qat_crypto_bank_init(sc, qcb); + if (error) + return error; + } + + mtx_init(&qcy->qcy_crypto_mtx, "qcy crypto", NULL, MTX_DEF); + + ctx = device_get_sysctl_ctx(sc->sc_dev); + oid = device_get_sysctl_tree(sc->sc_dev); + children = SYSCTL_CHILDREN(oid); + oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics"); + children = SYSCTL_CHILDREN(oid); + + sc->sc_gcm_aad_restarts = counter_u64_alloc(M_WAITOK); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_restarts", + CTLFLAG_RD, &sc->sc_gcm_aad_restarts, + "GCM requests deferred due to AAD size change"); + sc->sc_gcm_aad_updates = counter_u64_alloc(M_WAITOK); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "gcm_aad_updates", + CTLFLAG_RD, &sc->sc_gcm_aad_updates, + "GCM requests that required session state update"); + sc->sc_ring_full_restarts = counter_u64_alloc(M_WAITOK); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ring_full", + CTLFLAG_RD, &sc->sc_ring_full_restarts, + "Requests deferred due to in-flight max reached"); + + return 0; +} + +static void +qat_crypto_deinit(struct qat_softc *sc) +{ + struct qat_crypto *qcy = &sc->sc_crypto; + struct qat_crypto_bank *qcb; + int bank; + + if (qcy->qcy_banks != NULL) { + for (bank = 0; bank < qcy->qcy_num_banks; bank++) { + qcb = &qcy->qcy_banks[bank]; + qat_crypto_bank_deinit(sc, qcb); + } + qat_free_mem(qcy->qcy_banks); + mtx_destroy(&qcy->qcy_crypto_mtx); + } +} + +static int +qat_crypto_start(struct qat_softc *sc) +{ + struct qat_crypto *qcy; + + qcy = &sc->sc_crypto; + qcy->qcy_cid = crypto_get_driverid(sc->sc_dev, + sizeof(struct qat_session), CRYPTOCAP_F_HARDWARE); + if (qcy->qcy_cid < 0) { + device_printf(sc->sc_dev, + "could not get opencrypto driver id\n"); + return ENOENT; + } + + return 0; +} + +static void +qat_crypto_stop(struct qat_softc *sc) +{ + struct qat_crypto *qcy; + + qcy = &sc->sc_crypto; + if (qcy->qcy_cid >= 0) + (void)crypto_unregister_all(qcy->qcy_cid); +} + +static int +qat_crypto_sym_rxintr(struct qat_softc *sc, void *arg, void *msg) +{ + char icv[QAT_SYM_HASH_BUFFER_LEN]; + struct qat_crypto_bank *qcb = arg; + struct qat_crypto *qcy; + struct qat_session *qs; + struct qat_sym_cookie *qsc; + struct qat_sym_bulk_cookie *qsbc; + struct cryptop *crp; + int error; + uint16_t auth_sz; + bool blocked; + + qsc = *(void **)((uintptr_t)msg + sc->sc_hw.qhw_crypto_opaque_offset); + + qsbc = &qsc->u.qsc_bulk_cookie; + qcy = qsbc->qsbc_crypto; + qs = qsbc->qsbc_session; + crp = qsbc->qsbc_cb_tag; + + bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap); + + error = 0; + if ((auth_sz = qs->qs_auth_mlen) != 0) { + if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { + crypto_copydata(crp, crp->crp_digest_start, + auth_sz, icv); + if (timingsafe_bcmp(icv, qsc->qsc_auth_res, + auth_sz) != 0) { + error = EBADMSG; + } + } else { + crypto_copyback(crp, crp->crp_digest_start, + auth_sz, qsc->qsc_auth_res); + } + } + + qat_crypto_free_sym_cookie(qcb, qsc); + + blocked = false; + mtx_lock(&qs->qs_session_mtx); + MPASS(qs->qs_status & QAT_SESSION_STATUS_ACTIVE); + qs->qs_inflight--; + if (__predict_false(qs->qs_need_wakeup && qs->qs_inflight == 0)) { + blocked = true; + qs->qs_need_wakeup = false; + } + mtx_unlock(&qs->qs_session_mtx); + + crp->crp_etype = error; + crypto_done(crp); + + if (blocked) + crypto_unblock(qcy->qcy_cid, CRYPTO_SYMQ); + + return 1; +} + +static int +qat_probesession(device_t dev, const struct crypto_session_params *csp) +{ + if (csp->csp_cipher_alg == CRYPTO_AES_XTS && + qat_lookup(dev)->qatp_chip == QAT_CHIP_C2XXX) { + /* + * AES-XTS is not supported by the NanoQAT. + */ + return EINVAL; + } + + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return EINVAL; + break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return EINVAL; + break; + default: + return EINVAL; + } + break; + case CSP_MODE_DIGEST: + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512: + case CRYPTO_SHA2_512_HMAC: + break; + case CRYPTO_AES_NIST_GMAC: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return EINVAL; + break; + default: + return EINVAL; + } + break; + case CSP_MODE_AEAD: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_NIST_GCM_16: + if (csp->csp_ivlen != AES_GCM_IV_LEN) + return EINVAL; + break; + default: + return EINVAL; + } + break; + case CSP_MODE_ETA: + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + case CRYPTO_AES_ICM: + if (csp->csp_ivlen != AES_BLOCK_LEN) + return EINVAL; + break; + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return EINVAL; + break; + default: + return EINVAL; + } + break; + default: + return EINVAL; + } + break; + default: + return EINVAL; + } + + return CRYPTODEV_PROBE_HARDWARE; +} + +static int +qat_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct qat_crypto *qcy; + struct qat_dmamem *qdm; + struct qat_session *qs; + struct qat_softc *sc; + struct qat_crypto_desc *ddesc, *edesc; + int error, slices; + + sc = device_get_softc(dev); + qs = crypto_get_driver_session(cses); + qcy = &sc->sc_crypto; + + qdm = &qs->qs_desc_mem; + error = qat_alloc_dmamem(sc, qdm, QAT_MAXSEG, + sizeof(struct qat_crypto_desc) * 2, QAT_OPTIMAL_ALIGN); + if (error != 0) + return error; + + mtx_init(&qs->qs_session_mtx, "qs session", NULL, MTX_DEF); + qs->qs_aad_length = -1; + + qs->qs_dec_desc = ddesc = qdm->qdm_dma_vaddr; + qs->qs_enc_desc = edesc = ddesc + 1; + + ddesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr; + ddesc->qcd_hash_state_paddr = ddesc->qcd_desc_paddr + + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); + edesc->qcd_desc_paddr = qdm->qdm_dma_seg.ds_addr + + sizeof(struct qat_crypto_desc); + edesc->qcd_hash_state_paddr = edesc->qcd_desc_paddr + + offsetof(struct qat_crypto_desc, qcd_hash_state_prefix_buf); + + qs->qs_status = QAT_SESSION_STATUS_ACTIVE; + qs->qs_inflight = 0; + + qs->qs_cipher_key = csp->csp_cipher_key; + qs->qs_cipher_klen = csp->csp_cipher_klen; + qs->qs_auth_key = csp->csp_auth_key; + qs->qs_auth_klen = csp->csp_auth_klen; + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); + qs->qs_cipher_mode = HW_CIPHER_CBC_MODE; + break; + case CRYPTO_AES_ICM: + qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); + qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; + break; + case CRYPTO_AES_XTS: + qs->qs_cipher_algo = + qat_aes_cipher_algo(csp->csp_cipher_klen / 2); + qs->qs_cipher_mode = HW_CIPHER_XTS_MODE; + break; + case CRYPTO_AES_NIST_GCM_16: + qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_cipher_klen); + qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; + qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; + qs->qs_auth_mode = HW_AUTH_MODE1; + break; + case 0: + break; + default: + panic("%s: unhandled cipher algorithm %d", __func__, + csp->csp_cipher_alg); + } + + switch (csp->csp_auth_alg) { + case CRYPTO_SHA1_HMAC: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; + qs->qs_auth_mode = HW_AUTH_MODE1; + break; + case CRYPTO_SHA1: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA1; + qs->qs_auth_mode = HW_AUTH_MODE0; + break; + case CRYPTO_SHA2_256_HMAC: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; + qs->qs_auth_mode = HW_AUTH_MODE1; + break; + case CRYPTO_SHA2_256: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA256; + qs->qs_auth_mode = HW_AUTH_MODE0; + break; + case CRYPTO_SHA2_384_HMAC: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; + qs->qs_auth_mode = HW_AUTH_MODE1; + break; + case CRYPTO_SHA2_384: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA384; + qs->qs_auth_mode = HW_AUTH_MODE0; + break; + case CRYPTO_SHA2_512_HMAC: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; + qs->qs_auth_mode = HW_AUTH_MODE1; + break; + case CRYPTO_SHA2_512: + qs->qs_auth_algo = HW_AUTH_ALGO_SHA512; + qs->qs_auth_mode = HW_AUTH_MODE0; + break; + case CRYPTO_AES_NIST_GMAC: + qs->qs_cipher_algo = qat_aes_cipher_algo(csp->csp_auth_klen); + qs->qs_cipher_mode = HW_CIPHER_CTR_MODE; + qs->qs_auth_algo = HW_AUTH_ALGO_GALOIS_128; + qs->qs_auth_mode = HW_AUTH_MODE1; + + qs->qs_cipher_key = qs->qs_auth_key; + qs->qs_cipher_klen = qs->qs_auth_klen; + break; + case 0: + break; + default: + panic("%s: unhandled auth algorithm %d", __func__, + csp->csp_auth_alg); + } + + slices = 0; + switch (csp->csp_mode) { + case CSP_MODE_AEAD: + case CSP_MODE_ETA: + /* auth then decrypt */ + ddesc->qcd_slices[0] = FW_SLICE_AUTH; + ddesc->qcd_slices[1] = FW_SLICE_CIPHER; + ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; + ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; + /* encrypt then auth */ + edesc->qcd_slices[0] = FW_SLICE_CIPHER; + edesc->qcd_slices[1] = FW_SLICE_AUTH; + edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; + edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; + slices = 2; + break; + case CSP_MODE_CIPHER: + /* decrypt */ + ddesc->qcd_slices[0] = FW_SLICE_CIPHER; + ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; + ddesc->qcd_cmd_id = FW_LA_CMD_CIPHER; + /* encrypt */ + edesc->qcd_slices[0] = FW_SLICE_CIPHER; + edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; + edesc->qcd_cmd_id = FW_LA_CMD_CIPHER; + slices = 1; + break; + case CSP_MODE_DIGEST: + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { + /* auth then decrypt */ + ddesc->qcd_slices[0] = FW_SLICE_AUTH; + ddesc->qcd_slices[1] = FW_SLICE_CIPHER; + ddesc->qcd_cipher_dir = HW_CIPHER_DECRYPT; + ddesc->qcd_cmd_id = FW_LA_CMD_HASH_CIPHER; + /* encrypt then auth */ + edesc->qcd_slices[0] = FW_SLICE_CIPHER; + edesc->qcd_slices[1] = FW_SLICE_AUTH; + edesc->qcd_cipher_dir = HW_CIPHER_ENCRYPT; + edesc->qcd_cmd_id = FW_LA_CMD_CIPHER_HASH; + slices = 2; + } else { + ddesc->qcd_slices[0] = FW_SLICE_AUTH; + ddesc->qcd_cmd_id = FW_LA_CMD_AUTH; + edesc->qcd_slices[0] = FW_SLICE_AUTH; + edesc->qcd_cmd_id = FW_LA_CMD_AUTH; + slices = 1; + } + break; + default: + panic("%s: unhandled crypto algorithm %d, %d", __func__, + csp->csp_cipher_alg, csp->csp_auth_alg); + } + ddesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; + edesc->qcd_slices[slices] = FW_SLICE_DRAM_WR; + + qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, ddesc); + qcy->qcy_sc->sc_hw.qhw_crypto_setup_desc(qcy, qs, edesc); + + if (csp->csp_auth_mlen != 0) + qs->qs_auth_mlen = csp->csp_auth_mlen; + else + qs->qs_auth_mlen = edesc->qcd_auth_sz; + + /* Compute the GMAC by specifying a null cipher payload. */ + if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) + ddesc->qcd_cmd_id = edesc->qcd_cmd_id = FW_LA_CMD_AUTH; + + return 0; +} + +static void +qat_crypto_clear_desc(struct qat_crypto_desc *desc) +{ + explicit_bzero(desc->qcd_content_desc, sizeof(desc->qcd_content_desc)); + explicit_bzero(desc->qcd_hash_state_prefix_buf, + sizeof(desc->qcd_hash_state_prefix_buf)); + explicit_bzero(desc->qcd_req_cache, sizeof(desc->qcd_req_cache)); +} + +static void +qat_freesession(device_t dev, crypto_session_t cses) +{ + struct qat_session *qs; + + qs = crypto_get_driver_session(cses); + KASSERT(qs->qs_inflight == 0, + ("%s: session %p has requests in flight", __func__, qs)); + + qat_crypto_clear_desc(qs->qs_enc_desc); + qat_crypto_clear_desc(qs->qs_dec_desc); + qat_free_dmamem(device_get_softc(dev), &qs->qs_desc_mem); + mtx_destroy(&qs->qs_session_mtx); +} + +static int +qat_process(device_t dev, struct cryptop *crp, int hint) +{ + struct qat_crypto *qcy; + struct qat_crypto_bank *qcb; + struct qat_crypto_desc const *desc; + struct qat_session *qs; + struct qat_softc *sc; + struct qat_sym_cookie *qsc; + struct qat_sym_bulk_cookie *qsbc; + int error; + + sc = device_get_softc(dev); + qcy = &sc->sc_crypto; + qs = crypto_get_driver_session(crp->crp_session); + qsc = NULL; + + if (__predict_false(crypto_buffer_len(&crp->crp_buf) > QAT_MAXLEN)) { + error = E2BIG; + goto fail1; + } + + mtx_lock(&qs->qs_session_mtx); + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { + if (crp->crp_aad_length > QAT_GCM_AAD_SIZE_MAX) { + error = E2BIG; + mtx_unlock(&qs->qs_session_mtx); + goto fail1; + } + + /* + * The firmware interface for GCM annoyingly requires the AAD + * size to be stored in the session's content descriptor, which + * is not really meant to be updated after session + * initialization. For IPSec the AAD size is fixed so this is + * not much of a problem in practice, but we have to catch AAD + * size updates here so that the device code can safely update + * the session's recorded AAD size. + */ + if (__predict_false(crp->crp_aad_length != qs->qs_aad_length)) { + if (qs->qs_inflight == 0) { + if (qs->qs_aad_length != -1) { + counter_u64_add(sc->sc_gcm_aad_updates, + 1); + } + qs->qs_aad_length = crp->crp_aad_length; + } else { + qs->qs_need_wakeup = true; + mtx_unlock(&qs->qs_session_mtx); + counter_u64_add(sc->sc_gcm_aad_restarts, 1); + error = ERESTART; + goto fail1; + } + } + } + qs->qs_inflight++; + mtx_unlock(&qs->qs_session_mtx); + + qcb = qat_crypto_select_bank(qcy); + + qsc = qat_crypto_alloc_sym_cookie(qcb); + if (qsc == NULL) { + error = ENOBUFS; + goto fail2; + } + + if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) + desc = qs->qs_enc_desc; + else + desc = qs->qs_dec_desc; + + error = qat_crypto_load(qs, qsc, desc, crp); + if (error != 0) + goto fail2; + + qsbc = &qsc->u.qsc_bulk_cookie; + qsbc->qsbc_crypto = qcy; + qsbc->qsbc_session = qs; + qsbc->qsbc_cb_tag = crp; + + sc->sc_hw.qhw_crypto_setup_req_params(qcb, qs, desc, qsc, crp); + + bus_dmamap_sync(qsc->qsc_buf_dma_tag, qsc->qsc_buf_dmamap, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + bus_dmamap_sync(qsc->qsc_self_dma_tag, qsc->qsc_self_dmamap, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + + error = qat_etr_put_msg(sc, qcb->qcb_sym_tx, + (uint32_t *)qsbc->qsbc_msg); + if (error) + goto fail2; + + return 0; + +fail2: + if (qsc) + qat_crypto_free_sym_cookie(qcb, qsc); + mtx_lock(&qs->qs_session_mtx); + qs->qs_inflight--; + mtx_unlock(&qs->qs_session_mtx); +fail1: + crp->crp_etype = error; + crypto_done(crp); + return 0; +} + +static device_method_t qat_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, qat_probe), + DEVMETHOD(device_attach, qat_attach), + DEVMETHOD(device_detach, qat_detach), + + /* Cryptodev interface */ + DEVMETHOD(cryptodev_probesession, qat_probesession), + DEVMETHOD(cryptodev_newsession, qat_newsession), + DEVMETHOD(cryptodev_freesession, qat_freesession), + DEVMETHOD(cryptodev_process, qat_process), + + DEVMETHOD_END +}; + +static devclass_t qat_devclass; + +static driver_t qat_driver = { + .name = "qat", + .methods = qat_methods, + .size = sizeof(struct qat_softc), +}; + +DRIVER_MODULE(qat, pci, qat_driver, qat_devclass, 0, 0); +MODULE_VERSION(qat, 1); +MODULE_DEPEND(qat, crypto, 1, 1, 1); +MODULE_DEPEND(qat, pci, 1, 1, 1); Index: head/sys/dev/qat/qat_ae.c =================================================================== --- head/sys/dev/qat/qat_ae.c +++ head/sys/dev/qat/qat_ae.c @@ -0,0 +1,3456 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_ae.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qatvar.h" +#include "qat_aevar.h" + +static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, + uint32_t); +static int qat_ae_read_4(struct qat_softc *, u_char, bus_size_t, + uint32_t *); +static int qat_ae_write_4(struct qat_softc *, u_char, bus_size_t, + uint32_t); +static void qat_ae_ctx_indr_write(struct qat_softc *, u_char, uint32_t, + bus_size_t, uint32_t); +static int qat_ae_ctx_indr_read(struct qat_softc *, u_char, uint32_t, + bus_size_t, uint32_t *); + +static u_short qat_aereg_get_10bit_addr(enum aereg_type, u_short); +static int qat_aereg_rel_data_write(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, uint32_t); +static int qat_aereg_rel_data_read(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, uint32_t *); +static int qat_aereg_rel_rdxfer_write(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, uint32_t); +static int qat_aereg_rel_wrxfer_write(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, uint32_t); +static int qat_aereg_rel_nn_write(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, uint32_t); +static int qat_aereg_abs_to_rel(struct qat_softc *, u_char, u_short, + u_short *, u_char *); +static int qat_aereg_abs_data_write(struct qat_softc *, u_char, + enum aereg_type, u_short, uint32_t); + +static void qat_ae_enable_ctx(struct qat_softc *, u_char, u_int); +static void qat_ae_disable_ctx(struct qat_softc *, u_char, u_int); +static void qat_ae_write_ctx_mode(struct qat_softc *, u_char, u_char); +static void qat_ae_write_nn_mode(struct qat_softc *, u_char, u_char); +static void qat_ae_write_lm_mode(struct qat_softc *, u_char, + enum aereg_type, u_char); +static void qat_ae_write_shared_cs_mode0(struct qat_softc *, u_char, + u_char); +static void qat_ae_write_shared_cs_mode(struct qat_softc *, u_char, u_char); +static int qat_ae_set_reload_ustore(struct qat_softc *, u_char, u_int, int, + u_int); + +static enum qat_ae_status qat_ae_get_status(struct qat_softc *, u_char); +static int qat_ae_is_active(struct qat_softc *, u_char); +static int qat_ae_wait_num_cycles(struct qat_softc *, u_char, int, int); + +static int qat_ae_clear_reset(struct qat_softc *); +static int qat_ae_check(struct qat_softc *); +static int qat_ae_reset_timestamp(struct qat_softc *); +static void qat_ae_clear_xfer(struct qat_softc *); +static int qat_ae_clear_gprs(struct qat_softc *); + +static void qat_ae_get_shared_ustore_ae(u_char, u_char *); +static u_int qat_ae_ucode_parity64(uint64_t); +static uint64_t qat_ae_ucode_set_ecc(uint64_t); +static int qat_ae_ucode_write(struct qat_softc *, u_char, u_int, u_int, + const uint64_t *); +static int qat_ae_ucode_read(struct qat_softc *, u_char, u_int, u_int, + uint64_t *); +static u_int qat_ae_concat_ucode(uint64_t *, u_int, u_int, u_int, u_int *); +static int qat_ae_exec_ucode(struct qat_softc *, u_char, u_char, + uint64_t *, u_int, int, u_int, u_int *); +static int qat_ae_exec_ucode_init_lm(struct qat_softc *, u_char, u_char, + int *, uint64_t *, u_int, + u_int *, u_int *, u_int *, u_int *, u_int *); +static int qat_ae_restore_init_lm_gprs(struct qat_softc *, u_char, u_char, + u_int, u_int, u_int, u_int, u_int); +static int qat_ae_get_inst_num(int); +static int qat_ae_batch_put_lm(struct qat_softc *, u_char, + struct qat_ae_batch_init_list *, size_t); +static int qat_ae_write_pc(struct qat_softc *, u_char, u_int, u_int); + +static u_int qat_aefw_csum(char *, int); +static const char *qat_aefw_uof_string(struct qat_softc *, size_t); +static struct uof_chunk_hdr *qat_aefw_uof_find_chunk(struct qat_softc *, + const char *, struct uof_chunk_hdr *); + +static int qat_aefw_load_mof(struct qat_softc *); +static void qat_aefw_unload_mof(struct qat_softc *); +static int qat_aefw_load_mmp(struct qat_softc *); +static void qat_aefw_unload_mmp(struct qat_softc *); + +static int qat_aefw_mof_find_uof0(struct qat_softc *, + struct mof_uof_hdr *, struct mof_uof_chunk_hdr *, + u_int, size_t, const char *, + size_t *, void **); +static int qat_aefw_mof_find_uof(struct qat_softc *); +static int qat_aefw_mof_parse(struct qat_softc *); + +static int qat_aefw_uof_parse_image(struct qat_softc *, + struct qat_uof_image *, struct uof_chunk_hdr *uch); +static int qat_aefw_uof_parse_images(struct qat_softc *); +static int qat_aefw_uof_parse(struct qat_softc *); + +static int qat_aefw_alloc_auth_dmamem(struct qat_softc *, char *, size_t, + struct qat_dmamem *); +static int qat_aefw_auth(struct qat_softc *, struct qat_dmamem *); +static int qat_aefw_suof_load(struct qat_softc *sc, + struct qat_dmamem *dma); +static int qat_aefw_suof_parse_image(struct qat_softc *, + struct qat_suof_image *, struct suof_chunk_hdr *); +static int qat_aefw_suof_parse(struct qat_softc *); +static int qat_aefw_suof_write(struct qat_softc *); + +static int qat_aefw_uof_assign_image(struct qat_softc *, struct qat_ae *, + struct qat_uof_image *); +static int qat_aefw_uof_init_ae(struct qat_softc *, u_char); +static int qat_aefw_uof_init(struct qat_softc *); + +static int qat_aefw_init_memory_one(struct qat_softc *, + struct uof_init_mem *); +static void qat_aefw_free_lm_init(struct qat_softc *, u_char); +static int qat_aefw_init_ustore(struct qat_softc *); +static int qat_aefw_init_reg(struct qat_softc *, u_char, u_char, + enum aereg_type, u_short, u_int); +static int qat_aefw_init_reg_sym_expr(struct qat_softc *, u_char, + struct qat_uof_image *); +static int qat_aefw_init_memory(struct qat_softc *); +static int qat_aefw_init_globals(struct qat_softc *); +static uint64_t qat_aefw_get_uof_inst(struct qat_softc *, + struct qat_uof_page *, u_int); +static int qat_aefw_do_pagein(struct qat_softc *, u_char, + struct qat_uof_page *); +static int qat_aefw_uof_write_one(struct qat_softc *, + struct qat_uof_image *); +static int qat_aefw_uof_write(struct qat_softc *); + +static int +qat_ae_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, + uint32_t value) +{ + int times = TIMEOUT_AE_CSR; + + do { + qat_ae_local_write_4(sc, ae, offset, value); + if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & + LOCAL_CSR_STATUS_STATUS) == 0) + return 0; + + } while (times--); + + device_printf(sc->sc_dev, + "couldn't write AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); + return EFAULT; +} + +static int +qat_ae_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset, + uint32_t *value) +{ + int times = TIMEOUT_AE_CSR; + uint32_t v; + + do { + v = qat_ae_local_read_4(sc, ae, offset); + if ((qat_ae_local_read_4(sc, ae, LOCAL_CSR_STATUS) & + LOCAL_CSR_STATUS_STATUS) == 0) { + *value = v; + return 0; + } + } while (times--); + + device_printf(sc->sc_dev, + "couldn't read AE CSR: ae 0x%hhx offset 0x%lx\n", ae, (long)offset); + return EFAULT; +} + +static void +qat_ae_ctx_indr_write(struct qat_softc *sc, u_char ae, uint32_t ctx_mask, + bus_size_t offset, uint32_t value) +{ + int ctx; + uint32_t ctxptr; + + MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || + offset == FUTURE_COUNT_SIGNAL_INDIRECT || + offset == CTX_STS_INDIRECT || + offset == CTX_WAKEUP_EVENTS_INDIRECT || + offset == CTX_SIG_EVENTS_INDIRECT || + offset == LM_ADDR_0_INDIRECT || + offset == LM_ADDR_1_INDIRECT || + offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || + offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); + + qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); + for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { + if ((ctx_mask & (1 << ctx)) == 0) + continue; + qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); + qat_ae_write_4(sc, ae, offset, value); + } + qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); +} + +static int +qat_ae_ctx_indr_read(struct qat_softc *sc, u_char ae, uint32_t ctx, + bus_size_t offset, uint32_t *value) +{ + int error; + uint32_t ctxptr; + + MPASS(offset == CTX_FUTURE_COUNT_INDIRECT || + offset == FUTURE_COUNT_SIGNAL_INDIRECT || + offset == CTX_STS_INDIRECT || + offset == CTX_WAKEUP_EVENTS_INDIRECT || + offset == CTX_SIG_EVENTS_INDIRECT || + offset == LM_ADDR_0_INDIRECT || + offset == LM_ADDR_1_INDIRECT || + offset == INDIRECT_LM_ADDR_0_BYTE_INDEX || + offset == INDIRECT_LM_ADDR_1_BYTE_INDEX); + + /* save the ctx ptr */ + qat_ae_read_4(sc, ae, CSR_CTX_POINTER, &ctxptr); + if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != + (ctx & CSR_CTX_POINTER_CONTEXT)) + qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctx); + + error = qat_ae_read_4(sc, ae, offset, value); + + /* restore ctx ptr */ + if ((ctxptr & CSR_CTX_POINTER_CONTEXT) != + (ctx & CSR_CTX_POINTER_CONTEXT)) + qat_ae_write_4(sc, ae, CSR_CTX_POINTER, ctxptr); + + return error; +} + +static u_short +qat_aereg_get_10bit_addr(enum aereg_type regtype, u_short reg) +{ + u_short addr; + + switch (regtype) { + case AEREG_GPA_ABS: + case AEREG_GPB_ABS: + addr = (reg & 0x7f) | 0x80; + break; + case AEREG_GPA_REL: + case AEREG_GPB_REL: + addr = reg & 0x1f; + break; + case AEREG_SR_RD_REL: + case AEREG_SR_WR_REL: + case AEREG_SR_REL: + addr = 0x180 | (reg & 0x1f); + break; + case AEREG_SR_INDX: + addr = 0x140 | ((reg & 0x3) << 1); + break; + case AEREG_DR_RD_REL: + case AEREG_DR_WR_REL: + case AEREG_DR_REL: + addr = 0x1c0 | (reg & 0x1f); + break; + case AEREG_DR_INDX: + addr = 0x100 | ((reg & 0x3) << 1); + break; + case AEREG_NEIGH_INDX: + addr = 0x241 | ((reg & 0x3) << 1); + break; + case AEREG_NEIGH_REL: + addr = 0x280 | (reg & 0x1f); + break; + case AEREG_LMEM0: + addr = 0x200; + break; + case AEREG_LMEM1: + addr = 0x220; + break; + case AEREG_NO_DEST: + addr = 0x300 | (reg & 0xff); + break; + default: + addr = AEREG_BAD_REGADDR; + break; + } + return (addr); +} + +static int +qat_aereg_rel_data_write(struct qat_softc *sc, u_char ae, u_char ctx, + enum aereg_type regtype, u_short relreg, uint32_t value) +{ + uint16_t srchi, srclo, destaddr, data16hi, data16lo; + uint64_t inst[] = { + 0x0F440000000ull, /* immed_w1[reg, val_hi16] */ + 0x0F040000000ull, /* immed_w0[reg, val_lo16] */ + 0x0F0000C0300ull, /* nop */ + 0x0E000010000ull /* ctx_arb[kill] */ + }; + const int ninst = nitems(inst); + const int imm_w1 = 0, imm_w0 = 1; + unsigned int ctxen; + uint16_t mask; + + /* This logic only works for GPRs and LM index registers, + not NN or XFER registers! */ + MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || + regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); + + if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL)) { + /* determine the context mode */ + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { + /* 4-ctx mode */ + if (ctx & 0x1) + return EINVAL; + mask = 0x1f; + } else { + /* 8-ctx mode */ + mask = 0x0f; + } + if (relreg & ~mask) + return EINVAL; + } + if ((destaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == + AEREG_BAD_REGADDR) { + return EINVAL; + } + + data16lo = 0xffff & value; + data16hi = 0xffff & (value >> 16); + srchi = qat_aereg_get_10bit_addr(AEREG_NO_DEST, + (uint16_t)(0xff & data16hi)); + srclo = qat_aereg_get_10bit_addr(AEREG_NO_DEST, + (uint16_t)(0xff & data16lo)); + + switch (regtype) { + case AEREG_GPA_REL: /* A rel source */ + inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | + ((srchi & 0x3ff) << 10) | (destaddr & 0x3ff); + inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | + ((srclo & 0x3ff) << 10) | (destaddr & 0x3ff); + break; + default: + inst[imm_w1] = inst[imm_w1] | ((data16hi >> 8) << 20) | + ((destaddr & 0x3ff) << 10) | (srchi & 0x3ff); + inst[imm_w0] = inst[imm_w0] | ((data16lo >> 8) << 20) | + ((destaddr & 0x3ff) << 10) | (srclo & 0x3ff); + break; + } + + return qat_ae_exec_ucode(sc, ae, ctx, inst, ninst, 1, ninst * 5, NULL); +} + +static int +qat_aereg_rel_data_read(struct qat_softc *sc, u_char ae, u_char ctx, + enum aereg_type regtype, u_short relreg, uint32_t *value) +{ + uint64_t inst, savucode; + uint32_t ctxen, misc, nmisc, savctx, ctxarbctl, ulo, uhi; + u_int uaddr, ustore_addr; + int error; + u_short mask, regaddr; + u_char nae; + + MPASS(regtype == AEREG_GPA_REL || regtype == AEREG_GPB_REL || + regtype == AEREG_SR_REL || regtype == AEREG_SR_RD_REL || + regtype == AEREG_DR_REL || regtype == AEREG_DR_RD_REL || + regtype == AEREG_LMEM0 || regtype == AEREG_LMEM1); + + if ((regtype == AEREG_GPA_REL) || (regtype == AEREG_GPB_REL) || + (regtype == AEREG_SR_REL) || (regtype == AEREG_SR_RD_REL) || + (regtype == AEREG_DR_REL) || (regtype == AEREG_DR_RD_REL)) + { + /* determine the context mode */ + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { + /* 4-ctx mode */ + if (ctx & 0x1) + return EINVAL; + mask = 0x1f; + } else { + /* 8-ctx mode */ + mask = 0x0f; + } + if (relreg & ~mask) + return EINVAL; + } + if ((regaddr = qat_aereg_get_10bit_addr(regtype, relreg)) == + AEREG_BAD_REGADDR) { + return EINVAL; + } + + /* instruction -- alu[--, --, B, reg] */ + switch (regtype) { + case AEREG_GPA_REL: + /* A rel source */ + inst = 0xA070000000ull | (regaddr & 0x3ff); + break; + default: + inst = (0xA030000000ull | ((regaddr & 0x3ff) << 10)); + break; + } + + /* backup shared control store bit, and force AE to + * none-shared mode before executing ucode snippet */ + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); + if (misc & AE_MISC_CONTROL_SHARE_CS) { + qat_ae_get_shared_ustore_ae(ae, &nae); + if ((1 << nae) & sc->sc_ae_mask && qat_ae_is_active(sc, nae)) + return EBUSY; + } + + nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); + + /* read current context */ + qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); + qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); + + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + /* prevent clearing the W1C bits: the breakpoint bit, + ECC error bit, and Parity error bit */ + ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; + + /* change the context */ + if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + ctx & ACTIVE_CTX_STATUS_ACNO); + /* save a ustore location */ + if ((error = qat_ae_ucode_read(sc, ae, 0, 1, &savucode)) != 0) { + /* restore AE_MISC_CONTROL csr */ + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); + + /* restore the context */ + if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + savctx & ACTIVE_CTX_STATUS_ACNO); + } + qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); + + return (error); + } + + /* turn off ustore parity */ + qat_ae_write_4(sc, ae, CTX_ENABLES, + ctxen & (~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE)); + + /* save ustore-addr csr */ + qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); + + /* write the ALU instruction to ustore, enable ecs bit */ + uaddr = 0 | USTORE_ADDRESS_ECS; + + /* set the uaddress */ + qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); + inst = qat_ae_ucode_set_ecc(inst); + + ulo = (uint32_t)(inst & 0xffffffff); + uhi = (uint32_t)(inst >> 32); + + qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); + + /* this will auto increment the address */ + qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); + + /* set the uaddress */ + qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); + + /* delay for at least 8 cycles */ + qat_ae_wait_num_cycles(sc, ae, 0x8, 0); + + /* read ALU output -- the instruction should have been executed + prior to clearing the ECS in putUwords */ + qat_ae_read_4(sc, ae, ALU_OUT, value); + + /* restore ustore-addr csr */ + qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); + + /* restore the ustore */ + error = qat_ae_ucode_write(sc, ae, 0, 1, &savucode); + + /* restore the context */ + if (ctx != (savctx & ACTIVE_CTX_STATUS_ACNO)) { + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + savctx & ACTIVE_CTX_STATUS_ACNO); + } + + qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); + + /* restore AE_MISC_CONTROL csr */ + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); + + qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); + + return error; +} + +static int +qat_aereg_rel_rdxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, + enum aereg_type regtype, u_short relreg, uint32_t value) +{ + bus_size_t addr; + int error; + uint32_t ctxen; + u_short mask; + u_short dr_offset; + + MPASS(regtype == AEREG_SR_REL || regtype == AEREG_DR_REL || + regtype == AEREG_SR_RD_REL || regtype == AEREG_DR_RD_REL); + + error = qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { + if (ctx & 0x1) { + device_printf(sc->sc_dev, + "bad ctx argument in 4-ctx mode,ctx=0x%x\n", ctx); + return EINVAL; + } + mask = 0x1f; + dr_offset = 0x20; + + } else { + mask = 0x0f; + dr_offset = 0x10; + } + + if (relreg & ~mask) + return EINVAL; + + addr = relreg + (ctx << 0x5); + + switch (regtype) { + case AEREG_SR_REL: + case AEREG_SR_RD_REL: + qat_ae_xfer_write_4(sc, ae, addr, value); + break; + case AEREG_DR_REL: + case AEREG_DR_RD_REL: + qat_ae_xfer_write_4(sc, ae, addr + dr_offset, value); + break; + default: + error = EINVAL; + } + + return error; +} + +static int +qat_aereg_rel_wrxfer_write(struct qat_softc *sc, u_char ae, u_char ctx, + enum aereg_type regtype, u_short relreg, uint32_t value) +{ + + panic("notyet"); + + return 0; +} + +static int +qat_aereg_rel_nn_write(struct qat_softc *sc, u_char ae, u_char ctx, + enum aereg_type regtype, u_short relreg, uint32_t value) +{ + + panic("notyet"); + + return 0; +} + +static int +qat_aereg_abs_to_rel(struct qat_softc *sc, u_char ae, + u_short absreg, u_short *relreg, u_char *ctx) +{ + uint32_t ctxen; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { + /* 4-ctx mode */ + *relreg = absreg & 0x1f; + *ctx = (absreg >> 0x4) & 0x6; + } else { + /* 8-ctx mode */ + *relreg = absreg & 0x0f; + *ctx = (absreg >> 0x4) & 0x7; + } + + return 0; +} + +static int +qat_aereg_abs_data_write(struct qat_softc *sc, u_char ae, + enum aereg_type regtype, u_short absreg, uint32_t value) +{ + int error; + u_short relreg; + u_char ctx; + + qat_aereg_abs_to_rel(sc, ae, absreg, &relreg, &ctx); + + switch (regtype) { + case AEREG_GPA_ABS: + MPASS(absreg < MAX_GPR_REG); + error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, + relreg, value); + break; + case AEREG_GPB_ABS: + MPASS(absreg < MAX_GPR_REG); + error = qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, + relreg, value); + break; + case AEREG_DR_RD_ABS: + MPASS(absreg < MAX_XFER_REG); + error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_DR_RD_REL, + relreg, value); + break; + case AEREG_SR_RD_ABS: + MPASS(absreg < MAX_XFER_REG); + error = qat_aereg_rel_rdxfer_write(sc, ae, ctx, AEREG_SR_RD_REL, + relreg, value); + break; + case AEREG_DR_WR_ABS: + MPASS(absreg < MAX_XFER_REG); + error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_DR_WR_REL, + relreg, value); + break; + case AEREG_SR_WR_ABS: + MPASS(absreg < MAX_XFER_REG); + error = qat_aereg_rel_wrxfer_write(sc, ae, ctx, AEREG_SR_WR_REL, + relreg, value); + break; + case AEREG_NEIGH_ABS: + MPASS(absreg < MAX_NN_REG); + if (absreg >= MAX_NN_REG) + return EINVAL; + error = qat_aereg_rel_nn_write(sc, ae, ctx, AEREG_NEIGH_REL, + relreg, value); + break; + default: + panic("Invalid Register Type"); + } + + return error; +} + +static void +qat_ae_enable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) +{ + uint32_t ctxen; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; + + if (ctxen & CTX_ENABLES_INUSE_CONTEXTS) { + ctx_mask &= 0x55; + } else { + ctx_mask &= 0xff; + } + + ctxen |= __SHIFTIN(ctx_mask, CTX_ENABLES_ENABLE); + qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); +} + +static void +qat_ae_disable_ctx(struct qat_softc *sc, u_char ae, u_int ctx_mask) +{ + uint32_t ctxen; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; + ctxen &= ~(__SHIFTIN(ctx_mask & AE_ALL_CTX, CTX_ENABLES_ENABLE)); + qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); +} + +static void +qat_ae_write_ctx_mode(struct qat_softc *sc, u_char ae, u_char mode) +{ + uint32_t val, nval; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &val); + val &= CTX_ENABLES_IGNORE_W1C_MASK; + + if (mode == 4) + nval = val | CTX_ENABLES_INUSE_CONTEXTS; + else + nval = val & ~CTX_ENABLES_INUSE_CONTEXTS; + + if (val != nval) + qat_ae_write_4(sc, ae, CTX_ENABLES, nval); +} + +static void +qat_ae_write_nn_mode(struct qat_softc *sc, u_char ae, u_char mode) +{ + uint32_t val, nval; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &val); + val &= CTX_ENABLES_IGNORE_W1C_MASK; + + if (mode) + nval = val | CTX_ENABLES_NN_MODE; + else + nval = val & ~CTX_ENABLES_NN_MODE; + + if (val != nval) + qat_ae_write_4(sc, ae, CTX_ENABLES, nval); +} + +static void +qat_ae_write_lm_mode(struct qat_softc *sc, u_char ae, + enum aereg_type lm, u_char mode) +{ + uint32_t val, nval; + uint32_t bit; + + qat_ae_read_4(sc, ae, CTX_ENABLES, &val); + val &= CTX_ENABLES_IGNORE_W1C_MASK; + + switch (lm) { + case AEREG_LMEM0: + bit = CTX_ENABLES_LMADDR_0_GLOBAL; + break; + case AEREG_LMEM1: + bit = CTX_ENABLES_LMADDR_1_GLOBAL; + break; + default: + panic("invalid lmem reg type"); + break; + } + + if (mode) + nval = val | bit; + else + nval = val & ~bit; + + if (val != nval) + qat_ae_write_4(sc, ae, CTX_ENABLES, nval); +} + +static void +qat_ae_write_shared_cs_mode0(struct qat_softc *sc, u_char ae, u_char mode) +{ + uint32_t val, nval; + + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); + + if (mode == 1) + nval = val | AE_MISC_CONTROL_SHARE_CS; + else + nval = val & ~AE_MISC_CONTROL_SHARE_CS; + + if (val != nval) + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nval); +} + +static void +qat_ae_write_shared_cs_mode(struct qat_softc *sc, u_char ae, u_char mode) +{ + u_char nae; + + qat_ae_get_shared_ustore_ae(ae, &nae); + + qat_ae_write_shared_cs_mode0(sc, ae, mode); + + if ((sc->sc_ae_mask & (1 << nae))) { + qat_ae_write_shared_cs_mode0(sc, nae, mode); + } +} + +static int +qat_ae_set_reload_ustore(struct qat_softc *sc, u_char ae, + u_int reload_size, int shared_mode, u_int ustore_dram_addr) +{ + uint32_t val, cs_reload; + + switch (reload_size) { + case 0: + cs_reload = 0x0; + break; + case QAT_2K: + cs_reload = 0x1; + break; + case QAT_4K: + cs_reload = 0x2; + break; + case QAT_8K: + cs_reload = 0x3; + break; + default: + return EINVAL; + } + + if (cs_reload) + QAT_AE(sc, ae).qae_ustore_dram_addr = ustore_dram_addr; + + QAT_AE(sc, ae).qae_reload_size = reload_size; + + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); + val &= ~(AE_MISC_CONTROL_ONE_CTX_RELOAD | + AE_MISC_CONTROL_CS_RELOAD | AE_MISC_CONTROL_SHARE_CS); + val |= __SHIFTIN(cs_reload, AE_MISC_CONTROL_CS_RELOAD) | + __SHIFTIN(shared_mode, AE_MISC_CONTROL_ONE_CTX_RELOAD); + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); + + return 0; +} + +static enum qat_ae_status +qat_ae_get_status(struct qat_softc *sc, u_char ae) +{ + int error; + uint32_t val = 0; + + error = qat_ae_read_4(sc, ae, CTX_ENABLES, &val); + if (error || val & CTX_ENABLES_ENABLE) + return QAT_AE_ENABLED; + + qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); + if (val & ACTIVE_CTX_STATUS_ABO) + return QAT_AE_ACTIVE; + + return QAT_AE_DISABLED; +} + + +static int +qat_ae_is_active(struct qat_softc *sc, u_char ae) +{ + uint32_t val; + + if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) + return 1; + + qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &val); + if (val & ACTIVE_CTX_STATUS_ABO) + return 1; + else + return 0; +} + +/* returns 1 if actually waited for specified number of cycles */ +static int +qat_ae_wait_num_cycles(struct qat_softc *sc, u_char ae, int cycles, int check) +{ + uint32_t cnt, actx; + int pcnt, ccnt, elapsed, times; + + qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); + pcnt = cnt & 0xffff; + + times = TIMEOUT_AE_CHECK; + do { + qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); + ccnt = cnt & 0xffff; + + elapsed = ccnt - pcnt; + if (elapsed == 0) { + times--; + } + if (times <= 0) { + device_printf(sc->sc_dev, + "qat_ae_wait_num_cycles timeout\n"); + return -1; + } + + if (elapsed < 0) + elapsed += 0x10000; + + if (elapsed >= CYCLES_FROM_READY2EXE && check) { + if (qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, + &actx) == 0) { + if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) + return 0; + } + } + } while (cycles > elapsed); + + if (check && qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &actx) == 0) { + if ((actx & ACTIVE_CTX_STATUS_ABO) == 0) + return 0; + } + + return 1; +} + +int +qat_ae_init(struct qat_softc *sc) +{ + int error; + uint32_t mask, val = 0; + u_char ae; + + /* XXX adf_initSysMemInfo */ + + /* XXX Disable clock gating for some chip if debug mode */ + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + struct qat_ae *qae = &sc->sc_ae[ae]; + if (!(mask & 1)) + continue; + + qae->qae_ustore_size = USTORE_SIZE; + + qae->qae_free_addr = 0; + qae->qae_free_size = USTORE_SIZE; + qae->qae_live_ctx_mask = AE_ALL_CTX; + qae->qae_ustore_dram_addr = 0; + qae->qae_reload_size = 0; + } + + /* XXX Enable attention interrupt */ + + error = qat_ae_clear_reset(sc); + if (error) + return error; + + qat_ae_clear_xfer(sc); + + if (!sc->sc_hw.qhw_fw_auth) { + error = qat_ae_clear_gprs(sc); + if (error) + return error; + } + + /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_ae_read_4(sc, ae, SIGNATURE_ENABLE, &val); + val |= 0x1; + qat_ae_write_4(sc, ae, SIGNATURE_ENABLE, val); + } + + error = qat_ae_clear_reset(sc); + if (error) + return error; + + /* XXX XXX XXX Clean MMP memory if mem scrub is supported */ + /* halMem_ScrubMMPMemory */ + + return 0; +} + +int +qat_ae_start(struct qat_softc *sc) +{ + int error; + u_char ae; + + for (ae = 0; ae < sc->sc_ae_num; ae++) { + if ((sc->sc_ae_mask & (1 << ae)) == 0) + continue; + + error = qat_aefw_start(sc, ae, 0xff); + if (error) + return error; + } + + return 0; +} + +void +qat_ae_cluster_intr(void *arg) +{ + /* Nothing to implement until we support SRIOV. */ + printf("qat_ae_cluster_intr\n"); +} + +static int +qat_ae_clear_reset(struct qat_softc *sc) +{ + int error; + uint32_t times, reset, clock, reg, mask; + u_char ae; + + reset = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); + reset &= ~(__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK)); + reset &= ~(__SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)); + times = TIMEOUT_AE_RESET; + do { + qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_RESET, reset); + if ((times--) == 0) { + device_printf(sc->sc_dev, "couldn't reset AEs\n"); + return EBUSY; + } + reg = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_RESET); + } while ((__SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_RESET_AE_MASK) | + __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_RESET_ACCEL_MASK)) + & reg); + + /* Enable clock for AE and QAT */ + clock = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_CLK_EN); + clock |= __SHIFTIN(sc->sc_ae_mask, CAP_GLOBAL_CTL_CLK_EN_AE_MASK); + clock |= __SHIFTIN(sc->sc_accel_mask, CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK); + qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_CLK_EN, clock); + + error = qat_ae_check(sc); + if (error) + return error; + + /* + * Set undefined power-up/reset states to reasonable default values... + * just to make sure we're starting from a known point + */ + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + + /* init the ctx_enable */ + qat_ae_write_4(sc, ae, CTX_ENABLES, + CTX_ENABLES_INIT); + + /* initialize the PCs */ + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_STS_INDIRECT, + UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); + + /* init the ctx_arb */ + qat_ae_write_4(sc, ae, CTX_ARB_CNTL, + CTX_ARB_CNTL_INIT); + + /* enable cc */ + qat_ae_write_4(sc, ae, CC_ENABLE, + CC_ENABLE_INIT); + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_WAKEUP_EVENTS_INDIRECT, + CTX_WAKEUP_EVENTS_INDIRECT_INIT); + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_SIG_EVENTS_INDIRECT, + CTX_SIG_EVENTS_INDIRECT_INIT); + } + + if ((sc->sc_ae_mask != 0) && + sc->sc_flags & QAT_FLAG_ESRAM_ENABLE_AUTO_INIT) { + /* XXX XXX XXX init eSram only when this is boot time */ + } + + if ((sc->sc_ae_mask != 0) && + sc->sc_flags & QAT_FLAG_SHRAM_WAIT_READY) { + /* XXX XXX XXX wait shram to complete initialization */ + } + + qat_ae_reset_timestamp(sc); + + return 0; +} + +static int +qat_ae_check(struct qat_softc *sc) +{ + int error, times, ae; + uint32_t cnt, pcnt, mask; + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + + times = TIMEOUT_AE_CHECK; + error = qat_ae_read_4(sc, ae, PROFILE_COUNT, &cnt); + if (error) { + device_printf(sc->sc_dev, + "couldn't access AE %d CSR\n", ae); + return error; + } + pcnt = cnt & 0xffff; + + while (1) { + error = qat_ae_read_4(sc, ae, + PROFILE_COUNT, &cnt); + if (error) { + device_printf(sc->sc_dev, + "couldn't access AE %d CSR\n", ae); + return error; + } + cnt &= 0xffff; + if (cnt == pcnt) + times--; + else + break; + if (times <= 0) { + device_printf(sc->sc_dev, + "AE %d CSR is useless\n", ae); + return EFAULT; + } + } + } + + return 0; +} + +static int +qat_ae_reset_timestamp(struct qat_softc *sc) +{ + uint32_t misc, mask; + u_char ae; + + /* stop the timestamp timers */ + misc = qat_cap_global_read_4(sc, CAP_GLOBAL_CTL_MISC); + if (misc & CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN) { + qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, + misc & (~CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN)); + } + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_ae_write_4(sc, ae, TIMESTAMP_LOW, 0); + qat_ae_write_4(sc, ae, TIMESTAMP_HIGH, 0); + } + + /* start timestamp timers */ + qat_cap_global_write_4(sc, CAP_GLOBAL_CTL_MISC, + misc | CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN); + + return 0; +} + +static void +qat_ae_clear_xfer(struct qat_softc *sc) +{ + u_int mask, reg; + u_char ae; + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + + for (reg = 0; reg < MAX_GPR_REG; reg++) { + qat_aereg_abs_data_write(sc, ae, AEREG_SR_RD_ABS, + reg, 0); + qat_aereg_abs_data_write(sc, ae, AEREG_DR_RD_ABS, + reg, 0); + } + } +} + +static int +qat_ae_clear_gprs(struct qat_softc *sc) +{ + uint32_t val; + uint32_t saved_ctx = 0; + int times = TIMEOUT_AE_CHECK, rv; + u_char ae; + u_int mask; + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + + /* turn off share control store bit */ + val = qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &val); + val &= ~AE_MISC_CONTROL_SHARE_CS; + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, val); + + /* turn off ucode parity */ + /* make sure nn_mode is set to self */ + qat_ae_read_4(sc, ae, CTX_ENABLES, &val); + val &= CTX_ENABLES_IGNORE_W1C_MASK; + val |= CTX_ENABLES_NN_MODE; + val &= ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE; + qat_ae_write_4(sc, ae, CTX_ENABLES, val); + + /* copy instructions to ustore */ + qat_ae_ucode_write(sc, ae, 0, nitems(ae_clear_gprs_inst), + ae_clear_gprs_inst); + + /* set PC */ + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_STS_INDIRECT, + UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); + + /* save current context */ + qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &saved_ctx); + /* change the active context */ + /* start the context from ctx 0 */ + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, 0); + + /* wakeup-event voluntary */ + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_WAKEUP_EVENTS_INDIRECT, + CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); + /* clean signals */ + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_SIG_EVENTS_INDIRECT, 0); + qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); + + qat_ae_enable_ctx(sc, ae, AE_ALL_CTX); + } + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + /* wait for AE to finish */ + do { + rv = qat_ae_wait_num_cycles(sc, ae, AE_EXEC_CYCLE, 1); + } while (rv && times--); + if (times <= 0) { + device_printf(sc->sc_dev, + "qat_ae_clear_gprs timeout"); + return ETIMEDOUT; + } + qat_ae_disable_ctx(sc, ae, AE_ALL_CTX); + /* change the active context */ + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + saved_ctx & ACTIVE_CTX_STATUS_ACNO); + /* init the ctx_enable */ + qat_ae_write_4(sc, ae, CTX_ENABLES, CTX_ENABLES_INIT); + /* initialize the PCs */ + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_STS_INDIRECT, UPC_MASK & CTX_STS_INDIRECT_UPC_INIT); + /* init the ctx_arb */ + qat_ae_write_4(sc, ae, CTX_ARB_CNTL, CTX_ARB_CNTL_INIT); + /* enable cc */ + qat_ae_write_4(sc, ae, CC_ENABLE, CC_ENABLE_INIT); + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, + CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_INIT); + qat_ae_ctx_indr_write(sc, ae, AE_ALL_CTX, CTX_SIG_EVENTS_INDIRECT, + CTX_SIG_EVENTS_INDIRECT_INIT); + } + + return 0; +} + +static void +qat_ae_get_shared_ustore_ae(u_char ae, u_char *nae) +{ + if (ae & 0x1) + *nae = ae - 1; + else + *nae = ae + 1; +} + +static u_int +qat_ae_ucode_parity64(uint64_t ucode) +{ + + ucode ^= ucode >> 1; + ucode ^= ucode >> 2; + ucode ^= ucode >> 4; + ucode ^= ucode >> 8; + ucode ^= ucode >> 16; + ucode ^= ucode >> 32; + + return ((u_int)(ucode & 1)); +} + +static uint64_t +qat_ae_ucode_set_ecc(uint64_t ucode) +{ + static const uint64_t + bit0mask=0xff800007fffULL, bit1mask=0x1f801ff801fULL, + bit2mask=0xe387e0781e1ULL, bit3mask=0x7cb8e388e22ULL, + bit4mask=0xaf5b2c93244ULL, bit5mask=0xf56d5525488ULL, + bit6mask=0xdaf69a46910ULL; + + /* clear the ecc bits */ + ucode &= ~(0x7fULL << USTORE_ECC_BIT_0); + + ucode |= (uint64_t)qat_ae_ucode_parity64(bit0mask & ucode) << + USTORE_ECC_BIT_0; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit1mask & ucode) << + USTORE_ECC_BIT_1; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit2mask & ucode) << + USTORE_ECC_BIT_2; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit3mask & ucode) << + USTORE_ECC_BIT_3; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit4mask & ucode) << + USTORE_ECC_BIT_4; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit5mask & ucode) << + USTORE_ECC_BIT_5; + ucode |= (uint64_t)qat_ae_ucode_parity64(bit6mask & ucode) << + USTORE_ECC_BIT_6; + + return (ucode); +} + +static int +qat_ae_ucode_write(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, + const uint64_t *ucode) +{ + uint64_t tmp; + uint32_t ustore_addr, ulo, uhi; + int i; + + qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); + uaddr |= USTORE_ADDRESS_ECS; + + qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); + for (i = 0; i < ninst; i++) { + tmp = qat_ae_ucode_set_ecc(ucode[i]); + ulo = (uint32_t)(tmp & 0xffffffff); + uhi = (uint32_t)(tmp >> 32); + + qat_ae_write_4(sc, ae, USTORE_DATA_LOWER, ulo); + /* this will auto increment the address */ + qat_ae_write_4(sc, ae, USTORE_DATA_UPPER, uhi); + } + qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); + + return 0; +} + +static int +qat_ae_ucode_read(struct qat_softc *sc, u_char ae, u_int uaddr, u_int ninst, + uint64_t *ucode) +{ + uint32_t misc, ustore_addr, ulo, uhi; + u_int ii; + u_char nae; + + if (qat_ae_get_status(sc, ae) != QAT_AE_DISABLED) + return EBUSY; + + /* determine whether it neighbour AE runs in shared control store + * status */ + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); + if (misc & AE_MISC_CONTROL_SHARE_CS) { + qat_ae_get_shared_ustore_ae(ae, &nae); + if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) + return EBUSY; + } + + /* if reloadable, then get it all from dram-ustore */ + if (__SHIFTOUT(misc, AE_MISC_CONTROL_CS_RELOAD)) + panic("notyet"); /* XXX getReloadUwords */ + + /* disable SHARE_CS bit to workaround silicon bug */ + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc & 0xfffffffb); + + MPASS(uaddr + ninst <= USTORE_SIZE); + + /* save ustore-addr csr */ + qat_ae_read_4(sc, ae, USTORE_ADDRESS, &ustore_addr); + + uaddr |= USTORE_ADDRESS_ECS; /* enable ecs bit */ + for (ii = 0; ii < ninst; ii++) { + qat_ae_write_4(sc, ae, USTORE_ADDRESS, uaddr); + + uaddr++; + qat_ae_read_4(sc, ae, USTORE_DATA_LOWER, &ulo); + qat_ae_read_4(sc, ae, USTORE_DATA_UPPER, &uhi); + ucode[ii] = uhi; + ucode[ii] = (ucode[ii] << 32) | ulo; + } + + /* restore SHARE_CS bit to workaround silicon bug */ + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); + qat_ae_write_4(sc, ae, USTORE_ADDRESS, ustore_addr); + + return 0; +} + +static u_int +qat_ae_concat_ucode(uint64_t *ucode, u_int ninst, u_int size, u_int addr, + u_int *value) +{ + const uint64_t *inst_arr; + u_int ninst0, curvalue; + int ii, vali, fixup, usize = 0; + + if (size == 0) + return 0; + + ninst0 = ninst; + vali = 0; + curvalue = value[vali++]; + + switch (size) { + case 0x1: + inst_arr = ae_inst_1b; + usize = nitems(ae_inst_1b); + break; + case 0x2: + inst_arr = ae_inst_2b; + usize = nitems(ae_inst_2b); + break; + case 0x3: + inst_arr = ae_inst_3b; + usize = nitems(ae_inst_3b); + break; + default: + inst_arr = ae_inst_4b; + usize = nitems(ae_inst_4b); + break; + } + + fixup = ninst; + for (ii = 0; ii < usize; ii++) + ucode[ninst++] = inst_arr[ii]; + + INSERT_IMMED_GPRA_CONST(ucode[fixup], (addr)); + fixup++; + INSERT_IMMED_GPRA_CONST(ucode[fixup], 0); + fixup++; + INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); + fixup++; + INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); + /* XXX fixup++ ? */ + + if (size <= 0x4) + return (ninst - ninst0); + + size -= sizeof(u_int); + while (size >= sizeof(u_int)) { + curvalue = value[vali++]; + fixup = ninst; + ucode[ninst++] = ae_inst_4b[0x2]; + ucode[ninst++] = ae_inst_4b[0x3]; + ucode[ninst++] = ae_inst_4b[0x8]; + INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 16)); + fixup++; + INSERT_IMMED_GPRB_CONST(ucode[fixup], (curvalue >> 0)); + /* XXX fixup++ ? */ + + addr += sizeof(u_int); + size -= sizeof(u_int); + } + /* call this function recusive when the left size less than 4 */ + ninst += + qat_ae_concat_ucode(ucode, ninst, size, addr, value + vali); + + return (ninst - ninst0); +} + +static int +qat_ae_exec_ucode(struct qat_softc *sc, u_char ae, u_char ctx, + uint64_t *ucode, u_int ninst, int cond_code_off, u_int max_cycles, + u_int *endpc) +{ + int error = 0, share_cs = 0; + uint64_t savucode[MAX_EXEC_INST]; + uint32_t indr_lm_addr_0, indr_lm_addr_1; + uint32_t indr_lm_addr_byte_0, indr_lm_addr_byte_1; + uint32_t indr_future_cnt_sig; + uint32_t indr_sig, active_sig; + uint32_t wakeup_ev, savpc, savcc, savctx, ctxarbctl; + uint32_t misc, nmisc, ctxen; + u_char nae; + + MPASS(ninst <= USTORE_SIZE); + + if (qat_ae_is_active(sc, ae)) + return EBUSY; + + /* save current LM addr */ + qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_0_INDIRECT, &indr_lm_addr_0); + qat_ae_ctx_indr_read(sc, ae, ctx, LM_ADDR_1_INDIRECT, &indr_lm_addr_1); + qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, + &indr_lm_addr_byte_0); + qat_ae_ctx_indr_read(sc, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, + &indr_lm_addr_byte_1); + + /* backup shared control store bit, and force AE to + none-shared mode before executing ucode snippet */ + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); + if (misc & AE_MISC_CONTROL_SHARE_CS) { + share_cs = 1; + qat_ae_get_shared_ustore_ae(ae, &nae); + if ((sc->sc_ae_mask & (1 << nae)) && qat_ae_is_active(sc, nae)) + return EBUSY; + } + nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); + + /* save current states: */ + if (ninst <= MAX_EXEC_INST) { + error = qat_ae_ucode_read(sc, ae, 0, ninst, savucode); + if (error) { + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, misc); + return error; + } + } + + /* save wakeup-events */ + qat_ae_ctx_indr_read(sc, ae, ctx, CTX_WAKEUP_EVENTS_INDIRECT, + &wakeup_ev); + /* save PC */ + qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, &savpc); + savpc &= UPC_MASK; + + /* save ctx enables */ + qat_ae_read_4(sc, ae, CTX_ENABLES, &ctxen); + ctxen &= CTX_ENABLES_IGNORE_W1C_MASK; + /* save conditional-code */ + qat_ae_read_4(sc, ae, CC_ENABLE, &savcc); + /* save current context */ + qat_ae_read_4(sc, ae, ACTIVE_CTX_STATUS, &savctx); + qat_ae_read_4(sc, ae, CTX_ARB_CNTL, &ctxarbctl); + + /* save indirect csrs */ + qat_ae_ctx_indr_read(sc, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, + &indr_future_cnt_sig); + qat_ae_ctx_indr_read(sc, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &indr_sig); + qat_ae_read_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, &active_sig); + + /* turn off ucode parity */ + qat_ae_write_4(sc, ae, CTX_ENABLES, + ctxen & ~CTX_ENABLES_CNTL_STORE_PARITY_ENABLE); + + /* copy instructions to ustore */ + qat_ae_ucode_write(sc, ae, 0, ninst, ucode); + /* set PC */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, 0); + /* change the active context */ + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + ctx & ACTIVE_CTX_STATUS_ACNO); + + if (cond_code_off) { + /* disable conditional-code*/ + qat_ae_write_4(sc, ae, CC_ENABLE, savcc & 0xffffdfff); + } + + /* wakeup-event voluntary */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, + CTX_WAKEUP_EVENTS_INDIRECT, CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY); + + /* clean signals */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, 0); + qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, 0); + + /* enable context */ + qat_ae_enable_ctx(sc, ae, 1 << ctx); + + /* wait for it to finish */ + if (qat_ae_wait_num_cycles(sc, ae, max_cycles, 1) != 0) + error = ETIMEDOUT; + + /* see if we need to get the current PC */ + if (endpc != NULL) { + uint32_t ctx_status; + + qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, + &ctx_status); + *endpc = ctx_status & UPC_MASK; + } +#if 0 + { + uint32_t ctx_status; + + qat_ae_ctx_indr_read(sc, ae, ctx, CTX_STS_INDIRECT, + &ctx_status); + printf("%s: endpc 0x%08x\n", __func__, + ctx_status & UPC_MASK); + } +#endif + + /* retore to previous states: */ + /* disable context */ + qat_ae_disable_ctx(sc, ae, 1 << ctx); + if (ninst <= MAX_EXEC_INST) { + /* instructions */ + qat_ae_ucode_write(sc, ae, 0, ninst, savucode); + } + /* wakeup-events */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_WAKEUP_EVENTS_INDIRECT, + wakeup_ev); + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_STS_INDIRECT, savpc); + + /* only restore shared control store bit, + other bit might be changed by AE code snippet */ + qat_ae_read_4(sc, ae, AE_MISC_CONTROL, &misc); + if (share_cs) + nmisc = misc | AE_MISC_CONTROL_SHARE_CS; + else + nmisc = misc & ~AE_MISC_CONTROL_SHARE_CS; + qat_ae_write_4(sc, ae, AE_MISC_CONTROL, nmisc); + /* conditional-code */ + qat_ae_write_4(sc, ae, CC_ENABLE, savcc); + /* change the active context */ + qat_ae_write_4(sc, ae, ACTIVE_CTX_STATUS, + savctx & ACTIVE_CTX_STATUS_ACNO); + /* restore the nxt ctx to run */ + qat_ae_write_4(sc, ae, CTX_ARB_CNTL, ctxarbctl); + /* restore current LM addr */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_0_INDIRECT, + indr_lm_addr_0); + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, LM_ADDR_1_INDIRECT, + indr_lm_addr_1); + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, + indr_lm_addr_byte_0); + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, + indr_lm_addr_byte_1); + + /* restore indirect csrs */ + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, FUTURE_COUNT_SIGNAL_INDIRECT, + indr_future_cnt_sig); + qat_ae_ctx_indr_write(sc, ae, 1 << ctx, CTX_SIG_EVENTS_INDIRECT, + indr_sig); + qat_ae_write_4(sc, ae, CTX_SIG_EVENTS_ACTIVE, active_sig); + + /* ctx-enables */ + qat_ae_write_4(sc, ae, CTX_ENABLES, ctxen); + + return error; +} + +static int +qat_ae_exec_ucode_init_lm(struct qat_softc *sc, u_char ae, u_char ctx, + int *first_exec, uint64_t *ucode, u_int ninst, + u_int *gpr_a0, u_int *gpr_a1, u_int *gpr_a2, u_int *gpr_b0, u_int *gpr_b1) +{ + + if (*first_exec) { + qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); + qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); + qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); + qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); + qat_aereg_rel_data_read(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); + *first_exec = 0; + } + + return qat_ae_exec_ucode(sc, ae, ctx, ucode, ninst, 1, ninst * 5, NULL); +} + +static int +qat_ae_restore_init_lm_gprs(struct qat_softc *sc, u_char ae, u_char ctx, + u_int gpr_a0, u_int gpr_a1, u_int gpr_a2, u_int gpr_b0, u_int gpr_b1) +{ + qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 0, gpr_a0); + qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 1, gpr_a1); + qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPA_REL, 2, gpr_a2); + qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 0, gpr_b0); + qat_aereg_rel_data_write(sc, ae, ctx, AEREG_GPB_REL, 1, gpr_b1); + + return 0; +} + +static int +qat_ae_get_inst_num(int lmsize) +{ + int ninst, left; + + if (lmsize == 0) + return 0; + + left = lmsize % sizeof(u_int); + + if (left) { + ninst = nitems(ae_inst_1b) + + qat_ae_get_inst_num(lmsize - left); + } else { + /* 3 instruction is needed for further code */ + ninst = (lmsize - sizeof(u_int)) * 3 / 4 + nitems(ae_inst_4b); + } + + return (ninst); +} + +static int +qat_ae_batch_put_lm(struct qat_softc *sc, u_char ae, + struct qat_ae_batch_init_list *qabi_list, size_t nqabi) +{ + struct qat_ae_batch_init *qabi; + size_t alloc_ninst, ninst; + uint64_t *ucode; + u_int gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1; + int insnsz, error = 0, execed = 0, first_exec = 1; + + if (STAILQ_FIRST(qabi_list) == NULL) + return 0; + + alloc_ninst = min(USTORE_SIZE, nqabi); + ucode = qat_alloc_mem(sizeof(uint64_t) * alloc_ninst); + + ninst = 0; + STAILQ_FOREACH(qabi, qabi_list, qabi_next) { + insnsz = qat_ae_get_inst_num(qabi->qabi_size); + if (insnsz + ninst > alloc_ninst) { + /* add ctx_arb[kill] */ + ucode[ninst++] = 0x0E000010000ull; + execed = 1; + + error = qat_ae_exec_ucode_init_lm(sc, ae, 0, + &first_exec, ucode, ninst, + &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); + if (error) { + qat_ae_restore_init_lm_gprs(sc, ae, 0, + gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); + qat_free_mem(ucode); + return error; + } + /* run microExec to execute the microcode */ + ninst = 0; + } + ninst += qat_ae_concat_ucode(ucode, ninst, + qabi->qabi_size, qabi->qabi_addr, qabi->qabi_value); + } + + if (ninst > 0) { + ucode[ninst++] = 0x0E000010000ull; + execed = 1; + + error = qat_ae_exec_ucode_init_lm(sc, ae, 0, + &first_exec, ucode, ninst, + &gpr_a0, &gpr_a1, &gpr_a2, &gpr_b0, &gpr_b1); + } + if (execed) { + qat_ae_restore_init_lm_gprs(sc, ae, 0, + gpr_a0, gpr_a1, gpr_a2, gpr_b0, gpr_b1); + } + + qat_free_mem(ucode); + + return error; +} + +static int +qat_ae_write_pc(struct qat_softc *sc, u_char ae, u_int ctx_mask, u_int upc) +{ + + if (qat_ae_is_active(sc, ae)) + return EBUSY; + + qat_ae_ctx_indr_write(sc, ae, ctx_mask, CTX_STS_INDIRECT, + UPC_MASK & upc); + return 0; +} + +static inline u_int +qat_aefw_csum_calc(u_int reg, int ch) +{ + int i; + u_int topbit = CRC_BITMASK(CRC_WIDTH - 1); + u_int inbyte = (u_int)((reg >> 0x18) ^ ch); + + reg ^= inbyte << (CRC_WIDTH - 0x8); + for (i = 0; i < 0x8; i++) { + if (reg & topbit) + reg = (reg << 1) ^ CRC_POLY; + else + reg <<= 1; + } + + return (reg & CRC_WIDTHMASK(CRC_WIDTH)); +} + +static u_int +qat_aefw_csum(char *buf, int size) +{ + u_int csum = 0; + + while (size--) { + csum = qat_aefw_csum_calc(csum, *buf++); + } + + return csum; +} + +static const char * +qat_aefw_uof_string(struct qat_softc *sc, size_t offset) +{ + if (offset >= sc->sc_aefw_uof.qafu_str_tab_size) + return NULL; + if (sc->sc_aefw_uof.qafu_str_tab == NULL) + return NULL; + + return (const char *)((uintptr_t)sc->sc_aefw_uof.qafu_str_tab + offset); +} + +static struct uof_chunk_hdr * +qat_aefw_uof_find_chunk(struct qat_softc *sc, + const char *id, struct uof_chunk_hdr *cur) +{ + struct uof_obj_hdr *uoh = sc->sc_aefw_uof.qafu_obj_hdr; + struct uof_chunk_hdr *uch; + int i; + + uch = (struct uof_chunk_hdr *)(uoh + 1); + for (i = 0; i < uoh->uoh_num_chunks; i++, uch++) { + if (uch->uch_offset + uch->uch_size > sc->sc_aefw_uof.qafu_size) + return NULL; + + if (cur < uch && !strncmp(uch->uch_id, id, UOF_OBJ_ID_LEN)) + return uch; + } + + return NULL; +} + +static int +qat_aefw_load_mof(struct qat_softc *sc) +{ + const struct firmware *fw; + + fw = firmware_get(sc->sc_hw.qhw_mof_fwname); + if (fw == NULL) { + device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", + sc->sc_hw.qhw_mof_fwname); + return ENXIO; + } + + sc->sc_fw_mof = qat_alloc_mem(fw->datasize); + sc->sc_fw_mof_size = fw->datasize; + memcpy(sc->sc_fw_mof, fw->data, fw->datasize); + firmware_put(fw, FIRMWARE_UNLOAD); + return 0; +} + +static void +qat_aefw_unload_mof(struct qat_softc *sc) +{ + if (sc->sc_fw_mof != NULL) { + qat_free_mem(sc->sc_fw_mof); + sc->sc_fw_mof = NULL; + } +} + +static int +qat_aefw_load_mmp(struct qat_softc *sc) +{ + const struct firmware *fw; + + fw = firmware_get(sc->sc_hw.qhw_mmp_fwname); + if (fw == NULL) { + device_printf(sc->sc_dev, "couldn't load MOF firmware %s\n", + sc->sc_hw.qhw_mmp_fwname); + return ENXIO; + } + + sc->sc_fw_mmp = qat_alloc_mem(fw->datasize); + sc->sc_fw_mmp_size = fw->datasize; + memcpy(sc->sc_fw_mmp, fw->data, fw->datasize); + firmware_put(fw, FIRMWARE_UNLOAD); + return 0; +} + +static void +qat_aefw_unload_mmp(struct qat_softc *sc) +{ + if (sc->sc_fw_mmp != NULL) { + qat_free_mem(sc->sc_fw_mmp); + sc->sc_fw_mmp = NULL; + } +} + +static int +qat_aefw_mof_find_uof0(struct qat_softc *sc, + struct mof_uof_hdr *muh, struct mof_uof_chunk_hdr *head, + u_int nchunk, size_t size, const char *id, + size_t *fwsize, void **fwptr) +{ + int i; + char *uof_name; + + for (i = 0; i < nchunk; i++) { + struct mof_uof_chunk_hdr *much = &head[i]; + + if (strncmp(much->much_id, id, MOF_OBJ_ID_LEN)) + return EINVAL; + + if (much->much_offset + much->much_size > size) + return EINVAL; + + if (sc->sc_mof.qmf_sym_size <= much->much_name) + return EINVAL; + + uof_name = (char *)((uintptr_t)sc->sc_mof.qmf_sym + + much->much_name); + + if (!strcmp(uof_name, sc->sc_fw_uof_name)) { + *fwptr = (void *)((uintptr_t)muh + + (uintptr_t)much->much_offset); + *fwsize = (size_t)much->much_size; + return 0; + } + } + + return ENOENT; +} + +static int +qat_aefw_mof_find_uof(struct qat_softc *sc) +{ + struct mof_uof_hdr *uof_hdr, *suof_hdr; + u_int nuof_chunks = 0, nsuof_chunks = 0; + int error; + + uof_hdr = sc->sc_mof.qmf_uof_objs; + suof_hdr = sc->sc_mof.qmf_suof_objs; + + if (uof_hdr != NULL) { + if (uof_hdr->muh_max_chunks < uof_hdr->muh_num_chunks) { + return EINVAL; + } + nuof_chunks = uof_hdr->muh_num_chunks; + } + if (suof_hdr != NULL) { + if (suof_hdr->muh_max_chunks < suof_hdr->muh_num_chunks) + return EINVAL; + nsuof_chunks = suof_hdr->muh_num_chunks; + } + + if (nuof_chunks + nsuof_chunks == 0) + return EINVAL; + + if (uof_hdr != NULL) { + error = qat_aefw_mof_find_uof0(sc, uof_hdr, + (struct mof_uof_chunk_hdr *)(uof_hdr + 1), nuof_chunks, + sc->sc_mof.qmf_uof_objs_size, UOF_IMAG, + &sc->sc_fw_uof_size, &sc->sc_fw_uof); + if (error && error != ENOENT) + return error; + } + + if (suof_hdr != NULL) { + error = qat_aefw_mof_find_uof0(sc, suof_hdr, + (struct mof_uof_chunk_hdr *)(suof_hdr + 1), nsuof_chunks, + sc->sc_mof.qmf_suof_objs_size, SUOF_IMAG, + &sc->sc_fw_suof_size, &sc->sc_fw_suof); + if (error && error != ENOENT) + return error; + } + + if (sc->sc_fw_uof == NULL && sc->sc_fw_suof == NULL) + return ENOENT; + + return 0; +} + +static int +qat_aefw_mof_parse(struct qat_softc *sc) +{ + const struct mof_file_hdr *mfh; + const struct mof_file_chunk_hdr *mfch; + size_t size; + u_int csum; + int error, i; + + size = sc->sc_fw_mof_size; + + if (size < sizeof(struct mof_file_hdr)) + return EINVAL; + size -= sizeof(struct mof_file_hdr); + + mfh = sc->sc_fw_mof; + + if (mfh->mfh_fid != MOF_FID) + return EINVAL; + + csum = qat_aefw_csum((char *)((uintptr_t)sc->sc_fw_mof + + offsetof(struct mof_file_hdr, mfh_min_ver)), + sc->sc_fw_mof_size - + offsetof(struct mof_file_hdr, mfh_min_ver)); + if (mfh->mfh_csum != csum) + return EINVAL; + + if (mfh->mfh_min_ver != MOF_MIN_VER || + mfh->mfh_maj_ver != MOF_MAJ_VER) + return EINVAL; + + if (mfh->mfh_max_chunks < mfh->mfh_num_chunks) + return EINVAL; + + if (size < sizeof(struct mof_file_chunk_hdr) * mfh->mfh_num_chunks) + return EINVAL; + mfch = (const struct mof_file_chunk_hdr *)(mfh + 1); + + for (i = 0; i < mfh->mfh_num_chunks; i++, mfch++) { + if (mfch->mfch_offset + mfch->mfch_size > sc->sc_fw_mof_size) + return EINVAL; + + if (!strncmp(mfch->mfch_id, SYM_OBJS, MOF_OBJ_ID_LEN)) { + if (sc->sc_mof.qmf_sym != NULL) + return EINVAL; + + sc->sc_mof.qmf_sym = + (void *)((uintptr_t)sc->sc_fw_mof + + (uintptr_t)mfch->mfch_offset + sizeof(u_int)); + sc->sc_mof.qmf_sym_size = + *(u_int *)((uintptr_t)sc->sc_fw_mof + + (uintptr_t)mfch->mfch_offset); + + if (sc->sc_mof.qmf_sym_size % sizeof(u_int) != 0) + return EINVAL; + if (mfch->mfch_size != sc->sc_mof.qmf_sym_size + + sizeof(u_int) || mfch->mfch_size == 0) + return EINVAL; + if (*(char *)((uintptr_t)sc->sc_mof.qmf_sym + + sc->sc_mof.qmf_sym_size - 1) != '\0') + return EINVAL; + + } else if (!strncmp(mfch->mfch_id, UOF_OBJS, MOF_OBJ_ID_LEN)) { + if (sc->sc_mof.qmf_uof_objs != NULL) + return EINVAL; + + sc->sc_mof.qmf_uof_objs = + (void *)((uintptr_t)sc->sc_fw_mof + + (uintptr_t)mfch->mfch_offset); + sc->sc_mof.qmf_uof_objs_size = mfch->mfch_size; + + } else if (!strncmp(mfch->mfch_id, SUOF_OBJS, MOF_OBJ_ID_LEN)) { + if (sc->sc_mof.qmf_suof_objs != NULL) + return EINVAL; + + sc->sc_mof.qmf_suof_objs = + (void *)((uintptr_t)sc->sc_fw_mof + + (uintptr_t)mfch->mfch_offset); + sc->sc_mof.qmf_suof_objs_size = mfch->mfch_size; + } + } + + if (sc->sc_mof.qmf_sym == NULL || + (sc->sc_mof.qmf_uof_objs == NULL && + sc->sc_mof.qmf_suof_objs == NULL)) + return EINVAL; + + error = qat_aefw_mof_find_uof(sc); + if (error) + return error; + return 0; +} + +static int +qat_aefw_uof_parse_image(struct qat_softc *sc, + struct qat_uof_image *qui, struct uof_chunk_hdr *uch) +{ + struct uof_image *image; + struct uof_code_page *page; + uintptr_t base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; + size_t lim = uch->uch_offset + uch->uch_size, size; + int i, p; + + size = uch->uch_size; + if (size < sizeof(struct uof_image)) + return EINVAL; + size -= sizeof(struct uof_image); + + qui->qui_image = image = + (struct uof_image *)(base + uch->uch_offset); + +#define ASSIGN_OBJ_TAB(np, typep, type, base, off, lim) \ +do { \ + u_int nent; \ + nent = ((struct uof_obj_table *)((base) + (off)))->uot_nentries;\ + if ((lim) < off + sizeof(struct uof_obj_table) + \ + sizeof(type) * nent) \ + return EINVAL; \ + *(np) = nent; \ + if (nent > 0) \ + *(typep) = (type)((struct uof_obj_table *) \ + ((base) + (off)) + 1); \ + else \ + *(typep) = NULL; \ +} while (0) + + ASSIGN_OBJ_TAB(&qui->qui_num_ae_reg, &qui->qui_ae_reg, + struct uof_ae_reg *, base, image->ui_reg_tab, lim); + ASSIGN_OBJ_TAB(&qui->qui_num_init_reg_sym, &qui->qui_init_reg_sym, + struct uof_init_reg_sym *, base, image->ui_init_reg_sym_tab, lim); + ASSIGN_OBJ_TAB(&qui->qui_num_sbreak, &qui->qui_sbreak, + struct qui_sbreak *, base, image->ui_sbreak_tab, lim); + + if (size < sizeof(struct uof_code_page) * image->ui_num_pages) + return EINVAL; + if (nitems(qui->qui_pages) < image->ui_num_pages) + return EINVAL; + + page = (struct uof_code_page *)(image + 1); + + for (p = 0; p < image->ui_num_pages; p++, page++) { + struct qat_uof_page *qup = &qui->qui_pages[p]; + struct uof_code_area *uca; + + qup->qup_page_num = page->ucp_page_num; + qup->qup_def_page = page->ucp_def_page; + qup->qup_page_region = page->ucp_page_region; + qup->qup_beg_vaddr = page->ucp_beg_vaddr; + qup->qup_beg_paddr = page->ucp_beg_paddr; + + ASSIGN_OBJ_TAB(&qup->qup_num_uc_var, &qup->qup_uc_var, + struct uof_uword_fixup *, base, + page->ucp_uc_var_tab, lim); + ASSIGN_OBJ_TAB(&qup->qup_num_imp_var, &qup->qup_imp_var, + struct uof_import_var *, base, + page->ucp_imp_var_tab, lim); + ASSIGN_OBJ_TAB(&qup->qup_num_imp_expr, &qup->qup_imp_expr, + struct uof_uword_fixup *, base, + page->ucp_imp_expr_tab, lim); + ASSIGN_OBJ_TAB(&qup->qup_num_neigh_reg, &qup->qup_neigh_reg, + struct uof_uword_fixup *, base, + page->ucp_neigh_reg_tab, lim); + + if (lim < page->ucp_code_area + sizeof(struct uof_code_area)) + return EINVAL; + + uca = (struct uof_code_area *)(base + page->ucp_code_area); + qup->qup_num_micro_words = uca->uca_num_micro_words; + + ASSIGN_OBJ_TAB(&qup->qup_num_uw_blocks, &qup->qup_uw_blocks, + struct qat_uof_uword_block *, base, + uca->uca_uword_block_tab, lim); + + for (i = 0; i < qup->qup_num_uw_blocks; i++) { + u_int uwordoff = ((struct uof_uword_block *)( + &qup->qup_uw_blocks[i]))->uub_uword_offset; + + if (lim < uwordoff) + return EINVAL; + + qup->qup_uw_blocks[i].quub_micro_words = + (base + uwordoff); + } + } + +#undef ASSIGN_OBJ_TAB + + return 0; +} + +static int +qat_aefw_uof_parse_images(struct qat_softc *sc) +{ + struct uof_chunk_hdr *uch = NULL; + u_int assigned_ae; + int i, error; + + for (i = 0; i < MAX_NUM_AE * MAX_AE_CTX; i++) { + uch = qat_aefw_uof_find_chunk(sc, UOF_IMAG, uch); + if (uch == NULL) + break; + + if (i >= nitems(sc->sc_aefw_uof.qafu_imgs)) + return ENOENT; + + error = qat_aefw_uof_parse_image(sc, &sc->sc_aefw_uof.qafu_imgs[i], uch); + if (error) + return error; + + sc->sc_aefw_uof.qafu_num_imgs++; + } + + assigned_ae = 0; + for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { + assigned_ae |= sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned; + } + + return 0; +} + +static int +qat_aefw_uof_parse(struct qat_softc *sc) +{ + struct uof_file_hdr *ufh; + struct uof_file_chunk_hdr *ufch; + struct uof_obj_hdr *uoh; + struct uof_chunk_hdr *uch; + void *uof = NULL; + size_t size, uof_size, hdr_size; + uintptr_t base; + u_int csum; + int i; + + size = sc->sc_fw_uof_size; + if (size < MIN_UOF_SIZE) + return EINVAL; + size -= sizeof(struct uof_file_hdr); + + ufh = sc->sc_fw_uof; + + if (ufh->ufh_id != UOF_FID) + return EINVAL; + if (ufh->ufh_min_ver != UOF_MIN_VER || ufh->ufh_maj_ver != UOF_MAJ_VER) + return EINVAL; + + if (ufh->ufh_max_chunks < ufh->ufh_num_chunks) + return EINVAL; + if (size < sizeof(struct uof_file_chunk_hdr) * ufh->ufh_num_chunks) + return EINVAL; + ufch = (struct uof_file_chunk_hdr *)(ufh + 1); + + uof_size = 0; + for (i = 0; i < ufh->ufh_num_chunks; i++, ufch++) { + if (ufch->ufch_offset + ufch->ufch_size > sc->sc_fw_uof_size) + return EINVAL; + + if (!strncmp(ufch->ufch_id, UOF_OBJS, UOF_OBJ_ID_LEN)) { + if (uof != NULL) + return EINVAL; + + uof = + (void *)((uintptr_t)sc->sc_fw_uof + + ufch->ufch_offset); + uof_size = ufch->ufch_size; + + csum = qat_aefw_csum(uof, uof_size); + if (csum != ufch->ufch_csum) + return EINVAL; + } + } + + if (uof == NULL) + return ENOENT; + + size = uof_size; + if (size < sizeof(struct uof_obj_hdr)) + return EINVAL; + size -= sizeof(struct uof_obj_hdr); + + uoh = uof; + + if (size < sizeof(struct uof_chunk_hdr) * uoh->uoh_num_chunks) + return EINVAL; + + /* Check if the UOF objects are compatible with the chip */ + if ((uoh->uoh_cpu_type & sc->sc_hw.qhw_prod_type) == 0) + return ENOTSUP; + + if (uoh->uoh_min_cpu_ver > sc->sc_rev || + uoh->uoh_max_cpu_ver < sc->sc_rev) + return ENOTSUP; + + sc->sc_aefw_uof.qafu_size = uof_size; + sc->sc_aefw_uof.qafu_obj_hdr = uoh; + + base = (uintptr_t)sc->sc_aefw_uof.qafu_obj_hdr; + + /* map uof string-table */ + uch = qat_aefw_uof_find_chunk(sc, UOF_STRT, NULL); + if (uch != NULL) { + hdr_size = offsetof(struct uof_str_tab, ust_strings); + sc->sc_aefw_uof.qafu_str_tab = + (void *)(base + uch->uch_offset + hdr_size); + sc->sc_aefw_uof.qafu_str_tab_size = uch->uch_size - hdr_size; + } + + /* get ustore mem inits table -- should be only one */ + uch = qat_aefw_uof_find_chunk(sc, UOF_IMEM, NULL); + if (uch != NULL) { + if (uch->uch_size < sizeof(struct uof_obj_table)) + return EINVAL; + sc->sc_aefw_uof.qafu_num_init_mem = ((struct uof_obj_table *)(base + + uch->uch_offset))->uot_nentries; + if (sc->sc_aefw_uof.qafu_num_init_mem) { + sc->sc_aefw_uof.qafu_init_mem = + (struct uof_init_mem *)(base + uch->uch_offset + + sizeof(struct uof_obj_table)); + sc->sc_aefw_uof.qafu_init_mem_size = + uch->uch_size - sizeof(struct uof_obj_table); + } + } + + uch = qat_aefw_uof_find_chunk(sc, UOF_MSEG, NULL); + if (uch != NULL) { + if (uch->uch_size < sizeof(struct uof_obj_table) + + sizeof(struct uof_var_mem_seg)) + return EINVAL; + sc->sc_aefw_uof.qafu_var_mem_seg = + (struct uof_var_mem_seg *)(base + uch->uch_offset + + sizeof(struct uof_obj_table)); + } + + return qat_aefw_uof_parse_images(sc); +} + +static int +qat_aefw_suof_parse_image(struct qat_softc *sc, struct qat_suof_image *qsi, + struct suof_chunk_hdr *sch) +{ + struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; + struct simg_ae_mode *ae_mode; + u_int maj_ver; + + qsi->qsi_simg_buf = qafs->qafs_suof_buf + sch->sch_offset + + sizeof(struct suof_obj_hdr); + qsi->qsi_simg_len = + ((struct suof_obj_hdr *) + (qafs->qafs_suof_buf + sch->sch_offset))->soh_img_length; + + qsi->qsi_css_header = qsi->qsi_simg_buf; + qsi->qsi_css_key = qsi->qsi_css_header + sizeof(struct css_hdr); + qsi->qsi_css_signature = qsi->qsi_css_key + + CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; + qsi->qsi_css_simg = qsi->qsi_css_signature + CSS_SIGNATURE_LEN; + + ae_mode = (struct simg_ae_mode *)qsi->qsi_css_simg; + qsi->qsi_ae_mask = ae_mode->sam_ae_mask; + qsi->qsi_simg_name = (u_long)&ae_mode->sam_simg_name; + qsi->qsi_appmeta_data = (u_long)&ae_mode->sam_appmeta_data; + qsi->qsi_fw_type = ae_mode->sam_fw_type; + + if (ae_mode->sam_dev_type != sc->sc_hw.qhw_prod_type) + return EINVAL; + + maj_ver = (QAT_PID_MAJOR_REV | (sc->sc_rev & QAT_PID_MINOR_REV)) & 0xff; + if ((maj_ver > ae_mode->sam_devmax_ver) || + (maj_ver < ae_mode->sam_devmin_ver)) { + return EINVAL; + } + + return 0; +} + +static int +qat_aefw_suof_parse(struct qat_softc *sc) +{ + struct suof_file_hdr *sfh; + struct suof_chunk_hdr *sch; + struct qat_aefw_suof *qafs = &sc->sc_aefw_suof; + struct qat_suof_image *qsi; + size_t size; + u_int csum; + int ae0_img = MAX_AE; + int i, error; + + size = sc->sc_fw_suof_size; + if (size < sizeof(struct suof_file_hdr)) + return EINVAL; + + sfh = sc->sc_fw_suof; + + if (sfh->sfh_file_id != SUOF_FID) + return EINVAL; + if (sfh->sfh_fw_type != 0) + return EINVAL; + if (sfh->sfh_num_chunks <= 1) + return EINVAL; + if (sfh->sfh_min_ver != SUOF_MIN_VER || + sfh->sfh_maj_ver != SUOF_MAJ_VER) + return EINVAL; + + csum = qat_aefw_csum((char *)&sfh->sfh_min_ver, + size - offsetof(struct suof_file_hdr, sfh_min_ver)); + if (csum != sfh->sfh_check_sum) + return EINVAL; + + size -= sizeof(struct suof_file_hdr); + + qafs->qafs_file_id = SUOF_FID; + qafs->qafs_suof_buf = sc->sc_fw_suof; + qafs->qafs_suof_size = sc->sc_fw_suof_size; + qafs->qafs_check_sum = sfh->sfh_check_sum; + qafs->qafs_min_ver = sfh->sfh_min_ver; + qafs->qafs_maj_ver = sfh->sfh_maj_ver; + qafs->qafs_fw_type = sfh->sfh_fw_type; + + if (size < sizeof(struct suof_chunk_hdr)) + return EINVAL; + sch = (struct suof_chunk_hdr *)(sfh + 1); + size -= sizeof(struct suof_chunk_hdr); + + if (size < sizeof(struct suof_str_tab)) + return EINVAL; + size -= offsetof(struct suof_str_tab, sst_strings); + + qafs->qafs_sym_size = ((struct suof_str_tab *) + (qafs->qafs_suof_buf + sch->sch_offset))->sst_tab_length; + if (size < qafs->qafs_sym_size) + return EINVAL; + qafs->qafs_sym_str = qafs->qafs_suof_buf + sch->sch_offset + + offsetof(struct suof_str_tab, sst_strings); + + qafs->qafs_num_simgs = sfh->sfh_num_chunks - 1; + if (qafs->qafs_num_simgs == 0) + return EINVAL; + + qsi = qat_alloc_mem( + sizeof(struct qat_suof_image) * qafs->qafs_num_simgs); + qafs->qafs_simg = qsi; + + for (i = 0; i < qafs->qafs_num_simgs; i++) { + error = qat_aefw_suof_parse_image(sc, &qsi[i], &sch[i + 1]); + if (error) + return error; + if ((qsi[i].qsi_ae_mask & 0x1) != 0) + ae0_img = i; + } + + if (ae0_img != qafs->qafs_num_simgs - 1) { + struct qat_suof_image last_qsi; + + memcpy(&last_qsi, &qsi[qafs->qafs_num_simgs - 1], + sizeof(struct qat_suof_image)); + memcpy(&qsi[qafs->qafs_num_simgs - 1], &qsi[ae0_img], + sizeof(struct qat_suof_image)); + memcpy(&qsi[ae0_img], &last_qsi, + sizeof(struct qat_suof_image)); + } + + return 0; +} + +static int +qat_aefw_alloc_auth_dmamem(struct qat_softc *sc, char *image, size_t size, + struct qat_dmamem *dma) +{ + struct css_hdr *css = (struct css_hdr *)image; + struct auth_chunk *auth_chunk; + struct fw_auth_desc *auth_desc; + size_t mapsize, simg_offset = sizeof(struct auth_chunk); + bus_size_t bus_addr; + uintptr_t virt_addr; + int error; + + if (size > AE_IMG_OFFSET + CSS_MAX_IMAGE_LEN) + return EINVAL; + + mapsize = (css->css_fw_type == CSS_AE_FIRMWARE) ? + CSS_AE_SIMG_LEN + simg_offset : + size + CSS_FWSK_PAD_LEN + simg_offset; + error = qat_alloc_dmamem(sc, dma, 1, mapsize, PAGE_SIZE); + if (error) + return error; + + memset(dma->qdm_dma_vaddr, 0, mapsize); + + auth_chunk = dma->qdm_dma_vaddr; + auth_chunk->ac_chunk_size = mapsize; + auth_chunk->ac_chunk_bus_addr = dma->qdm_dma_seg.ds_addr; + + virt_addr = (uintptr_t)dma->qdm_dma_vaddr; + virt_addr += simg_offset; + bus_addr = auth_chunk->ac_chunk_bus_addr; + bus_addr += simg_offset; + + auth_desc = &auth_chunk->ac_fw_auth_desc; + auth_desc->fad_css_hdr_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_css_hdr_low = bus_addr; + + memcpy((void *)virt_addr, image, sizeof(struct css_hdr)); + /* pub key */ + virt_addr += sizeof(struct css_hdr); + bus_addr += sizeof(struct css_hdr); + image += sizeof(struct css_hdr); + + auth_desc->fad_fwsk_pub_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_fwsk_pub_low = bus_addr; + + memcpy((void *)virt_addr, image, CSS_FWSK_MODULUS_LEN); + memset((void *)(virt_addr + CSS_FWSK_MODULUS_LEN), 0, CSS_FWSK_PAD_LEN); + memcpy((void *)(virt_addr + CSS_FWSK_MODULUS_LEN + CSS_FWSK_PAD_LEN), + image + CSS_FWSK_MODULUS_LEN, sizeof(uint32_t)); + + virt_addr += CSS_FWSK_PUB_LEN; + bus_addr += CSS_FWSK_PUB_LEN; + image += CSS_FWSK_MODULUS_LEN + CSS_FWSK_EXPONENT_LEN; + + auth_desc->fad_signature_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_signature_low = bus_addr; + + memcpy((void *)virt_addr, image, CSS_SIGNATURE_LEN); + + virt_addr += CSS_SIGNATURE_LEN; + bus_addr += CSS_SIGNATURE_LEN; + image += CSS_SIGNATURE_LEN; + + auth_desc->fad_img_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_img_low = bus_addr; + auth_desc->fad_img_len = size - AE_IMG_OFFSET; + + memcpy((void *)virt_addr, image, auth_desc->fad_img_len); + + if (css->css_fw_type == CSS_AE_FIRMWARE) { + auth_desc->fad_img_ae_mode_data_high = auth_desc->fad_img_high; + auth_desc->fad_img_ae_mode_data_low = auth_desc->fad_img_low; + + bus_addr += sizeof(struct simg_ae_mode); + + auth_desc->fad_img_ae_init_data_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_img_ae_init_data_low = bus_addr; + + bus_addr += SIMG_AE_INIT_SEQ_LEN; + + auth_desc->fad_img_ae_insts_high = (uint64_t)bus_addr >> 32; + auth_desc->fad_img_ae_insts_low = bus_addr; + } else { + auth_desc->fad_img_ae_insts_high = auth_desc->fad_img_high; + auth_desc->fad_img_ae_insts_low = auth_desc->fad_img_low; + } + + bus_dmamap_sync(dma->qdm_dma_tag, dma->qdm_dma_map, + BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); + + return 0; +} + +static int +qat_aefw_auth(struct qat_softc *sc, struct qat_dmamem *dma) +{ + bus_addr_t addr; + uint32_t fcu, sts; + int retry = 0; + + addr = dma->qdm_dma_seg.ds_addr; + qat_cap_global_write_4(sc, FCU_DRAM_ADDR_HI, (uint64_t)addr >> 32); + qat_cap_global_write_4(sc, FCU_DRAM_ADDR_LO, addr); + qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_AUTH); + + do { + DELAY(FW_AUTH_WAIT_PERIOD * 1000); + fcu = qat_cap_global_read_4(sc, FCU_STATUS); + sts = __SHIFTOUT(fcu, FCU_STATUS_STS); + if (sts == FCU_STATUS_STS_VERI_FAIL) + goto fail; + if (fcu & FCU_STATUS_AUTHFWLD && + sts == FCU_STATUS_STS_VERI_DONE) { + return 0; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + +fail: + device_printf(sc->sc_dev, + "firmware authentication error: status 0x%08x retry %d\n", + fcu, retry); + return EINVAL; +} + +static int +qat_aefw_suof_load(struct qat_softc *sc, struct qat_dmamem *dma) +{ + struct simg_ae_mode *ae_mode; + uint32_t fcu, sts, loaded; + u_int mask; + u_char ae; + int retry = 0; + + ae_mode = (struct simg_ae_mode *)((uintptr_t)dma->qdm_dma_vaddr + + sizeof(struct auth_chunk) + sizeof(struct css_hdr) + + CSS_FWSK_PUB_LEN + CSS_SIGNATURE_LEN); + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + if (!((ae_mode->sam_ae_mask >> ae) & 0x1)) + continue; + if (qat_ae_is_active(sc, ae)) { + device_printf(sc->sc_dev, "AE %d is active\n", ae); + return EINVAL; + } + qat_cap_global_write_4(sc, FCU_CTRL, + FCU_CTRL_CMD_LOAD | __SHIFTIN(ae, FCU_CTRL_AE)); + do { + DELAY(FW_AUTH_WAIT_PERIOD * 1000); + fcu = qat_cap_global_read_4(sc, FCU_STATUS); + sts = __SHIFTOUT(fcu, FCU_STATUS_STS); + loaded = __SHIFTOUT(fcu, FCU_STATUS_LOADED_AE); + if (sts == FCU_STATUS_STS_LOAD_DONE && + (loaded & (1 << ae))) { + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + + if (retry > FW_AUTH_MAX_RETRY) { + device_printf(sc->sc_dev, + "firmware load timeout: status %08x\n", fcu); + return EINVAL; + } + } + + return 0; +} + +static int +qat_aefw_suof_write(struct qat_softc *sc) +{ + struct qat_suof_image *qsi = NULL; + int i, error = 0; + + for (i = 0; i < sc->sc_aefw_suof.qafs_num_simgs; i++) { + qsi = &sc->sc_aefw_suof.qafs_simg[i]; + error = qat_aefw_alloc_auth_dmamem(sc, qsi->qsi_simg_buf, + qsi->qsi_simg_len, &qsi->qsi_dma); + if (error) + return error; + error = qat_aefw_auth(sc, &qsi->qsi_dma); + if (error) + goto fail; + error = qat_aefw_suof_load(sc, &qsi->qsi_dma); + if (error) + goto fail; + + qat_free_dmamem(sc, &qsi->qsi_dma); + } + qat_free_mem(sc->sc_aefw_suof.qafs_simg); + + return 0; +fail: + if (qsi != NULL) + qat_free_dmamem(sc, &qsi->qsi_dma); + return error; +} + +static int +qat_aefw_uof_assign_image(struct qat_softc *sc, struct qat_ae *qae, + struct qat_uof_image *qui) +{ + struct qat_ae_slice *slice; + int i, npages, nregions; + + if (qae->qae_num_slices >= nitems(qae->qae_slices)) + return ENOENT; + + if (qui->qui_image->ui_ae_mode & + (AE_MODE_RELOAD_CTX_SHARED | AE_MODE_SHARED_USTORE)) { + /* XXX */ + device_printf(sc->sc_dev, + "shared ae mode is not supported yet\n"); + return ENOTSUP; + } + + qae->qae_shareable_ustore = 0; /* XXX */ + qae->qae_effect_ustore_size = USTORE_SIZE; + + slice = &qae->qae_slices[qae->qae_num_slices]; + + slice->qas_image = qui; + slice->qas_assigned_ctx_mask = qui->qui_image->ui_ctx_assigned; + + nregions = qui->qui_image->ui_num_page_regions; + npages = qui->qui_image->ui_num_pages; + + if (nregions > nitems(slice->qas_regions)) + return ENOENT; + if (npages > nitems(slice->qas_pages)) + return ENOENT; + + for (i = 0; i < nregions; i++) { + STAILQ_INIT(&slice->qas_regions[i].qar_waiting_pages); + } + for (i = 0; i < npages; i++) { + struct qat_ae_page *page = &slice->qas_pages[i]; + int region; + + page->qap_page = &qui->qui_pages[i]; + region = page->qap_page->qup_page_region; + if (region >= nregions) + return EINVAL; + + page->qap_region = &slice->qas_regions[region]; + } + + qae->qae_num_slices++; + + return 0; +} + +static int +qat_aefw_uof_init_ae(struct qat_softc *sc, u_char ae) +{ + struct uof_image *image; + struct qat_ae *qae = &(QAT_AE(sc, ae)); + int s; + u_char nn_mode; + + for (s = 0; s < qae->qae_num_slices; s++) { + if (qae->qae_slices[s].qas_image == NULL) + continue; + + image = qae->qae_slices[s].qas_image->qui_image; + qat_ae_write_ctx_mode(sc, ae, + __SHIFTOUT(image->ui_ae_mode, AE_MODE_CTX_MODE)); + + nn_mode = __SHIFTOUT(image->ui_ae_mode, AE_MODE_NN_MODE); + if (nn_mode != AE_MODE_NN_MODE_DONTCARE) + qat_ae_write_nn_mode(sc, ae, nn_mode); + + qat_ae_write_lm_mode(sc, ae, AEREG_LMEM0, + __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM0)); + qat_ae_write_lm_mode(sc, ae, AEREG_LMEM1, + __SHIFTOUT(image->ui_ae_mode, AE_MODE_LMEM1)); + + qat_ae_write_shared_cs_mode(sc, ae, + __SHIFTOUT(image->ui_ae_mode, AE_MODE_SHARED_USTORE)); + qat_ae_set_reload_ustore(sc, ae, image->ui_reloadable_size, + __SHIFTOUT(image->ui_ae_mode, AE_MODE_RELOAD_CTX_SHARED), + qae->qae_reloc_ustore_dram); + } + + return 0; +} + +static int +qat_aefw_uof_init(struct qat_softc *sc) +{ + int ae, i, error; + uint32_t mask; + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + struct qat_ae *qae; + + if (!(mask & 1)) + continue; + + qae = &(QAT_AE(sc, ae)); + + for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { + if ((sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_ae_assigned & + (1 << ae)) == 0) + continue; + + error = qat_aefw_uof_assign_image(sc, qae, + &sc->sc_aefw_uof.qafu_imgs[i]); + if (error) + return error; + } + + /* XXX UcLo_initNumUwordUsed */ + + qae->qae_reloc_ustore_dram = UINT_MAX; /* XXX */ + + error = qat_aefw_uof_init_ae(sc, ae); + if (error) + return error; + } + + return 0; +} + +int +qat_aefw_load(struct qat_softc *sc) +{ + int error; + + error = qat_aefw_load_mof(sc); + if (error) + return error; + + error = qat_aefw_load_mmp(sc); + if (error) + return error; + + error = qat_aefw_mof_parse(sc); + if (error) { + device_printf(sc->sc_dev, "couldn't parse mof: %d\n", error); + return error; + } + + if (sc->sc_hw.qhw_fw_auth) { + error = qat_aefw_suof_parse(sc); + if (error) { + device_printf(sc->sc_dev, "couldn't parse suof: %d\n", + error); + return error; + } + + error = qat_aefw_suof_write(sc); + if (error) { + device_printf(sc->sc_dev, + "could not write firmware: %d\n", error); + return error; + } + + } else { + error = qat_aefw_uof_parse(sc); + if (error) { + device_printf(sc->sc_dev, "couldn't parse uof: %d\n", + error); + return error; + } + + error = qat_aefw_uof_init(sc); + if (error) { + device_printf(sc->sc_dev, + "couldn't init for aefw: %d\n", error); + return error; + } + + error = qat_aefw_uof_write(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not write firmware: %d\n", error); + return error; + } + } + + return 0; +} + +void +qat_aefw_unload(struct qat_softc *sc) +{ + qat_aefw_unload_mmp(sc); + qat_aefw_unload_mof(sc); +} + +int +qat_aefw_start(struct qat_softc *sc, u_char ae, u_int ctx_mask) +{ + uint32_t fcu; + int retry = 0; + + if (sc->sc_hw.qhw_fw_auth) { + qat_cap_global_write_4(sc, FCU_CTRL, FCU_CTRL_CMD_START); + do { + DELAY(FW_AUTH_WAIT_PERIOD * 1000); + fcu = qat_cap_global_read_4(sc, FCU_STATUS); + if (fcu & FCU_STATUS_DONE) + return 0; + } while (retry++ < FW_AUTH_MAX_RETRY); + + device_printf(sc->sc_dev, + "firmware start timeout: status %08x\n", fcu); + return EINVAL; + } else { + qat_ae_ctx_indr_write(sc, ae, (~ctx_mask) & AE_ALL_CTX, + CTX_WAKEUP_EVENTS_INDIRECT, + CTX_WAKEUP_EVENTS_INDIRECT_SLEEP); + qat_ae_enable_ctx(sc, ae, ctx_mask); + } + + return 0; +} + +static int +qat_aefw_init_memory_one(struct qat_softc *sc, struct uof_init_mem *uim) +{ + struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; + struct qat_ae_batch_init_list *qabi_list; + struct uof_mem_val_attr *memattr; + size_t *curinit; + u_long ael; + int i; + const char *sym; + char *ep; + + memattr = (struct uof_mem_val_attr *)(uim + 1); + + switch (uim->uim_region) { + case LMEM_REGION: + if ((uim->uim_addr + uim->uim_num_bytes) > MAX_LMEM_REG * 4) { + device_printf(sc->sc_dev, + "Invalid lmem addr or bytes\n"); + return ENOBUFS; + } + if (uim->uim_scope != UOF_SCOPE_LOCAL) + return EINVAL; + sym = qat_aefw_uof_string(sc, uim->uim_sym_name); + ael = strtoul(sym, &ep, 10); + if (ep == sym || ael > MAX_AE) + return EINVAL; + if ((sc->sc_ae_mask & (1 << ael)) == 0) + return 0; /* ae is fused out */ + + curinit = &qafu->qafu_num_lm_init[ael]; + qabi_list = &qafu->qafu_lm_init[ael]; + + for (i = 0; i < uim->uim_num_val_attr; i++, memattr++) { + struct qat_ae_batch_init *qabi; + + qabi = qat_alloc_mem(sizeof(struct qat_ae_batch_init)); + if (*curinit == 0) + STAILQ_INIT(qabi_list); + STAILQ_INSERT_TAIL(qabi_list, qabi, qabi_next); + + qabi->qabi_ae = (u_int)ael; + qabi->qabi_addr = + uim->uim_addr + memattr->umva_byte_offset; + qabi->qabi_value = &memattr->umva_value; + qabi->qabi_size = 4; + qafu->qafu_num_lm_init_inst[ael] += + qat_ae_get_inst_num(qabi->qabi_size); + (*curinit)++; + if (*curinit >= MAX_LMEM_REG) { + device_printf(sc->sc_dev, + "Invalid lmem val attr\n"); + return ENOBUFS; + } + } + break; + case SRAM_REGION: + case DRAM_REGION: + case DRAM1_REGION: + case SCRATCH_REGION: + case UMEM_REGION: + /* XXX */ + /* fallthrough */ + default: + device_printf(sc->sc_dev, + "unsupported memory region to init: %d\n", + uim->uim_region); + return ENOTSUP; + } + + return 0; +} + +static void +qat_aefw_free_lm_init(struct qat_softc *sc, u_char ae) +{ + struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; + struct qat_ae_batch_init *qabi; + + while ((qabi = STAILQ_FIRST(&qafu->qafu_lm_init[ae])) != NULL) { + STAILQ_REMOVE_HEAD(&qafu->qafu_lm_init[ae], qabi_next); + qat_free_mem(qabi); + } + + qafu->qafu_num_lm_init[ae] = 0; + qafu->qafu_num_lm_init_inst[ae] = 0; +} + +static int +qat_aefw_init_ustore(struct qat_softc *sc) +{ + uint64_t *fill; + uint32_t dont_init; + int a, i, p; + int error = 0; + int usz, end, start; + u_char ae, nae; + + fill = qat_alloc_mem(MAX_USTORE * sizeof(uint64_t)); + + for (a = 0; a < sc->sc_aefw_uof.qafu_num_imgs; a++) { + struct qat_uof_image *qui = &sc->sc_aefw_uof.qafu_imgs[a]; + struct uof_image *ui = qui->qui_image; + + for (i = 0; i < MAX_USTORE; i++) + memcpy(&fill[i], ui->ui_fill_pattern, sizeof(uint64_t)); + /* + * Compute do_not_init value as a value that will not be equal + * to fill data when cast to an int + */ + dont_init = 0; + if (dont_init == (uint32_t)fill[0]) + dont_init = 0xffffffff; + + for (p = 0; p < ui->ui_num_pages; p++) { + struct qat_uof_page *qup = &qui->qui_pages[p]; + if (!qup->qup_def_page) + continue; + + for (i = qup->qup_beg_paddr; + i < qup->qup_beg_paddr + qup->qup_num_micro_words; + i++ ) { + fill[i] = (uint64_t)dont_init; + } + } + + for (ae = 0; ae < sc->sc_ae_num; ae++) { + MPASS(ae < UOF_MAX_NUM_OF_AE); + if ((ui->ui_ae_assigned & (1 << ae)) == 0) + continue; + + if (QAT_AE(sc, ae).qae_shareable_ustore && (ae & 1)) { + qat_ae_get_shared_ustore_ae(ae, &nae); + if (ui->ui_ae_assigned & (1 << ae)) + continue; + } + usz = QAT_AE(sc, ae).qae_effect_ustore_size; + + /* initialize the areas not going to be overwritten */ + end = -1; + do { + /* find next uword that needs to be initialized */ + for (start = end + 1; start < usz; start++) { + if ((uint32_t)fill[start] != dont_init) + break; + } + /* see if there are no more such uwords */ + if (start >= usz) + break; + for (end = start + 1; end < usz; end++) { + if ((uint32_t)fill[end] == dont_init) + break; + } + if (QAT_AE(sc, ae).qae_shareable_ustore) { + error = ENOTSUP; /* XXX */ + goto out; + } else { + error = qat_ae_ucode_write(sc, ae, + start, end - start, &fill[start]); + if (error) { + goto out; + } + } + + } while (end < usz); + } + } + +out: + qat_free_mem(fill); + return error; +} + +static int +qat_aefw_init_reg(struct qat_softc *sc, u_char ae, u_char ctx_mask, + enum aereg_type regtype, u_short regaddr, u_int value) +{ + int error = 0; + u_char ctx; + + switch (regtype) { + case AEREG_GPA_REL: + case AEREG_GPB_REL: + case AEREG_SR_REL: + case AEREG_SR_RD_REL: + case AEREG_SR_WR_REL: + case AEREG_DR_REL: + case AEREG_DR_RD_REL: + case AEREG_DR_WR_REL: + case AEREG_NEIGH_REL: + /* init for all valid ctx */ + for (ctx = 0; ctx < MAX_AE_CTX; ctx++) { + if ((ctx_mask & (1 << ctx)) == 0) + continue; + error = qat_aereg_rel_data_write(sc, ae, ctx, regtype, + regaddr, value); + } + break; + case AEREG_GPA_ABS: + case AEREG_GPB_ABS: + case AEREG_SR_ABS: + case AEREG_SR_RD_ABS: + case AEREG_SR_WR_ABS: + case AEREG_DR_ABS: + case AEREG_DR_RD_ABS: + case AEREG_DR_WR_ABS: + error = qat_aereg_abs_data_write(sc, ae, regtype, + regaddr, value); + break; + default: + error = EINVAL; + break; + } + + return error; +} + +static int +qat_aefw_init_reg_sym_expr(struct qat_softc *sc, u_char ae, + struct qat_uof_image *qui) +{ + u_int i, expres; + u_char ctx_mask; + + for (i = 0; i < qui->qui_num_init_reg_sym; i++) { + struct uof_init_reg_sym *uirs = &qui->qui_init_reg_sym[i]; + + if (uirs->uirs_value_type == EXPR_VAL) { + /* XXX */ + device_printf(sc->sc_dev, + "does not support initializing EXPR_VAL\n"); + return ENOTSUP; + } else { + expres = uirs->uirs_value; + } + + switch (uirs->uirs_init_type) { + case INIT_REG: + if (__SHIFTOUT(qui->qui_image->ui_ae_mode, + AE_MODE_CTX_MODE) == MAX_AE_CTX) { + ctx_mask = 0xff; /* 8-ctx mode */ + } else { + ctx_mask = 0x55; /* 4-ctx mode */ + } + qat_aefw_init_reg(sc, ae, ctx_mask, + (enum aereg_type)uirs->uirs_reg_type, + (u_short)uirs->uirs_addr_offset, expres); + break; + case INIT_REG_CTX: + if (__SHIFTOUT(qui->qui_image->ui_ae_mode, + AE_MODE_CTX_MODE) == MAX_AE_CTX) { + ctx_mask = 0xff; /* 8-ctx mode */ + } else { + ctx_mask = 0x55; /* 4-ctx mode */ + } + if (((1 << uirs->uirs_ctx) & ctx_mask) == 0) + return EINVAL; + qat_aefw_init_reg(sc, ae, 1 << uirs->uirs_ctx, + (enum aereg_type)uirs->uirs_reg_type, + (u_short)uirs->uirs_addr_offset, expres); + break; + case INIT_EXPR: + case INIT_EXPR_ENDIAN_SWAP: + default: + device_printf(sc->sc_dev, + "does not support initializing init_type %d\n", + uirs->uirs_init_type); + return ENOTSUP; + } + } + + return 0; +} + +static int +qat_aefw_init_memory(struct qat_softc *sc) +{ + struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; + size_t uimsz, initmemsz = qafu->qafu_init_mem_size; + struct uof_init_mem *uim; + int error, i; + u_char ae; + + uim = qafu->qafu_init_mem; + for (i = 0; i < qafu->qafu_num_init_mem; i++) { + uimsz = sizeof(struct uof_init_mem) + + sizeof(struct uof_mem_val_attr) * uim->uim_num_val_attr; + if (uimsz > initmemsz) { + device_printf(sc->sc_dev, + "invalid uof_init_mem or uof_mem_val_attr size\n"); + return EINVAL; + } + + if (uim->uim_num_bytes > 0) { + error = qat_aefw_init_memory_one(sc, uim); + if (error) { + device_printf(sc->sc_dev, + "Could not init ae memory: %d\n", error); + return error; + } + } + uim = (struct uof_init_mem *)((uintptr_t)uim + uimsz); + initmemsz -= uimsz; + } + + /* run Batch put LM API */ + for (ae = 0; ae < MAX_AE; ae++) { + error = qat_ae_batch_put_lm(sc, ae, &qafu->qafu_lm_init[ae], + qafu->qafu_num_lm_init_inst[ae]); + if (error) + device_printf(sc->sc_dev, "Could not put lm\n"); + + qat_aefw_free_lm_init(sc, ae); + } + + error = qat_aefw_init_ustore(sc); + + /* XXX run Batch put LM API */ + + return error; +} + +static int +qat_aefw_init_globals(struct qat_softc *sc) +{ + struct qat_aefw_uof *qafu = &sc->sc_aefw_uof; + int error, i, p, s; + u_char ae; + + /* initialize the memory segments */ + if (qafu->qafu_num_init_mem > 0) { + error = qat_aefw_init_memory(sc); + if (error) + return error; + } else { + error = qat_aefw_init_ustore(sc); + if (error) + return error; + } + + /* XXX bind import variables with ivd values */ + + /* XXX bind the uC global variables + * local variables will done on-the-fly */ + for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { + for (p = 0; p < sc->sc_aefw_uof.qafu_imgs[i].qui_image->ui_num_pages; p++) { + struct qat_uof_page *qup = + &sc->sc_aefw_uof.qafu_imgs[i].qui_pages[p]; + if (qup->qup_num_uw_blocks && + (qup->qup_num_uc_var || qup->qup_num_imp_var)) { + device_printf(sc->sc_dev, + "not support uC global variables\n"); + return ENOTSUP; + } + } + } + + for (ae = 0; ae < sc->sc_ae_num; ae++) { + struct qat_ae *qae = &(QAT_AE(sc, ae)); + + for (s = 0; s < qae->qae_num_slices; s++) { + struct qat_ae_slice *qas = &qae->qae_slices[s]; + + if (qas->qas_image == NULL) + continue; + + error = + qat_aefw_init_reg_sym_expr(sc, ae, qas->qas_image); + if (error) + return error; + } + } + + return 0; +} + +static uint64_t +qat_aefw_get_uof_inst(struct qat_softc *sc, struct qat_uof_page *qup, + u_int addr) +{ + uint64_t uinst = 0; + u_int i; + + /* find the block */ + for (i = 0; i < qup->qup_num_uw_blocks; i++) { + struct qat_uof_uword_block *quub = &qup->qup_uw_blocks[i]; + + if ((addr >= quub->quub_start_addr) && + (addr <= (quub->quub_start_addr + + (quub->quub_num_words - 1)))) { + /* unpack n bytes and assigned to the 64-bit uword value. + note: the microwords are stored as packed bytes. + */ + addr -= quub->quub_start_addr; + addr *= AEV2_PACKED_UWORD_BYTES; + memcpy(&uinst, + (void *)((uintptr_t)quub->quub_micro_words + addr), + AEV2_PACKED_UWORD_BYTES); + uinst = uinst & UWORD_MASK; + + return uinst; + } + } + + return INVLD_UWORD; +} + +static int +qat_aefw_do_pagein(struct qat_softc *sc, u_char ae, struct qat_uof_page *qup) +{ + struct qat_ae *qae = &(QAT_AE(sc, ae)); + uint64_t fill, *ucode_cpybuf; + u_int error, i, upaddr, uraddr, ninst, cpylen; + + if (qup->qup_num_uc_var || qup->qup_num_neigh_reg || + qup->qup_num_imp_var || qup->qup_num_imp_expr) { + device_printf(sc->sc_dev, + "does not support fixup locals\n"); + return ENOTSUP; + } + + ucode_cpybuf = qat_alloc_mem(UWORD_CPYBUF_SIZE * sizeof(uint64_t)); + + /* XXX get fill-pattern from an image -- they are all the same */ + memcpy(&fill, sc->sc_aefw_uof.qafu_imgs[0].qui_image->ui_fill_pattern, + sizeof(uint64_t)); + + upaddr = qup->qup_beg_paddr; + uraddr = 0; + ninst = qup->qup_num_micro_words; + while (ninst > 0) { + cpylen = min(ninst, UWORD_CPYBUF_SIZE); + + /* load the buffer */ + for (i = 0; i < cpylen; i++) { + /* keep below code structure in case there are + * different handling for shared secnarios */ + if (!qae->qae_shareable_ustore) { + /* qat_aefw_get_uof_inst() takes an address that + * is relative to the start of the page. + * So we don't need to add in the physical + * offset of the page. */ + if (qup->qup_page_region != 0) { + /* XXX */ + device_printf(sc->sc_dev, + "region != 0 is not supported\n"); + qat_free_mem(ucode_cpybuf); + return ENOTSUP; + } else { + /* for mixing case, it should take + * physical address */ + ucode_cpybuf[i] = qat_aefw_get_uof_inst( + sc, qup, upaddr + i); + if (ucode_cpybuf[i] == INVLD_UWORD) { + /* fill hole in the uof */ + ucode_cpybuf[i] = fill; + } + } + } else { + /* XXX */ + qat_free_mem(ucode_cpybuf); + return ENOTSUP; + } + } + + /* copy the buffer to ustore */ + if (!qae->qae_shareable_ustore) { + error = qat_ae_ucode_write(sc, ae, upaddr, cpylen, + ucode_cpybuf); + if (error) + return error; + } else { + /* XXX */ + qat_free_mem(ucode_cpybuf); + return ENOTSUP; + } + upaddr += cpylen; + uraddr += cpylen; + ninst -= cpylen; + } + + qat_free_mem(ucode_cpybuf); + + return 0; +} + +static int +qat_aefw_uof_write_one(struct qat_softc *sc, struct qat_uof_image *qui) +{ + struct uof_image *ui = qui->qui_image; + struct qat_ae_page *qap; + u_int s, p, c; + int error; + u_char ae, ctx_mask; + + if (__SHIFTOUT(ui->ui_ae_mode, AE_MODE_CTX_MODE) == MAX_AE_CTX) + ctx_mask = 0xff; /* 8-ctx mode */ + else + ctx_mask = 0x55; /* 4-ctx mode */ + + /* load the default page and set assigned CTX PC + * to the entrypoint address */ + for (ae = 0; ae < sc->sc_ae_num; ae++) { + struct qat_ae *qae = &(QAT_AE(sc, ae)); + struct qat_ae_slice *qas; + u_int metadata; + + MPASS(ae < UOF_MAX_NUM_OF_AE); + + if ((ui->ui_ae_assigned & (1 << ae)) == 0) + continue; + + /* find the slice to which this image is assigned */ + for (s = 0; s < qae->qae_num_slices; s++) { + qas = &qae->qae_slices[s]; + if (ui->ui_ctx_assigned & qas->qas_assigned_ctx_mask) + break; + } + if (s >= qae->qae_num_slices) + continue; + + qas = &qae->qae_slices[s]; + + for (p = 0; p < ui->ui_num_pages; p++) { + qap = &qas->qas_pages[p]; + + /* Only load pages loaded by default */ + if (!qap->qap_page->qup_def_page) + continue; + + error = qat_aefw_do_pagein(sc, ae, qap->qap_page); + if (error) + return error; + } + + metadata = qas->qas_image->qui_image->ui_app_metadata; + if (metadata != 0xffffffff && bootverbose) { + device_printf(sc->sc_dev, + "loaded firmware: %s\n", + qat_aefw_uof_string(sc, metadata)); + } + + /* Assume starting page is page 0 */ + qap = &qas->qas_pages[0]; + for (c = 0; c < MAX_AE_CTX; c++) { + if (ctx_mask & (1 << c)) + qas->qas_cur_pages[c] = qap; + else + qas->qas_cur_pages[c] = NULL; + } + + /* set the live context */ + qae->qae_live_ctx_mask = ui->ui_ctx_assigned; + + /* set context PC to the image entrypoint address */ + error = qat_ae_write_pc(sc, ae, ui->ui_ctx_assigned, + ui->ui_entry_address); + if (error) + return error; + } + + /* XXX store the checksum for convenience */ + + return 0; +} + +static int +qat_aefw_uof_write(struct qat_softc *sc) +{ + int error = 0; + int i; + + error = qat_aefw_init_globals(sc); + if (error) { + device_printf(sc->sc_dev, + "Could not initialize globals\n"); + return error; + } + + for (i = 0; i < sc->sc_aefw_uof.qafu_num_imgs; i++) { + error = qat_aefw_uof_write_one(sc, + &sc->sc_aefw_uof.qafu_imgs[i]); + if (error) + break; + } + + /* XXX UcLo_computeFreeUstore */ + + return error; +} Index: head/sys/dev/qat/qat_aevar.h =================================================================== --- head/sys/dev/qat/qat_aevar.h +++ head/sys/dev/qat/qat_aevar.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_aevar.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_AEVAR_H_ +#define _DEV_PCI_QAT_AEVAR_H_ + +int qat_ae_init(struct qat_softc *); +int qat_ae_start(struct qat_softc *); +void qat_ae_cluster_intr(void *); + +int qat_aefw_load(struct qat_softc *); +void qat_aefw_unload(struct qat_softc *); +int qat_aefw_start(struct qat_softc *, u_char, u_int); + +#endif Index: head/sys/dev/qat/qat_c2xxx.c =================================================================== --- head/sys/dev/qat/qat_c2xxx.c +++ head/sys/dev/qat/qat_c2xxx.c @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c2xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_c2xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw15reg.h" +#include "qat_c2xxxreg.h" +#include "qatvar.h" +#include "qat_hw15var.h" + +static uint32_t +qat_c2xxx_get_accel_mask(struct qat_softc *sc) +{ + uint32_t fusectl; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + + return ((~fusectl) & ACCEL_MASK_C2XXX); +} + +static uint32_t +qat_c2xxx_get_ae_mask(struct qat_softc *sc) +{ + uint32_t fusectl; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + if (fusectl & ( + FUSECTL_C2XXX_PKE_DISABLE | + FUSECTL_C2XXX_ATH_DISABLE | + FUSECTL_C2XXX_CPH_DISABLE)) { + return 0; + } else { + if ((~fusectl & AE_MASK_C2XXX) == 0x3) { + /* + * With both AEs enabled we get spurious completions on + * ETR rings. Work around that for now by simply + * disabling the second AE. + */ + device_printf(sc->sc_dev, "disabling second AE\n"); + fusectl |= 0x2; + } + return ((~fusectl) & AE_MASK_C2XXX); + } +} + +static enum qat_sku +qat_c2xxx_get_sku(struct qat_softc *sc) +{ + uint32_t fusectl; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + + switch (sc->sc_ae_num) { + case 1: + if (fusectl & FUSECTL_C2XXX_LOW_SKU) + return QAT_SKU_3; + else if (fusectl & FUSECTL_C2XXX_MID_SKU) + return QAT_SKU_2; + break; + case MAX_AE_C2XXX: + return QAT_SKU_1; + } + + return QAT_SKU_UNKNOWN; +} + +static uint32_t +qat_c2xxx_get_accel_cap(struct qat_softc *sc) +{ + return QAT_ACCEL_CAP_CRYPTO_SYMMETRIC | + QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC | + QAT_ACCEL_CAP_CIPHER | + QAT_ACCEL_CAP_AUTHENTICATION; +} + +static const char * +qat_c2xxx_get_fw_uof_name(struct qat_softc *sc) +{ + if (sc->sc_rev < QAT_REVID_C2XXX_B0) + return AE_FW_UOF_NAME_C2XXX_A0; + + /* QAT_REVID_C2XXX_B0 and QAT_REVID_C2XXX_C0 */ + return AE_FW_UOF_NAME_C2XXX_B0; +} + +static void +qat_c2xxx_enable_intr(struct qat_softc *sc) +{ + + qat_misc_write_4(sc, EP_SMIA_C2XXX, EP_SMIA_MASK_C2XXX); +} + +static void +qat_c2xxx_init_etr_intr(struct qat_softc *sc, int bank) +{ + /* + * For now, all rings within the bank are setup such that the generation + * of flag interrupts will be triggered when ring leaves the empty + * state. Note that in order for the ring interrupt to generate an IRQ + * the interrupt must also be enabled for the ring. + */ + qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL, + ETR_INT_SRCSEL_MASK_0_C2XXX); + qat_etr_bank_write_4(sc, bank, ETR_INT_SRCSEL_2, + ETR_INT_SRCSEL_MASK_X_C2XXX); +} + +const struct qat_hw qat_hw_c2xxx = { + .qhw_sram_bar_id = BAR_SRAM_ID_C2XXX, + .qhw_misc_bar_id = BAR_PMISC_ID_C2XXX, + .qhw_etr_bar_id = BAR_ETR_ID_C2XXX, + .qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C2XXX, + .qhw_ae_offset = AE_OFFSET_C2XXX, + .qhw_ae_local_offset = AE_LOCAL_OFFSET_C2XXX, + .qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C2XXX, + .qhw_num_banks = ETR_MAX_BANKS_C2XXX, + .qhw_num_ap_banks = ETR_MAX_AP_BANKS_C2XXX, + .qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK, + .qhw_num_accel = MAX_ACCEL_C2XXX, + .qhw_num_engines = MAX_AE_C2XXX, + .qhw_tx_rx_gap = ETR_TX_RX_GAP_C2XXX, + .qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C2XXX, + .qhw_msix_ae_vec_gap = MSIX_AE_VEC_GAP_C2XXX, + .qhw_fw_auth = false, + .qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW15, + .qhw_fw_resp_size = FW_REQ_DEFAULT_SZ_HW15, + .qhw_ring_asym_tx = 2, + .qhw_ring_asym_rx = 3, + .qhw_ring_sym_tx = 4, + .qhw_ring_sym_rx = 5, + .qhw_mof_fwname = AE_FW_MOF_NAME_C2XXX, + .qhw_mmp_fwname = AE_FW_MMP_NAME_C2XXX, + .qhw_prod_type = AE_FW_PROD_TYPE_C2XXX, + .qhw_get_accel_mask = qat_c2xxx_get_accel_mask, + .qhw_get_ae_mask = qat_c2xxx_get_ae_mask, + .qhw_get_sku = qat_c2xxx_get_sku, + .qhw_get_accel_cap = qat_c2xxx_get_accel_cap, + .qhw_get_fw_uof_name = qat_c2xxx_get_fw_uof_name, + .qhw_enable_intr = qat_c2xxx_enable_intr, + .qhw_init_etr_intr = qat_c2xxx_init_etr_intr, + .qhw_init_admin_comms = qat_adm_ring_init, + .qhw_send_admin_init = qat_adm_ring_send_init, + .qhw_crypto_setup_desc = qat_hw15_crypto_setup_desc, + .qhw_crypto_setup_req_params = qat_hw15_crypto_setup_req_params, + .qhw_crypto_opaque_offset = + offsetof(struct fw_la_resp, comn_resp.opaque_data), +}; Index: head/sys/dev/qat/qat_c2xxxreg.h =================================================================== --- head/sys/dev/qat/qat_c2xxxreg.h +++ head/sys/dev/qat/qat_c2xxxreg.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c2xxxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_C2XXXREG_H_ +#define _DEV_PCI_QAT_C2XXXREG_H_ + +/* PCI revision IDs */ +#define QAT_REVID_C2XXX_A0 0x00 +#define QAT_REVID_C2XXX_B0 0x02 +#define QAT_REVID_C2XXX_C0 0x03 + +/* Max number of accelerators and engines */ +#define MAX_ACCEL_C2XXX 1 +#define MAX_AE_C2XXX 2 + +/* PCIe BAR index */ +#define BAR_SRAM_ID_C2XXX NO_PCI_REG +#define BAR_PMISC_ID_C2XXX 0 +#define BAR_ETR_ID_C2XXX 1 + +#define ACCEL_MASK_C2XXX 0x1 +#define AE_MASK_C2XXX 0x3 + +#define MSIX_AE_VEC_GAP_C2XXX 8 + +/* PCIe configuration space registers */ +/* PESRAM: 512K eSRAM */ +#define BAR_PESRAM_C2XXX NO_PCI_REG +#define BAR_PESRAM_SIZE_C2XXX 0 + +/* + * PMISC: 16K CAP, 16K Scratch, 32K SSU(QATs), + * 32K AE CSRs and transfer registers, 8K CHAP/PMU, + * 4K EP CSRs, 4K MSI-X Tables + */ +#define BAR_PMISC_C2XXX 0x18 +#define BAR_PMISC_SIZE_C2XXX 0x20000 /* 128K */ + +/* PETRINGCSR: 8K 16 bundles of ET Ring CSRs */ +#define BAR_PETRINGCSR_C2XXX 0x20 +#define BAR_PETRINGCSR_SIZE_C2XXX 0x4000 /* 16K */ + +/* Fuse Control */ +#define FUSECTL_C2XXX_PKE_DISABLE (1 << 6) +#define FUSECTL_C2XXX_ATH_DISABLE (1 << 5) +#define FUSECTL_C2XXX_CPH_DISABLE (1 << 4) +#define FUSECTL_C2XXX_LOW_SKU (1 << 3) +#define FUSECTL_C2XXX_MID_SKU (1 << 2) +#define FUSECTL_C2XXX_AE1_DISABLE (1 << 1) + +/* SINT: Signal Target Raw Interrupt Register */ +#define EP_SINTPF_C2XXX 0x1A024 + +/* SMIA: Signal Target IA Mask Register */ +#define EP_SMIA_C2XXX 0x1A028 +#define EP_SMIA_BUNDLES_IRQ_MASK_C2XXX 0xFF +#define EP_SMIA_AE_IRQ_MASK_C2XXX 0x10000 +#define EP_SMIA_MASK_C2XXX \ + (EP_SMIA_BUNDLES_IRQ_MASK_C2XXX | EP_SMIA_AE_IRQ_MASK_C2XXX) + +#define EP_RIMISCCTL_C2XXX 0x1A0C4 +#define EP_RIMISCCTL_MASK_C2XXX 0x40000000 + +#define PFCGCIOSFPRIR_REG_C2XXX 0x2C0 +#define PFCGCIOSFPRIR_MASK_C2XXX 0XFFFF7FFF + +/* BAR sub-regions */ +#define PESRAM_BAR_C2XXX NO_PCI_REG +#define PESRAM_OFFSET_C2XXX 0x0 +#define PESRAM_SIZE_C2XXX 0x0 +#define CAP_GLOBAL_BAR_C2XXX BAR_PMISC_C2XXX +#define CAP_GLOBAL_OFFSET_C2XXX 0x00000 +#define CAP_GLOBAL_SIZE_C2XXX 0x04000 +#define CAP_HASH_OFFSET 0x900 +#define SCRATCH_BAR_C2XXX NO_PCI_REG +#define SCRATCH_OFFSET_C2XXX NO_REG_OFFSET +#define SCRATCH_SIZE_C2XXX 0x0 +#define SSU_BAR_C2XXX BAR_PMISC_C2XXX +#define SSU_OFFSET_C2XXX 0x08000 +#define SSU_SIZE_C2XXX 0x08000 +#define AE_BAR_C2XXX BAR_PMISC_C2XXX +#define AE_OFFSET_C2XXX 0x10000 +#define AE_LOCAL_OFFSET_C2XXX 0x10800 +#define PMU_BAR_C2XXX NO_PCI_REG +#define PMU_OFFSET_C2XXX NO_REG_OFFSET +#define PMU_SIZE_C2XXX 0x0 +#define EP_BAR_C2XXX BAR_PMISC_C2XXX +#define EP_OFFSET_C2XXX 0x1A000 +#define EP_SIZE_C2XXX 0x01000 +#define MSIX_TAB_BAR_C2XXX NO_PCI_REG /* mapped by pci(9) */ +#define MSIX_TAB_OFFSET_C2XXX 0x1B000 +#define MSIX_TAB_SIZE_C2XXX 0x01000 +#define PETRINGCSR_BAR_C2XXX BAR_PETRINGCSR_C2XXX +#define PETRINGCSR_OFFSET_C2XXX 0x0 +#define PETRINGCSR_SIZE_C2XXX 0x0 /* use size of BAR */ + +/* ETR */ +#define ETR_MAX_BANKS_C2XXX 8 +#define ETR_MAX_ET_RINGS_C2XXX \ + (ETR_MAX_BANKS_C2XXX * ETR_MAX_RINGS_PER_BANK_C2XXX) +#define ETR_MAX_AP_BANKS_C2XXX 4 + +#define ETR_TX_RX_GAP_C2XXX 1 +#define ETR_TX_RINGS_MASK_C2XXX 0x51 + +#define ETR_BUNDLE_SIZE_C2XXX 0x0200 + +/* Initial bank Interrupt Source mask */ +#define ETR_INT_SRCSEL_MASK_0_C2XXX 0x4444444CUL +#define ETR_INT_SRCSEL_MASK_X_C2XXX 0x44444444UL + +/* AE firmware */ +#define AE_FW_PROD_TYPE_C2XXX 0x00800000 +#define AE_FW_MOF_NAME_C2XXX "mof_firmware_c2xxx" +#define AE_FW_MMP_NAME_C2XXX "mmp_firmware_c2xxx" +#define AE_FW_UOF_NAME_C2XXX_A0 "icp_qat_nae.uof" +#define AE_FW_UOF_NAME_C2XXX_B0 "icp_qat_nae_b0.uof" + +#endif Index: head/sys/dev/qat/qat_c3xxx.c =================================================================== --- head/sys/dev/qat/qat_c3xxx.c +++ head/sys/dev/qat/qat_c3xxx.c @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c3xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_c3xxx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw17reg.h" +#include "qat_c3xxxreg.h" +#include "qatvar.h" +#include "qat_hw17var.h" + +static uint32_t +qat_c3xxx_get_accel_mask(struct qat_softc *sc) +{ + uint32_t fusectl, strap; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4); + + return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_C3XXX) & + ACCEL_MASK_C3XXX); +} + +static uint32_t +qat_c3xxx_get_ae_mask(struct qat_softc *sc) +{ + uint32_t fusectl, me_strap, me_disable, ssms_disabled; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4); + + /* If SSMs are disabled, then disable the corresponding MEs */ + ssms_disabled = (~qat_c3xxx_get_accel_mask(sc)) & ACCEL_MASK_C3XXX; + me_disable = 0x3; + while (ssms_disabled) { + if (ssms_disabled & 1) + me_strap |= me_disable; + ssms_disabled >>= 1; + me_disable <<= 2; + } + + return (~(fusectl | me_strap)) & AE_MASK_C3XXX; +} + +static enum qat_sku +qat_c3xxx_get_sku(struct qat_softc *sc) +{ + switch (sc->sc_ae_num) { + case MAX_AE_C3XXX: + return QAT_SKU_4; + } + + return QAT_SKU_UNKNOWN; +} + +static uint32_t +qat_c3xxx_get_accel_cap(struct qat_softc *sc) +{ + uint32_t cap, legfuse, strap; + + legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C3XXX, 4); + + cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC + + QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC + + QAT_ACCEL_CAP_CIPHER + + QAT_ACCEL_CAP_AUTHENTICATION + + QAT_ACCEL_CAP_COMPRESSION + + QAT_ACCEL_CAP_ZUC + + QAT_ACCEL_CAP_SHA3; + + if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) { + cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC; + cap &= ~QAT_ACCEL_CAP_CIPHER; + } + if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE) + cap &= ~QAT_ACCEL_CAP_AUTHENTICATION; + if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE) + cap &= ~QAT_ACCEL_CAP_ZUC; + + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_C3XXX) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_C3XXX) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + + return cap; +} + +static const char * +qat_c3xxx_get_fw_uof_name(struct qat_softc *sc) +{ + + return AE_FW_UOF_NAME_C3XXX; +} + +static void +qat_c3xxx_enable_intr(struct qat_softc *sc) +{ + + /* Enable bundle and misc interrupts */ + qat_misc_write_4(sc, SMIAPF0_C3XXX, SMIA0_MASK_C3XXX); + qat_misc_write_4(sc, SMIAPF1_C3XXX, SMIA1_MASK_C3XXX); +} + +/* Worker thread to service arbiter mappings */ +static uint32_t thrd_to_arb_map[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static void +qat_c3xxx_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config) +{ + int i; + + for (i = 1; i < MAX_AE_C3XXX; i++) { + if ((~sc->sc_ae_mask) & (1 << i)) + thrd_to_arb_map[i] = 0; + } + *arb_map_config = thrd_to_arb_map; +} + +static void +qat_c3xxx_enable_error_interrupts(struct qat_softc *sc) +{ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_C3XXX); /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_C3XXX); /* ME4-ME5 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_C3XXX); /* SSM2 */ + + /* Reset everything except VFtoPF1_16. */ + qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_C3XXX); + + /* RI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, RICPPINTCTL_C3XXX, RICPP_EN_C3XXX); + + /* TI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, TICPPINTCTL_C3XXX, TICPP_EN_C3XXX); + + /* Enable CFC Error interrupts and logging. */ + qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_C3XXX, CPP_CFC_UE_C3XXX); +} + +static void +qat_c3xxx_disable_error_interrupts(struct qat_softc *sc) +{ + /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_C3XXX | ERRMSK0_CERR_C3XXX); + /* ME4-ME5 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_C3XXX | ERRMSK1_CERR_C3XXX); + /* CPP Push Pull, RI, TI, SSM0-SSM1, CFC */ + qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_C3XXX); + /* SSM2 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_C3XXX); +} + +static void +qat_c3xxx_enable_error_correction(struct qat_softc *sc) +{ + u_int i, mask; + + /* Enable Accel Engine error detection & correction */ + for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_C3XXX(i), + ENABLE_AE_ECC_ERR_C3XXX); + qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_C3XXX(i), + ENABLE_AE_ECC_PARITY_CORR_C3XXX); + } + + /* Enable shared memory error detection & correction */ + for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + + qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_C3XXX); + qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_C3XXX); + qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_C3XXX); + } + + qat_c3xxx_enable_error_interrupts(sc); +} + +const struct qat_hw qat_hw_c3xxx = { + .qhw_sram_bar_id = BAR_SRAM_ID_C3XXX, + .qhw_misc_bar_id = BAR_PMISC_ID_C3XXX, + .qhw_etr_bar_id = BAR_ETR_ID_C3XXX, + .qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C3XXX, + .qhw_ae_offset = AE_OFFSET_C3XXX, + .qhw_ae_local_offset = AE_LOCAL_OFFSET_C3XXX, + .qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C3XXX, + .qhw_num_banks = ETR_MAX_BANKS_C3XXX, + .qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK, + .qhw_num_accel = MAX_ACCEL_C3XXX, + .qhw_num_engines = MAX_AE_C3XXX, + .qhw_tx_rx_gap = ETR_TX_RX_GAP_C3XXX, + .qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C3XXX, + .qhw_clock_per_sec = CLOCK_PER_SEC_C3XXX, + .qhw_fw_auth = true, + .qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17, + .qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17, + .qhw_ring_asym_tx = 0, + .qhw_ring_asym_rx = 8, + .qhw_ring_sym_tx = 2, + .qhw_ring_sym_rx = 10, + .qhw_mof_fwname = AE_FW_MOF_NAME_C3XXX, + .qhw_mmp_fwname = AE_FW_MMP_NAME_C3XXX, + .qhw_prod_type = AE_FW_PROD_TYPE_C3XXX, + .qhw_get_accel_mask = qat_c3xxx_get_accel_mask, + .qhw_get_ae_mask = qat_c3xxx_get_ae_mask, + .qhw_get_sku = qat_c3xxx_get_sku, + .qhw_get_accel_cap = qat_c3xxx_get_accel_cap, + .qhw_get_fw_uof_name = qat_c3xxx_get_fw_uof_name, + .qhw_enable_intr = qat_c3xxx_enable_intr, + .qhw_init_admin_comms = qat_adm_mailbox_init, + .qhw_send_admin_init = qat_adm_mailbox_send_init, + .qhw_init_arb = qat_arb_init, + .qhw_get_arb_mapping = qat_c3xxx_get_arb_mapping, + .qhw_enable_error_correction = qat_c3xxx_enable_error_correction, + .qhw_disable_error_interrupts = qat_c3xxx_disable_error_interrupts, + .qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer, + .qhw_check_slice_hang = qat_check_slice_hang, + .qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc, + .qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params, + .qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data), +}; Index: head/sys/dev/qat/qat_c3xxxreg.h =================================================================== --- head/sys/dev/qat/qat_c3xxxreg.h +++ head/sys/dev/qat/qat_c3xxxreg.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c3xxxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_C3XXXREG_H_ +#define _DEV_PCI_QAT_C3XXXREG_H_ + +/* Max number of accelerators and engines */ +#define MAX_ACCEL_C3XXX 3 +#define MAX_AE_C3XXX 6 + +/* PCIe BAR index */ +#define BAR_SRAM_ID_C3XXX NO_PCI_REG +#define BAR_PMISC_ID_C3XXX 0 +#define BAR_ETR_ID_C3XXX 1 + +/* BAR PMISC sub-regions */ +#define AE_OFFSET_C3XXX 0x20000 +#define AE_LOCAL_OFFSET_C3XXX 0x20800 +#define CAP_GLOBAL_OFFSET_C3XXX 0x30000 + +#define SOFTSTRAP_REG_C3XXX 0x2EC +#define SOFTSTRAP_SS_POWERGATE_CY_C3XXX __BIT(23) +#define SOFTSTRAP_SS_POWERGATE_PKE_C3XXX __BIT(24) + +#define ACCEL_REG_OFFSET_C3XXX 16 +#define ACCEL_MASK_C3XXX 0x7 +#define AE_MASK_C3XXX 0x3F + +#define SMIAPF0_C3XXX 0x3A028 +#define SMIAPF1_C3XXX 0x3A030 +#define SMIA0_MASK_C3XXX 0xFFFF +#define SMIA1_MASK_C3XXX 0x1 + +/* Error detection and correction */ +#define AE_CTX_ENABLES_C3XXX(i) ((i) * 0x1000 + 0x20818) +#define AE_MISC_CONTROL_C3XXX(i) ((i) * 0x1000 + 0x20960) +#define ENABLE_AE_ECC_ERR_C3XXX __BIT(28) +#define ENABLE_AE_ECC_PARITY_CORR_C3XXX (__BIT(24) | __BIT(12)) +#define ERRSSMSH_EN_C3XXX __BIT(3) +/* BIT(2) enables the logging of push/pull data errors. */ +#define PPERR_EN_C3XXX (__BIT(2)) + +/* Mask for VF2PF interrupts */ +#define VF2PF1_16_C3XXX (0xFFFF << 9) +#define ERRSOU3_VF2PF_C3XXX(errsou3) (((errsou3) & 0x01FFFE00) >> 9) +#define ERRMSK3_VF2PF_C3XXX(vf_mask) (((vf_mask) & 0xFFFF) << 9) + +/* Masks for correctable error interrupts. */ +#define ERRMSK0_CERR_C3XXX (__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0)) +#define ERRMSK1_CERR_C3XXX (__BIT(8) | __BIT(0)) +#define ERRMSK5_CERR_C3XXX (0) + +/* Masks for uncorrectable error interrupts. */ +#define ERRMSK0_UERR_C3XXX (__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1)) +#define ERRMSK1_UERR_C3XXX (__BIT(9) | __BIT(1)) +#define ERRMSK3_UERR_C3XXX (__BIT(6) | __BIT(5) | __BIT(4) | __BIT(3) | \ + __BIT(2) | __BIT(0)) +#define ERRMSK5_UERR_C3XXX (__BIT(16)) + +/* RI CPP control */ +#define RICPPINTCTL_C3XXX (0x3A000 + 0x110) +/* + * BIT(2) enables error detection and reporting on the RI Parity Error. + * BIT(1) enables error detection and reporting on the RI CPP Pull interface. + * BIT(0) enables error detection and reporting on the RI CPP Push interface. + */ +#define RICPP_EN_C3XXX (__BIT(2) | __BIT(1) | __BIT(0)) + +/* TI CPP control */ +#define TICPPINTCTL_C3XXX (0x3A400 + 0x138) +/* + * BIT(3) enables error detection and reporting on the ETR Parity Error. + * BIT(2) enables error detection and reporting on the TI Parity Error. + * BIT(1) enables error detection and reporting on the TI CPP Pull interface. + * BIT(0) enables error detection and reporting on the TI CPP Push interface. + */ +#define TICPP_EN_C3XXX \ + (__BIT(3) | __BIT(2) | __BIT(1) | __BIT(0)) + +/* CFC Uncorrectable Errors */ +#define CPP_CFC_ERR_CTRL_C3XXX (0x30000 + 0xC00) +/* + * BIT(1) enables interrupt. + * BIT(0) enables detecting and logging of push/pull data errors. + */ +#define CPP_CFC_UE_C3XXX (__BIT(1) | __BIT(0)) + +#define SLICEPWRDOWN_C3XXX(i) ((i) * 0x4000 + 0x2C) +/* Enabling PKE4-PKE0. */ +#define MMP_PWR_UP_MSK_C3XXX \ + (__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16)) + +/* CPM Uncorrectable Errors */ +#define INTMASKSSM_C3XXX(i) ((i) * 0x4000 + 0x0) +/* Disabling interrupts for correctable errors. */ +#define INTMASKSSM_UERR_C3XXX \ + (__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1)) + +/* MMP */ +/* BIT(3) enables correction. */ +#define CERRSSMMMP_EN_C3XXX (__BIT(3)) + +/* BIT(3) enables logging. */ +#define UERRSSMMMP_EN_C3XXX (__BIT(3)) + +/* ETR */ +#define ETR_MAX_BANKS_C3XXX 16 +#define ETR_TX_RX_GAP_C3XXX 8 +#define ETR_TX_RINGS_MASK_C3XXX 0xFF +#define ETR_BUNDLE_SIZE_C3XXX 0x1000 + +/* AE firmware */ +#define AE_FW_PROD_TYPE_C3XXX 0x02000000 +#define AE_FW_MOF_NAME_C3XXX "qat_c3xxx" +#define AE_FW_MMP_NAME_C3XXX "qat_c3xxx_mmp" +#define AE_FW_UOF_NAME_C3XXX "icp_qat_ae.suof" + +/* Clock frequency */ +#define CLOCK_PER_SEC_C3XXX (685 * 1000000 / 16) + +#endif Index: head/sys/dev/qat/qat_c62x.c =================================================================== --- head/sys/dev/qat/qat_c62x.c +++ head/sys/dev/qat/qat_c62x.c @@ -0,0 +1,314 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c62x.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_c62x.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw17reg.h" +#include "qat_c62xreg.h" +#include "qatvar.h" +#include "qat_hw17var.h" + +static uint32_t +qat_c62x_get_accel_mask(struct qat_softc *sc) +{ + uint32_t fusectl, strap; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4); + + return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_C62X) & + ACCEL_MASK_C62X); +} + +static uint32_t +qat_c62x_get_ae_mask(struct qat_softc *sc) +{ + uint32_t fusectl, me_strap, me_disable, ssms_disabled; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4); + + /* If SSMs are disabled, then disable the corresponding MEs */ + ssms_disabled = (~qat_c62x_get_accel_mask(sc)) & ACCEL_MASK_C62X; + me_disable = 0x3; + while (ssms_disabled) { + if (ssms_disabled & 1) + me_strap |= me_disable; + ssms_disabled >>= 1; + me_disable <<= 2; + } + + return (~(fusectl | me_strap)) & AE_MASK_C62X; +} + +static enum qat_sku +qat_c62x_get_sku(struct qat_softc *sc) +{ + switch (sc->sc_ae_num) { + case 8: + return QAT_SKU_2; + case MAX_AE_C62X: + return QAT_SKU_4; + } + + return QAT_SKU_UNKNOWN; +} + +static uint32_t +qat_c62x_get_accel_cap(struct qat_softc *sc) +{ + uint32_t cap, legfuse, strap; + + legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_C62X, 4); + + cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC + + QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC + + QAT_ACCEL_CAP_CIPHER + + QAT_ACCEL_CAP_AUTHENTICATION + + QAT_ACCEL_CAP_COMPRESSION + + QAT_ACCEL_CAP_ZUC + + QAT_ACCEL_CAP_SHA3; + + if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) { + cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC; + cap &= ~QAT_ACCEL_CAP_CIPHER; + } + if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE) + cap &= ~QAT_ACCEL_CAP_AUTHENTICATION; + if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE) + cap &= ~QAT_ACCEL_CAP_ZUC; + + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_C62X) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_C62X) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + + return cap; +} + +static const char * +qat_c62x_get_fw_uof_name(struct qat_softc *sc) +{ + + return AE_FW_UOF_NAME_C62X; +} + +static void +qat_c62x_enable_intr(struct qat_softc *sc) +{ + + /* Enable bundle and misc interrupts */ + qat_misc_write_4(sc, SMIAPF0_C62X, SMIA0_MASK_C62X); + qat_misc_write_4(sc, SMIAPF1_C62X, SMIA1_MASK_C62X); +} + +/* Worker thread to service arbiter mappings */ +static uint32_t thrd_to_arb_map[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static void +qat_c62x_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config) +{ + int i; + + for (i = 1; i < MAX_AE_C62X; i++) { + if ((~sc->sc_ae_mask) & (1 << i)) + thrd_to_arb_map[i] = 0; + } + *arb_map_config = thrd_to_arb_map; +} + +static void +qat_c62x_enable_error_interrupts(struct qat_softc *sc) +{ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_C62X); /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_C62X); /* ME4-ME7 */ + qat_misc_write_4(sc, ERRMSK4, ERRMSK4_CERR_C62X); /* ME8-ME9 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_C62X); /* SSM2-SSM4 */ + + /* Reset everything except VFtoPF1_16. */ + qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_C62X); + /* Disable Secure RAM correctable error interrupt */ + qat_misc_read_write_or_4(sc, ERRMSK3, ERRMSK3_CERR_C62X); + + /* RI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, RICPPINTCTL_C62X, RICPP_EN_C62X); + + /* TI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, TICPPINTCTL_C62X, TICPP_EN_C62X); + + /* Enable CFC Error interrupts and logging. */ + qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_C62X, CPP_CFC_UE_C62X); + + /* Enable SecureRAM to fix and log Correctable errors */ + qat_misc_write_4(sc, SECRAMCERR_C62X, SECRAM_CERR_C62X); + + /* Enable SecureRAM Uncorrectable error interrupts and logging */ + qat_misc_write_4(sc, SECRAMUERR, SECRAM_UERR_C62X); + + /* Enable Push/Pull Misc Uncorrectable error interrupts and logging */ + qat_misc_write_4(sc, CPPMEMTGTERR, TGT_UERR_C62X); +} + +static void +qat_c62x_disable_error_interrupts(struct qat_softc *sc) +{ + /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_C62X | ERRMSK0_CERR_C62X); + /* ME4-ME7 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_C62X | ERRMSK1_CERR_C62X); + /* Secure RAM, CPP Push Pull, RI, TI, SSM0-SSM1, CFC */ + qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_C62X | ERRMSK3_CERR_C62X); + /* ME8-ME9 */ + qat_misc_write_4(sc, ERRMSK4, ERRMSK4_UERR_C62X | ERRMSK4_CERR_C62X); + /* SSM2-SSM4 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_C62X | ERRMSK5_CERR_C62X); +} + +static void +qat_c62x_enable_error_correction(struct qat_softc *sc) +{ + u_int i, mask; + + /* Enable Accel Engine error detection & correction */ + for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_C62X(i), + ENABLE_AE_ECC_ERR_C62X); + qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_C62X(i), + ENABLE_AE_ECC_PARITY_CORR_C62X); + } + + /* Enable shared memory error detection & correction */ + for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + + qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_C62X); + qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_C62X); + qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_C62X); + } + + qat_c62x_enable_error_interrupts(sc); +} + +const struct qat_hw qat_hw_c62x = { + .qhw_sram_bar_id = BAR_SRAM_ID_C62X, + .qhw_misc_bar_id = BAR_PMISC_ID_C62X, + .qhw_etr_bar_id = BAR_ETR_ID_C62X, + .qhw_cap_global_offset = CAP_GLOBAL_OFFSET_C62X, + .qhw_ae_offset = AE_OFFSET_C62X, + .qhw_ae_local_offset = AE_LOCAL_OFFSET_C62X, + .qhw_etr_bundle_size = ETR_BUNDLE_SIZE_C62X, + .qhw_num_banks = ETR_MAX_BANKS_C62X, + .qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK, + .qhw_num_accel = MAX_ACCEL_C62X, + .qhw_num_engines = MAX_AE_C62X, + .qhw_tx_rx_gap = ETR_TX_RX_GAP_C62X, + .qhw_tx_rings_mask = ETR_TX_RINGS_MASK_C62X, + .qhw_clock_per_sec = CLOCK_PER_SEC_C62X, + .qhw_fw_auth = true, + .qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17, + .qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17, + .qhw_ring_asym_tx = 0, + .qhw_ring_asym_rx = 8, + .qhw_ring_sym_tx = 2, + .qhw_ring_sym_rx = 10, + .qhw_mof_fwname = AE_FW_MOF_NAME_C62X, + .qhw_mmp_fwname = AE_FW_MMP_NAME_C62X, + .qhw_prod_type = AE_FW_PROD_TYPE_C62X, + .qhw_get_accel_mask = qat_c62x_get_accel_mask, + .qhw_get_ae_mask = qat_c62x_get_ae_mask, + .qhw_get_sku = qat_c62x_get_sku, + .qhw_get_accel_cap = qat_c62x_get_accel_cap, + .qhw_get_fw_uof_name = qat_c62x_get_fw_uof_name, + .qhw_enable_intr = qat_c62x_enable_intr, + .qhw_init_admin_comms = qat_adm_mailbox_init, + .qhw_send_admin_init = qat_adm_mailbox_send_init, + .qhw_init_arb = qat_arb_init, + .qhw_get_arb_mapping = qat_c62x_get_arb_mapping, + .qhw_enable_error_correction = qat_c62x_enable_error_correction, + .qhw_disable_error_interrupts = qat_c62x_disable_error_interrupts, + .qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer, + .qhw_check_slice_hang = qat_check_slice_hang, + .qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc, + .qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params, + .qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data), +}; Index: head/sys/dev/qat/qat_c62xreg.h =================================================================== --- head/sys/dev/qat/qat_c62xreg.h +++ head/sys/dev/qat/qat_c62xreg.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_c62xreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_C62XREG_H_ +#define _DEV_PCI_QAT_C62XREG_H_ + +/* Max number of accelerators and engines */ +#define MAX_ACCEL_C62X 5 +#define MAX_AE_C62X 10 + +/* PCIe BAR index */ +#define BAR_SRAM_ID_C62X 0 +#define BAR_PMISC_ID_C62X 1 +#define BAR_ETR_ID_C62X 2 + +/* BAR PMISC sub-regions */ +#define AE_OFFSET_C62X 0x20000 +#define AE_LOCAL_OFFSET_C62X 0x20800 +#define CAP_GLOBAL_OFFSET_C62X 0x30000 + +#define SOFTSTRAP_REG_C62X 0x2EC +#define SOFTSTRAP_SS_POWERGATE_CY_C62X __BIT(23) +#define SOFTSTRAP_SS_POWERGATE_PKE_C62X __BIT(24) + +#define ACCEL_REG_OFFSET_C62X 16 +#define ACCEL_MASK_C62X 0x1F +#define AE_MASK_C62X 0x3FF + +#define SMIAPF0_C62X 0x3A028 +#define SMIAPF1_C62X 0x3A030 +#define SMIA0_MASK_C62X 0xFFFF +#define SMIA1_MASK_C62X 0x1 + +/* Error detection and correction */ +#define AE_CTX_ENABLES_C62X(i) ((i) * 0x1000 + 0x20818) +#define AE_MISC_CONTROL_C62X(i) ((i) * 0x1000 + 0x20960) +#define ENABLE_AE_ECC_ERR_C62X __BIT(28) +#define ENABLE_AE_ECC_PARITY_CORR_C62X (__BIT(24) | __BIT(12)) +#define ERRSSMSH_EN_C62X __BIT(3) +/* BIT(2) enables the logging of push/pull data errors. */ +#define PPERR_EN_C62X (__BIT(2)) + +/* Mask for VF2PF interrupts */ +#define VF2PF1_16_C62X (0xFFFF << 9) +#define ERRSOU3_VF2PF_C62X(errsou3) (((errsou3) & 0x01FFFE00) >> 9) +#define ERRMSK3_VF2PF_C62X(vf_mask) (((vf_mask) & 0xFFFF) << 9) + +/* Masks for correctable error interrupts. */ +#define ERRMSK0_CERR_C62X (__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0)) +#define ERRMSK1_CERR_C62X (__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0)) +#define ERRMSK3_CERR_C62X (__BIT(7)) +#define ERRMSK4_CERR_C62X (__BIT(8) | __BIT(0)) +#define ERRMSK5_CERR_C62X (0) + +/* Masks for uncorrectable error interrupts. */ +#define ERRMSK0_UERR_C62X (__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1)) +#define ERRMSK1_UERR_C62X (__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1)) +#define ERRMSK3_UERR_C62X (__BIT(8) | __BIT(6) | __BIT(5) | __BIT(4) | \ + __BIT(3) | __BIT(2) | __BIT(0)) +#define ERRMSK4_UERR_C62X (__BIT(9) | __BIT(1)) +#define ERRMSK5_UERR_C62X (__BIT(18) | __BIT(17) | __BIT(16)) + +/* RI CPP control */ +#define RICPPINTCTL_C62X (0x3A000 + 0x110) +/* + * BIT(2) enables error detection and reporting on the RI Parity Error. + * BIT(1) enables error detection and reporting on the RI CPP Pull interface. + * BIT(0) enables error detection and reporting on the RI CPP Push interface. + */ +#define RICPP_EN_C62X (__BIT(2) | __BIT(1) | __BIT(0)) + +/* TI CPP control */ +#define TICPPINTCTL_C62X (0x3A400 + 0x138) +/* + * BIT(3) enables error detection and reporting on the ETR Parity Error. + * BIT(2) enables error detection and reporting on the TI Parity Error. + * BIT(1) enables error detection and reporting on the TI CPP Pull interface. + * BIT(0) enables error detection and reporting on the TI CPP Push interface. + */ +#define TICPP_EN_C62X \ + (__BIT(4) | __BIT(3) | __BIT(2) | __BIT(1) | __BIT(0)) + +/* CFC Uncorrectable Errors */ +#define CPP_CFC_ERR_CTRL_C62X (0x30000 + 0xC00) +/* + * BIT(1) enables interrupt. + * BIT(0) enables detecting and logging of push/pull data errors. + */ +#define CPP_CFC_UE_C62X (__BIT(1) | __BIT(0)) + +/* Correctable SecureRAM Error Reg */ +#define SECRAMCERR_C62X (0x3AC00 + 0x00) +/* BIT(3) enables fixing and logging of correctable errors. */ +#define SECRAM_CERR_C62X (__BIT(3)) + +/* Uncorrectable SecureRAM Error Reg */ +/* + * BIT(17) enables interrupt. + * BIT(3) enables detecting and logging of uncorrectable errors. + */ +#define SECRAM_UERR_C62X (__BIT(17) | __BIT(3)) + +/* Miscellaneous Memory Target Errors Register */ +/* + * BIT(3) enables detecting and logging push/pull data errors. + * BIT(2) enables interrupt. + */ +#define TGT_UERR_C62X (__BIT(3) | __BIT(2)) + + +#define SLICEPWRDOWN_C62X(i) ((i) * 0x4000 + 0x2C) +/* Enabling PKE4-PKE0. */ +#define MMP_PWR_UP_MSK_C62X \ + (__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16)) + +/* CPM Uncorrectable Errors */ +#define INTMASKSSM_C62X(i) ((i) * 0x4000 + 0x0) +/* Disabling interrupts for correctable errors. */ +#define INTMASKSSM_UERR_C62X \ + (__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1)) + +/* MMP */ +/* BIT(3) enables correction. */ +#define CERRSSMMMP_EN_C62X (__BIT(3)) + +/* BIT(3) enables logging. */ +#define UERRSSMMMP_EN_C62X (__BIT(3)) + +/* ETR */ +#define ETR_MAX_BANKS_C62X 16 +#define ETR_TX_RX_GAP_C62X 8 +#define ETR_TX_RINGS_MASK_C62X 0xFF +#define ETR_BUNDLE_SIZE_C62X 0x1000 + +/* AE firmware */ +#define AE_FW_PROD_TYPE_C62X 0x01000000 +#define AE_FW_MOF_NAME_C62X "qat_c62x" +#define AE_FW_MMP_NAME_C62X "qat_c62x_mmp" +#define AE_FW_UOF_NAME_C62X "icp_qat_ae.suof" + +/* Clock frequency */ +#define CLOCK_PER_SEC_C62X (685 * 1000000 / 16) + +#endif Index: head/sys/dev/qat/qat_d15xx.c =================================================================== --- head/sys/dev/qat/qat_d15xx.c +++ head/sys/dev/qat/qat_d15xx.c @@ -0,0 +1,314 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_d15xx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_d15xx.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw17reg.h" +#include "qat_d15xxreg.h" +#include "qatvar.h" +#include "qat_hw17var.h" + +static uint32_t +qat_d15xx_get_accel_mask(struct qat_softc *sc) +{ + uint32_t fusectl, strap; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4); + + return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_D15XX) & + ACCEL_MASK_D15XX); +} + +static uint32_t +qat_d15xx_get_ae_mask(struct qat_softc *sc) +{ + uint32_t fusectl, me_strap, me_disable, ssms_disabled; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + me_strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4); + + /* If SSMs are disabled, then disable the corresponding MEs */ + ssms_disabled = (~qat_d15xx_get_accel_mask(sc)) & ACCEL_MASK_D15XX; + me_disable = 0x3; + while (ssms_disabled) { + if (ssms_disabled & 1) + me_strap |= me_disable; + ssms_disabled >>= 1; + me_disable <<= 2; + } + + return (~(fusectl | me_strap)) & AE_MASK_D15XX; +} + +static enum qat_sku +qat_d15xx_get_sku(struct qat_softc *sc) +{ + switch (sc->sc_ae_num) { + case 8: + return QAT_SKU_2; + case MAX_AE_D15XX: + return QAT_SKU_4; + } + + return QAT_SKU_UNKNOWN; +} + +static uint32_t +qat_d15xx_get_accel_cap(struct qat_softc *sc) +{ + uint32_t cap, legfuse, strap; + + legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_D15XX, 4); + + cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC + + QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC + + QAT_ACCEL_CAP_CIPHER + + QAT_ACCEL_CAP_AUTHENTICATION + + QAT_ACCEL_CAP_COMPRESSION + + QAT_ACCEL_CAP_ZUC + + QAT_ACCEL_CAP_SHA3; + + if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) { + cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC; + cap &= ~QAT_ACCEL_CAP_CIPHER; + } + if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE) + cap &= ~QAT_ACCEL_CAP_AUTHENTICATION; + if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE) + cap &= ~QAT_ACCEL_CAP_ZUC; + + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_PKE_D15XX) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if ((strap | legfuse) & SOFTSTRAP_SS_POWERGATE_CY_D15XX) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + + return cap; +} + +static const char * +qat_d15xx_get_fw_uof_name(struct qat_softc *sc) +{ + + return AE_FW_UOF_NAME_D15XX; +} + +static void +qat_d15xx_enable_intr(struct qat_softc *sc) +{ + + /* Enable bundle and misc interrupts */ + qat_misc_write_4(sc, SMIAPF0_D15XX, SMIA0_MASK_D15XX); + qat_misc_write_4(sc, SMIAPF1_D15XX, SMIA1_MASK_D15XX); +} + +/* Worker thread to service arbiter mappings */ +static uint32_t thrd_to_arb_map[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static void +qat_d15xx_get_arb_mapping(struct qat_softc *sc, const uint32_t **arb_map_config) +{ + int i; + + for (i = 1; i < MAX_AE_D15XX; i++) { + if ((~sc->sc_ae_mask) & (1 << i)) + thrd_to_arb_map[i] = 0; + } + *arb_map_config = thrd_to_arb_map; +} + +static void +qat_d15xx_enable_error_interrupts(struct qat_softc *sc) +{ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_CERR_D15XX); /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_CERR_D15XX); /* ME4-ME7 */ + qat_misc_write_4(sc, ERRMSK4, ERRMSK4_CERR_D15XX); /* ME8-ME9 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_CERR_D15XX); /* SSM2-SSM4 */ + + /* Reset everything except VFtoPF1_16. */ + qat_misc_read_write_and_4(sc, ERRMSK3, VF2PF1_16_D15XX); + /* Disable Secure RAM correctable error interrupt */ + qat_misc_read_write_or_4(sc, ERRMSK3, ERRMSK3_CERR_D15XX); + + /* RI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, RICPPINTCTL_D15XX, RICPP_EN_D15XX); + + /* TI CPP bus interface error detection and reporting. */ + qat_misc_write_4(sc, TICPPINTCTL_D15XX, TICPP_EN_D15XX); + + /* Enable CFC Error interrupts and logging. */ + qat_misc_write_4(sc, CPP_CFC_ERR_CTRL_D15XX, CPP_CFC_UE_D15XX); + + /* Enable SecureRAM to fix and log Correctable errors */ + qat_misc_write_4(sc, SECRAMCERR_D15XX, SECRAM_CERR_D15XX); + + /* Enable SecureRAM Uncorrectable error interrupts and logging */ + qat_misc_write_4(sc, SECRAMUERR, SECRAM_UERR_D15XX); + + /* Enable Push/Pull Misc Uncorrectable error interrupts and logging */ + qat_misc_write_4(sc, CPPMEMTGTERR, TGT_UERR_D15XX); +} + +static void +qat_d15xx_disable_error_interrupts(struct qat_softc *sc) +{ + /* ME0-ME3 */ + qat_misc_write_4(sc, ERRMSK0, ERRMSK0_UERR_D15XX | ERRMSK0_CERR_D15XX); + /* ME4-ME7 */ + qat_misc_write_4(sc, ERRMSK1, ERRMSK1_UERR_D15XX | ERRMSK1_CERR_D15XX); + /* Secure RAM, CPP Push Pull, RI, TI, SSM0-SSM1, CFC */ + qat_misc_write_4(sc, ERRMSK3, ERRMSK3_UERR_D15XX | ERRMSK3_CERR_D15XX); + /* ME8-ME9 */ + qat_misc_write_4(sc, ERRMSK4, ERRMSK4_UERR_D15XX | ERRMSK4_CERR_D15XX); + /* SSM2-SSM4 */ + qat_misc_write_4(sc, ERRMSK5, ERRMSK5_UERR_D15XX | ERRMSK5_CERR_D15XX); +} + +static void +qat_d15xx_enable_error_correction(struct qat_softc *sc) +{ + u_int i, mask; + + /* Enable Accel Engine error detection & correction */ + for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_D15XX(i), + ENABLE_AE_ECC_ERR_D15XX); + qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_D15XX(i), + ENABLE_AE_ECC_PARITY_CORR_D15XX); + } + + /* Enable shared memory error detection & correction */ + for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + + qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_D15XX); + qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_D15XX); + qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_D15XX); + } + + qat_d15xx_enable_error_interrupts(sc); +} + +const struct qat_hw qat_hw_d15xx = { + .qhw_sram_bar_id = BAR_SRAM_ID_D15XX, + .qhw_misc_bar_id = BAR_PMISC_ID_D15XX, + .qhw_etr_bar_id = BAR_ETR_ID_D15XX, + .qhw_cap_global_offset = CAP_GLOBAL_OFFSET_D15XX, + .qhw_ae_offset = AE_OFFSET_D15XX, + .qhw_ae_local_offset = AE_LOCAL_OFFSET_D15XX, + .qhw_etr_bundle_size = ETR_BUNDLE_SIZE_D15XX, + .qhw_num_banks = ETR_MAX_BANKS_D15XX, + .qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK, + .qhw_num_accel = MAX_ACCEL_D15XX, + .qhw_num_engines = MAX_AE_D15XX, + .qhw_tx_rx_gap = ETR_TX_RX_GAP_D15XX, + .qhw_tx_rings_mask = ETR_TX_RINGS_MASK_D15XX, + .qhw_clock_per_sec = CLOCK_PER_SEC_D15XX, + .qhw_fw_auth = true, + .qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17, + .qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17, + .qhw_ring_asym_tx = 0, + .qhw_ring_asym_rx = 8, + .qhw_ring_sym_tx = 2, + .qhw_ring_sym_rx = 10, + .qhw_mof_fwname = AE_FW_MOF_NAME_D15XX, + .qhw_mmp_fwname = AE_FW_MMP_NAME_D15XX, + .qhw_prod_type = AE_FW_PROD_TYPE_D15XX, + .qhw_get_accel_mask = qat_d15xx_get_accel_mask, + .qhw_get_ae_mask = qat_d15xx_get_ae_mask, + .qhw_get_sku = qat_d15xx_get_sku, + .qhw_get_accel_cap = qat_d15xx_get_accel_cap, + .qhw_get_fw_uof_name = qat_d15xx_get_fw_uof_name, + .qhw_enable_intr = qat_d15xx_enable_intr, + .qhw_init_admin_comms = qat_adm_mailbox_init, + .qhw_send_admin_init = qat_adm_mailbox_send_init, + .qhw_init_arb = qat_arb_init, + .qhw_get_arb_mapping = qat_d15xx_get_arb_mapping, + .qhw_enable_error_correction = qat_d15xx_enable_error_correction, + .qhw_disable_error_interrupts = qat_d15xx_disable_error_interrupts, + .qhw_set_ssm_wdtimer = qat_set_ssm_wdtimer, + .qhw_check_slice_hang = qat_check_slice_hang, + .qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc, + .qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params, + .qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data), +}; Index: head/sys/dev/qat/qat_d15xxreg.h =================================================================== --- head/sys/dev/qat/qat_d15xxreg.h +++ head/sys/dev/qat/qat_d15xxreg.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_d15xxreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_D15XXREG_H_ +#define _DEV_PCI_QAT_D15XXREG_H_ + +/* Max number of accelerators and engines */ +#define MAX_ACCEL_D15XX 5 +#define MAX_AE_D15XX 10 + +/* PCIe BAR index */ +#define BAR_SRAM_ID_D15XX 0 +#define BAR_PMISC_ID_D15XX 1 +#define BAR_ETR_ID_D15XX 2 + +/* BAR PMISC sub-regions */ +#define AE_OFFSET_D15XX 0x20000 +#define AE_LOCAL_OFFSET_D15XX 0x20800 +#define CAP_GLOBAL_OFFSET_D15XX 0x30000 + +#define SOFTSTRAP_REG_D15XX 0x2EC +#define SOFTSTRAP_SS_POWERGATE_CY_D15XX __BIT(23) +#define SOFTSTRAP_SS_POWERGATE_PKE_D15XX __BIT(24) + +#define ACCEL_REG_OFFSET_D15XX 16 +#define ACCEL_MASK_D15XX 0x1F +#define AE_MASK_D15XX 0x3FF + +#define SMIAPF0_D15XX 0x3A028 +#define SMIAPF1_D15XX 0x3A030 +#define SMIA0_MASK_D15XX 0xFFFF +#define SMIA1_MASK_D15XX 0x1 + +/* Error detection and correction */ +#define AE_CTX_ENABLES_D15XX(i) ((i) * 0x1000 + 0x20818) +#define AE_MISC_CONTROL_D15XX(i) ((i) * 0x1000 + 0x20960) +#define ENABLE_AE_ECC_ERR_D15XX __BIT(28) +#define ENABLE_AE_ECC_PARITY_CORR_D15XX (__BIT(24) | __BIT(12)) +#define ERRSSMSH_EN_D15XX __BIT(3) +/* BIT(2) enables the logging of push/pull data errors. */ +#define PPERR_EN_D15XX (__BIT(2)) + +/* Mask for VF2PF interrupts */ +#define VF2PF1_16_D15XX (0xFFFF << 9) +#define ERRSOU3_VF2PF_D15XX(errsou3) (((errsou3) & 0x01FFFE00) >> 9) +#define ERRMSK3_VF2PF_D15XX(vf_mask) (((vf_mask) & 0xFFFF) << 9) + +/* Masks for correctable error interrupts. */ +#define ERRMSK0_CERR_D15XX (__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0)) +#define ERRMSK1_CERR_D15XX (__BIT(24) | __BIT(16) | __BIT(8) | __BIT(0)) +#define ERRMSK3_CERR_D15XX (__BIT(7)) +#define ERRMSK4_CERR_D15XX (__BIT(8) | __BIT(0)) +#define ERRMSK5_CERR_D15XX (0) + +/* Masks for uncorrectable error interrupts. */ +#define ERRMSK0_UERR_D15XX (__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1)) +#define ERRMSK1_UERR_D15XX (__BIT(25) | __BIT(17) | __BIT(9) | __BIT(1)) +#define ERRMSK3_UERR_D15XX (__BIT(8) | __BIT(6) | __BIT(5) | __BIT(4) | \ + __BIT(3) | __BIT(2) | __BIT(0)) +#define ERRMSK4_UERR_D15XX (__BIT(9) | __BIT(1)) +#define ERRMSK5_UERR_D15XX (__BIT(18) | __BIT(17) | __BIT(16)) + +/* RI CPP control */ +#define RICPPINTCTL_D15XX (0x3A000 + 0x110) +/* + * BIT(2) enables error detection and reporting on the RI Parity Error. + * BIT(1) enables error detection and reporting on the RI CPP Pull interface. + * BIT(0) enables error detection and reporting on the RI CPP Push interface. + */ +#define RICPP_EN_D15XX (__BIT(2) | __BIT(1) | __BIT(0)) + +/* TI CPP control */ +#define TICPPINTCTL_D15XX (0x3A400 + 0x138) +/* + * BIT(3) enables error detection and reporting on the ETR Parity Error. + * BIT(2) enables error detection and reporting on the TI Parity Error. + * BIT(1) enables error detection and reporting on the TI CPP Pull interface. + * BIT(0) enables error detection and reporting on the TI CPP Push interface. + */ +#define TICPP_EN_D15XX \ + (__BIT(4) | __BIT(3) | __BIT(2) | __BIT(1) | __BIT(0)) + +/* CFC Uncorrectable Errors */ +#define CPP_CFC_ERR_CTRL_D15XX (0x30000 + 0xC00) +/* + * BIT(1) enables interrupt. + * BIT(0) enables detecting and logging of push/pull data errors. + */ +#define CPP_CFC_UE_D15XX (__BIT(1) | __BIT(0)) + +/* Correctable SecureRAM Error Reg */ +#define SECRAMCERR_D15XX (0x3AC00 + 0x00) +/* BIT(3) enables fixing and logging of correctable errors. */ +#define SECRAM_CERR_D15XX (__BIT(3)) + +/* Uncorrectable SecureRAM Error Reg */ +/* + * BIT(17) enables interrupt. + * BIT(3) enables detecting and logging of uncorrectable errors. + */ +#define SECRAM_UERR_D15XX (__BIT(17) | __BIT(3)) + +/* Miscellaneous Memory Target Errors Register */ +/* + * BIT(3) enables detecting and logging push/pull data errors. + * BIT(2) enables interrupt. + */ +#define TGT_UERR_D15XX (__BIT(3) | __BIT(2)) + + +#define SLICEPWRDOWN_D15XX(i) ((i) * 0x4000 + 0x2C) +/* Enabling PKE4-PKE0. */ +#define MMP_PWR_UP_MSK_D15XX \ + (__BIT(20) | __BIT(19) | __BIT(18) | __BIT(17) | __BIT(16)) + +/* CPM Uncorrectable Errors */ +#define INTMASKSSM_D15XX(i) ((i) * 0x4000 + 0x0) +/* Disabling interrupts for correctable errors. */ +#define INTMASKSSM_UERR_D15XX \ + (__BIT(11) | __BIT(9) | __BIT(7) | __BIT(5) | __BIT(3) | __BIT(1)) + +/* MMP */ +/* BIT(3) enables correction. */ +#define CERRSSMMMP_EN_D15XX (__BIT(3)) + +/* BIT(3) enables logging. */ +#define UERRSSMMMP_EN_D15XX (__BIT(3)) + +/* ETR */ +#define ETR_MAX_BANKS_D15XX 16 +#define ETR_TX_RX_GAP_D15XX 8 +#define ETR_TX_RINGS_MASK_D15XX 0xFF +#define ETR_BUNDLE_SIZE_D15XX 0x1000 + +/* AE firmware */ +#define AE_FW_PROD_TYPE_D15XX 0x01000000 +#define AE_FW_MOF_NAME_D15XX "qat_d15xx" +#define AE_FW_MMP_NAME_D15XX "qat_d15xx_mmp" +#define AE_FW_UOF_NAME_D15XX "icp_qat_ae.suof" + +/* Clock frequency */ +#define CLOCK_PER_SEC_D15XX (685 * 1000000 / 16) + +#endif Index: head/sys/dev/qat/qat_dh895xcc.c =================================================================== --- head/sys/dev/qat/qat_dh895xcc.c +++ head/sys/dev/qat/qat_dh895xcc.c @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: BSD-2-Clause AND BSD-3-Clause */ +/* + * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 - 2020 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qatvar.h" +#include "qat_hw17reg.h" +#include "qat_hw17var.h" +#include "qat_dh895xccreg.h" + +static uint32_t +qat_dh895xcc_get_accel_mask(struct qat_softc *sc) +{ + uint32_t fusectl, strap; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_DH895XCC, 4); + + return (((~(fusectl | strap)) >> ACCEL_REG_OFFSET_DH895XCC) & + ACCEL_MASK_DH895XCC); +} + +static uint32_t +qat_dh895xcc_get_ae_mask(struct qat_softc *sc) +{ + uint32_t fusectl, strap; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + strap = pci_read_config(sc->sc_dev, SOFTSTRAP_REG_DH895XCC, 4); + + return (~(fusectl | strap)) & AE_MASK_DH895XCC; +} + +static enum qat_sku +qat_dh895xcc_get_sku(struct qat_softc *sc) +{ + uint32_t fusectl, sku; + + fusectl = pci_read_config(sc->sc_dev, FUSECTL_REG, 4); + sku = (fusectl & FUSECTL_SKU_MASK_DH895XCC) >> + FUSECTL_SKU_SHIFT_DH895XCC; + switch (sku) { + case FUSECTL_SKU_1_DH895XCC: + return QAT_SKU_1; + case FUSECTL_SKU_2_DH895XCC: + return QAT_SKU_2; + case FUSECTL_SKU_3_DH895XCC: + return QAT_SKU_3; + case FUSECTL_SKU_4_DH895XCC: + return QAT_SKU_4; + default: + return QAT_SKU_UNKNOWN; + } +} + +static uint32_t +qat_dh895xcc_get_accel_cap(struct qat_softc *sc) +{ + uint32_t cap, legfuse; + + legfuse = pci_read_config(sc->sc_dev, LEGFUSE_REG, 4); + + cap = QAT_ACCEL_CAP_CRYPTO_SYMMETRIC + + QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC + + QAT_ACCEL_CAP_CIPHER + + QAT_ACCEL_CAP_AUTHENTICATION + + QAT_ACCEL_CAP_COMPRESSION + + QAT_ACCEL_CAP_ZUC + + QAT_ACCEL_CAP_SHA3; + + if (legfuse & LEGFUSE_ACCEL_MASK_CIPHER_SLICE) { + cap &= ~QAT_ACCEL_CAP_CRYPTO_SYMMETRIC; + cap &= ~QAT_ACCEL_CAP_CIPHER; + } + if (legfuse & LEGFUSE_ACCEL_MASK_AUTH_SLICE) + cap &= ~QAT_ACCEL_CAP_AUTHENTICATION; + if (legfuse & LEGFUSE_ACCEL_MASK_PKE_SLICE) + cap &= ~QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC; + if (legfuse & LEGFUSE_ACCEL_MASK_COMPRESS_SLICE) + cap &= ~QAT_ACCEL_CAP_COMPRESSION; + if (legfuse & LEGFUSE_ACCEL_MASK_EIA3_SLICE) + cap &= ~QAT_ACCEL_CAP_ZUC; + + return cap; +} + +static const char * +qat_dh895xcc_get_fw_uof_name(struct qat_softc *sc) +{ + return AE_FW_UOF_NAME_DH895XCC; +} + +static void +qat_dh895xcc_enable_intr(struct qat_softc *sc) +{ + /* Enable bundle and misc interrupts */ + qat_misc_write_4(sc, SMIAPF0_DH895XCC, SMIA0_MASK_DH895XCC); + qat_misc_write_4(sc, SMIAPF1_DH895XCC, SMIA1_MASK_DH895XCC); +} + +/* Worker thread to service arbiter mappings based on dev SKUs */ +static uint32_t thrd_to_arb_map_sku4[] = { + 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, +}; + +static uint32_t thrd_to_arb_map_sku6[] = { + 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, +}; + +static void +qat_dh895xcc_get_arb_mapping(struct qat_softc *sc, + const uint32_t **arb_map_config) +{ + uint32_t *map, sku; + int i; + + sku = qat_dh895xcc_get_sku(sc); + switch (sku) { + case QAT_SKU_1: + map = thrd_to_arb_map_sku4; + break; + case QAT_SKU_2: + case QAT_SKU_4: + map = thrd_to_arb_map_sku6; + break; + default: + *arb_map_config = NULL; + return; + } + + for (i = 1; i < MAX_AE_DH895XCC; i++) { + if ((~sc->sc_ae_mask) & (1 << i)) + map[i] = 0; + } + *arb_map_config = map; +} + +static void +qat_dh895xcc_enable_error_correction(struct qat_softc *sc) +{ + uint32_t mask; + u_int i; + + /* Enable Accel Engine error detection & correction */ + for (i = 0, mask = sc->sc_ae_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_misc_read_write_or_4(sc, AE_CTX_ENABLES_DH895XCC(i), + ENABLE_AE_ECC_ERR_DH895XCC); + qat_misc_read_write_or_4(sc, AE_MISC_CONTROL_DH895XCC(i), + ENABLE_AE_ECC_PARITY_CORR_DH895XCC); + } + + /* Enable shared memory error detection & correction */ + for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + + qat_misc_read_write_or_4(sc, UERRSSMSH(i), ERRSSMSH_EN_DH895XCC); + qat_misc_read_write_or_4(sc, CERRSSMSH(i), ERRSSMSH_EN_DH895XCC); + qat_misc_read_write_or_4(sc, PPERR(i), PPERR_EN_DH895XCC); + } +} + +const struct qat_hw qat_hw_dh895xcc = { + .qhw_sram_bar_id = BAR_SRAM_ID_DH895XCC, + .qhw_misc_bar_id = BAR_PMISC_ID_DH895XCC, + .qhw_etr_bar_id = BAR_ETR_ID_DH895XCC, + .qhw_cap_global_offset = CAP_GLOBAL_OFFSET_DH895XCC, + .qhw_ae_offset = AE_OFFSET_DH895XCC, + .qhw_ae_local_offset = AE_LOCAL_OFFSET_DH895XCC, + .qhw_etr_bundle_size = ETR_BUNDLE_SIZE_DH895XCC, + .qhw_num_banks = ETR_MAX_BANKS_DH895XCC, + .qhw_num_rings_per_bank = ETR_MAX_RINGS_PER_BANK, + .qhw_num_accel = MAX_ACCEL_DH895XCC, + .qhw_num_engines = MAX_AE_DH895XCC, + .qhw_tx_rx_gap = ETR_TX_RX_GAP_DH895XCC, + .qhw_tx_rings_mask = ETR_TX_RINGS_MASK_DH895XCC, + .qhw_clock_per_sec = CLOCK_PER_SEC_DH895XCC, + .qhw_fw_auth = false, + .qhw_fw_req_size = FW_REQ_DEFAULT_SZ_HW17, + .qhw_fw_resp_size = FW_RESP_DEFAULT_SZ_HW17, + .qhw_ring_asym_tx = 0, + .qhw_ring_asym_rx = 8, + .qhw_ring_sym_tx = 2, + .qhw_ring_sym_rx = 10, + .qhw_mof_fwname = AE_FW_MOF_NAME_DH895XCC, + .qhw_mmp_fwname = AE_FW_MMP_NAME_DH895XCC, + .qhw_prod_type = AE_FW_PROD_TYPE_DH895XCC, + .qhw_get_accel_mask = qat_dh895xcc_get_accel_mask, + .qhw_get_ae_mask = qat_dh895xcc_get_ae_mask, + .qhw_get_sku = qat_dh895xcc_get_sku, + .qhw_get_accel_cap = qat_dh895xcc_get_accel_cap, + .qhw_get_fw_uof_name = qat_dh895xcc_get_fw_uof_name, + .qhw_enable_intr = qat_dh895xcc_enable_intr, + .qhw_init_admin_comms = qat_adm_mailbox_init, + .qhw_send_admin_init = qat_adm_mailbox_send_init, + .qhw_init_arb = qat_arb_init, + .qhw_get_arb_mapping = qat_dh895xcc_get_arb_mapping, + .qhw_enable_error_correction = qat_dh895xcc_enable_error_correction, + .qhw_check_slice_hang = qat_check_slice_hang, + .qhw_crypto_setup_desc = qat_hw17_crypto_setup_desc, + .qhw_crypto_setup_req_params = qat_hw17_crypto_setup_req_params, + .qhw_crypto_opaque_offset = offsetof(struct fw_la_resp, opaque_data), +}; Index: head/sys/dev/qat/qat_dh895xccreg.h =================================================================== --- head/sys/dev/qat/qat_dh895xccreg.h +++ head/sys/dev/qat/qat_dh895xccreg.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014-2020 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_DH895XCCREG_H_ +#define _DEV_PCI_QAT_DH895XCCREG_H_ + +/* Max number of accelerators and engines */ +#define MAX_ACCEL_DH895XCC 6 +#define MAX_AE_DH895XCC 12 + +/* PCIe BAR index */ +#define BAR_SRAM_ID_DH895XCC 0 +#define BAR_PMISC_ID_DH895XCC 1 +#define BAR_ETR_ID_DH895XCC 2 + +/* BAR PMISC sub-regions */ +#define AE_OFFSET_DH895XCC 0x20000 +#define AE_LOCAL_OFFSET_DH895XCC 0x20800 +#define CAP_GLOBAL_OFFSET_DH895XCC 0x30000 + +#define SOFTSTRAP_REG_DH895XCC 0x2EC + +#define FUSECTL_SKU_MASK_DH895XCC 0x300000 +#define FUSECTL_SKU_SHIFT_DH895XCC 20 +#define FUSECTL_SKU_1_DH895XCC 0 +#define FUSECTL_SKU_2_DH895XCC 1 +#define FUSECTL_SKU_3_DH895XCC 2 +#define FUSECTL_SKU_4_DH895XCC 3 + +#define ACCEL_REG_OFFSET_DH895XCC 13 +#define ACCEL_MASK_DH895XCC 0x3F +#define AE_MASK_DH895XCC 0xFFF + +#define SMIAPF0_DH895XCC 0x3A028 +#define SMIAPF1_DH895XCC 0x3A030 +#define SMIA0_MASK_DH895XCC 0xFFFFFFFF +#define SMIA1_MASK_DH895XCC 0x1 + +/* Error detection and correction */ +#define AE_CTX_ENABLES_DH895XCC(i) ((i) * 0x1000 + 0x20818) +#define AE_MISC_CONTROL_DH895XCC(i) ((i) * 0x1000 + 0x20960) +#define ENABLE_AE_ECC_ERR_DH895XCC __BIT(28) +#define ENABLE_AE_ECC_PARITY_CORR_DH895XCC (__BIT(24) | __BIT(12)) +#define ERRSSMSH_EN_DH895XCC __BIT(3) +/* BIT(2) enables the logging of push/pull data errors. */ +#define PPERR_EN_DH895XCC (__BIT(2)) + +/* ETR */ +#define ETR_MAX_BANKS_DH895XCC 32 +#define ETR_TX_RX_GAP_DH895XCC 8 +#define ETR_TX_RINGS_MASK_DH895XCC 0xFF +#define ETR_BUNDLE_SIZE_DH895XCC 0x1000 + +/* AE firmware */ +#define AE_FW_PROD_TYPE_DH895XCC 0x00400000 +#define AE_FW_MOF_NAME_DH895XCC "qat_895xcc" +#define AE_FW_MMP_NAME_DH895XCC "qat_895xcc_mmp" +#define AE_FW_UOF_NAME_DH895XCC "icp_qat_ae.uof" + +/* Clock frequency */ +#define CLOCK_PER_SEC_DH895XCC (685 * 1000000 / 16) + +#endif Index: head/sys/dev/qat/qat_hw15.c =================================================================== --- head/sys/dev/qat/qat_hw15.c +++ head/sys/dev/qat/qat_hw15.c @@ -0,0 +1,953 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw15reg.h" +#include "qatvar.h" +#include "qat_hw15var.h" + +static int qat_adm_ring_init_ring_table(struct qat_softc *); +static void qat_adm_ring_build_slice_mask(uint16_t *, uint32_t, uint32_t); +static void qat_adm_ring_build_shram_mask(uint64_t *, uint32_t, uint32_t); +static int qat_adm_ring_build_ring_table(struct qat_softc *, uint32_t); +static int qat_adm_ring_build_init_msg(struct qat_softc *, + struct fw_init_req *, enum fw_init_cmd_id, uint32_t, + struct qat_accel_init_cb *); +static int qat_adm_ring_send_init_msg_sync(struct qat_softc *, + enum fw_init_cmd_id, uint32_t); +static int qat_adm_ring_send_init_msg(struct qat_softc *, + enum fw_init_cmd_id); +static int qat_adm_ring_intr(struct qat_softc *, void *, void *); + +void +qat_msg_req_type_populate(struct arch_if_req_hdr *msg, enum arch_if_req type, + uint32_t rxring) +{ + + memset(msg, 0, sizeof(struct arch_if_req_hdr)); + msg->flags = ARCH_IF_FLAGS_VALID_FLAG | + ARCH_IF_FLAGS_RESP_RING_TYPE_ET | ARCH_IF_FLAGS_RESP_TYPE_S; + msg->req_type = type; + msg->resp_pipe_id = rxring; +} + +void +qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *msg, bus_addr_t desc_paddr, + uint8_t hdrsz, uint8_t hwblksz, uint16_t comn_req_flags, uint32_t flow_id) +{ + struct fw_comn_req_hdr *hdr = &msg->comn_hdr; + + hdr->comn_req_flags = comn_req_flags; + hdr->content_desc_params_sz = hwblksz; + hdr->content_desc_hdr_sz = hdrsz; + hdr->content_desc_addr = desc_paddr; + msg->flow_id = flow_id; +} + +void +qat_msg_service_cmd_populate(struct fw_la_bulk_req *msg, enum fw_la_cmd_id cmdid, + uint16_t cmd_flags) +{ + msg->comn_la_req.la_cmd_id = cmdid; + msg->comn_la_req.u.la_flags = cmd_flags; +} + +void +qat_msg_cmn_mid_populate(struct fw_comn_req_mid *msg, void *cookie, + uint64_t src, uint64_t dst) +{ + + msg->opaque_data = (uint64_t)(uintptr_t)cookie; + msg->src_data_addr = src; + if (dst == 0) + msg->dest_data_addr = src; + else + msg->dest_data_addr = dst; +} + +void +qat_msg_req_params_populate(struct fw_la_bulk_req *msg, + bus_addr_t req_params_paddr, uint8_t req_params_sz) +{ + msg->req_params_addr = req_params_paddr; + msg->comn_la_req.u1.req_params_blk_sz = req_params_sz / 8; +} + +void +qat_msg_cmn_footer_populate(union fw_comn_req_ftr *msg, uint64_t next_addr) +{ + msg->next_request_addr = next_addr; +} + +void +qat_msg_params_populate(struct fw_la_bulk_req *msg, + struct qat_crypto_desc *desc, uint8_t req_params_sz, + uint16_t service_cmd_flags, uint16_t comn_req_flags) +{ + qat_msg_cmn_hdr_populate(msg, desc->qcd_desc_paddr, + desc->qcd_hdr_sz, desc->qcd_hw_blk_sz, comn_req_flags, 0); + qat_msg_service_cmd_populate(msg, desc->qcd_cmd_id, service_cmd_flags); + qat_msg_cmn_mid_populate(&msg->comn_mid, NULL, 0, 0); + qat_msg_req_params_populate(msg, 0, req_params_sz); + qat_msg_cmn_footer_populate(&msg->comn_ftr, 0); +} + +static int +qat_adm_ring_init_ring_table(struct qat_softc *sc) +{ + struct qat_admin_rings *qadr = &sc->sc_admin_rings; + + if (sc->sc_ae_num == 1) { + qadr->qadr_cya_ring_tbl = + &qadr->qadr_master_ring_tbl[0]; + qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; + } else if (sc->sc_ae_num == 2 || sc->sc_ae_num == 4) { + qadr->qadr_cya_ring_tbl = + &qadr->qadr_master_ring_tbl[0]; + qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; + qadr->qadr_cyb_ring_tbl = + &qadr->qadr_master_ring_tbl[1]; + qadr->qadr_srv_mask[1] = QAT_SERVICE_CRYPTO_B; + } + + return 0; +} + +int +qat_adm_ring_init(struct qat_softc *sc) +{ + struct qat_admin_rings *qadr = &sc->sc_admin_rings; + int error, i, j; + + error = qat_alloc_dmamem(sc, &qadr->qadr_dma, 1, PAGE_SIZE, PAGE_SIZE); + if (error) + return error; + + qadr->qadr_master_ring_tbl = qadr->qadr_dma.qdm_dma_vaddr; + + MPASS(sc->sc_ae_num * + sizeof(struct fw_init_ring_table) <= PAGE_SIZE); + + /* Initialize the Master Ring Table */ + for (i = 0; i < sc->sc_ae_num; i++) { + struct fw_init_ring_table *firt = + &qadr->qadr_master_ring_tbl[i]; + + for (j = 0; j < INIT_RING_TABLE_SZ; j++) { + struct fw_init_ring_params *firp = + &firt->firt_bulk_rings[j]; + + firp->firp_reserved = 0; + firp->firp_curr_weight = QAT_DEFAULT_RING_WEIGHT; + firp->firp_init_weight = QAT_DEFAULT_RING_WEIGHT; + firp->firp_ring_pvl = QAT_DEFAULT_PVL; + } + memset(firt->firt_ring_mask, 0, sizeof(firt->firt_ring_mask)); + } + + error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_TX, + ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_req_size, + NULL, NULL, "admin_tx", &qadr->qadr_admin_tx); + if (error) + return error; + + error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_RX, + ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_resp_size, + qat_adm_ring_intr, qadr, "admin_rx", &qadr->qadr_admin_rx); + if (error) + return error; + + /* + * Finally set up the service indices into the Master Ring Table + * and convenient ring table pointers for each service enabled. + * Only the Admin rings are initialized. + */ + error = qat_adm_ring_init_ring_table(sc); + if (error) + return error; + + /* + * Calculate the number of active AEs per QAT + * needed for Shram partitioning. + */ + for (i = 0; i < sc->sc_ae_num; i++) { + if (qadr->qadr_srv_mask[i]) + qadr->qadr_active_aes_per_accel++; + } + + return 0; +} + +static void +qat_adm_ring_build_slice_mask(uint16_t *slice_mask, uint32_t srv_mask, + uint32_t init_shram) +{ + uint16_t shram = 0, comn_req = 0; + + if (init_shram) + shram = COMN_REQ_SHRAM_INIT_REQUIRED; + + if (srv_mask & QAT_SERVICE_CRYPTO_A) + comn_req |= COMN_REQ_CY0_ONLY(shram); + if (srv_mask & QAT_SERVICE_CRYPTO_B) + comn_req |= COMN_REQ_CY1_ONLY(shram); + + *slice_mask = comn_req; +} + +static void +qat_adm_ring_build_shram_mask(uint64_t *shram_mask, uint32_t active_aes, + uint32_t ae) +{ + *shram_mask = 0; + + if (active_aes == 1) { + *shram_mask = ~(*shram_mask); + } else if (active_aes == 2) { + if (ae == 1) + *shram_mask = ((~(*shram_mask)) & 0xffffffff); + else + *shram_mask = ((~(*shram_mask)) & 0xffffffff00000000ull); + } else if (active_aes == 3) { + if (ae == 0) + *shram_mask = ((~(*shram_mask)) & 0x7fffff); + else if (ae == 1) + *shram_mask = ((~(*shram_mask)) & 0x3fffff800000ull); + else + *shram_mask = ((~(*shram_mask)) & 0xffffc00000000000ull); + } else { + panic("Only three services are supported in current version"); + } +} + +static int +qat_adm_ring_build_ring_table(struct qat_softc *sc, uint32_t ae) +{ + struct qat_admin_rings *qadr = &sc->sc_admin_rings; + struct fw_init_ring_table *tbl; + struct fw_init_ring_params *param; + uint8_t srv_mask = sc->sc_admin_rings.qadr_srv_mask[ae]; + + if ((srv_mask & QAT_SERVICE_CRYPTO_A)) { + tbl = qadr->qadr_cya_ring_tbl; + } else if ((srv_mask & QAT_SERVICE_CRYPTO_B)) { + tbl = qadr->qadr_cyb_ring_tbl; + } else { + device_printf(sc->sc_dev, + "Invalid execution engine %d\n", ae); + return EINVAL; + } + + param = &tbl->firt_bulk_rings[sc->sc_hw.qhw_ring_sym_tx]; + param->firp_curr_weight = QAT_HI_PRIO_RING_WEIGHT; + param->firp_init_weight = QAT_HI_PRIO_RING_WEIGHT; + FW_INIT_RING_MASK_SET(tbl, sc->sc_hw.qhw_ring_sym_tx); + + return 0; +} + +static int +qat_adm_ring_build_init_msg(struct qat_softc *sc, + struct fw_init_req *initmsg, enum fw_init_cmd_id cmd, uint32_t ae, + struct qat_accel_init_cb *cb) +{ + struct fw_init_set_ae_info_hdr *aehdr; + struct fw_init_set_ae_info *aeinfo; + struct fw_init_set_ring_info_hdr *ringhdr; + struct fw_init_set_ring_info *ringinfo; + int init_shram = 0, tgt_id, cluster_id; + uint32_t srv_mask; + + srv_mask = sc->sc_admin_rings.qadr_srv_mask[ + ae % sc->sc_ae_num]; + + memset(initmsg, 0, sizeof(struct fw_init_req)); + + qat_msg_req_type_populate(&initmsg->comn_hdr.arch_if, + ARCH_IF_REQ_QAT_FW_INIT, + sc->sc_admin_rings.qadr_admin_rx->qr_ring_id); + + qat_msg_cmn_mid_populate(&initmsg->comn_mid, cb, 0, 0); + + switch (cmd) { + case FW_INIT_CMD_SET_AE_INFO: + if (ae % sc->sc_ae_num == 0) + init_shram = 1; + if (ae >= sc->sc_ae_num) { + tgt_id = 1; + cluster_id = 1; + } else { + cluster_id = 0; + if (sc->sc_ae_mask) + tgt_id = 0; + else + tgt_id = 1; + } + aehdr = &initmsg->u.set_ae_info; + aeinfo = &initmsg->u1.set_ae_info; + + aehdr->init_cmd_id = cmd; + /* XXX that does not support sparse ae_mask */ + aehdr->init_trgt_id = ae; + aehdr->init_ring_cluster_id = cluster_id; + aehdr->init_qat_id = tgt_id; + + qat_adm_ring_build_slice_mask(&aehdr->init_slice_mask, srv_mask, + init_shram); + + qat_adm_ring_build_shram_mask(&aeinfo->init_shram_mask, + sc->sc_admin_rings.qadr_active_aes_per_accel, + ae % sc->sc_ae_num); + + break; + case FW_INIT_CMD_SET_RING_INFO: + ringhdr = &initmsg->u.set_ring_info; + ringinfo = &initmsg->u1.set_ring_info; + + ringhdr->init_cmd_id = cmd; + /* XXX that does not support sparse ae_mask */ + ringhdr->init_trgt_id = ae; + + /* XXX */ + qat_adm_ring_build_ring_table(sc, + ae % sc->sc_ae_num); + + ringhdr->init_ring_tbl_sz = sizeof(struct fw_init_ring_table); + + ringinfo->init_ring_table_ptr = + sc->sc_admin_rings.qadr_dma.qdm_dma_seg.ds_addr + + ((ae % sc->sc_ae_num) * + sizeof(struct fw_init_ring_table)); + + break; + default: + return ENOTSUP; + } + + return 0; +} + +static int +qat_adm_ring_send_init_msg_sync(struct qat_softc *sc, + enum fw_init_cmd_id cmd, uint32_t ae) +{ + struct fw_init_req initmsg; + struct qat_accel_init_cb cb; + int error; + + error = qat_adm_ring_build_init_msg(sc, &initmsg, cmd, ae, &cb); + if (error) + return error; + + error = qat_etr_put_msg(sc, sc->sc_admin_rings.qadr_admin_tx, + (uint32_t *)&initmsg); + if (error) + return error; + + error = tsleep(&cb, PZERO, "qat_init", hz * 3 / 2); + if (error) { + device_printf(sc->sc_dev, + "Timed out initialization firmware: %d\n", error); + return error; + } + if (cb.qaic_status) { + device_printf(sc->sc_dev, "Failed to initialize firmware\n"); + return EIO; + } + + return error; +} + +static int +qat_adm_ring_send_init_msg(struct qat_softc *sc, + enum fw_init_cmd_id cmd) +{ + struct qat_admin_rings *qadr = &sc->sc_admin_rings; + uint32_t error, ae; + + for (ae = 0; ae < sc->sc_ae_num; ae++) { + uint8_t srv_mask = qadr->qadr_srv_mask[ae]; + switch (cmd) { + case FW_INIT_CMD_SET_AE_INFO: + case FW_INIT_CMD_SET_RING_INFO: + if (!srv_mask) + continue; + break; + case FW_INIT_CMD_TRNG_ENABLE: + case FW_INIT_CMD_TRNG_DISABLE: + if (!(srv_mask & QAT_SERVICE_CRYPTO_A)) + continue; + break; + default: + return ENOTSUP; + } + + error = qat_adm_ring_send_init_msg_sync(sc, cmd, ae); + if (error) + return error; + } + + return 0; +} + +int +qat_adm_ring_send_init(struct qat_softc *sc) +{ + int error; + + error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_AE_INFO); + if (error) + return error; + + error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_RING_INFO); + if (error) + return error; + + return 0; +} + +static int +qat_adm_ring_intr(struct qat_softc *sc, void *arg, void *msg) +{ + struct arch_if_resp_hdr *resp; + struct fw_init_resp *init_resp; + struct qat_accel_init_cb *init_cb; + int handled = 0; + + resp = (struct arch_if_resp_hdr *)msg; + + switch (resp->resp_type) { + case ARCH_IF_REQ_QAT_FW_INIT: + init_resp = (struct fw_init_resp *)msg; + init_cb = (struct qat_accel_init_cb *) + (uintptr_t)init_resp->comn_resp.opaque_data; + init_cb->qaic_status = + __SHIFTOUT(init_resp->comn_resp.comn_status, + COMN_RESP_INIT_ADMIN_STATUS); + wakeup(init_cb); + break; + default: + device_printf(sc->sc_dev, + "unknown resp type %d\n", resp->resp_type); + break; + } + + return handled; +} + +static inline uint16_t +qat_hw15_get_comn_req_flags(uint8_t ae) +{ + if (ae == 0) { + return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | + COMN_REQ_AUTH0_SLICE_REQUIRED | + COMN_REQ_CIPHER0_SLICE_REQUIRED; + } else { + return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | + COMN_REQ_AUTH1_SLICE_REQUIRED | + COMN_REQ_CIPHER1_SLICE_REQUIRED; + } +} + +static uint32_t +qat_hw15_crypto_setup_cipher_desc(struct qat_crypto_desc *desc, + struct qat_session *qs, struct fw_cipher_hdr *cipher_hdr, + uint32_t hw_blk_offset, enum fw_slice next_slice) +{ + desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ; + + cipher_hdr->state_padding_sz = 0; + cipher_hdr->key_sz = qs->qs_cipher_klen / 8; + + cipher_hdr->state_sz = desc->qcd_cipher_blk_sz / 8; + + cipher_hdr->next_id = next_slice; + cipher_hdr->curr_id = FW_SLICE_CIPHER; + cipher_hdr->offset = hw_blk_offset / 8; + cipher_hdr->resrvd = 0; + + return sizeof(struct hw_cipher_config) + qs->qs_cipher_klen; +} + +static void +qat_hw15_crypto_setup_cipher_config(const struct qat_crypto_desc *desc, + const struct qat_session *qs, const struct cryptop *crp, + struct hw_cipher_config *cipher_config) +{ + const uint8_t *key; + uint8_t *cipher_key; + + cipher_config->val = qat_crypto_load_cipher_session(desc, qs); + cipher_config->reserved = 0; + + cipher_key = (uint8_t *)(cipher_config + 1); + if (crp != NULL && crp->crp_cipher_key != NULL) + key = crp->crp_cipher_key; + else + key = qs->qs_cipher_key; + memcpy(cipher_key, key, qs->qs_cipher_klen); +} + +static uint32_t +qat_hw15_crypto_setup_auth_desc(struct qat_crypto_desc *desc, + struct qat_session *qs, struct fw_auth_hdr *auth_hdr, + uint32_t ctrl_blk_offset, uint32_t hw_blk_offset, + enum fw_slice next_slice) +{ + const struct qat_sym_hash_def *hash_def; + + (void)qat_crypto_load_auth_session(desc, qs, &hash_def); + + auth_hdr->next_id = next_slice; + auth_hdr->curr_id = FW_SLICE_AUTH; + auth_hdr->offset = hw_blk_offset / 8; + auth_hdr->resrvd = 0; + + auth_hdr->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED; + auth_hdr->u.inner_prefix_sz = 0; + auth_hdr->outer_prefix_sz = 0; + auth_hdr->final_sz = hash_def->qshd_alg->qshai_digest_len; + auth_hdr->inner_state1_sz = + roundup(hash_def->qshd_qat->qshqi_state1_len, 8); + auth_hdr->inner_res_sz = hash_def->qshd_alg->qshai_digest_len; + auth_hdr->inner_state2_sz = + roundup(hash_def->qshd_qat->qshqi_state2_len, 8); + auth_hdr->inner_state2_off = auth_hdr->offset + + ((sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz) / 8); + + auth_hdr->outer_config_off = 0; + auth_hdr->outer_state1_sz = 0; + auth_hdr->outer_res_sz = 0; + auth_hdr->outer_prefix_off = 0; + + desc->qcd_auth_sz = hash_def->qshd_alg->qshai_sah->hashsize; + desc->qcd_state_storage_sz = (sizeof(struct hw_auth_counter) + + roundup(hash_def->qshd_alg->qshai_state_size, 8)) / 8; + desc->qcd_gcm_aad_sz_offset1 = desc->qcd_auth_offset + + sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz + + AES_BLOCK_LEN; + desc->qcd_gcm_aad_sz_offset2 = ctrl_blk_offset + + offsetof(struct fw_auth_hdr, u.aad_sz); + + return sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz + + auth_hdr->inner_state2_sz; +} + +static void +qat_hw15_crypto_setup_auth_setup(const struct qat_crypto_desc *desc, + const struct qat_session *qs, const struct cryptop *crp, + struct hw_auth_setup *auth_setup) +{ + const struct qat_sym_hash_def *hash_def; + const uint8_t *key; + uint8_t *state1, *state2; + uint32_t state_sz, state1_sz, state2_sz, state1_pad_len, state2_pad_len; + + auth_setup->auth_config.config = qat_crypto_load_auth_session(desc, qs, + &hash_def); + auth_setup->auth_config.reserved = 0; + + auth_setup->auth_counter.counter = + htobe32(hash_def->qshd_qat->qshqi_auth_counter); + auth_setup->auth_counter.reserved = 0; + + state1 = (uint8_t *)(auth_setup + 1); + state2 = state1 + roundup(hash_def->qshd_qat->qshqi_state1_len, 8); + switch (qs->qs_auth_algo) { + case HW_AUTH_ALGO_GALOIS_128: + qat_crypto_gmac_precompute(desc, qs->qs_cipher_key, + qs->qs_cipher_klen, hash_def, state2); + break; + case HW_AUTH_ALGO_SHA1: + state_sz = hash_def->qshd_alg->qshai_state_size; + state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); + state2_sz = roundup(hash_def->qshd_qat->qshqi_state2_len, 8); + if (qs->qs_auth_mode == HW_AUTH_MODE1) { + state1_pad_len = state1_sz - state_sz; + state2_pad_len = state2_sz - state_sz; + if (state1_pad_len > 0) + memset(state1 + state_sz, 0, state1_pad_len); + if (state2_pad_len > 0) + memset(state2 + state_sz, 0, state2_pad_len); + } + /* FALLTHROUGH */ + case HW_AUTH_ALGO_SHA256: + case HW_AUTH_ALGO_SHA384: + case HW_AUTH_ALGO_SHA512: + switch (qs->qs_auth_mode) { + case HW_AUTH_MODE0: + memcpy(state1, hash_def->qshd_alg->qshai_init_state, + state1_sz); + /* Override for mode 0 hashes. */ + auth_setup->auth_counter.counter = 0; + break; + case HW_AUTH_MODE1: + if (crp != NULL && crp->crp_auth_key != NULL) + key = crp->crp_auth_key; + else + key = qs->qs_auth_key; + if (key != NULL) { + qat_crypto_hmac_precompute(desc, key, + qs->qs_auth_klen, hash_def, state1, state2); + } + break; + default: + panic("%s: unhandled auth mode %d", __func__, + qs->qs_auth_mode); + } + break; + default: + panic("%s: unhandled auth algorithm %d", __func__, + qs->qs_auth_algo); + } +} + +void +qat_hw15_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs, + struct qat_crypto_desc *desc) +{ + struct fw_cipher_hdr *cipher_hdr; + struct fw_auth_hdr *auth_hdr; + struct fw_la_bulk_req *req_cache; + struct hw_auth_setup *auth_setup; + struct hw_cipher_config *cipher_config; + uint32_t ctrl_blk_sz, ctrl_blk_offset, hw_blk_offset; + int i; + uint16_t la_cmd_flags; + uint8_t req_params_sz; + uint8_t *ctrl_blk_ptr, *hw_blk_ptr; + + ctrl_blk_sz = 0; + if (qs->qs_cipher_algo != HW_CIPHER_ALGO_NULL) + ctrl_blk_sz += sizeof(struct fw_cipher_hdr); + if (qs->qs_auth_algo != HW_AUTH_ALGO_NULL) + ctrl_blk_sz += sizeof(struct fw_auth_hdr); + + ctrl_blk_ptr = desc->qcd_content_desc; + ctrl_blk_offset = 0; + hw_blk_ptr = ctrl_blk_ptr + ctrl_blk_sz; + hw_blk_offset = 0; + + la_cmd_flags = 0; + req_params_sz = 0; + for (i = 0; i < MAX_FW_SLICE; i++) { + switch (desc->qcd_slices[i]) { + case FW_SLICE_CIPHER: + cipher_hdr = (struct fw_cipher_hdr *)(ctrl_blk_ptr + + ctrl_blk_offset); + cipher_config = (struct hw_cipher_config *)(hw_blk_ptr + + hw_blk_offset); + desc->qcd_cipher_offset = ctrl_blk_sz + hw_blk_offset; + hw_blk_offset += qat_hw15_crypto_setup_cipher_desc(desc, + qs, cipher_hdr, hw_blk_offset, + desc->qcd_slices[i + 1]); + qat_hw15_crypto_setup_cipher_config(desc, qs, NULL, + cipher_config); + ctrl_blk_offset += sizeof(struct fw_cipher_hdr); + req_params_sz += sizeof(struct fw_la_cipher_req_params); + break; + case FW_SLICE_AUTH: + auth_hdr = (struct fw_auth_hdr *)(ctrl_blk_ptr + + ctrl_blk_offset); + auth_setup = (struct hw_auth_setup *)(hw_blk_ptr + + hw_blk_offset); + desc->qcd_auth_offset = ctrl_blk_sz + hw_blk_offset; + hw_blk_offset += qat_hw15_crypto_setup_auth_desc(desc, + qs, auth_hdr, ctrl_blk_offset, hw_blk_offset, + desc->qcd_slices[i + 1]); + qat_hw15_crypto_setup_auth_setup(desc, qs, NULL, + auth_setup); + ctrl_blk_offset += sizeof(struct fw_auth_hdr); + req_params_sz += sizeof(struct fw_la_auth_req_params); + la_cmd_flags |= LA_FLAGS_RET_AUTH_RES; + /* no digest verify */ + break; + case FW_SLICE_DRAM_WR: + i = MAX_FW_SLICE; /* end of chain */ + break; + default: + MPASS(0); + break; + } + } + + desc->qcd_hdr_sz = ctrl_blk_offset / 8; + desc->qcd_hw_blk_sz = hw_blk_offset / 8; + + req_cache = (struct fw_la_bulk_req *)desc->qcd_req_cache; + qat_msg_req_type_populate( + &req_cache->comn_hdr.arch_if, + ARCH_IF_REQ_QAT_FW_LA, 0); + + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) + la_cmd_flags |= LA_FLAGS_PROTO_GCM | LA_FLAGS_GCM_IV_LEN_FLAG; + else + la_cmd_flags |= LA_FLAGS_PROTO_NO; + + qat_msg_params_populate(req_cache, desc, req_params_sz, + la_cmd_flags, 0); + + bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, + qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); +} + +static void +qat_hw15_crypto_req_setkey(const struct qat_crypto_desc *desc, + const struct qat_session *qs, struct qat_sym_cookie *qsc, + struct fw_la_bulk_req *bulk_req, struct cryptop *crp) +{ + struct hw_auth_setup *auth_setup; + struct hw_cipher_config *cipher_config; + uint8_t *cdesc; + int i; + + cdesc = qsc->qsc_content_desc; + memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE); + for (i = 0; i < MAX_FW_SLICE; i++) { + switch (desc->qcd_slices[i]) { + case FW_SLICE_CIPHER: + cipher_config = (struct hw_cipher_config *) + (cdesc + desc->qcd_cipher_offset); + qat_hw15_crypto_setup_cipher_config(desc, qs, crp, + cipher_config); + break; + case FW_SLICE_AUTH: + auth_setup = (struct hw_auth_setup *) + (cdesc + desc->qcd_auth_offset); + qat_hw15_crypto_setup_auth_setup(desc, qs, crp, + auth_setup); + break; + case FW_SLICE_DRAM_WR: + i = MAX_FW_SLICE; /* end of chain */ + break; + default: + MPASS(0); + } + } + + bulk_req->comn_hdr.content_desc_addr = qsc->qsc_content_desc_paddr; +} + +void +qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *qcb, + struct qat_session *qs, struct qat_crypto_desc const *desc, + struct qat_sym_cookie *qsc, struct cryptop *crp) +{ + struct qat_sym_bulk_cookie *qsbc; + struct fw_la_bulk_req *bulk_req; + struct fw_la_cipher_req_params *cipher_req; + struct fw_la_auth_req_params *auth_req; + bus_addr_t digest_paddr; + uint8_t *aad_szp2, *req_params_ptr; + uint32_t aad_sz, *aad_szp1; + enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id; + enum fw_slice next_slice; + + qsbc = &qsc->u.qsc_bulk_cookie; + + bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg; + memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE); + bulk_req->comn_hdr.arch_if.resp_pipe_id = qcb->qcb_sym_rx->qr_ring_id; + bulk_req->comn_hdr.comn_req_flags = + qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2); + bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr; + bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr; + bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr; + bulk_req->comn_ftr.next_request_addr = 0; + bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc; + if (__predict_false(crp->crp_cipher_key != NULL || + crp->crp_auth_key != NULL)) { + qat_hw15_crypto_req_setkey(desc, qs, qsc, bulk_req, crp); + } + + digest_paddr = 0; + if (desc->qcd_auth_sz != 0) + digest_paddr = qsc->qsc_auth_res_paddr; + + req_params_ptr = qsbc->qsbc_req_params_buf; + memset(req_params_ptr, 0, sizeof(qsbc->qsbc_req_params_buf)); + + /* + * The SG list layout is a bit different for GCM and GMAC, it's simpler + * to handle those cases separately. + */ + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { + cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr; + auth_req = (struct fw_la_auth_req_params *) + (req_params_ptr + sizeof(struct fw_la_cipher_req_params)); + + cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8; + cipher_req->curr_id = FW_SLICE_CIPHER; + if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH) + cipher_req->next_id = FW_SLICE_DRAM_WR; + else + cipher_req->next_id = FW_SLICE_AUTH; + cipher_req->state_address = qsc->qsc_iv_buf_paddr; + + if (cmd_id != FW_LA_CMD_AUTH) { + /* + * Don't fill out the cipher block if we're doing GMAC + * only. + */ + cipher_req->cipher_off = 0; + cipher_req->cipher_len = crp->crp_payload_length; + } + auth_req->curr_id = FW_SLICE_AUTH; + if (cmd_id == FW_LA_CMD_HASH_CIPHER || cmd_id == FW_LA_CMD_AUTH) + auth_req->next_id = FW_SLICE_CIPHER; + else + auth_req->next_id = FW_SLICE_DRAM_WR; + + auth_req->auth_res_address = digest_paddr; + auth_req->auth_res_sz = desc->qcd_auth_sz; + + auth_req->auth_off = 0; + auth_req->auth_len = crp->crp_payload_length; + + auth_req->hash_state_sz = + roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN) >> 3; + auth_req->u1.aad_addr = crp->crp_aad_length > 0 ? + qsc->qsc_gcm_aad_paddr : 0; + + /* + * Update the hash state block if necessary. This only occurs + * when the AAD length changes between requests in a session and + * is synchronized by qat_process(). + */ + aad_sz = htobe32(crp->crp_aad_length); + aad_szp1 = (uint32_t *)( + __DECONST(uint8_t *, desc->qcd_content_desc) + + desc->qcd_gcm_aad_sz_offset1); + aad_szp2 = __DECONST(uint8_t *, desc->qcd_content_desc) + + desc->qcd_gcm_aad_sz_offset2; + if (__predict_false(*aad_szp1 != aad_sz)) { + *aad_szp1 = aad_sz; + *aad_szp2 = (uint8_t)roundup2(crp->crp_aad_length, + QAT_AES_GCM_AAD_ALIGN); + bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, + qs->qs_desc_mem.qdm_dma_map, + BUS_DMASYNC_PREWRITE); + } + } else { + cipher_req = (struct fw_la_cipher_req_params *)req_params_ptr; + if (cmd_id != FW_LA_CMD_AUTH) { + if (cmd_id == FW_LA_CMD_CIPHER || + cmd_id == FW_LA_CMD_HASH_CIPHER) + next_slice = FW_SLICE_DRAM_WR; + else + next_slice = FW_SLICE_AUTH; + + cipher_req->cipher_state_sz = + desc->qcd_cipher_blk_sz / 8; + + cipher_req->curr_id = FW_SLICE_CIPHER; + cipher_req->next_id = next_slice; + + cipher_req->cipher_off = crp->crp_aad_length == 0 ? 0 : + crp->crp_payload_start - crp->crp_aad_start; + cipher_req->cipher_len = crp->crp_payload_length; + cipher_req->state_address = qsc->qsc_iv_buf_paddr; + } + if (cmd_id != FW_LA_CMD_CIPHER) { + if (cmd_id == FW_LA_CMD_AUTH) + auth_req = (struct fw_la_auth_req_params *) + req_params_ptr; + else + auth_req = (struct fw_la_auth_req_params *) + (cipher_req + 1); + if (cmd_id == FW_LA_CMD_HASH_CIPHER) + next_slice = FW_SLICE_CIPHER; + else + next_slice = FW_SLICE_DRAM_WR; + + auth_req->curr_id = FW_SLICE_AUTH; + auth_req->next_id = next_slice; + + auth_req->auth_res_address = digest_paddr; + auth_req->auth_res_sz = desc->qcd_auth_sz; + + auth_req->auth_len = + crp->crp_payload_length + crp->crp_aad_length; + auth_req->auth_off = 0; + + auth_req->hash_state_sz = 0; + auth_req->u1.prefix_addr = desc->qcd_hash_state_paddr + + desc->qcd_state_storage_sz; + } + } +} Index: head/sys/dev/qat/qat_hw15reg.h =================================================================== --- head/sys/dev/qat/qat_hw15reg.h +++ head/sys/dev/qat/qat_hw15reg.h @@ -0,0 +1,635 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw15reg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_HW15REG_H_ +#define _DEV_PCI_QAT_HW15REG_H_ + +/* Default message size in bytes */ +#define FW_REQ_DEFAULT_SZ_HW15 64 +#define FW_RESP_DEFAULT_SZ_HW15 64 + +#define ADMIN_RING_SIZE 256 +#define RING_NUM_ADMIN_TX 0 +#define RING_NUM_ADMIN_RX 1 + +/* -------------------------------------------------------------------------- */ +/* accel */ + +#define ARCH_IF_FLAGS_VALID_FLAG __BIT(7) +#define ARCH_IF_FLAGS_RESP_RING_TYPE __BITS(4, 3) +#define ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT 3 +#define ARCH_IF_FLAGS_RESP_RING_TYPE_SCRATCH (0 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_RING_TYPE_NN (1 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_RING_TYPE_ET (2 << ARCH_IF_FLAGS_RESP_RING_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_TYPE __BITS(2, 0) +#define ARCH_IF_FLAGS_RESP_TYPE_SHIFT 0 +#define ARCH_IF_FLAGS_RESP_TYPE_A (0 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_TYPE_B (1 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_TYPE_C (2 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT) +#define ARCH_IF_FLAGS_RESP_TYPE_S (3 << ARCH_IF_FLAGS_RESP_TYPE_SHIFT) + +enum arch_if_req { + ARCH_IF_REQ_NULL, /* NULL request type */ + + /* QAT-AE Service Request Type IDs - 01 to 20 */ + ARCH_IF_REQ_QAT_FW_INIT, /* QAT-FW Initialization Request */ + ARCH_IF_REQ_QAT_FW_ADMIN, /* QAT-FW Administration Request */ + ARCH_IF_REQ_QAT_FW_PKE, /* QAT-FW PKE Request */ + ARCH_IF_REQ_QAT_FW_LA, /* QAT-FW Lookaside Request */ + ARCH_IF_REQ_QAT_FW_IPSEC, /* QAT-FW IPSec Request */ + ARCH_IF_REQ_QAT_FW_SSL, /* QAT-FW SSL Request */ + ARCH_IF_REQ_QAT_FW_DMA, /* QAT-FW DMA Request */ + ARCH_IF_REQ_QAT_FW_STORAGE, /* QAT-FW Storage Request */ + ARCH_IF_REQ_QAT_FW_COMPRESS, /* QAT-FW Compression Request */ + ARCH_IF_REQ_QAT_FW_PATMATCH, /* QAT-FW Pattern Matching Request */ + + /* IP Service (Range Match and Exception) Blocks Request Type IDs 21 - 30 */ + ARCH_IF_REQ_RM_FLOW_MISS = 21, /* RM flow miss request */ + ARCH_IF_REQ_RM_FLOW_TIMER_EXP, /* RM flow timer exp Request */ + ARCH_IF_REQ_IP_SERVICES_RFC_LOOKUP_UPDATE, /* RFC Lookup request */ + ARCH_IF_REQ_IP_SERVICES_CONFIG_UPDATE, /* Config Update request */ + ARCH_IF_REQ_IP_SERVICES_FCT_CONFIG, /* FCT Config request */ + ARCH_IF_REQ_IP_SERVICES_NEXT_HOP_TIMER_EXPIRY, /* NH Timer expiry request */ + ARCH_IF_REQ_IP_SERVICES_EXCEPTION, /* Exception processign request */ + ARCH_IF_REQ_IP_SERVICES_STACK_DRIVER, /* Send to SD request */ + ARCH_IF_REQ_IP_SERVICES_ACTION_HANDLER, /* Send to AH request */ + ARCH_IF_REQ_IP_SERVICES_EVENT_HANDLER, /* Send to EH request */ + ARCH_IF_REQ_DELIMITER /* End delimiter */ +}; + +struct arch_if_req_hdr { + uint8_t resp_dest_id; + /* Opaque identifier passed from the request to response to allow + * response handler perform any further processing */ + uint8_t resp_pipe_id; + /* Response pipe to write the response associated with this request to */ + uint8_t req_type; + /* Definition of the service described by the request */ + uint8_t flags; + /* Request and response control flags */ +}; + +struct arch_if_resp_hdr { + uint8_t dest_id; + /* Opaque identifier passed from the request to response to allow + * response handler perform any further processing */ + uint8_t serv_id; + /* Definition of the service id generating the response */ + uint8_t resp_type; + /* Definition of the service described by the request */ + uint8_t flags; + /* Request and response control flags */ +}; + +struct fw_comn_req_hdr { + struct arch_if_req_hdr arch_if; + /* Common arch fields used by all ICP interface requests. Remaining + * fields are specific to the common QAT FW service. */ + uint16_t comn_req_flags; + /* Flags used to describe common processing required by the request and + * the meaning of parameters in it i.e. differentiating between a buffer + * descriptor and a flat buffer pointer in the source (src) and destination + * (dest) data address fields. Full definition of the fields is given + * below */ + uint8_t content_desc_params_sz; + /* Size of the content descriptor parameters in quad words. These + * parameters describe the session setup configuration info for the + * slices that this request relies upon i.e. the configuration word and + * cipher key needed by the cipher slice if there is a request for cipher + * processing. The format of the parameters are contained in icp_qat_hw.h + * and vary depending on the algorithm and mode being used. It is the + * clients responsibility to ensure this structure is correctly packed */ + uint8_t content_desc_hdr_sz; + /* Size of the content descriptor header in quad words. This information + * is read into the QAT AE xfr registers */ + uint64_t content_desc_addr; + /* Address of the content descriptor containing both the content header + * the size of which is defined by content_desc_hdr_sz followed by the + * content parameters whose size is described bycontent_desc_params_sz + */ +}; + +struct fw_comn_req_mid { + uint64_t opaque_data; + /* Opaque data passed unmodified from the request to response messages + * by firmware (fw) */ + uint64_t src_data_addr; + /* Generic definition of the source data supplied to the QAT AE. The + * common flags are used to further describe the attributes of this + * field */ + uint64_t dest_data_addr; + /* Generic definition of the destination data supplied to the QAT AE. + * The common flags are used to further describe the attributes of this + * field */ +}; + +union fw_comn_req_ftr { + uint64_t next_request_addr; + /* Overloaded field, for stateful requests, this field is the pointer to + next request descriptor */ + struct { + uint32_t src_length; + /* Length of source flat buffer incase src buffer type is flat */ + uint32_t dst_length; + /* Length of source flat buffer incase dst buffer type is flat */ + } s; +}; + +union fw_comn_error { + struct { + uint8_t resrvd; /* 8 bit reserved field */ + uint8_t comn_err_code; /**< 8 bit common error code */ + } s; + /* Structure which is used for non-compression responses */ + + struct { + uint8_t xlat_err_code; /* 8 bit translator error field */ + uint8_t cmp_err_code; /* 8 bit compression error field */ + } s1; + /* Structure which is used for compression responses */ +}; + +struct fw_comn_resp_hdr { + struct arch_if_resp_hdr arch_if; + /* Common arch fields used by all ICP interface response messages. The + * remaining fields are specific to the QAT FW */ + union fw_comn_error comn_error; + /* This field is overloaded to allow for one 8 bit common error field + * or two 8 bit error fields from compression and translator */ + uint8_t comn_status; + /* Status field which specifies which slice(s) report an error */ + uint8_t serv_cmd_id; + /* For services that define multiple commands this field represents the + * command. If only 1 command is supported then this field will be 0 */ + uint64_t opaque_data; + /* Opaque data passed from the request to the response message */ +}; + + +#define RING_MASK_TABLE_ENTRY_LOG_SZ (5) + +#define FW_INIT_RING_MASK_SET(table, id) \ + table->firt_ring_mask[id >> RING_MASK_TABLE_ENTRY_LOG_SZ] =\ + table->firt_ring_mask[id >> RING_MASK_TABLE_ENTRY_LOG_SZ] | \ + (1 << (id & 0x1f)) + +struct fw_init_ring_params { + uint8_t firp_curr_weight; /* Current ring weight (working copy), + * has to be equal to init_weight */ + uint8_t firp_init_weight; /* Initial ring weight: -1 ... 0 + * -1 is equal to FF, -2 is equal to FE, + * the weighting uses negative logic + * where FF means poll the ring once, + * -2 is poll the ring twice, + * 0 is poll the ring 255 times */ + uint8_t firp_ring_pvl; /* Ring Privilege Level. */ + uint8_t firp_reserved; /* Reserved field which must be set + * to 0 by the client */ +}; + +#define INIT_RING_TABLE_SZ 128 +#define INIT_RING_TABLE_LW_SZ 4 + +struct fw_init_ring_table { + struct fw_init_ring_params firt_bulk_rings[INIT_RING_TABLE_SZ]; + /* array of ring parameters */ + uint32_t firt_ring_mask[INIT_RING_TABLE_LW_SZ]; + /* Structure to hold the bit masks for + * 128 rings. */ +}; + +struct fw_init_set_ae_info_hdr { + uint16_t init_slice_mask; /* Init time flags to set the ownership of the slices */ + uint16_t resrvd; /* Reserved field and must be set to 0 by the client */ + uint8_t init_qat_id; /* Init time qat id described in the request */ + uint8_t init_ring_cluster_id; /* Init time ring cluster Id */ + uint8_t init_trgt_id; /* Init time target AE id described in the request */ + uint8_t init_cmd_id; /* Init time command that is described in the request */ +}; + +struct fw_init_set_ae_info { + uint64_t init_shram_mask; /* Init time shram mask to set the page ownership in page pool of AE*/ + uint64_t resrvd; /* Reserved field and must be set to 0 by the client */ +}; + +struct fw_init_set_ring_info_hdr { + uint32_t resrvd; /* Reserved field and must be set to 0 by the client */ + uint16_t init_ring_tbl_sz; /* Init time information to state size of the ring table */ + uint8_t init_trgt_id; /* Init time target AE id described in the request */ + uint8_t init_cmd_id; /* Init time command that is described in the request */ +}; + +struct fw_init_set_ring_info { + uint64_t init_ring_table_ptr; /* Pointer to weighting information for 128 rings */ + uint64_t resrvd; /* Reserved field and must be set to 0 by the client */ +}; + +struct fw_init_trng_hdr { + uint32_t resrvd; /* Reserved field and must be set to 0 by the client */ + union { + uint8_t resrvd; /* Reserved field set to if cmd type is trng disable */ + uint8_t init_trng_cfg_sz; /* Size of the trng config word in QW*/ + } u; + uint8_t resrvd1; /* Reserved field and must be set to 0 by the client */ + uint8_t init_trgt_id; /* Init time target AE id described in the request */ + uint8_t init_cmd_id; /* Init time command that is described in the request */ +}; + +struct fw_init_trng { + union { + uint64_t resrvd; /* Reserved field set to 0 if cmd type is trng disable */ + uint64_t init_trng_cfg_ptr; /* Pointer to TRNG Slice config word*/ + } u; + uint64_t resrvd; /* Reserved field and must be set to 0 by the client */ +}; + +struct fw_init_req { + struct fw_comn_req_hdr comn_hdr; /* Common request header */ + union { + struct fw_init_set_ae_info_hdr set_ae_info; + /* INIT SET_AE_INFO request header structure */ + struct fw_init_set_ring_info_hdr set_ring_info; + /* INIT SET_RING_INFO request header structure */ + struct fw_init_trng_hdr init_trng; + /* INIT TRNG ENABLE/DISABLE request header structure */ + } u; + struct fw_comn_req_mid comn_mid; /* Common request middle section */ + union { + struct fw_init_set_ae_info set_ae_info; + /* INIT SET_AE_INFO request data structure */ + struct fw_init_set_ring_info set_ring_info; + /* INIT SET_RING_INFO request data structure */ + struct fw_init_trng init_trng; + /* INIT TRNG ENABLE/DISABLE request data structure */ + } u1; +}; + +enum fw_init_cmd_id { + FW_INIT_CMD_SET_AE_INFO, /* Setup AE Info command type */ + FW_INIT_CMD_SET_RING_INFO, /* Setup Ring Info command type */ + FW_INIT_CMD_TRNG_ENABLE, /* TRNG Enable command type */ + FW_INIT_CMD_TRNG_DISABLE, /* TRNG Disable command type */ + FW_INIT_CMD_DELIMITER /* Delimiter type */ +}; + +struct fw_init_resp { + struct fw_comn_resp_hdr comn_resp; /* Common interface response */ + uint8_t resrvd[64 - sizeof(struct fw_comn_resp_hdr)]; + /* XXX FW_RESP_DEFAULT_SZ_HW15 */ + /* Reserved padding out to the default response size */ +}; + +/* -------------------------------------------------------------------------- */ +/* look aside */ + +#define COMN_REQ_ORD UINT16_C(0x8000) +#define COMN_REQ_ORD_SHIFT 15 +#define COMN_REQ_ORD_NONE (0 << COMN_REQ_ORD_SHIFT) +#define COMN_REQ_ORD_STRICT (1 << COMN_REQ_ORD_SHIFT) +#define COMN_REQ_PTR_TYPE UINT16_C(0x4000) +#define COMN_REQ_PTR_TYPE_SHIFT 14 +#define COMN_REQ_PTR_TYPE_FLAT (0 << COMN_REQ_PTR_TYPE_SHIFT) +#define COMN_REQ_PTR_TYPE_SGL (1 << COMN_REQ_PTR_TYPE_SHIFT) +#define COMN_REQ_RESERVED UINT16_C(0x2000) +#define COMN_REQ_SHRAM_INIT UINT16_C(0x1000) +#define COMN_REQ_SHRAM_INIT_SHIFT 12 +#define COMN_REQ_SHRAM_INIT_REQUIRED (1 << COMN_REQ_SHRAM_INIT_SHIFT) +#define COMN_REQ_REGEX_SLICE UINT16_C(0x0800) +#define COMN_REQ_REGEX_SLICE_SHIFT 11 +#define COMN_REQ_REGEX_SLICE_REQUIRED (1 << COMN_REQ_REGEX_SLICE_SHIFT) +#define COMN_REQ_XLAT_SLICE UINT16_C(0x0400) +#define COMN_REQ_XLAT_SLICE_SHIFT 10 +#define COMN_REQ_XLAT_SLICE_REQUIRED (1 << COMN_REQ_XLAT_SLICE_SHIFT) +#define COMN_REQ_CPR_SLICE UINT16_C(0x0200) +#define COMN_REQ_CPR_SLICE_SHIFT 9 +#define COMN_REQ_CPR_SLICE_REQUIRED (1 << COMN_REQ_CPR_SLICE_SHIFT) +#define COMN_REQ_BULK_SLICE UINT16_C(0x0100) +#define COMN_REQ_BULK_SLICE_SHIFT 8 +#define COMN_REQ_BULK_SLICE_REQUIRED (1 << COMN_REQ_BULK_SLICE_SHIFT) +#define COMN_REQ_STORAGE_SLICE UINT16_C(0x0080) +#define COMN_REQ_STORAGE_SLICE_SHIFT 7 +#define COMN_REQ_STORAGE_SLICE_REQUIRED (1 << COMN_REQ_STORAGE_SLICE_SHIFT) +#define COMN_REQ_RND_SLICE UINT16_C(0x0040) +#define COMN_REQ_RND_SLICE_SHIFT 6 +#define COMN_REQ_RND_SLICE_REQUIRED (1 << COMN_REQ_RND_SLICE_SHIFT) +#define COMN_REQ_PKE1_SLICE UINT16_C(0x0020) +#define COMN_REQ_PKE1_SLICE_SHIFT 5 +#define COMN_REQ_PKE1_SLICE_REQUIRED (1 << COMN_REQ_PKE1_SLICE_SHIFT) +#define COMN_REQ_PKE0_SLICE UINT16_C(0x0010) +#define COMN_REQ_PKE0_SLICE_SHIFT 4 +#define COMN_REQ_PKE0_SLICE_REQUIRED (1 << COMN_REQ_PKE0_SLICE_SHIFT) +#define COMN_REQ_AUTH1_SLICE UINT16_C(0x0008) +#define COMN_REQ_AUTH1_SLICE_SHIFT 3 +#define COMN_REQ_AUTH1_SLICE_REQUIRED (1 << COMN_REQ_AUTH1_SLICE_SHIFT) +#define COMN_REQ_AUTH0_SLICE UINT16_C(0x0004) +#define COMN_REQ_AUTH0_SLICE_SHIFT 2 +#define COMN_REQ_AUTH0_SLICE_REQUIRED (1 << COMN_REQ_AUTH0_SLICE_SHIFT) +#define COMN_REQ_CIPHER1_SLICE UINT16_C(0x0002) +#define COMN_REQ_CIPHER1_SLICE_SHIFT 1 +#define COMN_REQ_CIPHER1_SLICE_REQUIRED (1 << COMN_REQ_CIPHER1_SLICE_SHIFT) +#define COMN_REQ_CIPHER0_SLICE UINT16_C(0x0001) +#define COMN_REQ_CIPHER0_SLICE_SHIFT 0 +#define COMN_REQ_CIPHER0_SLICE_REQUIRED (1 << COMN_REQ_CIPHER0_SLICE_SHIFT) + +#define COMN_REQ_CY0_ONLY(shram) \ + COMN_REQ_ORD_STRICT | \ + COMN_REQ_PTR_TYPE_FLAT | \ + (shram) | \ + COMN_REQ_RND_SLICE_REQUIRED | \ + COMN_REQ_PKE0_SLICE_REQUIRED | \ + COMN_REQ_AUTH0_SLICE_REQUIRED | \ + COMN_REQ_CIPHER0_SLICE_REQUIRED; +#define COMN_REQ_CY1_ONLY(shram) \ + COMN_REQ_ORD_STRICT | \ + COMN_REQ_PTR_TYPE_FLAT | \ + (shram) | \ + COMN_REQ_PKE1_SLICE_REQUIRED | \ + COMN_REQ_AUTH1_SLICE_REQUIRED | \ + COMN_REQ_CIPHER1_SLICE_REQUIRED; + +#define COMN_RESP_CRYPTO_STATUS __BIT(7) +#define COMN_RESP_PKE_STATUS __BIT(6) +#define COMN_RESP_CMP_STATUS __BIT(5) +#define COMN_RESP_XLAT_STATUS __BIT(4) +#define COMN_RESP_PM_STATUS __BIT(3) +#define COMN_RESP_INIT_ADMIN_STATUS __BIT(2) + +#define COMN_STATUS_FLAG_OK 0 +#define COMN_STATUS_FLAG_ERROR 1 + +struct fw_la_ssl_tls_common { + uint8_t out_len; /* Number of bytes of key material to output. */ + uint8_t label_len; /* Number of bytes of label for SSL and bytes + * for TLS key generation */ +}; + +struct fw_la_mgf_common { + uint8_t hash_len; + /* Number of bytes of hash output by the QAT per iteration */ + uint8_t seed_len; + /* Number of bytes of seed provided in src buffer for MGF1 */ +}; + +struct fw_cipher_hdr { + uint8_t state_sz; + /* State size in quad words of the cipher algorithm used in this session. + * Set to zero if the algorithm doesnt provide any state */ + uint8_t offset; + /* Quad word offset from the content descriptor parameters address i.e. + * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher + * processing */ + uint8_t curr_id; + /* Initialised with the cipher slice type */ + uint8_t next_id; + /* Set to the next slice to pass the ciphered data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * anymore slices after cipher */ + uint16_t resrvd; + /* Reserved padding byte to bring the struct to the word boundary. MUST be + * set to 0 */ + uint8_t state_padding_sz; + /* State padding size in quad words. Set to 0 if no padding is required. */ + uint8_t key_sz; + /* Key size in quad words of the cipher algorithm used in this session */ +}; + +struct fw_auth_hdr { + uint8_t hash_flags; + /* General flags defining the processing to perform. 0 is normal processing + * and 1 means there is a nested hash processing loop to go through */ + uint8_t offset; + /* Quad word offset from the content descriptor parameters address to the + * parameters for the auth processing */ + uint8_t curr_id; + /* Initialised with the auth slice type */ + uint8_t next_id; + /* Set to the next slice to pass data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * anymore slices after auth */ + union { + uint8_t inner_prefix_sz; + /* Size in bytes of the inner prefix data */ + uint8_t aad_sz; + /* Size in bytes of padded AAD data to prefix to the packet for CCM + * or GCM processing */ + } u; + + uint8_t outer_prefix_sz; + /* Size in bytes of outer prefix data */ + uint8_t final_sz; + /* Size in bytes of digest to be returned to the client if requested */ + uint8_t inner_res_sz; + /* Size in bytes of the digest from the inner hash algorithm */ + uint8_t resrvd; + /* This field is unused, assumed value is zero. */ + uint8_t inner_state1_sz; + /* Size in bytes of inner hash state1 data. Must be a qword multiple */ + uint8_t inner_state2_off; + /* Quad word offset from the content descriptor parameters pointer to the + * inner state2 value */ + uint8_t inner_state2_sz; + /* Size in bytes of inner hash state2 data. Must be a qword multiple */ + uint8_t outer_config_off; + /* Quad word offset from the content descriptor parameters pointer to the + * outer configuration information */ + uint8_t outer_state1_sz; + /* Size in bytes of the outer state1 value */ + uint8_t outer_res_sz; + /* Size in bytes of digest from the outer auth algorithm */ + uint8_t outer_prefix_off; + /* Quad word offset from the start of the inner prefix data to the outer + * prefix information. Should equal the rounded inner prefix size, converted + * to qwords */ +}; + +#define FW_AUTH_HDR_FLAG_DO_NESTED 1 +#define FW_AUTH_HDR_FLAG_NO_NESTED 0 + +struct fw_la_comn_req { + union { + uint16_t la_flags; + /* Definition of the common LA processing flags used for the + * bulk processing */ + union { + struct fw_la_ssl_tls_common ssl_tls_common; + /* For TLS or SSL Key Generation, this field is + * overloaded with ssl_tls common information */ + struct fw_la_mgf_common mgf_common; + /* For MGF Key Generation, this field is overloaded with + mgf information */ + } u; + } u; + + union { + uint8_t resrvd; + /* If not useRd by a request this field must be set to 0 */ + uint8_t tls_seed_len; + /* Byte Len of tls seed */ + uint8_t req_params_blk_sz; + /* For bulk processing this field represents the request + * parameters block size */ + uint8_t trng_cfg_sz; + /* This field is used for TRNG_ENABLE requests to indicate the + * size of the TRNG Slice configuration word. Size is in QW's */ + } u1; + uint8_t la_cmd_id; + /* Definition of the LA command defined by this request */ +}; + +#define LA_FLAGS_GCM_IV_LEN_FLAG __BIT(9) +#define LA_FLAGS_PROTO __BITS(8, 6) +#define LA_FLAGS_PROTO_SNOW_3G __SHIFTIN(4, LA_FLAGS_PROTO) +#define LA_FLAGS_PROTO_GCM __SHIFTIN(2, LA_FLAGS_PROTO) +#define LA_FLAGS_PROTO_CCM __SHIFTIN(1, LA_FLAGS_PROTO) +#define LA_FLAGS_PROTO_NO __SHIFTIN(0, LA_FLAGS_PROTO) +#define LA_FLAGS_DIGEST_IN_BUFFER __BIT(5) +#define LA_FLAGS_CMP_AUTH_RES __BIT(4) +#define LA_FLAGS_RET_AUTH_RES __BIT(3) +#define LA_FLAGS_UPDATE_STATE __BIT(2) +#define LA_FLAGS_PARTIAL __BITS(1, 0) + +struct fw_la_bulk_req { + struct fw_comn_req_hdr comn_hdr; + /* Common request header */ + uint32_t flow_id; + /* Field used by Firmware to limit the number of stateful requests + * for a session being processed at a given point of time */ + struct fw_la_comn_req comn_la_req; + /* Common LA request parameters */ + struct fw_comn_req_mid comn_mid; + /* Common request middle section */ + uint64_t req_params_addr; + /* Memory address of the request parameters */ + union fw_comn_req_ftr comn_ftr; + /* Common request footer */ +}; + +struct fw_la_resp { + struct fw_comn_resp_hdr comn_resp; + uint8_t resrvd[64 - sizeof(struct fw_comn_resp_hdr)]; + /* FW_RESP_DEFAULT_SZ_HW15 */ +}; + +struct fw_la_cipher_req_params { + uint8_t resrvd; + /* Reserved field and assumed set to 0 */ + uint8_t cipher_state_sz; + /* Number of quad words of state data for the cipher algorithm */ + uint8_t curr_id; + /* Initialised with the cipher slice type */ + uint8_t next_id; + /* Set to the next slice to pass the ciphered data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * anymore slices after cipher */ + uint16_t resrvd1; + /* Reserved field, should be set to zero*/ + uint8_t resrvd2; + /* Reserved field, should be set to zero*/ + uint8_t next_offset; + /* Offset in bytes to the next request parameter block */ + uint32_t cipher_off; + /* Byte offset from the start of packet to the cipher data region */ + uint32_t cipher_len; + /* Byte length of the cipher data region */ + uint64_t state_address; + /* Flat buffer address in memory of the cipher state information. Unused + * if the state size is 0 */ +}; + +struct fw_la_auth_req_params { + uint8_t auth_res_sz; + /* Size in quad words of digest information to validate */ + uint8_t hash_state_sz; + /* Number of quad words of inner and outer hash prefix data to process */ + uint8_t curr_id; + /* Initialised with the auth slice type */ + uint8_t next_id; + /* Set to the next slice to pass the auth data through. + * Set to ICP_QAT_FW_SLICE_NULL for in-place auth-only requests + * Set to ICP_QAT_FW_SLICE_DRAM_WR for all other request types + * if the data is not to go through anymore slices after auth */ + union { + uint16_t resrvd; + /* Reserved field should be set to zero for bulk services */ + uint16_t tls_secret_len; + /* Length of Secret information for TLS. */ + } u; + uint8_t resrvd; + /* Reserved field, should be set to zero*/ + uint8_t next_offset; + /* offset in bytes to the next request parameter block */ + uint32_t auth_off; + /* Byte offset from the start of packet to the auth data region */ + uint32_t auth_len; + /* Byte length of the auth data region */ + union { + uint64_t prefix_addr; + /* Address of the prefix information */ + uint64_t aad_addr; + /* Address of the AAD info in DRAM. Used for the CCM and GCM + * protocols */ + } u1; + uint64_t auth_res_address; + /* Address of the auth result information to validate or the location to + * writeback the digest information to */ +}; + +#endif Index: head/sys/dev/qat/qat_hw15var.h =================================================================== --- head/sys/dev/qat/qat_hw15var.h +++ head/sys/dev/qat/qat_hw15var.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw15var.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2013 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_HW15VAR_H_ +#define _DEV_PCI_QAT_HW15VAR_H_ + +CTASSERT(HASH_CONTENT_DESC_SIZE >= + sizeof(struct fw_auth_hdr) + MAX_HASH_SETUP_BLK_SZ); +CTASSERT(CIPHER_CONTENT_DESC_SIZE >= + sizeof(struct fw_cipher_hdr) + MAX_CIPHER_SETUP_BLK_SZ); +CTASSERT(CONTENT_DESC_MAX_SIZE >= + roundup(HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE, + QAT_OPTIMAL_ALIGN)); +CTASSERT(QAT_SYM_REQ_PARAMS_SIZE_PADDED >= + roundup(sizeof(struct fw_la_cipher_req_params) + + sizeof(struct fw_la_auth_req_params), QAT_OPTIMAL_ALIGN)); + +/* length of the 5 long words of the request that are stored in the session + * This is rounded up to 32 in order to use the fast memcopy function */ +#define QAT_HW15_SESSION_REQ_CACHE_SIZE (32) + +void qat_msg_req_type_populate(struct arch_if_req_hdr *, + enum arch_if_req, uint32_t); +void qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *, bus_addr_t, + uint8_t, uint8_t, uint16_t, uint32_t); +void qat_msg_service_cmd_populate(struct fw_la_bulk_req *, + enum fw_la_cmd_id, uint16_t); +void qat_msg_cmn_mid_populate(struct fw_comn_req_mid *, void *, + uint64_t , uint64_t); +void qat_msg_req_params_populate(struct fw_la_bulk_req *, bus_addr_t, + uint8_t); +void qat_msg_cmn_footer_populate(union fw_comn_req_ftr *, uint64_t); +void qat_msg_params_populate(struct fw_la_bulk_req *, + struct qat_crypto_desc *, uint8_t, uint16_t, + uint16_t); + + +int qat_adm_ring_init(struct qat_softc *); +int qat_adm_ring_send_init(struct qat_softc *); + +void qat_hw15_crypto_setup_desc(struct qat_crypto *, + struct qat_session *, struct qat_crypto_desc *); +void qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *, + struct qat_session *, struct qat_crypto_desc const *, + struct qat_sym_cookie *, struct cryptop *); + +#endif Index: head/sys/dev/qat/qat_hw17.c =================================================================== --- head/sys/dev/qat/qat_hw17.c +++ head/sys/dev/qat/qat_hw17.c @@ -0,0 +1,662 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); +#if 0 +__KERNEL_RCSID(0, "$NetBSD: qat_hw17.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); +#endif + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include "qatreg.h" +#include "qat_hw17reg.h" +#include "qatvar.h" +#include "qat_hw17var.h" + +int qat_adm_mailbox_put_msg_sync(struct qat_softc *, uint32_t, + void *, void *); +int qat_adm_mailbox_send(struct qat_softc *, + struct fw_init_admin_req *, struct fw_init_admin_resp *); +int qat_adm_mailbox_send_init_me(struct qat_softc *); +int qat_adm_mailbox_send_hb_timer(struct qat_softc *); +int qat_adm_mailbox_send_fw_status(struct qat_softc *); +int qat_adm_mailbox_send_constants(struct qat_softc *); + +int +qat_adm_mailbox_init(struct qat_softc *sc) +{ + uint64_t addr; + int error; + struct qat_dmamem *qdm; + + error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_dma, 1, + PAGE_SIZE, PAGE_SIZE); + if (error) + return error; + + qdm = &sc->sc_admin_comms.qadc_const_tbl_dma; + error = qat_alloc_dmamem(sc, qdm, 1, PAGE_SIZE, PAGE_SIZE); + if (error) + return error; + + memcpy(qdm->qdm_dma_vaddr, + mailbox_const_tab, sizeof(mailbox_const_tab)); + + bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, + BUS_DMASYNC_PREWRITE); + + error = qat_alloc_dmamem(sc, &sc->sc_admin_comms.qadc_hb_dma, 1, + PAGE_SIZE, PAGE_SIZE); + if (error) + return error; + + addr = (uint64_t)sc->sc_admin_comms.qadc_dma.qdm_dma_seg.ds_addr; + qat_misc_write_4(sc, ADMINMSGUR, addr >> 32); + qat_misc_write_4(sc, ADMINMSGLR, addr); + + return 0; +} + +int +qat_adm_mailbox_put_msg_sync(struct qat_softc *sc, uint32_t ae, + void *in, void *out) +{ + struct qat_dmamem *qdm; + uint32_t mailbox; + bus_size_t mb_offset = MAILBOX_BASE + (ae * MAILBOX_STRIDE); + int offset = ae * ADMINMSG_LEN * 2; + int times, received; + uint8_t *buf = (uint8_t *)sc->sc_admin_comms.qadc_dma.qdm_dma_vaddr + offset; + + mailbox = qat_misc_read_4(sc, mb_offset); + if (mailbox == 1) + return EAGAIN; + + qdm = &sc->sc_admin_comms.qadc_dma; + memcpy(buf, in, ADMINMSG_LEN); + bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + qat_misc_write_4(sc, mb_offset, 1); + + received = 0; + for (times = 0; times < 50; times++) { + DELAY(20000); + if (qat_misc_read_4(sc, mb_offset) == 0) { + received = 1; + break; + } + } + if (received) { + bus_dmamap_sync(qdm->qdm_dma_tag, qdm->qdm_dma_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + memcpy(out, buf + ADMINMSG_LEN, ADMINMSG_LEN); + } else { + device_printf(sc->sc_dev, + "Failed to send admin msg to accelerator\n"); + } + + return received ? 0 : EFAULT; +} + +int +qat_adm_mailbox_send(struct qat_softc *sc, + struct fw_init_admin_req *req, struct fw_init_admin_resp *resp) +{ + int error; + uint32_t mask; + uint8_t ae; + + for (ae = 0, mask = sc->sc_ae_mask; mask; ae++, mask >>= 1) { + if (!(mask & 1)) + continue; + + error = qat_adm_mailbox_put_msg_sync(sc, ae, req, resp); + if (error) + return error; + if (resp->init_resp_hdr.status) { + device_printf(sc->sc_dev, + "Failed to send admin msg: cmd %d\n", + req->init_admin_cmd_id); + return EFAULT; + } + } + + return 0; +} + +int +qat_adm_mailbox_send_init_me(struct qat_softc *sc) +{ + struct fw_init_admin_req req; + struct fw_init_admin_resp resp; + + memset(&req, 0, sizeof(req)); + req.init_admin_cmd_id = FW_INIT_ME; + + return qat_adm_mailbox_send(sc, &req, &resp); +} + +int +qat_adm_mailbox_send_hb_timer(struct qat_softc *sc) +{ + struct fw_init_admin_req req; + struct fw_init_admin_resp resp; + + memset(&req, 0, sizeof(req)); + req.init_admin_cmd_id = FW_HEARTBEAT_TIMER_SET; + + req.init_cfg_ptr = sc->sc_admin_comms.qadc_hb_dma.qdm_dma_seg.ds_addr; + req.heartbeat_ticks = + sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_HB_INTERVAL; + + return qat_adm_mailbox_send(sc, &req, &resp); +} + +int +qat_adm_mailbox_send_fw_status(struct qat_softc *sc) +{ + int error; + struct fw_init_admin_req req; + struct fw_init_admin_resp resp; + + memset(&req, 0, sizeof(req)); + req.init_admin_cmd_id = FW_STATUS_GET; + + error = qat_adm_mailbox_send(sc, &req, &resp); + if (error) + return error; + + return 0; +} + +int +qat_adm_mailbox_send_constants(struct qat_softc *sc) +{ + struct fw_init_admin_req req; + struct fw_init_admin_resp resp; + + memset(&req, 0, sizeof(req)); + req.init_admin_cmd_id = FW_CONSTANTS_CFG; + + req.init_cfg_sz = 1024; + req.init_cfg_ptr = + sc->sc_admin_comms.qadc_const_tbl_dma.qdm_dma_seg.ds_addr; + + return qat_adm_mailbox_send(sc, &req, &resp); +} + +int +qat_adm_mailbox_send_init(struct qat_softc *sc) +{ + int error; + + error = qat_adm_mailbox_send_init_me(sc); + if (error) + return error; + + error = qat_adm_mailbox_send_hb_timer(sc); + if (error) + return error; + + error = qat_adm_mailbox_send_fw_status(sc); + if (error) + return error; + + return qat_adm_mailbox_send_constants(sc); +} + +int +qat_arb_init(struct qat_softc *sc) +{ + uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; + uint32_t arb, i; + const uint32_t *thd_2_arb_cfg; + + /* Service arb configured for 32 bytes responses and + * ring flow control check enabled. */ + for (arb = 0; arb < MAX_ARB; arb++) + qat_arb_sarconfig_write_4(sc, arb, arb_cfg); + + /* Map worker threads to service arbiters */ + sc->sc_hw.qhw_get_arb_mapping(sc, &thd_2_arb_cfg); + + if (!thd_2_arb_cfg) + return EINVAL; + + for (i = 0; i < sc->sc_hw.qhw_num_engines; i++) + qat_arb_wrk_2_ser_map_write_4(sc, i, *(thd_2_arb_cfg + i)); + + return 0; +} + +int +qat_set_ssm_wdtimer(struct qat_softc *sc) +{ + uint32_t timer; + u_int mask; + int i; + + timer = sc->sc_hw.qhw_clock_per_sec / 1000 * QAT_SSM_WDT; + for (i = 0, mask = sc->sc_accel_mask; mask; i++, mask >>= 1) { + if (!(mask & 1)) + continue; + qat_misc_write_4(sc, SSMWDT(i), timer); + qat_misc_write_4(sc, SSMWDTPKE(i), timer); + } + + return 0; +} + +int +qat_check_slice_hang(struct qat_softc *sc) +{ + int handled = 0; + + return handled; +} + +static uint32_t +qat_hw17_crypto_setup_cipher_ctrl(struct qat_crypto_desc *desc, + struct qat_session *qs, uint32_t cd_blk_offset, + struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice) +{ + struct fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = + (struct fw_cipher_cd_ctrl_hdr *)&req_tmpl->cd_ctrl; + + desc->qcd_cipher_blk_sz = HW_AES_BLK_SZ; + desc->qcd_cipher_offset = cd_blk_offset; + + cipher_cd_ctrl->cipher_state_sz = desc->qcd_cipher_blk_sz >> 3; + cipher_cd_ctrl->cipher_key_sz = qs->qs_cipher_klen >> 3; + cipher_cd_ctrl->cipher_cfg_offset = cd_blk_offset >> 3; + FW_COMN_CURR_ID_SET(cipher_cd_ctrl, FW_SLICE_CIPHER); + FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, next_slice); + + return roundup(sizeof(struct hw_cipher_config) + qs->qs_cipher_klen, 8); +} + +static void +qat_hw17_crypto_setup_cipher_cdesc(const struct qat_crypto_desc *desc, + const struct qat_session *qs, const struct cryptop *crp, + union hw_cipher_algo_blk *cipher) +{ + const uint8_t *key; + + cipher->max.cipher_config.val = + qat_crypto_load_cipher_session(desc, qs); + if (crp != NULL && crp->crp_cipher_key != NULL) + key = crp->crp_cipher_key; + else + key = qs->qs_cipher_key; + memcpy(cipher->max.key, key, qs->qs_cipher_klen); +} + +static uint32_t +qat_hw17_crypto_setup_auth_ctrl(struct qat_crypto_desc *desc, + struct qat_session *qs, uint32_t cd_blk_offset, + struct fw_la_bulk_req *req_tmpl, enum fw_slice next_slice) +{ + struct fw_auth_cd_ctrl_hdr *auth_cd_ctrl = + (struct fw_auth_cd_ctrl_hdr *)&req_tmpl->cd_ctrl; + struct qat_sym_hash_def const *hash_def; + + (void)qat_crypto_load_auth_session(desc, qs, &hash_def); + + auth_cd_ctrl->hash_cfg_offset = cd_blk_offset >> 3; + auth_cd_ctrl->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED; + auth_cd_ctrl->inner_res_sz = hash_def->qshd_alg->qshai_digest_len; + auth_cd_ctrl->final_sz = hash_def->qshd_alg->qshai_sah->hashsize; + + auth_cd_ctrl->inner_state1_sz = + roundup(hash_def->qshd_qat->qshqi_state1_len, 8); + auth_cd_ctrl->inner_state2_sz = + roundup(hash_def->qshd_qat->qshqi_state2_len, 8); + auth_cd_ctrl->inner_state2_offset = + auth_cd_ctrl->hash_cfg_offset + + ((sizeof(struct hw_auth_setup) + + auth_cd_ctrl->inner_state1_sz) >> 3); + + FW_COMN_CURR_ID_SET(auth_cd_ctrl, FW_SLICE_AUTH); + FW_COMN_NEXT_ID_SET(auth_cd_ctrl, next_slice); + + desc->qcd_auth_sz = auth_cd_ctrl->final_sz; + desc->qcd_auth_offset = cd_blk_offset; + desc->qcd_gcm_aad_sz_offset1 = + cd_blk_offset + offsetof(union hw_auth_algo_blk, max.state1) + + auth_cd_ctrl->inner_state1_sz + AES_BLOCK_LEN; + + return roundup(auth_cd_ctrl->inner_state1_sz + + auth_cd_ctrl->inner_state2_sz + + sizeof(struct hw_auth_setup), 8); +} + +static void +qat_hw17_crypto_setup_auth_cdesc(const struct qat_crypto_desc *desc, + const struct qat_session *qs, const struct cryptop *crp, + union hw_auth_algo_blk *auth) +{ + struct qat_sym_hash_def const *hash_def; + uint8_t inner_state1_sz, *state1, *state2; + const uint8_t *key; + + auth->max.inner_setup.auth_config.config = + qat_crypto_load_auth_session(desc, qs, &hash_def); + auth->max.inner_setup.auth_counter.counter = + htobe32(hash_def->qshd_qat->qshqi_auth_counter); + inner_state1_sz = roundup(hash_def->qshd_qat->qshqi_state1_len, 8); + + state1 = auth->max.state1; + state2 = auth->max.state1 + inner_state1_sz; + switch (qs->qs_auth_algo) { + case HW_AUTH_ALGO_GALOIS_128: + key = NULL; + if (crp != NULL && crp->crp_cipher_key != NULL) + key = crp->crp_cipher_key; + else if (qs->qs_cipher_key != NULL) + key = qs->qs_cipher_key; + if (key != NULL) { + qat_crypto_gmac_precompute(desc, key, + qs->qs_cipher_klen, hash_def, state2); + } + break; + case HW_AUTH_ALGO_SHA1: + case HW_AUTH_ALGO_SHA256: + case HW_AUTH_ALGO_SHA384: + case HW_AUTH_ALGO_SHA512: + switch (qs->qs_auth_mode) { + case HW_AUTH_MODE0: + memcpy(state1, hash_def->qshd_alg->qshai_init_state, + inner_state1_sz); + /* Override for mode 0 hashes. */ + auth->max.inner_setup.auth_counter.counter = 0; + break; + case HW_AUTH_MODE1: + if (crp != NULL && crp->crp_auth_key != NULL) + key = crp->crp_auth_key; + else + key = qs->qs_auth_key; + if (key != NULL) { + qat_crypto_hmac_precompute(desc, key, + qs->qs_auth_klen, hash_def, state1, state2); + } + break; + default: + panic("%s: unhandled auth mode %d", __func__, + qs->qs_auth_mode); + } + break; + default: + panic("%s: unhandled auth algorithm %d", __func__, + qs->qs_auth_algo); + } +} + +static void +qat_hw17_init_comn_req_hdr(struct qat_crypto_desc *desc, + struct fw_la_bulk_req *req) +{ + union fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + struct fw_comn_req_hdr *req_hdr = &req->comn_hdr; + + req_hdr->service_cmd_id = desc->qcd_cmd_id; + req_hdr->hdr_flags = FW_COMN_VALID; + req_hdr->service_type = FW_COMN_REQ_CPM_FW_LA; + req_hdr->comn_req_flags = FW_COMN_FLAGS_BUILD( + COMN_CD_FLD_TYPE_64BIT_ADR, COMN_PTR_TYPE_SGL); + req_hdr->serv_specif_flags = 0; + cd_pars->s.content_desc_addr = desc->qcd_desc_paddr; +} + +void +qat_hw17_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs, + struct qat_crypto_desc *desc) +{ + union hw_cipher_algo_blk *cipher; + union hw_auth_algo_blk *auth; + struct fw_la_bulk_req *req_tmpl; + struct fw_comn_req_hdr *req_hdr; + uint32_t cd_blk_offset = 0; + int i; + uint8_t *cd_blk_ptr; + + req_tmpl = (struct fw_la_bulk_req *)desc->qcd_req_cache; + req_hdr = &req_tmpl->comn_hdr; + cd_blk_ptr = desc->qcd_content_desc; + + memset(req_tmpl, 0, sizeof(struct fw_la_bulk_req)); + qat_hw17_init_comn_req_hdr(desc, req_tmpl); + + for (i = 0; i < MAX_FW_SLICE; i++) { + switch (desc->qcd_slices[i]) { + case FW_SLICE_CIPHER: + cipher = (union hw_cipher_algo_blk *)(cd_blk_ptr + + cd_blk_offset); + cd_blk_offset += qat_hw17_crypto_setup_cipher_ctrl(desc, + qs, cd_blk_offset, req_tmpl, + desc->qcd_slices[i + 1]); + qat_hw17_crypto_setup_cipher_cdesc(desc, qs, NULL, + cipher); + break; + case FW_SLICE_AUTH: + auth = (union hw_auth_algo_blk *)(cd_blk_ptr + + cd_blk_offset); + cd_blk_offset += qat_hw17_crypto_setup_auth_ctrl(desc, + qs, cd_blk_offset, req_tmpl, + desc->qcd_slices[i + 1]); + qat_hw17_crypto_setup_auth_cdesc(desc, qs, NULL, auth); + req_hdr->serv_specif_flags |= FW_LA_RET_AUTH_RES; + break; + case FW_SLICE_DRAM_WR: + i = MAX_FW_SLICE; /* end of chain */ + break; + default: + MPASS(0); + break; + } + } + + req_tmpl->cd_pars.s.content_desc_params_sz = + roundup(cd_blk_offset, QAT_OPTIMAL_ALIGN) >> 3; + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) + req_hdr->serv_specif_flags |= + FW_LA_PROTO_GCM | FW_LA_GCM_IV_LEN_12_OCTETS; + + bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, + qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); +} + +static void +qat_hw17_crypto_req_setkey(const struct qat_crypto_desc *desc, + const struct qat_session *qs, struct qat_sym_cookie *qsc, + struct fw_la_bulk_req *bulk_req, const struct cryptop *crp) +{ + union hw_auth_algo_blk *auth; + union hw_cipher_algo_blk *cipher; + uint8_t *cdesc; + int i; + + cdesc = qsc->qsc_content_desc; + memcpy(cdesc, desc->qcd_content_desc, CONTENT_DESC_MAX_SIZE); + for (i = 0; i < MAX_FW_SLICE; i++) { + switch (desc->qcd_slices[i]) { + case FW_SLICE_CIPHER: + cipher = (union hw_cipher_algo_blk *) + (cdesc + desc->qcd_cipher_offset); + qat_hw17_crypto_setup_cipher_cdesc(desc, qs, crp, + cipher); + break; + case FW_SLICE_AUTH: + auth = (union hw_auth_algo_blk *) + (cdesc + desc->qcd_auth_offset); + qat_hw17_crypto_setup_auth_cdesc(desc, qs, crp, auth); + break; + case FW_SLICE_DRAM_WR: + i = MAX_FW_SLICE; /* end of chain */ + break; + default: + MPASS(0); + } + } + + bulk_req->cd_pars.s.content_desc_addr = qsc->qsc_content_desc_paddr; +} + +void +qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *qcb __unused, + struct qat_session *qs, const struct qat_crypto_desc *desc, + struct qat_sym_cookie *qsc, struct cryptop *crp) +{ + struct qat_sym_bulk_cookie *qsbc; + struct fw_la_bulk_req *bulk_req; + struct fw_la_cipher_req_params *cipher_param; + struct fw_la_auth_req_params *auth_param; + bus_addr_t digest_paddr; + uint32_t aad_sz, *aad_szp; + uint8_t *req_params_ptr; + enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id; + + qsbc = &qsc->u.qsc_bulk_cookie; + bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg; + + memcpy(bulk_req, desc->qcd_req_cache, sizeof(struct fw_la_bulk_req)); + bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc; + bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr; + bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr; + if (__predict_false(crp->crp_cipher_key != NULL || + crp->crp_auth_key != NULL)) + qat_hw17_crypto_req_setkey(desc, qs, qsc, bulk_req, crp); + + digest_paddr = 0; + if (desc->qcd_auth_sz != 0) + digest_paddr = qsc->qsc_auth_res_paddr; + + req_params_ptr = (uint8_t *)&bulk_req->serv_specif_rqpars; + cipher_param = (struct fw_la_cipher_req_params *)req_params_ptr; + auth_param = (struct fw_la_auth_req_params *) + (req_params_ptr + sizeof(struct fw_la_cipher_req_params)); + + cipher_param->u.s.cipher_IV_ptr = qsc->qsc_iv_buf_paddr; + + /* + * The SG list layout is a bit different for GCM and GMAC, it's simpler + * to handle those cases separately. + */ + if (qs->qs_auth_algo == HW_AUTH_ALGO_GALOIS_128) { + if (cmd_id != FW_LA_CMD_AUTH) { + /* + * Don't fill out the cipher block if we're doing GMAC + * only. + */ + cipher_param->cipher_offset = 0; + cipher_param->cipher_length = crp->crp_payload_length; + } + auth_param->auth_off = 0; + auth_param->auth_len = crp->crp_payload_length; + auth_param->auth_res_addr = digest_paddr; + auth_param->auth_res_sz = desc->qcd_auth_sz; + auth_param->u1.aad_adr = + crp->crp_aad_length > 0 ? qsc->qsc_gcm_aad_paddr : 0; + auth_param->u2.aad_sz = + roundup2(crp->crp_aad_length, QAT_AES_GCM_AAD_ALIGN); + auth_param->hash_state_sz = auth_param->u2.aad_sz >> 3; + + /* + * Update the hash state block if necessary. This only occurs + * when the AAD length changes between requests in a session and + * is synchronized by qat_process(). + */ + aad_sz = htobe32(crp->crp_aad_length); + aad_szp = (uint32_t *)( + __DECONST(uint8_t *, desc->qcd_content_desc) + + desc->qcd_gcm_aad_sz_offset1); + if (__predict_false(*aad_szp != aad_sz)) { + *aad_szp = aad_sz; + bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, + qs->qs_desc_mem.qdm_dma_map, + BUS_DMASYNC_PREWRITE); + } + } else { + if (cmd_id != FW_LA_CMD_AUTH) { + cipher_param->cipher_offset = + crp->crp_aad_length == 0 ? 0 : + crp->crp_payload_start - crp->crp_aad_start; + cipher_param->cipher_length = crp->crp_payload_length; + } + if (cmd_id != FW_LA_CMD_CIPHER) { + auth_param->auth_off = 0; + auth_param->auth_len = + crp->crp_payload_length + crp->crp_aad_length; + auth_param->auth_res_addr = digest_paddr; + auth_param->auth_res_sz = desc->qcd_auth_sz; + auth_param->u1.aad_adr = 0; + auth_param->u2.aad_sz = 0; + auth_param->hash_state_sz = 0; + } + } +} Index: head/sys/dev/qat/qat_hw17reg.h =================================================================== --- head/sys/dev/qat/qat_hw17reg.h +++ head/sys/dev/qat/qat_hw17reg.h @@ -0,0 +1,2460 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw17reg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_HW17REG_H_ +#define _DEV_PCI_QAT_HW17REG_H_ + +/* Default message size in bytes */ +#define FW_REQ_DEFAULT_SZ_HW17 128 +#define FW_RESP_DEFAULT_SZ_HW17 32 + +/* -------------------------------------------------------------------------- */ +/* accel */ + +enum fw_init_admin_cmd_id { + FW_INIT_ME = 0, + FW_TRNG_ENABLE = 1, + FW_TRNG_DISABLE = 2, + FW_CONSTANTS_CFG = 3, + FW_STATUS_GET = 4, + FW_COUNTERS_GET = 5, + FW_LOOPBACK = 6, + FW_HEARTBEAT_SYNC = 7, + FW_HEARTBEAT_GET = 8, + FW_COMP_CAPABILITY_GET = 9, + FW_CRYPTO_CAPABILITY_GET = 10, + FW_HEARTBEAT_TIMER_SET = 13, +}; + +enum fw_init_admin_resp_status { + FW_INIT_RESP_STATUS_SUCCESS = 0, + FW_INIT_RESP_STATUS_FAIL = 1, + FW_INIT_RESP_STATUS_UNSUPPORTED = 4 +}; + +struct fw_init_admin_req { + uint16_t init_cfg_sz; + uint8_t resrvd1; + uint8_t init_admin_cmd_id; + uint32_t resrvd2; + uint64_t opaque_data; + uint64_t init_cfg_ptr; + + union { + struct { + uint16_t ibuf_size_in_kb; + uint16_t resrvd3; + }; + uint32_t heartbeat_ticks; + }; + + uint32_t resrvd4; +}; + +struct fw_init_admin_resp_hdr { + uint8_t flags; + uint8_t resrvd1; + uint8_t status; + uint8_t init_admin_cmd_id; +}; + +enum fw_init_admin_init_flag { + FW_INIT_FLAG_PKE_DISABLED = 0 +}; + +struct fw_init_admin_fw_capability_resp_hdr { + uint16_t reserved; + uint8_t status; + uint8_t init_admin_cmd_id; +}; + +struct fw_init_admin_capability_resp { + struct fw_init_admin_fw_capability_resp_hdr init_resp_hdr; + uint32_t extended_features; + uint64_t opaque_data; + union { + struct { + uint16_t compression_algos; + uint16_t checksum_algos; + uint32_t deflate_capabilities; + uint32_t resrvd1; + uint32_t lzs_capabilities; + } compression; + struct { + uint32_t cipher_algos; + uint32_t hash_algos; + uint16_t keygen_algos; + uint16_t other; + uint16_t public_key_algos; + uint16_t prime_algos; + } crypto; + }; +}; + +struct fw_init_admin_resp_pars { + union { + uint32_t resrvd1[4]; + struct { + uint32_t version_patch_num; + uint8_t context_id; + uint8_t ae_id; + uint16_t resrvd1; + uint64_t resrvd2; + } s1; + struct { + uint64_t req_rec_count; + uint64_t resp_sent_count; + } s2; + } u; +}; + +struct fw_init_admin_hb_cnt { + uint16_t resp_heartbeat_cnt; + uint16_t req_heartbeat_cnt; +}; + +#define QAT_NUM_THREADS 8 + +struct fw_init_admin_hb_stats { + struct fw_init_admin_hb_cnt stats[QAT_NUM_THREADS]; +}; + +struct fw_init_admin_resp { + struct fw_init_admin_resp_hdr init_resp_hdr; + union { + uint32_t resrvd2; + struct { + uint16_t version_minor_num; + uint16_t version_major_num; + } s; + } u; + uint64_t opaque_data; + struct fw_init_admin_resp_pars init_resp_pars; +}; + +#define FW_COMN_HEARTBEAT_OK 0 +#define FW_COMN_HEARTBEAT_BLOCKED 1 +#define FW_COMN_HEARTBEAT_FLAG_BITPOS 0 +#define FW_COMN_HEARTBEAT_FLAG_MASK 0x1 +#define FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE +#define FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \ + FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags) + +#define FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \ + FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val) + +#define FW_COMN_HEARTBEAT_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, \ + FW_COMN_HEARTBEAT_FLAG_BITPOS, \ + FW_COMN_HEARTBEAT_FLAG_MASK) + +/* -------------------------------------------------------------------------- */ + +/* Big assumptions that both bitpos and mask are constants */ +#define FIELD_SET(flags, val, bitpos, mask) \ + (flags) = \ + (((flags) & (~((mask) << (bitpos)))) | (((val) & (mask)) << (bitpos))) + +#define FIELD_GET(flags, bitpos, mask) (((flags) >> (bitpos)) & (mask)) + +#define FLAG_SET(flags, bitpos) (flags) = ((flags) | (1 << (bitpos))) + +#define FLAG_CLEAR(flags, bitpos) (flags) = ((flags) & (~(1 << (bitpos)))) + +#define FLAG_GET(flags, bitpos) (((flags) >> (bitpos)) & 1) + +/* Default request and response ring size in bytes */ +#define FW_REQ_DEFAULT_SZ 128 +#define FW_RESP_DEFAULT_SZ 32 + +#define FW_COMN_ONE_BYTE_SHIFT 8 +#define FW_COMN_SINGLE_BYTE_MASK 0xFF + +/* Common Request - Block sizes definitions in multiples of individual long + * words */ +#define FW_NUM_LONGWORDS_1 1 +#define FW_NUM_LONGWORDS_2 2 +#define FW_NUM_LONGWORDS_3 3 +#define FW_NUM_LONGWORDS_4 4 +#define FW_NUM_LONGWORDS_5 5 +#define FW_NUM_LONGWORDS_6 6 +#define FW_NUM_LONGWORDS_7 7 +#define FW_NUM_LONGWORDS_10 10 +#define FW_NUM_LONGWORDS_13 13 + +/* Definition of the associated service Id for NULL service type. + Note: the response is expected to use FW_COMN_RESP_SERV_CPM_FW */ +#define FW_NULL_REQ_SERV_ID 1 + +/* + * Definition of the firmware interface service users, for + * responses. + * Enumeration which is used to indicate the ids of the services + * for responses using the external firmware interfaces. + */ + +enum fw_comn_resp_serv_id { + FW_COMN_RESP_SERV_NULL, /* NULL service id type */ + FW_COMN_RESP_SERV_CPM_FW, /* CPM FW Service ID */ + FW_COMN_RESP_SERV_DELIMITER /* Delimiter service id type */ +}; + +/* + * Definition of the request types + * Enumeration which is used to indicate the ids of the request + * types used in each of the external firmware interfaces + */ + +enum fw_comn_request_id { + FW_COMN_REQ_NULL = 0, /* NULL request type */ + FW_COMN_REQ_CPM_FW_PKE = 3, /* CPM FW PKE Request */ + FW_COMN_REQ_CPM_FW_LA = 4, /* CPM FW Lookaside Request */ + FW_COMN_REQ_CPM_FW_DMA = 7, /* CPM FW DMA Request */ + FW_COMN_REQ_CPM_FW_COMP = 9, /* CPM FW Compression Request */ + FW_COMN_REQ_DELIMITER /* End delimiter */ + +}; + +/* + * Definition of the common QAT FW request content descriptor field - + * points to the content descriptor parameters or itself contains service- + * specific data. Also specifies content descriptor parameter size. + * Contains reserved fields. + * Common section of the request used across all of the services exposed + * by the QAT FW. Each of the services inherit these common fields + */ +union fw_comn_req_hdr_cd_pars { + /* LWs 2-5 */ + struct + { + uint64_t content_desc_addr; + /* Address of the content descriptor */ + + uint16_t content_desc_resrvd1; + /* Content descriptor reserved field */ + + uint8_t content_desc_params_sz; + /* Size of the content descriptor parameters in quad words. These + * parameters describe the session setup configuration info for the + * slices that this request relies upon i.e. the configuration word and + * cipher key needed by the cipher slice if there is a request for + * cipher processing. */ + + uint8_t content_desc_hdr_resrvd2; + /* Content descriptor reserved field */ + + uint32_t content_desc_resrvd3; + /* Content descriptor reserved field */ + } s; + + struct + { + uint32_t serv_specif_fields[FW_NUM_LONGWORDS_4]; + + } s1; + +}; + +/* + * Definition of the common QAT FW request middle block. + * Common section of the request used across all of the services exposed + * by the QAT FW. Each of the services inherit these common fields + */ +struct fw_comn_req_mid +{ + /* LWs 6-13 */ + uint64_t opaque_data; + /* Opaque data passed unmodified from the request to response messages by + * firmware (fw) */ + + uint64_t src_data_addr; + /* Generic definition of the source data supplied to the QAT AE. The + * common flags are used to further describe the attributes of this + * field */ + + uint64_t dest_data_addr; + /* Generic definition of the destination data supplied to the QAT AE. The + * common flags are used to further describe the attributes of this + * field */ + + uint32_t src_length; + /* Length of source flat buffer incase src buffer + * type is flat */ + + uint32_t dst_length; + /* Length of source flat buffer incase dst buffer + * type is flat */ + +}; + +/* + * Definition of the common QAT FW request content descriptor control + * block. + * + * Service specific section of the request used across all of the services + * exposed by the QAT FW. Each of the services populates this block + * uniquely. Refer to the service-specific header structures e.g. + * 'fw_cipher_hdr_s' (for Cipher) etc. + */ +struct fw_comn_req_cd_ctrl +{ + /* LWs 27-31 */ + uint32_t content_desc_ctrl_lw[FW_NUM_LONGWORDS_5]; + +}; + +/* + * Definition of the common QAT FW request header. + * Common section of the request used across all of the services exposed + * by the QAT FW. Each of the services inherit these common fields. The + * reserved field of 7 bits and the service command Id field are all + * service-specific fields, along with the service specific flags. + */ +struct fw_comn_req_hdr +{ + /* LW0 */ + uint8_t resrvd1; + /* reserved field */ + + uint8_t service_cmd_id; + /* Service Command Id - this field is service-specific + * Please use service-specific command Id here e.g.Crypto Command Id + * or Compression Command Id etc. */ + + uint8_t service_type; + /* Service type */ + + uint8_t hdr_flags; + /* This represents a flags field for the Service Request. + * The most significant bit is the 'valid' flag and the only + * one used. All remaining bit positions are unused and + * are therefore reserved and need to be set to 0. */ + + /* LW1 */ + uint16_t serv_specif_flags; + /* Common Request service-specific flags + * e.g. Symmetric Crypto Command Flags */ + + uint16_t comn_req_flags; + /* Common Request Flags consisting of + * - 14 reserved bits, + * - 1 Content Descriptor field type bit and + * - 1 Source/destination pointer type bit */ + +}; + +/* + * Definition of the common QAT FW request parameter field. + * + * Service specific section of the request used across all of the services + * exposed by the QAT FW. Each of the services populates this block + * uniquely. Refer to service-specific header structures e.g. + * 'fw_comn_req_cipher_rqpars_s' (for Cipher) etc. + * + */ +struct fw_comn_req_rqpars +{ + /* LWs 14-26 */ + uint32_t serv_specif_rqpars_lw[FW_NUM_LONGWORDS_13]; + +}; + +/* + * Definition of the common request structure with service specific + * fields + * This is a definition of the full qat request structure used by all + * services. Each service is free to use the service fields in its own + * way. This struct is useful as a message passing argument before the + * service contained within the request is determined. + */ +struct fw_comn_req +{ + /* LWs 0-1 */ + struct fw_comn_req_hdr comn_hdr; + /* Common request header */ + + /* LWs 2-5 */ + union fw_comn_req_hdr_cd_pars cd_pars; + /* Common Request content descriptor field which points either to a + * content descriptor + * parameter block or contains the service-specific data itself. */ + + /* LWs 6-13 */ + struct fw_comn_req_mid comn_mid; + /* Common request middle section */ + + /* LWs 14-26 */ + struct fw_comn_req_rqpars serv_specif_rqpars; + /* Common request service-specific parameter field */ + + /* LWs 27-31 */ + struct fw_comn_req_cd_ctrl cd_ctrl; + /* Common request content descriptor control block - + * this field is service-specific */ + +}; + +/* + * Error code field + * + * Overloaded field with 8 bit common error field or two + * 8 bit compression error fields for compression and translator slices + */ +union fw_comn_error { + struct + { + uint8_t resrvd; + /* 8 bit reserved field */ + + uint8_t comn_err_code; + /* 8 bit common error code */ + + } s; + /* Structure which is used for non-compression responses */ + + struct + { + uint8_t xlat_err_code; + /* 8 bit translator error field */ + + uint8_t cmp_err_code; + /* 8 bit compression error field */ + + } s1; + /* Structure which is used for compression responses */ + +}; + +/* + * Definition of the common QAT FW response header. + * This section of the response is common across all of the services + * that generate a firmware interface response + */ +struct fw_comn_resp_hdr +{ + /* LW0 */ + uint8_t resrvd1; + /* Reserved field - this field is service-specific - + * Note: The Response Destination Id has been removed + * from first QWord */ + + uint8_t service_id; + /* Service Id returned by service block */ + + uint8_t response_type; + /* Response type - copied from the request to + * the response message */ + + uint8_t hdr_flags; + /* This represents a flags field for the Response. + * Bit<7> = 'valid' flag + * Bit<6> = 'CNV' flag indicating that CNV was executed + * on the current request + * Bit<5> = 'CNVNR' flag indicating that a recovery happened + * on the current request following a CNV error + * All remaining bits are unused and are therefore reserved. + * They must to be set to 0. + */ + + /* LW 1 */ + union fw_comn_error comn_error; + /* This field is overloaded to allow for one 8 bit common error field + * or two 8 bit error fields from compression and translator */ + + uint8_t comn_status; + /* Status field which specifies which slice(s) report an error */ + + uint8_t cmd_id; + /* Command Id - passed from the request to the response message */ + +}; + +/* + * Definition of the common response structure with service specific + * fields + * This is a definition of the full qat response structure used by all + * services. + */ +struct fw_comn_resp +{ + /* LWs 0-1 */ + struct fw_comn_resp_hdr comn_hdr; + /* Common header fields */ + + /* LWs 2-3 */ + uint64_t opaque_data; + /* Opaque data passed from the request to the response message */ + + /* LWs 4-7 */ + uint32_t resrvd[FW_NUM_LONGWORDS_4]; + /* Reserved */ + +}; + +/* Common QAT FW request header - structure of LW0 + * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + + * | Bit | 31 | 30 - 24 | 21 - 16 | 15 - 8 | 7 - 0 | + * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + + * | Flags | V | Reserved | Serv Type | Serv Cmd Id | Reserved | + * + ===== + ---- + ----------- + ----------- + ----------- + ----------- + + */ + +#define FW_COMN_VALID __BIT(7) + +/* Common QAT FW response header - structure of LW0 + * + ===== + --- + --- + ----- + ----- + --------- + ----------- + ----- + + * | Bit | 31 | 30 | 29 | 28-24 | 21 - 16 | 15 - 8 | 7-0 | + * + ===== + --- + ----+ ----- + ----- + --------- + ----------- + ----- + + * | Flags | V | CNV | CNVNR | Rsvd | Serv Type | Serv Cmd Id | Rsvd | + * + ===== + --- + --- + ----- + ----- + --------- + ----------- + ----- + */ +/* Macros defining the bit position and mask of 'CNV' flag + * within the hdr_flags field of LW0 (service response only) */ +#define FW_COMN_CNV_FLAG_BITPOS 6 +#define FW_COMN_CNV_FLAG_MASK 0x1 + +/* Macros defining the bit position and mask of CNVNR flag + * within the hdr_flags field of LW0 (service response only) */ +#define FW_COMN_CNVNR_FLAG_BITPOS 5 +#define FW_COMN_CNVNR_FLAG_MASK 0x1 + +/* + * Macro for extraction of Service Type Field + * + * struct fw_comn_req_hdr Structure 'fw_comn_req_hdr_t' + * to extract the Service Type Field + */ +#define FW_COMN_OV_SRV_TYPE_GET(fw_comn_req_hdr_t) \ + fw_comn_req_hdr_t.service_type + +/* + * Macro for setting of Service Type Field + * + * 'fw_comn_req_hdr_t' structure to set the Service + * Type Field + * val Value of the Service Type Field + */ +#define FW_COMN_OV_SRV_TYPE_SET(fw_comn_req_hdr_t, val) \ + fw_comn_req_hdr_t.service_type = val + +/* + * Macro for extraction of Service Command Id Field + * + * struct fw_comn_req_hdr Structure 'fw_comn_req_hdr_t' + * to extract the Service Command Id Field + */ +#define FW_COMN_OV_SRV_CMD_ID_GET(fw_comn_req_hdr_t) \ + fw_comn_req_hdr_t.service_cmd_id + +/* + * Macro for setting of Service Command Id Field + * + * 'fw_comn_req_hdr_t' structure to set the + * Service Command Id Field + * val Value of the Service Command Id Field + */ +#define FW_COMN_OV_SRV_CMD_ID_SET(fw_comn_req_hdr_t, val) \ + fw_comn_req_hdr_t.service_cmd_id = val + +/* + * Extract the valid flag from the request or response's header flags. + * + * hdr_t Request or Response 'hdr_t' structure to extract the valid bit + * from the 'hdr_flags' field. + */ +#define FW_COMN_HDR_VALID_FLAG_GET(hdr_t) \ + FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags) + +/* + * Extract the CNVNR flag from the header flags in the response only. + * + * hdr_t Response 'hdr_t' structure to extract the CNVNR bit + * from the 'hdr_flags' field. + */ +#define FW_COMN_HDR_CNVNR_FLAG_GET(hdr_flags) \ + FIELD_GET(hdr_flags, \ + FW_COMN_CNVNR_FLAG_BITPOS, \ + FW_COMN_CNVNR_FLAG_MASK) + +/* + * Extract the CNV flag from the header flags in the response only. + * + * hdr_t Response 'hdr_t' structure to extract the CNV bit + * from the 'hdr_flags' field. + */ +#define FW_COMN_HDR_CNV_FLAG_GET(hdr_flags) \ + FIELD_GET(hdr_flags, \ + FW_COMN_CNV_FLAG_BITPOS, \ + FW_COMN_CNV_FLAG_MASK) + +/* + * Set the valid bit in the request's header flags. + * + * hdr_t Request or Response 'hdr_t' structure to set the valid bit + * val Value of the valid bit flag. + */ +#define FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val) \ + FW_COMN_VALID_FLAG_SET(hdr_t, val) + +/* + * Common macro to extract the valid flag from the header flags field + * within the header structure (request or response). + * + * hdr_t Structure (request or response) to extract the + * valid bit from the 'hdr_flags' field. + */ +#define FW_COMN_VALID_FLAG_GET(hdr_flags) \ + FIELD_GET(hdr_flags, \ + FW_COMN_VALID_FLAG_BITPOS, \ + FW_COMN_VALID_FLAG_MASK) + +/* + * Common macro to extract the remaining reserved flags from the header + * flags field within the header structure (request or response). + * + * hdr_t Structure (request or response) to extract the + * remaining bits from the 'hdr_flags' field (excluding the + * valid flag). + */ +#define FW_COMN_HDR_RESRVD_FLD_GET(hdr_flags) \ + (hdr_flags & FW_COMN_HDR_RESRVD_FLD_MASK) + +/* + * Common macro to set the valid bit in the header flags field within + * the header structure (request or response). + * + * hdr_t Structure (request or response) containing the header + * flags field, to allow the valid bit to be set. + * val Value of the valid bit flag. + */ +#define FW_COMN_VALID_FLAG_SET(hdr_t, val) \ + FIELD_SET((hdr_t.hdr_flags), \ + (val), \ + FW_COMN_VALID_FLAG_BITPOS, \ + FW_COMN_VALID_FLAG_MASK) + +/* + * Macro that must be used when building the common header flags. + * Note that all bits reserved field bits 0-6 (LW0) need to be forced to 0. + * + * ptr Value of the valid flag + */ + +#define FW_COMN_HDR_FLAGS_BUILD(valid) \ + (((valid)&FW_COMN_VALID_FLAG_MASK) \ + << FW_COMN_VALID_FLAG_BITPOS) + +/* + * Common Request Flags Definition + * The bit offsets below are within the flags field. These are NOT relative to + * the memory word. Unused fields e.g. reserved bits, must be zeroed. + * + * + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- + + * | Bits [15:8] | 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | + * + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- + + * | Flags[15:8] | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv | Rsv | + * + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- + + * | Bits [7:0] | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | + * + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- + + * | Flags [7:0] | Rsv | Rsv | Rsv | Rsv | Rsv | BnP | Cdt | Ptr | + * + ===== + ------ + --- + --- + --- + --- + --- + --- + --- + --- + + */ + +#define COMN_PTR_TYPE_BITPOS 0 +/* Common Request Flags - Starting bit position indicating + * Src&Dst Buffer Pointer type */ + +#define COMN_PTR_TYPE_MASK 0x1 +/* Common Request Flags - One bit mask used to determine + * Src&Dst Buffer Pointer type */ + +#define COMN_CD_FLD_TYPE_BITPOS 1 +/* Common Request Flags - Starting bit position indicating + * CD Field type */ + +#define COMN_CD_FLD_TYPE_MASK 0x1 +/* Common Request Flags - One bit mask used to determine + * CD Field type */ + +#define COMN_BNP_ENABLED_BITPOS 2 +/* Common Request Flags - Starting bit position indicating + * the source buffer contains batch of requests. if this + * bit is set, source buffer is type of Batch And Pack OpData List + * and the Ptr Type Bit only applies to Destination buffer. */ + +#define COMN_BNP_ENABLED_MASK 0x1 +/* Batch And Pack Enabled Flag Mask - One bit mask used to determine + * the source buffer is in Batch and Pack OpData Link List Mode. */ + +/* ========================================================================= */ +/* Pointer Type Flag definitions */ +/* ========================================================================= */ +#define COMN_PTR_TYPE_FLAT 0x0 +/* Constant value indicating Src&Dst Buffer Pointer type is flat + * If Batch and Pack mode is enabled, only applies to Destination buffer. */ + +#define COMN_PTR_TYPE_SGL 0x1 +/* Constant value indicating Src&Dst Buffer Pointer type is SGL type + * If Batch and Pack mode is enabled, only applies to Destination buffer. */ + +#define COMN_PTR_TYPE_BATCH 0x2 +/* Constant value indicating Src is a batch request + * and Dst Buffer Pointer type is SGL type */ + +/* ========================================================================= */ +/* CD Field Flag definitions */ +/* ========================================================================= */ +#define COMN_CD_FLD_TYPE_64BIT_ADR 0x0 +/* Constant value indicating CD Field contains 64-bit address */ + +#define COMN_CD_FLD_TYPE_16BYTE_DATA 0x1 +/* Constant value indicating CD Field contains 16 bytes of setup data */ + +/* ========================================================================= */ +/* Batch And Pack Enable/Disable Definitions */ +/* ========================================================================= */ +#define COMN_BNP_ENABLED 0x1 +/* Constant value indicating Source buffer will point to Batch And Pack OpData + * List */ + +#define COMN_BNP_DISABLED 0x0 +/* Constant value indicating Source buffer will point to Batch And Pack OpData + * List */ + +/* + * Macro that must be used when building the common request flags (for all + * requests but comp BnP). + * Note that all bits reserved field bits 2-15 (LW1) need to be forced to 0. + * + * ptr Value of the pointer type flag + * cdt Value of the cd field type flag +*/ +#define FW_COMN_FLAGS_BUILD(cdt, ptr) \ + ((((cdt)&COMN_CD_FLD_TYPE_MASK) << COMN_CD_FLD_TYPE_BITPOS) | \ + (((ptr)&COMN_PTR_TYPE_MASK) << COMN_PTR_TYPE_BITPOS)) + +/* + * Macro that must be used when building the common request flags for comp + * BnP service. + * Note that all bits reserved field bits 3-15 (LW1) need to be forced to 0. + * + * ptr Value of the pointer type flag + * cdt Value of the cd field type flag + * bnp Value of the bnp enabled flag + */ +#define FW_COMN_FLAGS_BUILD_BNP(cdt, ptr, bnp) \ + ((((cdt)&COMN_CD_FLD_TYPE_MASK) << COMN_CD_FLD_TYPE_BITPOS) | \ + (((ptr)&COMN_PTR_TYPE_MASK) << COMN_PTR_TYPE_BITPOS) | \ + (((bnp)&COMN_BNP_ENABLED_MASK) << COMN_BNP_ENABLED_BITPOS)) + +/* + * Macro for extraction of the pointer type bit from the common flags + * + * flags Flags to extract the pointer type bit from + */ +#define FW_COMN_PTR_TYPE_GET(flags) \ + FIELD_GET(flags, COMN_PTR_TYPE_BITPOS, COMN_PTR_TYPE_MASK) + +/* + * Macro for extraction of the cd field type bit from the common flags + * + * flags Flags to extract the cd field type type bit from + */ +#define FW_COMN_CD_FLD_TYPE_GET(flags) \ + FIELD_GET(flags, COMN_CD_FLD_TYPE_BITPOS, COMN_CD_FLD_TYPE_MASK) + +/* + * Macro for extraction of the bnp field type bit from the common flags + * + * flags Flags to extract the bnp field type type bit from + * + */ +#define FW_COMN_BNP_ENABLED_GET(flags) \ + FIELD_GET(flags, COMN_BNP_ENABLED_BITPOS, COMN_BNP_ENABLED_MASK) + +/* + * Macro for setting the pointer type bit in the common flags + * + * flags Flags in which Pointer Type bit will be set + * val Value of the bit to be set in flags + * + */ +#define FW_COMN_PTR_TYPE_SET(flags, val) \ + FIELD_SET(flags, val, COMN_PTR_TYPE_BITPOS, COMN_PTR_TYPE_MASK) + +/* + * Macro for setting the cd field type bit in the common flags + * + * flags Flags in which Cd Field Type bit will be set + * val Value of the bit to be set in flags + * + */ +#define FW_COMN_CD_FLD_TYPE_SET(flags, val) \ + FIELD_SET( \ + flags, val, COMN_CD_FLD_TYPE_BITPOS, COMN_CD_FLD_TYPE_MASK) + +/* + * Macro for setting the bnp field type bit in the common flags + * + * flags Flags in which Bnp Field Type bit will be set + * val Value of the bit to be set in flags + * + */ +#define FW_COMN_BNP_ENABLE_SET(flags, val) \ + FIELD_SET( \ + flags, val, COMN_BNP_ENABLED_BITPOS, COMN_BNP_ENABLED_MASK) + +/* + * Macros using the bit position and mask to set/extract the next + * and current id nibbles within the next_curr_id field of the + * content descriptor header block. Note that these are defined + * in the common header file, as they are used by compression, cipher + * and authentication. + * + * cd_ctrl_hdr_t Content descriptor control block header pointer. + * val Value of the field being set. + */ +#define FW_COMN_NEXT_ID_BITPOS 4 +#define FW_COMN_NEXT_ID_MASK 0xF0 +#define FW_COMN_CURR_ID_BITPOS 0 +#define FW_COMN_CURR_ID_MASK 0x0F + +#define FW_COMN_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_NEXT_ID_MASK) >> \ + (FW_COMN_NEXT_ID_BITPOS)) + +#define FW_COMN_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ + ((cd_ctrl_hdr_t)->next_curr_id) = \ + ((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_CURR_ID_MASK) | \ + ((val << FW_COMN_NEXT_ID_BITPOS) & \ + FW_COMN_NEXT_ID_MASK)) + +#define FW_COMN_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_CURR_ID_MASK) + +#define FW_COMN_CURR_ID_SET(cd_ctrl_hdr_t, val) \ + ((cd_ctrl_hdr_t)->next_curr_id) = \ + ((((cd_ctrl_hdr_t)->next_curr_id) & FW_COMN_NEXT_ID_MASK) | \ + ((val)&FW_COMN_CURR_ID_MASK)) + +/* + * Common Status Field Definition The bit offsets below are within the COMMON + * RESPONSE status field, assumed to be 8 bits wide. In the case of the PKE + * response (which follows the CPM 1.5 message format), the status field is 16 + * bits wide. + * The status flags are contained within the most significant byte and align + * with the diagram below. Please therefore refer to the service-specific PKE + * header file for the appropriate macro definition to extract the PKE status + * flag from the PKE response, which assumes that a word is passed to the + * macro. + * + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- + + * | Bit | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | + * + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- + + * | Flags | Crypto | Pke | Cmp | Xlat | EOLB | UnSupReq | Rsvd | XltWaApply | + * + ===== + ------ + --- + --- + ---- + ---- + -------- + ---- + ---------- + + * Note: + * For the service specific status bit definitions refer to service header files + * Eg. Crypto Status bit refers to Symmetric Crypto, Key Generation, and NRBG + * Requests' Status. Unused bits e.g. reserved bits need to have been forced to + * 0. + */ + +#define COMN_RESP_CRYPTO_STATUS_BITPOS 7 +/* Starting bit position indicating Response for Crypto service Flag */ + +#define COMN_RESP_CRYPTO_STATUS_MASK 0x1 +/* One bit mask used to determine Crypto status mask */ + +#define COMN_RESP_PKE_STATUS_BITPOS 6 +/* Starting bit position indicating Response for PKE service Flag */ + +#define COMN_RESP_PKE_STATUS_MASK 0x1 +/* One bit mask used to determine PKE status mask */ + +#define COMN_RESP_CMP_STATUS_BITPOS 5 +/* Starting bit position indicating Response for Compression service Flag */ + +#define COMN_RESP_CMP_STATUS_MASK 0x1 +/* One bit mask used to determine Compression status mask */ + +#define COMN_RESP_XLAT_STATUS_BITPOS 4 +/* Starting bit position indicating Response for Xlat service Flag */ + +#define COMN_RESP_XLAT_STATUS_MASK 0x1 +/* One bit mask used to determine Translator status mask */ + +#define COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3 +/* Starting bit position indicating the last block in a deflate stream for + the compression service Flag */ + +#define COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1 +/* One bit mask used to determine the last block in a deflate stream + status mask */ + +#define COMN_RESP_UNSUPPORTED_REQUEST_BITPOS 2 +/* Starting bit position indicating when an unsupported service request Flag */ + +#define COMN_RESP_UNSUPPORTED_REQUEST_MASK 0x1 +/* One bit mask used to determine the unsupported service request status mask */ + +#define COMN_RESP_XLT_WA_APPLIED_BITPOS 0 +/* Bit position indicating a firmware workaround was applied to translation */ + +#define COMN_RESP_XLT_WA_APPLIED_MASK 0x1 +/* One bit mask */ + +/* + * Macro that must be used when building the status + * for the common response + * + * crypto Value of the Crypto Service status flag + * comp Value of the Compression Service Status flag + * xlat Value of the Xlator Status flag + * eolb Value of the Compression End of Last Block Status flag + * unsupp Value of the Unsupported Request flag + * xlt_wa Value of the Translation WA marker + */ +#define FW_COMN_RESP_STATUS_BUILD( \ + crypto, pke, comp, xlat, eolb, unsupp, xlt_wa) \ + ((((crypto)&COMN_RESP_CRYPTO_STATUS_MASK) \ + << COMN_RESP_CRYPTO_STATUS_BITPOS) | \ + (((pke)&COMN_RESP_PKE_STATUS_MASK) \ + << COMN_RESP_PKE_STATUS_BITPOS) | \ + (((xlt_wa)&COMN_RESP_XLT_WA_APPLIED_MASK) \ + << COMN_RESP_XLT_WA_APPLIED_BITPOS) | \ + (((comp)&COMN_RESP_CMP_STATUS_MASK) \ + << COMN_RESP_CMP_STATUS_BITPOS) | \ + (((xlat)&COMN_RESP_XLAT_STATUS_MASK) \ + << COMN_RESP_XLAT_STATUS_BITPOS) | \ + (((eolb)&COMN_RESP_CMP_END_OF_LAST_BLK_MASK) \ + << COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS) | \ + (((unsupp)&COMN_RESP_UNSUPPORTED_REQUEST_BITPOS) \ + << COMN_RESP_UNSUPPORTED_REQUEST_MASK)) + +/* + * Macro for extraction of the Crypto bit from the status + * + * status Status to extract the status bit from + */ +#define FW_COMN_RESP_CRYPTO_STAT_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_CRYPTO_STATUS_BITPOS, \ + COMN_RESP_CRYPTO_STATUS_MASK) + +/* + * Macro for extraction of the PKE bit from the status + * + * status Status to extract the status bit from + */ +#define FW_COMN_RESP_PKE_STAT_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_PKE_STATUS_BITPOS, \ + COMN_RESP_PKE_STATUS_MASK) + +/* + * Macro for extraction of the Compression bit from the status + * + * status Status to extract the status bit from + */ +#define FW_COMN_RESP_CMP_STAT_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_CMP_STATUS_BITPOS, \ + COMN_RESP_CMP_STATUS_MASK) + +/* + * Macro for extraction of the Translator bit from the status + * + * status Status to extract the status bit from + */ +#define FW_COMN_RESP_XLAT_STAT_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_XLAT_STATUS_BITPOS, \ + COMN_RESP_XLAT_STATUS_MASK) + +/* + * Macro for extraction of the Translation Workaround Applied bit from the + * status + * + * status Status to extract the status bit from + */ +#define FW_COMN_RESP_XLT_WA_APPLIED_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_XLT_WA_APPLIED_BITPOS, \ + COMN_RESP_XLT_WA_APPLIED_MASK) + +/* + * Macro for extraction of the end of compression block bit from the + * status + * + * status + * Status to extract the status bit from + */ +#define FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS, \ + COMN_RESP_CMP_END_OF_LAST_BLK_MASK) + +/* + * Macro for extraction of the Unsupported request from the status + * + * status + * Status to extract the status bit from + */ +#define FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(status) \ + FIELD_GET(status, \ + COMN_RESP_UNSUPPORTED_REQUEST_BITPOS, \ + COMN_RESP_UNSUPPORTED_REQUEST_MASK) + +#define FW_COMN_STATUS_FLAG_OK 0 +/* Definition of successful processing of a request */ + +#define FW_COMN_STATUS_FLAG_ERROR 1 +/* Definition of erroneous processing of a request */ + +#define FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0 +/* Final Deflate block of a compression request not completed */ + +#define FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET 1 +/* Final Deflate block of a compression request completed */ + +#define ERR_CODE_NO_ERROR 0 +/* Error Code constant value for no error */ + +#define ERR_CODE_INVALID_BLOCK_TYPE -1 +/* Invalid block type (type == 3)*/ + +#define ERR_CODE_NO_MATCH_ONES_COMP -2 +/* Stored block length does not match one's complement */ + +#define ERR_CODE_TOO_MANY_LEN_OR_DIS -3 +/* Too many length or distance codes */ + +#define ERR_CODE_INCOMPLETE_LEN -4 +/* Code lengths codes incomplete */ + +#define ERR_CODE_RPT_LEN_NO_FIRST_LEN -5 +/* Repeat lengths with no first length */ + +#define ERR_CODE_RPT_GT_SPEC_LEN -6 +/* Repeat more than specified lengths */ + +#define ERR_CODE_INV_LIT_LEN_CODE_LEN -7 +/* Invalid lit/len code lengths */ + +#define ERR_CODE_INV_DIS_CODE_LEN -8 +/* Invalid distance code lengths */ + +#define ERR_CODE_INV_LIT_LEN_DIS_IN_BLK -9 +/* Invalid lit/len or distance code in fixed/dynamic block */ + +#define ERR_CODE_DIS_TOO_FAR_BACK -10 +/* Distance too far back in fixed or dynamic block */ + +/* Common Error code definitions */ +#define ERR_CODE_OVERFLOW_ERROR -11 +/* Error Code constant value for overflow error */ + +#define ERR_CODE_SOFT_ERROR -12 +/* Error Code constant value for soft error */ + +#define ERR_CODE_FATAL_ERROR -13 +/* Error Code constant value for hard/fatal error */ + +#define ERR_CODE_COMP_OUTPUT_CORRUPTION -14 +/* Error Code constant for compression output corruption */ + +#define ERR_CODE_HW_INCOMPLETE_FILE -15 +/* Error Code constant value for incomplete file hardware error */ + +#define ERR_CODE_SSM_ERROR -16 +/* Error Code constant value for error detected by SSM e.g. slice hang */ + +#define ERR_CODE_ENDPOINT_ERROR -17 +/* Error Code constant value for error detected by PCIe Endpoint, e.g. push + * data error */ + +#define ERR_CODE_CNV_ERROR -18 +/* Error Code constant value for cnv failure */ + +#define ERR_CODE_EMPTY_DYM_BLOCK -19 +/* Error Code constant value for submission of empty dynamic stored block to + * slice */ + +#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_HANDLE -20 +/* Error Code constant for invalid handle in kpt crypto service */ + +#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_HMAC_FAILED -21 +/* Error Code constant for failed hmac in kpt crypto service */ + +#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_WRAPPING_ALGO -22 +/* Error Code constant for invalid wrapping algo in kpt crypto service */ + +#define ERR_CODE_KPT_DRNG_SEED_NOT_LOAD -23 +/* Error Code constant for no drng seed is not loaded in kpt ecdsa signrs +/service */ + +#define FW_LA_ICV_VER_STATUS_PASS FW_COMN_STATUS_FLAG_OK +/* Status flag indicating that the ICV verification passed */ + +#define FW_LA_ICV_VER_STATUS_FAIL FW_COMN_STATUS_FLAG_ERROR +/* Status flag indicating that the ICV verification failed */ + +#define FW_LA_TRNG_STATUS_PASS FW_COMN_STATUS_FLAG_OK +/* Status flag indicating that the TRNG returned valid entropy data */ + +#define FW_LA_TRNG_STATUS_FAIL FW_COMN_STATUS_FLAG_ERROR +/* Status flag indicating that the TRNG Command Failed. */ + +/* -------------------------------------------------------------------------- */ + +/* + * Definition of the full bulk processing request structure. + * Used for hash, cipher, hash-cipher and authentication-encryption + * requests etc. + */ +struct fw_la_bulk_req +{ + /* LWs 0-1 */ + struct fw_comn_req_hdr comn_hdr; + /* Common request header - for Service Command Id, + * use service-specific Crypto Command Id. + * Service Specific Flags - use Symmetric Crypto Command Flags + * (all of cipher, auth, SSL3, TLS and MGF, + * excluding TRNG - field unused) */ + + /* LWs 2-5 */ + union fw_comn_req_hdr_cd_pars cd_pars; + /* Common Request content descriptor field which points either to a + * content descriptor + * parameter block or contains the service-specific data itself. */ + + /* LWs 6-13 */ + struct fw_comn_req_mid comn_mid; + /* Common request middle section */ + + /* LWs 14-26 */ + struct fw_comn_req_rqpars serv_specif_rqpars; + /* Common request service-specific parameter field */ + + /* LWs 27-31 */ + struct fw_comn_req_cd_ctrl cd_ctrl; + /* Common request content descriptor control block - + * this field is service-specific */ + +}; + +/* clang-format off */ + +/* + * LA BULK (SYMMETRIC CRYPTO) COMMAND FLAGS + * + * + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + + * | Bit | [15:13] | 12 | 11 | 10 | 7-9 | 6 | 5 | 4 | 3 | 2 | 1-0 | + * + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ------+ ----- + + * | Flags | Resvd Bits | ZUC | GcmIV |Digest | Prot | Cmp | Rtn | Upd | Ciph/ | CiphIV| Part- | + * | | =0 | Prot | Len | In Buf| flgs | Auth | Auth | State | Auth | Field | ial | + * + ===== + ---------- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ----- + ------+ ----- + + */ + +/* clang-format on */ + +/* Private defines */ + +#define FW_LA_ZUC_3G_PROTO __BIT(12) +/* Indicating ZUC processing for a encrypt command + * Must be set for Cipher-only, Cipher + Auth and Auth-only */ + +#define FW_LA_GCM_IV_LEN_12_OCTETS __BIT(11) +/* Indicates the IV Length for GCM protocol is 96 Bits (12 Octets) + * If set FW does the padding to compute CTR0 */ + +#define FW_LA_DIGEST_IN_BUFFER __BIT(10) +/* Flag representing that authentication digest is stored or is extracted + * from the source buffer. Auth Result Pointer will be ignored in this case. */ + +#define FW_LA_PROTO __BITS(7, 9) +#define FW_LA_PROTO_SNOW_3G __BIT(9) +/* Indicates SNOW_3G processing for a encrypt command */ +#define FW_LA_PROTO_GCM __BIT(8) +/* Indicates GCM processing for a auth_encrypt command */ +#define FW_LA_PROTO_CCM __BIT(7) +/* Indicates CCM processing for a auth_encrypt command */ +#define FW_LA_PROTO_NONE 0 +/* Indicates no specific protocol processing for the command */ + +#define FW_LA_CMP_AUTH_RES __BIT(6) +/* Flag representing the need to compare the auth result data to the expected + * value in DRAM at the auth_address. */ + +#define FW_LA_RET_AUTH_RES __BIT(5) +/* Flag representing the need to return the auth result data to dram after the + * request processing is complete */ + +#define FW_LA_UPDATE_STATE __BIT(4) +/* Flag representing the need to update the state data in dram after the + * request processing is complete */ + +#define FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP __BIT(3) +/* Flag representing Cipher/Auth Config Offset Type, where the offset + * is contained in SHRAM constants page. When the SHRAM constants page + * is not used for cipher/auth configuration, then the Content Descriptor + * pointer field must be a pointer (as opposed to a 16-byte key), since + * the block pointed to must contain both the slice config and the key */ + +#define FW_CIPH_IV_16BYTE_DATA __BIT(2) +/* Flag representing Cipher IV field contents as 16-byte data array + * Otherwise Cipher IV field contents via 64-bit pointer */ + +#define FW_LA_PARTIAL __BITS(0, 1) +#define FW_LA_PARTIAL_NONE 0 +/* Flag representing no need for partial processing condition i.e. + * entire packet processed in the current command */ +#define FW_LA_PARTIAL_START 1 +/* Flag representing the first chunk of the partial packet */ +#define FW_LA_PARTIAL_MID 3 +/* Flag representing a middle chunk of the partial packet */ +#define FW_LA_PARTIAL_END 2 +/* Flag representing the final/end chunk of the partial packet */ + +/* The table below defines the meaning of the prefix_addr & hash_state_sz in + * the case of partial processing. See the HLD for further details + * + * + ====== + ------------------------- + ----------------------- + + * | Parial | Prefix Addr | Hash State Sz | + * | State | | | + * + ====== + ------------------------- + ----------------------- + + * | FULL | Points to the prefix data | Prefix size as below. | + * | | | No update of state | + * + ====== + ------------------------- + ----------------------- + + * | SOP | Points to the prefix | = inner prefix rounded | + * | | data. State is updated | to qwrds + outer prefix | + * | | at prefix_addr - state_sz | rounded to qwrds. The | + * | | - 8 (counter size) | writeback state sz | + * | | | comes from the CD | + * + ====== + ------------------------- + ----------------------- + + * | MOP | Points to the state data | State size rounded to | + * | | Updated state written to | num qwrds + 8 (for the | + * | | same location | counter) + inner prefix | + * | | | rounded to qwrds + | + * | | | outer prefix rounded to | + * | | | qwrds. | + * + ====== + ------------------------- + ----------------------- + + * | EOP | Points to the state data | State size rounded to | + * | | | num qwrds + 8 (for the | + * | | | counter) + inner prefix | + * | | | rounded to qwrds + | + * | | | outer prefix rounded to | + * | | | qwrds. | + * + ====== + ------------------------- + ----------------------- + + * + * Notes: + * + * - If the EOP is set it is assumed that no state update is to be performed. + * However it is the clients responsibility to set the update_state flag + * correctly i.e. not set for EOP or Full packet cases. Only set for SOP and + * MOP with no EOP flag + * - The SOP take precedence over the MOP and EOP i.e. in the calculation of + * the address to writeback the state. + * - The prefix address must be on at least the 8 byte boundary + */ + +/* Macros for extracting field bits */ +/* + * Macro for extraction of the Cipher IV field contents (bit 2) + * + * flags Flags to extract the Cipher IV field contents + * + */ +#define FW_LA_CIPH_IV_FLD_FLAG_GET(flags) \ + FIELD_GET(flags, LA_CIPH_IV_FLD_BITPOS, LA_CIPH_IV_FLD_MASK) + +/* + * Macro for extraction of the Cipher/Auth Config + * offset type (bit 3) + * + * flags Flags to extract the Cipher/Auth Config offset type + * + */ +#define FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_GET(flags) \ + FIELD_GET(flags, \ + LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + LA_CIPH_AUTH_CFG_OFFSET_MASK) + +/* + * Macro for extraction of the ZUC protocol bit + * information (bit 11) + * + * flags Flags to extract the ZUC protocol bit + */ +#define FW_LA_ZUC_3G_PROTO_FLAG_GET(flags) \ + FIELD_GET(flags, \ + FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +/* + * Macro for extraction of the GCM IV Len is 12 Octets / 96 Bits + * information (bit 11) + * + * flags Flags to extract the GCM IV length + */ +#define FW_LA_GCM_IV_LEN_FLAG_GET(flags) \ + FIELD_GET( \ + flags, LA_GCM_IV_LEN_FLAG_BITPOS, LA_GCM_IV_LEN_FLAG_MASK) + +/* + * Macro for extraction of the LA protocol state (bits 9-7) + * + * flags Flags to extract the protocol state + */ +#define FW_LA_PROTO_GET(flags) \ + FIELD_GET(flags, LA_PROTO_BITPOS, LA_PROTO_MASK) + +/* + * Macro for extraction of the "compare auth" state (bit 6) + * + * flags Flags to extract the compare auth result state + * + */ +#define FW_LA_CMP_AUTH_GET(flags) \ + FIELD_GET(flags, LA_CMP_AUTH_RES_BITPOS, LA_CMP_AUTH_RES_MASK) + +/* + * Macro for extraction of the "return auth" state (bit 5) + * + * flags Flags to extract the return auth result state + * + */ +#define FW_LA_RET_AUTH_GET(flags) \ + FIELD_GET(flags, LA_RET_AUTH_RES_BITPOS, LA_RET_AUTH_RES_MASK) + +/* + * Macro for extraction of the "digest in buffer" state (bit 10) + * + * flags Flags to extract the digest in buffer state + * + */ +#define FW_LA_DIGEST_IN_BUFFER_GET(flags) \ + FIELD_GET( \ + flags, LA_DIGEST_IN_BUFFER_BITPOS, LA_DIGEST_IN_BUFFER_MASK) + +/* + * Macro for extraction of the update content state value. (bit 4) + * + * flags Flags to extract the update content state bit + */ +#define FW_LA_UPDATE_STATE_GET(flags) \ + FIELD_GET(flags, LA_UPDATE_STATE_BITPOS, LA_UPDATE_STATE_MASK) + +/* + * Macro for extraction of the "partial" packet state (bits 1-0) + * + * flags Flags to extract the partial state + */ +#define FW_LA_PARTIAL_GET(flags) \ + FIELD_GET(flags, LA_PARTIAL_BITPOS, LA_PARTIAL_MASK) + +/* Macros for setting field bits */ +/* + * Macro for setting the Cipher IV field contents + * + * flags Flags to set with the Cipher IV field contents + * val Field contents indicator value + */ +#define FW_LA_CIPH_IV_FLD_FLAG_SET(flags, val) \ + FIELD_SET( \ + flags, val, LA_CIPH_IV_FLD_BITPOS, LA_CIPH_IV_FLD_MASK) + +/* + * Macro for setting the Cipher/Auth Config + * offset type + * + * flags Flags to set the Cipher/Auth Config offset type + * val Offset type value + */ +#define FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(flags, val) \ + FIELD_SET(flags, \ + val, \ + LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + LA_CIPH_AUTH_CFG_OFFSET_MASK) + +/* + * Macro for setting the ZUC protocol flag + * + * flags Flags to set the ZUC protocol flag + * val Protocol value + */ +#define FW_LA_ZUC_3G_PROTO_FLAG_SET(flags, val) \ + FIELD_SET(flags, \ + val, \ + FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +/* + * Macro for setting the GCM IV length flag state + * + * flags Flags to set the GCM IV length flag state + * val Protocol value + */ +#define FW_LA_GCM_IV_LEN_FLAG_SET(flags, val) \ + FIELD_SET(flags, \ + val, \ + LA_GCM_IV_LEN_FLAG_BITPOS, \ + LA_GCM_IV_LEN_FLAG_MASK) + +/* + * Macro for setting the LA protocol flag state + * + * flags Flags to set the protocol state + * val Protocol value + */ +#define FW_LA_PROTO_SET(flags, val) \ + FIELD_SET(flags, val, LA_PROTO_BITPOS, LA_PROTO_MASK) + +/* + * Macro for setting the "compare auth" flag state + * + * flags Flags to set the compare auth result state + * val Compare Auth value + */ +#define FW_LA_CMP_AUTH_SET(flags, val) \ + FIELD_SET( \ + flags, val, LA_CMP_AUTH_RES_BITPOS, LA_CMP_AUTH_RES_MASK) + +/* + * Macro for setting the "return auth" flag state + * + * flags Flags to set the return auth result state + * val Return Auth value + */ +#define FW_LA_RET_AUTH_SET(flags, val) \ + FIELD_SET( \ + flags, val, LA_RET_AUTH_RES_BITPOS, LA_RET_AUTH_RES_MASK) + +/* + * Macro for setting the "digest in buffer" flag state + * + * flags Flags to set the digest in buffer state + * val Digest in buffer value + */ +#define FW_LA_DIGEST_IN_BUFFER_SET(flags, val) \ + FIELD_SET(flags, \ + val, \ + LA_DIGEST_IN_BUFFER_BITPOS, \ + LA_DIGEST_IN_BUFFER_MASK) + +/* + * Macro for setting the "update state" flag value + * + * flags Flags to set the update content state + * val Update Content State flag value + */ +#define FW_LA_UPDATE_STATE_SET(flags, val) \ + FIELD_SET( \ + flags, val, LA_UPDATE_STATE_BITPOS, LA_UPDATE_STATE_MASK) + +/* + * Macro for setting the "partial" packet flag state + * + * flags Flags to set the partial state + * val Partial state value + */ +#define FW_LA_PARTIAL_SET(flags, val) \ + FIELD_SET(flags, val, LA_PARTIAL_BITPOS, LA_PARTIAL_MASK) + +/* + * Definition of the Cipher header Content Descriptor pars block + * Definition of the cipher processing header cd pars block. + * The structure is a service-specific implementation of the common + * 'fw_comn_req_hdr_cd_pars_s' structure. + */ +union fw_cipher_req_hdr_cd_pars { + /* LWs 2-5 */ + struct + { + uint64_t content_desc_addr; + /* Address of the content descriptor */ + + uint16_t content_desc_resrvd1; + /* Content descriptor reserved field */ + + uint8_t content_desc_params_sz; + /* Size of the content descriptor parameters in quad words. These + * parameters describe the session setup configuration info for the + * slices that this request relies upon i.e. the configuration word and + * cipher key needed by the cipher slice if there is a request for + * cipher processing. */ + + uint8_t content_desc_hdr_resrvd2; + /* Content descriptor reserved field */ + + uint32_t content_desc_resrvd3; + /* Content descriptor reserved field */ + } s; + + struct + { + uint32_t cipher_key_array[FW_NUM_LONGWORDS_4]; + /* Cipher Key Array */ + + } s1; + +}; + +/* + * Definition of the Authentication header Content Descriptor pars block + * Definition of the authentication processing header cd pars block. + */ +/* Note: Authentication uses the common 'fw_comn_req_hdr_cd_pars_s' + * structure - similarly, it is also used by SSL3, TLS and MGF. Only cipher + * and cipher + authentication require service-specific implementations of + * the structure */ + +/* + * Definition of the Cipher + Auth header Content Descriptor pars block + * Definition of the cipher + auth processing header cd pars block. + * The structure is a service-specific implementation of the common + * 'fw_comn_req_hdr_cd_pars_s' structure. + */ +union fw_cipher_auth_req_hdr_cd_pars { + /* LWs 2-5 */ + struct + { + uint64_t content_desc_addr; + /* Address of the content descriptor */ + + uint16_t content_desc_resrvd1; + /* Content descriptor reserved field */ + + uint8_t content_desc_params_sz; + /* Size of the content descriptor parameters in quad words. These + * parameters describe the session setup configuration info for the + * slices that this request relies upon i.e. the configuration word and + * cipher key needed by the cipher slice if there is a request for + * cipher processing. */ + + uint8_t content_desc_hdr_resrvd2; + /* Content descriptor reserved field */ + + uint32_t content_desc_resrvd3; + /* Content descriptor reserved field */ + } s; + + struct + { + uint32_t cipher_key_array[FW_NUM_LONGWORDS_4]; + /* Cipher Key Array */ + + } sl; + +}; + +/* + * Cipher content descriptor control block (header) + * Definition of the service-specific cipher control block header + * structure. This header forms part of the content descriptor + * block incorporating LWs 27-31, as defined by the common base + * parameters structure. + */ +struct fw_cipher_cd_ctrl_hdr +{ + /* LW 27 */ + uint8_t cipher_state_sz; + /* State size in quad words of the cipher algorithm used in this session. + * Set to zero if the algorithm doesnt provide any state */ + + uint8_t cipher_key_sz; + /* Key size in quad words of the cipher algorithm used in this session */ + + uint8_t cipher_cfg_offset; + /* Quad word offset from the content descriptor parameters address i.e. + * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher + * processing */ + + uint8_t next_curr_id; + /* This field combines the next and current id (each four bits) - + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the ciphered data through. + * Set to FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after cipher. + * Current Id: Initialised with the cipher slice type */ + + /* LW 28 */ + uint8_t cipher_padding_sz; + /* State padding size in quad words. Set to 0 if no padding is required. + */ + + uint8_t resrvd1; + uint16_t resrvd2; + /* Reserved bytes to bring the struct to the word boundary, used by + * authentication. MUST be set to 0 */ + + /* LWs 29-31 */ + uint32_t resrvd3[FW_NUM_LONGWORDS_3]; + /* Reserved bytes used by authentication. MUST be set to 0 */ + +}; + +/* + * Authentication content descriptor control block (header) + * Definition of the service-specific authentication control block + * header structure. This header forms part of the content descriptor + * block incorporating LWs 27-31, as defined by the common base + * parameters structure, the first portion of which is reserved for + * cipher. + */ +struct fw_auth_cd_ctrl_hdr +{ + /* LW 27 */ + uint32_t resrvd1; + /* Reserved bytes, used by cipher only. MUST be set to 0 */ + + /* LW 28 */ + uint8_t resrvd2; + /* Reserved byte, used by cipher only. MUST be set to 0 */ + + uint8_t hash_flags; + /* General flags defining the processing to perform. 0 is normal + * processing + * and 1 means there is a nested hash processing loop to go through */ + + uint8_t hash_cfg_offset; + /* Quad word offset from the content descriptor parameters address to the + * parameters for the auth processing */ + + uint8_t next_curr_id; + /* This field combines the next and current id (each four bits) - + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the authentication data through. + * Set to FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after authentication. + * Current Id: Initialised with the authentication slice type */ + + /* LW 29 */ + uint8_t resrvd3; + /* Now a reserved field. MUST be set to 0 */ + + uint8_t outer_prefix_sz; + /* Size in bytes of outer prefix data */ + + uint8_t final_sz; + /* Size in bytes of digest to be returned to the client if requested */ + + uint8_t inner_res_sz; + /* Size in bytes of the digest from the inner hash algorithm */ + + /* LW 30 */ + uint8_t resrvd4; + /* Now a reserved field. MUST be set to zero. */ + + uint8_t inner_state1_sz; + /* Size in bytes of inner hash state1 data. Must be a qword multiple */ + + uint8_t inner_state2_offset; + /* Quad word offset from the content descriptor parameters pointer to the + * inner state2 value */ + + uint8_t inner_state2_sz; + /* Size in bytes of inner hash state2 data. Must be a qword multiple */ + + /* LW 31 */ + uint8_t outer_config_offset; + /* Quad word offset from the content descriptor parameters pointer to the + * outer configuration information */ + + uint8_t outer_state1_sz; + /* Size in bytes of the outer state1 value */ + + uint8_t outer_res_sz; + /* Size in bytes of digest from the outer auth algorithm */ + + uint8_t outer_prefix_offset; + /* Quad word offset from the start of the inner prefix data to the outer + * prefix information. Should equal the rounded inner prefix size, converted + * to qwords */ + +}; + +/* + * Cipher + Authentication content descriptor control block header + * Definition of both service-specific cipher + authentication control + * block header structures. This header forms part of the content + * descriptor block incorporating LWs 27-31, as defined by the common + * base parameters structure. + */ +struct fw_cipher_auth_cd_ctrl_hdr +{ + /* LW 27 */ + uint8_t cipher_state_sz; + /* State size in quad words of the cipher algorithm used in this session. + * Set to zero if the algorithm doesnt provide any state */ + + uint8_t cipher_key_sz; + /* Key size in quad words of the cipher algorithm used in this session */ + + uint8_t cipher_cfg_offset; + /* Quad word offset from the content descriptor parameters address i.e. + * (content_address + (cd_hdr_sz << 3)) to the parameters for the cipher + * processing */ + + uint8_t next_curr_id_cipher; + /* This field combines the next and current id (each four bits) - + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the ciphered data through. + * Set to FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after cipher. + * Current Id: Initialised with the cipher slice type */ + + /* LW 28 */ + uint8_t cipher_padding_sz; + /* State padding size in quad words. Set to 0 if no padding is required. + */ + + uint8_t hash_flags; + /* General flags defining the processing to perform. 0 is normal + * processing + * and 1 means there is a nested hash processing loop to go through */ + + uint8_t hash_cfg_offset; + /* Quad word offset from the content descriptor parameters address to the + * parameters for the auth processing */ + + uint8_t next_curr_id_auth; + /* This field combines the next and current id (each four bits) - + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the authentication data through. + * Set to FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after authentication. + * Current Id: Initialised with the authentication slice type */ + + /* LW 29 */ + uint8_t resrvd1; + /* Reserved field. MUST be set to 0 */ + + uint8_t outer_prefix_sz; + /* Size in bytes of outer prefix data */ + + uint8_t final_sz; + /* Size in bytes of digest to be returned to the client if requested */ + + uint8_t inner_res_sz; + /* Size in bytes of the digest from the inner hash algorithm */ + + /* LW 30 */ + uint8_t resrvd2; + /* Now a reserved field. MUST be set to zero. */ + + uint8_t inner_state1_sz; + /* Size in bytes of inner hash state1 data. Must be a qword multiple */ + + uint8_t inner_state2_offset; + /* Quad word offset from the content descriptor parameters pointer to the + * inner state2 value */ + + uint8_t inner_state2_sz; + /* Size in bytes of inner hash state2 data. Must be a qword multiple */ + + /* LW 31 */ + uint8_t outer_config_offset; + /* Quad word offset from the content descriptor parameters pointer to the + * outer configuration information */ + + uint8_t outer_state1_sz; + /* Size in bytes of the outer state1 value */ + + uint8_t outer_res_sz; + /* Size in bytes of digest from the outer auth algorithm */ + + uint8_t outer_prefix_offset; + /* Quad word offset from the start of the inner prefix data to the outer + * prefix information. Should equal the rounded inner prefix size, converted + * to qwords */ + +}; + +#define FW_AUTH_HDR_FLAG_DO_NESTED 1 +/* Definition of the hash_flags bit of the auth_hdr to indicate the request + * requires nested hashing */ + +#define FW_AUTH_HDR_FLAG_NO_NESTED 0 +/* Definition of the hash_flags bit of the auth_hdr for no nested hashing + * required */ + +#define FW_CCM_GCM_AAD_SZ_MAX 240 +/* Maximum size of AAD data allowed for CCM or GCM processing. AAD data size90 - + * is stored in 8-bit field and must be multiple of hash block size. 240 is + * largest value which satisfy both requirements.AAD_SZ_MAX is in byte units */ + +/* + * request parameter #defines + */ +#define FW_HASH_REQUEST_PARAMETERS_OFFSET \ + (sizeof(fw_la_cipher_req_params_t)) +/* Offset in bytes from the start of the request parameters block to the hash + * (auth) request parameters */ + +#define FW_CIPHER_REQUEST_PARAMETERS_OFFSET (0) +/* Offset in bytes from the start of the request parameters block to the cipher + * request parameters */ + +/* + * Definition of the cipher request parameters block + * + * Definition of the cipher processing request parameters block + * structure, which forms part of the block incorporating LWs 14-26, + * as defined by the common base parameters structure. + * Unused fields must be set to 0. + */ +struct fw_la_cipher_req_params { + /* LW 14 */ + uint32_t cipher_offset; + /* Cipher offset long word. */ + + /* LW 15 */ + uint32_t cipher_length; + /* Cipher length long word. */ + + /* LWs 16-19 */ + union { + uint32_t cipher_IV_array[FW_NUM_LONGWORDS_4]; + /* Cipher IV array */ + + struct + { + uint64_t cipher_IV_ptr; + /* Cipher IV pointer or Partial State Pointer */ + + uint64_t resrvd1; + /* reserved */ + + } s; + + } u; + +}; + +/* + * Definition of the auth request parameters block + * Definition of the authentication processing request parameters block + * structure, which forms part of the block incorporating LWs 14-26, + * as defined by the common base parameters structure. Note: + * This structure is used by TLS only. + */ +struct fw_la_auth_req_params { + /* LW 20 */ + uint32_t auth_off; + /* Byte offset from the start of packet to the auth data region */ + + /* LW 21 */ + uint32_t auth_len; + /* Byte length of the auth data region */ + + /* LWs 22-23 */ + union { + uint64_t auth_partial_st_prefix; + /* Address of the authentication partial state prefix + * information */ + + uint64_t aad_adr; + /* Address of the AAD info in DRAM. Used for the CCM and GCM + * protocols */ + + } u1; + + /* LWs 24-25 */ + uint64_t auth_res_addr; + /* Address of the authentication result information to validate or + * the location to which the digest information can be written back to */ + + /* LW 26 */ + union { + uint8_t inner_prefix_sz; + /* Size in bytes of the inner prefix data */ + + uint8_t aad_sz; + /* Size in bytes of padded AAD data to prefix to the packet for CCM + * or GCM processing */ + } u2; + + uint8_t resrvd1; + /* reserved */ + + uint8_t hash_state_sz; + /* Number of quad words of inner and outer hash prefix data to process + * Maximum size is 240 */ + + uint8_t auth_res_sz; + /* Size in bytes of the authentication result */ + +} __packed; + +/* + * Definition of the auth request parameters block + * Definition of the authentication processing request parameters block + * structure, which forms part of the block incorporating LWs 14-26, + * as defined by the common base parameters structure. Note: + * This structure is used by SSL3 and MGF1 only. All fields other than + * inner prefix/ AAD size are unused and therefore reserved. + */ +struct fw_la_auth_req_params_resrvd_flds { + /* LWs 20-25 */ + uint32_t resrvd[FW_NUM_LONGWORDS_6]; + + /* LW 26 */ + union { + uint8_t inner_prefix_sz; + /* Size in bytes of the inner prefix data */ + + uint8_t aad_sz; + /* Size in bytes of padded AAD data to prefix to the packet for CCM + * or GCM processing */ + } u2; + + uint8_t resrvd1; + /* reserved */ + + uint16_t resrvd2; + /* reserved */ +}; + +/* + * Definition of the shared fields within the parameter block + * containing SSL, TLS or MGF information. + * This structure defines the shared fields for SSL, TLS or MGF + * within the parameter block incorporating LWs 14-26, as defined + * by the common base parameters structure. + * Unused fields must be set to 0. + */ +struct fw_la_key_gen_common { + /* LW 14 */ + union { + /* SSL3 */ + uint16_t secret_lgth_ssl; + /* Length of Secret information for SSL. In the case of TLS the + * secret is supplied in the content descriptor */ + + /* MGF */ + uint16_t mask_length; + /* Size in bytes of the desired output mask for MGF1*/ + + /* TLS */ + uint16_t secret_lgth_tls; + /* TLS Secret length */ + + } u; + + union { + /* SSL3 */ + struct + { + uint8_t output_lgth_ssl; + /* Output length */ + + uint8_t label_lgth_ssl; + /* Label length */ + + } s1; + + /* MGF */ + struct + { + uint8_t hash_length; + /* Hash length */ + + uint8_t seed_length; + /* Seed length */ + + } s2; + + /* TLS */ + struct + { + uint8_t output_lgth_tls; + /* Output length */ + + uint8_t label_lgth_tls; + /* Label length */ + + } s3; + + } u1; + + /* LW 15 */ + union { + /* SSL3 */ + uint8_t iter_count; + /* Iteration count used by the SSL key gen request */ + + /* TLS */ + uint8_t tls_seed_length; + /* TLS Seed length */ + + uint8_t resrvd1; + /* Reserved field set to 0 for MGF1 */ + + } u2; + + uint8_t resrvd2; + uint16_t resrvd3; + /* Reserved space - unused */ + +}; + +/* + * Definition of the SSL3 request parameters block + * This structure contains the the SSL3 processing request parameters + * incorporating LWs 14-26, as defined by the common base + * parameters structure. Unused fields must be set to 0. + */ +struct fw_la_ssl3_req_params { + /* LWs 14-15 */ + struct fw_la_key_gen_common keygen_comn; + /* For other key gen processing these field holds ssl, tls or mgf + * parameters */ + + /* LW 16-25 */ + uint32_t resrvd[FW_NUM_LONGWORDS_10]; + /* Reserved */ + + /* LW 26 */ + union { + uint8_t inner_prefix_sz; + /* Size in bytes of the inner prefix data */ + + uint8_t aad_sz; + /* Size in bytes of padded AAD data to prefix to the packet for CCM + * or GCM processing */ + } u2; + + uint8_t resrvd1; + /* reserved */ + + uint16_t resrvd2; + /* reserved */ + +}; + +/* + * Definition of the MGF request parameters block + * This structure contains the the MGF processing request parameters + * incorporating LWs 14-26, as defined by the common base parameters + * structure. Unused fields must be set to 0. + */ +struct fw_la_mgf_req_params { + /* LWs 14-15 */ + struct fw_la_key_gen_common keygen_comn; + /* For other key gen processing these field holds ssl or mgf + * parameters */ + + /* LW 16-25 */ + uint32_t resrvd[FW_NUM_LONGWORDS_10]; + /* Reserved */ + + /* LW 26 */ + union { + uint8_t inner_prefix_sz; + /* Size in bytes of the inner prefix data */ + + uint8_t aad_sz; + /* Size in bytes of padded AAD data to prefix to the packet for CCM + * or GCM processing */ + } u2; + + uint8_t resrvd1; + /* reserved */ + + uint16_t resrvd2; + /* reserved */ + +}; + +/* + * Definition of the TLS request parameters block + * This structure contains the the TLS processing request parameters + * incorporating LWs 14-26, as defined by the common base parameters + * structure. Unused fields must be set to 0. + */ +struct fw_la_tls_req_params { + /* LWs 14-15 */ + struct fw_la_key_gen_common keygen_comn; + /* For other key gen processing these field holds ssl, tls or mgf + * parameters */ + + /* LW 16-19 */ + uint32_t resrvd[FW_NUM_LONGWORDS_4]; + /* Reserved */ + +}; + +/* + * Definition of the common QAT FW request middle block for TRNG. + * Common section of the request used across all of the services exposed + * by the QAT FW. Each of the services inherit these common fields. TRNG + * requires a specific implementation. + */ +struct fw_la_trng_req_mid { + /* LWs 6-13 */ + uint64_t opaque_data; + /* Opaque data passed unmodified from the request to response messages by + * firmware (fw) */ + + uint64_t resrvd1; + /* Reserved, unused for TRNG */ + + uint64_t dest_data_addr; + /* Generic definition of the destination data supplied to the QAT AE. The + * common flags are used to further describe the attributes of this + * field */ + + uint32_t resrvd2; + /* Reserved, unused for TRNG */ + + uint32_t entropy_length; + /* Size of the data in bytes to process. Used by the get_random + * command. Set to 0 for commands that dont need a length parameter */ + +}; + +/* + * Definition of the common LA QAT FW TRNG request + * Definition of the TRNG processing request type + */ +struct fw_la_trng_req { + /* LWs 0-1 */ + struct fw_comn_req_hdr comn_hdr; + /* Common request header */ + + /* LWs 2-5 */ + union fw_comn_req_hdr_cd_pars cd_pars; + /* Common Request content descriptor field which points either to a + * content descriptor + * parameter block or contains the service-specific data itself. */ + + /* LWs 6-13 */ + struct fw_la_trng_req_mid comn_mid; + /* TRNG request middle section - differs from the common mid-section */ + + /* LWs 14-26 */ + uint32_t resrvd1[FW_NUM_LONGWORDS_13]; + + /* LWs 27-31 */ + uint32_t resrvd2[FW_NUM_LONGWORDS_5]; + +}; + +/* + * Definition of the Lookaside Eagle Tail Response + * This is the response delivered to the ET rings by the Lookaside + * QAT FW service for all commands + */ +struct fw_la_resp { + /* LWs 0-1 */ + struct fw_comn_resp_hdr comn_resp; + /* Common interface response format see fw.h */ + + /* LWs 2-3 */ + uint64_t opaque_data; + /* Opaque data passed from the request to the response message */ + + /* LWs 4-7 */ + uint32_t resrvd[FW_NUM_LONGWORDS_4]; + /* Reserved */ + +}; + +/* + * Definition of the Lookaside TRNG Test Status Structure + * As an addition to FW_LA_TRNG_STATUS Pass or Fail information + * in common response fields, as a response to TRNG_TEST request, Test + * status, Counter for failed tests and 4 entropy counter values are + * sent + * Status of test status and the fail counts. + */ +struct fw_la_trng_test_result { + uint32_t test_status_info; + /* TRNG comparator health test status& Validity information + see Test Status Bit Fields below. */ + + uint32_t test_status_fail_count; + /* TRNG comparator health test status, 32bit fail counter */ + + uint64_t r_ent_ones_cnt; + /* Raw Entropy ones counter */ + + uint64_t r_ent_zeros_cnt; + /* Raw Entropy zeros counter */ + + uint64_t c_ent_ones_cnt; + /* Conditioned Entropy ones counter */ + + uint64_t c_ent_zeros_cnt; + /* Conditioned Entropy zeros counter */ + + uint64_t resrvd; + /* Reserved field must be set to zero */ + +}; + +/* + * Definition of the Lookaside SSL Key Material Input + * This struct defines the layout of input parameters for the + * SSL3 key generation (source flat buffer format) + */ +struct fw_la_ssl_key_material_input { + uint64_t seed_addr; + /* Pointer to seed */ + + uint64_t label_addr; + /* Pointer to label(s) */ + + uint64_t secret_addr; + /* Pointer to secret */ + +}; + +/* + * Definition of the Lookaside TLS Key Material Input + * This struct defines the layout of input parameters for the + * TLS key generation (source flat buffer format) + * NOTE: + * Secret state value (S split into S1 and S2 parts) is supplied via + * Content Descriptor. S1 is placed in an outer prefix buffer, and S2 + * inside the inner prefix buffer. + */ +struct fw_la_tls_key_material_input { + uint64_t seed_addr; + /* Pointer to seed */ + + uint64_t label_addr; + /* Pointer to label(s) */ + +}; + +/* + * Macros using the bit position and mask to set/extract the next + * and current id nibbles within the next_curr_id field of the + * content descriptor header block, ONLY FOR CIPHER + AUTH COMBINED. + * Note that for cipher only or authentication only, the common macros + * need to be used. These are defined in the 'fw.h' common header + * file, as they are used by compression, cipher and authentication. + * + * cd_ctrl_hdr_t Content descriptor control block header. + * val Value of the field being set. + */ +/* Cipher fields within Cipher + Authentication structure */ +#define FW_CIPHER_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ + FW_COMN_NEXT_ID_MASK) >> \ + (FW_COMN_NEXT_ID_BITPOS)) + +#define FW_CIPHER_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ + (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ + FW_COMN_CURR_ID_MASK) | \ + ((val << FW_COMN_NEXT_ID_BITPOS) & \ + FW_COMN_NEXT_ID_MASK)) + +#define FW_CIPHER_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_cipher) & FW_COMN_CURR_ID_MASK) + +#define FW_CIPHER_CURR_ID_SET(cd_ctrl_hdr_t, val) \ + (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ + FW_COMN_NEXT_ID_MASK) | \ + ((val)&FW_COMN_CURR_ID_MASK)) + +/* Authentication fields within Cipher + Authentication structure */ +#define FW_AUTH_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) & FW_COMN_NEXT_ID_MASK) >> \ + (FW_COMN_NEXT_ID_BITPOS)) + +#define FW_AUTH_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ + (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) & \ + FW_COMN_CURR_ID_MASK) | \ + ((val << FW_COMN_NEXT_ID_BITPOS) & \ + FW_COMN_NEXT_ID_MASK)) + +#define FW_AUTH_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_auth) & FW_COMN_CURR_ID_MASK) + +#define FW_AUTH_CURR_ID_SET(cd_ctrl_hdr_t, val) \ + (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) & \ + FW_COMN_NEXT_ID_MASK) | \ + ((val)&FW_COMN_CURR_ID_MASK)) + +/* Definitions of the bits in the test_status_info of the TRNG_TEST response. + * The values returned by the Lookaside service are given below + * The Test result and Test Fail Count values are only valid if the Test + * Results Valid (Tv) is set. + * + * TRNG Test Status Info + * + ===== + ------------------------------------------------ + --- + --- + + * | Bit | 31 - 2 | 1 | 0 | + * + ===== + ------------------------------------------------ + --- + --- + + * | Flags | RESERVED = 0 | Tv | Ts | + * + ===== + ------------------------------------------------------------ + + */ +/* + * Definition of the Lookaside TRNG Test Status Information received as + * a part of fw_la_trng_test_result_t + * + */ +#define FW_LA_TRNG_TEST_STATUS_TS_BITPOS 0 +/* TRNG Test Result t_status field bit pos definition. */ + +#define FW_LA_TRNG_TEST_STATUS_TS_MASK 0x1 +/* TRNG Test Result t_status field mask definition. */ + +#define FW_LA_TRNG_TEST_STATUS_TV_BITPOS 1 +/* TRNG Test Result test results valid field bit pos definition. */ + +#define FW_LA_TRNG_TEST_STATUS_TV_MASK 0x1 +/* TRNG Test Result test results valid field mask definition. */ + +/* + * Definition of the Lookaside TRNG test_status values. + * + * + */ +#define FW_LA_TRNG_TEST_STATUS_TV_VALID 1 +/* TRNG TEST Response Test Results Valid Value. */ + +#define FW_LA_TRNG_TEST_STATUS_TV_NOT_VALID 0 +/* TRNG TEST Response Test Results are NOT Valid Value. */ + +#define FW_LA_TRNG_TEST_STATUS_TS_NO_FAILS 1 +/* Value for TRNG Test status tests have NO FAILs Value. */ + +#define FW_LA_TRNG_TEST_STATUS_TS_HAS_FAILS 0 +/* Value for TRNG Test status tests have one or more FAILS Value. */ + +/* + * Macro for extraction of the Test Status Field returned in the response + * to TRNG TEST command. + * + * test_status 8 bit test_status value to extract the status bit + */ +#define FW_LA_TRNG_TEST_STATUS_TS_FLD_GET(test_status) \ + FIELD_GET(test_status, \ + FW_LA_TRNG_TEST_STATUS_TS_BITPOS, \ + FW_LA_TRNG_TEST_STATUS_TS_MASK) +/* + * Macro for extraction of the Test Results Valid Field returned in the + * response to TRNG TEST command. + * + * test_status 8 bit test_status value to extract the Tests + * Results valid bit + */ +#define FW_LA_TRNG_TEST_STATUS_TV_FLD_GET(test_status) \ + FIELD_GET(test_status, \ + FW_LA_TRNG_TEST_STATUS_TV_BITPOS, \ + FW_LA_TRNG_TEST_STATUS_TV_MASK) + +/* + * MGF Max supported input parameters + */ +#define FW_LA_MGF_SEED_LEN_MAX 255 +/* Maximum seed length for MGF1 request in bytes + * Typical values may be 48, 64, 128 bytes (or any). */ + +#define FW_LA_MGF_MASK_LEN_MAX 65528 +/* Maximum mask length for MGF1 request in bytes + * Typical values may be 8 (64-bit), 16 (128-bit). MUST be quad word multiple */ + +/* + * SSL Max supported input parameters + */ +#define FW_LA_SSL_SECRET_LEN_MAX 512 +/* Maximum secret length for SSL3 Key Gen request (bytes) */ + +#define FW_LA_SSL_ITERATES_LEN_MAX 16 +/* Maximum iterations for SSL3 Key Gen request (integer) */ + +#define FW_LA_SSL_LABEL_LEN_MAX 136 +/* Maximum label length for SSL3 Key Gen request (bytes) */ + +#define FW_LA_SSL_SEED_LEN_MAX 64 +/* Maximum seed length for SSL3 Key Gen request (bytes) */ + +#define FW_LA_SSL_OUTPUT_LEN_MAX 248 +/* Maximum output length for SSL3 Key Gen request (bytes) */ + +/* + * TLS Max supported input parameters + */ +#define FW_LA_TLS_SECRET_LEN_MAX 128 +/* Maximum secret length for TLS Key Gen request (bytes) */ + +#define FW_LA_TLS_V1_1_SECRET_LEN_MAX 128 +/* Maximum secret length for TLS Key Gen request (bytes) */ + +#define FW_LA_TLS_V1_2_SECRET_LEN_MAX 64 +/* Maximum secret length for TLS Key Gen request (bytes) */ + +#define FW_LA_TLS_LABEL_LEN_MAX 255 +/* Maximum label length for TLS Key Gen request (bytes) */ + +#define FW_LA_TLS_SEED_LEN_MAX 64 +/* Maximum seed length for TLS Key Gen request (bytes) */ + +#define FW_LA_TLS_OUTPUT_LEN_MAX 248 +/* Maximum output length for TLS Key Gen request (bytes) */ + +#endif Index: head/sys/dev/qat/qat_hw17var.h =================================================================== --- head/sys/dev/qat/qat_hw17var.h +++ head/sys/dev/qat/qat_hw17var.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qat_hw17var.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2014 Intel Corporation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QAT_HW17VAR_H_ +#define _DEV_PCI_QAT_HW17VAR_H_ + +CTASSERT(CONTENT_DESC_MAX_SIZE >= + roundup(sizeof(union hw_cipher_algo_blk), 8) + + roundup(sizeof(union hw_auth_algo_blk), 8)); + +int qat_adm_mailbox_init(struct qat_softc *); +int qat_adm_mailbox_send_init(struct qat_softc *); +int qat_arb_init(struct qat_softc *); +int qat_set_ssm_wdtimer(struct qat_softc *); +int qat_check_slice_hang(struct qat_softc *); + +void qat_hw17_crypto_setup_desc(struct qat_crypto *, + struct qat_session *, struct qat_crypto_desc *); +void qat_hw17_crypto_setup_req_params(struct qat_crypto_bank *, + struct qat_session *, struct qat_crypto_desc const *, + struct qat_sym_cookie *, struct cryptop *); + +#endif Index: head/sys/dev/qat/qatreg.h =================================================================== --- head/sys/dev/qat/qatreg.h +++ head/sys/dev/qat/qatreg.h @@ -0,0 +1,1582 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qatreg.h,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QATREG_H_ +#define _DEV_PCI_QATREG_H_ + +#define __BIT(__n) \ + (((uintmax_t)(__n) >= NBBY * sizeof(uintmax_t)) ? 0 : \ + ((uintmax_t)1 << (uintmax_t)((__n) & (NBBY * sizeof(uintmax_t) - 1)))) +#define __BITS(__m, __n) \ + ((__BIT(MAX((__m), (__n)) + 1) - 1) ^ (__BIT(MIN((__m), (__n))) - 1)) + +#define __LOWEST_SET_BIT(__mask) ((((__mask) - 1) & (__mask)) ^ (__mask)) +#define __SHIFTOUT(__x, __mask) (((__x) & (__mask)) / __LOWEST_SET_BIT(__mask)) +#define __SHIFTIN(__x, __mask) ((__x) * __LOWEST_SET_BIT(__mask)) + +/* Limits */ +#define MAX_NUM_AE 0x10 +#define MAX_NUM_ACCEL 6 +#define MAX_AE 0x18 +#define MAX_AE_CTX 8 +#define MAX_ARB 4 + +#define MAX_USTORE_PER_SEG 0x8000 /* 16k * 2 */ +#define MAX_USTORE MAX_USTORE_PER_SEG + +#define MAX_AE_PER_ACCEL 4 /* XXX */ +#define MAX_BANK_PER_ACCEL 16 /* XXX */ +#define MAX_RING_PER_BANK 16 + +#define MAX_XFER_REG 128 +#define MAX_GPR_REG 128 +#define MAX_NN_REG 128 +#define MAX_LMEM_REG 1024 +#define MAX_INP_STATE 16 +#define MAX_CAM_REG 16 +#define MAX_FIFO_QWADDR 160 + +#define MAX_EXEC_INST 100 +#define UWORD_CPYBUF_SIZE 1024 /* micro-store copy buffer (bytes) */ +#define INVLD_UWORD 0xffffffffffull /* invalid micro-instruction */ +#define AEV2_PACKED_UWORD_BYTES 6 /* version 2 packed uword size */ +#define UWORD_MASK 0xbffffffffffull /* micro-word mask without parity */ + +#define AE_ALL_CTX 0xff + +/* PCIe configuration space parameter */ +#define NO_PCI_REG (-1) +#define NO_REG_OFFSET 0 + +#define MAX_BARS 3 + +/* Fuse Control */ +#define FUSECTL_REG 0x40 +#define FUSECTL_MASK __BIT(31) + +#define LEGFUSE_REG 0x4c +#define LEGFUSE_ACCEL_MASK_CIPHER_SLICE __BIT(0) +#define LEGFUSE_ACCEL_MASK_AUTH_SLICE __BIT(1) +#define LEGFUSE_ACCEL_MASK_PKE_SLICE __BIT(2) +#define LEGFUSE_ACCEL_MASK_COMPRESS_SLICE __BIT(3) +#define LEGFUSE_ACCEL_MASK_LZS_SLICE __BIT(4) +#define LEGFUSE_ACCEL_MASK_EIA3_SLICE __BIT(5) +#define LEGFUSE_ACCEL_MASK_SHA3_SLICE __BIT(6) + +/* -------------------------------------------------------------------------- */ +/* PETRINGCSR region */ + +/* ETR parameters */ +#define ETR_MAX_RINGS_PER_BANK 16 + +/* ETR registers */ +#define ETR_RING_CONFIG 0x0000 +#define ETR_RING_LBASE 0x0040 +#define ETR_RING_UBASE 0x0080 +#define ETR_RING_HEAD_OFFSET 0x00C0 +#define ETR_RING_TAIL_OFFSET 0x0100 +#define ETR_RING_STAT 0x0140 +#define ETR_UO_STAT 0x0148 +#define ETR_E_STAT 0x014C +#define ETR_NE_STAT 0x0150 +#define ETR_NF_STAT 0x0154 +#define ETR_F_STAT 0x0158 +#define ETR_C_STAT 0x015C +#define ETR_INT_EN 0x016C +#define ETR_INT_REG 0x0170 +#define ETR_INT_SRCSEL 0x0174 +#define ETR_INT_SRCSEL_2 0x0178 +#define ETR_INT_COL_EN 0x017C +#define ETR_INT_COL_CTL 0x0180 +#define ETR_AP_NF_MASK 0x2000 +#define ETR_AP_NF_DEST 0x2020 +#define ETR_AP_NE_MASK 0x2040 +#define ETR_AP_NE_DEST 0x2060 +#define ETR_AP_DELAY 0x2080 + +/* ARB registers */ +#define ARB_OFFSET 0x30000 +#define ARB_REG_SIZE 0x4 +#define ARB_WTR_SIZE 0x20 +#define ARB_REG_SLOT 0x1000 +#define ARB_WTR_OFFSET 0x010 +#define ARB_RO_EN_OFFSET 0x090 +#define ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ARB_RINGSRVARBEN_OFFSET 0x19c + +/* Ring Config */ +#define ETR_RING_CONFIG_LATE_HEAD_POINTER_MODE __BIT(31) +#define ETR_RING_CONFIG_NEAR_FULL_WM __BITS(14, 10) +#define ETR_RING_CONFIG_NEAR_EMPTY_WM __BITS(9, 5) +#define ETR_RING_CONFIG_RING_SIZE __BITS(4, 0) + +#define ETR_RING_CONFIG_NEAR_WM_0 0x00 +#define ETR_RING_CONFIG_NEAR_WM_4 0x01 +#define ETR_RING_CONFIG_NEAR_WM_8 0x02 +#define ETR_RING_CONFIG_NEAR_WM_16 0x03 +#define ETR_RING_CONFIG_NEAR_WM_32 0x04 +#define ETR_RING_CONFIG_NEAR_WM_64 0x05 +#define ETR_RING_CONFIG_NEAR_WM_128 0x06 +#define ETR_RING_CONFIG_NEAR_WM_256 0x07 +#define ETR_RING_CONFIG_NEAR_WM_512 0x08 +#define ETR_RING_CONFIG_NEAR_WM_1K 0x09 +#define ETR_RING_CONFIG_NEAR_WM_2K 0x0A +#define ETR_RING_CONFIG_NEAR_WM_4K 0x0B +#define ETR_RING_CONFIG_NEAR_WM_8K 0x0C +#define ETR_RING_CONFIG_NEAR_WM_16K 0x0D +#define ETR_RING_CONFIG_NEAR_WM_32K 0x0E +#define ETR_RING_CONFIG_NEAR_WM_64K 0x0F +#define ETR_RING_CONFIG_NEAR_WM_128K 0x10 +#define ETR_RING_CONFIG_NEAR_WM_256K 0x11 +#define ETR_RING_CONFIG_NEAR_WM_512K 0x12 +#define ETR_RING_CONFIG_NEAR_WM_1M 0x13 +#define ETR_RING_CONFIG_NEAR_WM_2M 0x14 +#define ETR_RING_CONFIG_NEAR_WM_4M 0x15 + +#define ETR_RING_CONFIG_SIZE_64 0x00 +#define ETR_RING_CONFIG_SIZE_128 0x01 +#define ETR_RING_CONFIG_SIZE_256 0x02 +#define ETR_RING_CONFIG_SIZE_512 0x03 +#define ETR_RING_CONFIG_SIZE_1K 0x04 +#define ETR_RING_CONFIG_SIZE_2K 0x05 +#define ETR_RING_CONFIG_SIZE_4K 0x06 +#define ETR_RING_CONFIG_SIZE_8K 0x07 +#define ETR_RING_CONFIG_SIZE_16K 0x08 +#define ETR_RING_CONFIG_SIZE_32K 0x09 +#define ETR_RING_CONFIG_SIZE_64K 0x0A +#define ETR_RING_CONFIG_SIZE_128K 0x0B +#define ETR_RING_CONFIG_SIZE_256K 0x0C +#define ETR_RING_CONFIG_SIZE_512K 0x0D +#define ETR_RING_CONFIG_SIZE_1M 0x0E +#define ETR_RING_CONFIG_SIZE_2M 0x0F +#define ETR_RING_CONFIG_SIZE_4M 0x10 + +/* Default Ring Config is Nearly Full = Full and Nearly Empty = Empty */ +#define ETR_RING_CONFIG_BUILD(size) \ + (__SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0, \ + ETR_RING_CONFIG_NEAR_FULL_WM) | \ + __SHIFTIN(ETR_RING_CONFIG_NEAR_WM_0, \ + ETR_RING_CONFIG_NEAR_EMPTY_WM) | \ + __SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE)) + +/* Response Ring Configuration */ +#define ETR_RING_CONFIG_BUILD_RESP(size, wm_nf, wm_ne) \ + (__SHIFTIN((wm_nf), ETR_RING_CONFIG_NEAR_FULL_WM) | \ + __SHIFTIN((wm_ne), ETR_RING_CONFIG_NEAR_EMPTY_WM) | \ + __SHIFTIN((size), ETR_RING_CONFIG_RING_SIZE)) + +/* Ring Base */ +#define ETR_RING_BASE_BUILD(addr, size) \ + (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) + +#define ETR_INT_REG_CLEAR_MASK 0xffff + +/* Initial bank Interrupt Source mask */ +#define ETR_INT_SRCSEL_MASK 0x44444444UL + +#define ETR_INT_SRCSEL_NEXT_OFFSET 4 + +#define ETR_RINGS_PER_INT_SRCSEL 8 + +#define ETR_INT_COL_CTL_ENABLE __BIT(31) + +#define ETR_AP_NF_MASK_INIT 0xAAAAAAAA +#define ETR_AP_NE_MASK_INIT 0x55555555 + +/* Autopush destination AE bit */ +#define ETR_AP_DEST_ENABLE __BIT(7) +#define ETR_AP_DEST_AE __BITS(6, 2) +#define ETR_AP_DEST_MAILBOX __BITS(1, 0) + +/* Autopush destination enable bit */ + +/* Autopush CSR Offset */ +#define ETR_AP_BANK_OFFSET 4 + +/* Autopush maximum rings per bank */ +#define ETR_MAX_RINGS_PER_AP_BANK 32 + +/* Maximum mailbox per acclerator */ +#define ETR_MAX_MAILBOX_PER_ACCELERATOR 4 + +/* Maximum AEs per mailbox */ +#define ETR_MAX_AE_PER_MAILBOX 4 + +/* Macro to get the ring's autopush bank number */ +#define ETR_RING_AP_BANK_NUMBER(ring) ((ring) >> 5) + +/* Macro to get the ring's autopush mailbox number */ +#define ETR_RING_AP_MAILBOX_NUMBER(ring) \ + (ETR_RING_AP_BANK_NUMBER(ring) % ETR_MAX_MAILBOX_PER_ACCELERATOR) + +/* Macro to get the ring number in the autopush bank */ +#define ETR_RING_NUMBER_IN_AP_BANK(ring) \ + ((ring) % ETR_MAX_RINGS_PER_AP_BANK) + +#define ETR_RING_EMPTY_ENTRY_SIG (0x7F7F7F7F) + +/* -------------------------------------------------------------------------- */ +/* CAP_GLOBAL_CTL region */ + +#define FCU_CTRL 0x8c0 +#define FCU_CTRL_CMD_NOOP 0 +#define FCU_CTRL_CMD_AUTH 1 +#define FCU_CTRL_CMD_LOAD 2 +#define FCU_CTRL_CMD_START 3 +#define FCU_CTRL_AE __BITS(8, 31) + +#define FCU_STATUS 0x8c4 +#define FCU_STATUS_STS __BITS(0, 2) +#define FCU_STATUS_STS_NO 0 +#define FCU_STATUS_STS_VERI_DONE 1 +#define FCU_STATUS_STS_LOAD_DONE 2 +#define FCU_STATUS_STS_VERI_FAIL 3 +#define FCU_STATUS_STS_LOAD_FAIL 4 +#define FCU_STATUS_STS_BUSY 5 +#define FCU_STATUS_AUTHFWLD __BIT(8) +#define FCU_STATUS_DONE __BIT(9) +#define FCU_STATUS_LOADED_AE __BITS(22, 31) + +#define FCU_STATUS1 0x8c8 + +#define FCU_DRAM_ADDR_LO 0x8cc +#define FCU_DRAM_ADDR_HI 0x8d0 +#define FCU_RAMBASE_ADDR_HI 0x8d4 +#define FCU_RAMBASE_ADDR_LO 0x8d8 + +#define FW_AUTH_WAIT_PERIOD 10 +#define FW_AUTH_MAX_RETRY 300 + +#define CAP_GLOBAL_CTL_BASE 0xa00 +#define CAP_GLOBAL_CTL_MISC CAP_GLOBAL_CTL_BASE + 0x04 +#define CAP_GLOBAL_CTL_MISC_TIMESTAMP_EN __BIT(7) +#define CAP_GLOBAL_CTL_RESET CAP_GLOBAL_CTL_BASE + 0x0c +#define CAP_GLOBAL_CTL_RESET_MASK __BITS(31, 26) +#define CAP_GLOBAL_CTL_RESET_ACCEL_MASK __BITS(25, 20) +#define CAP_GLOBAL_CTL_RESET_AE_MASK __BITS(19, 0) +#define CAP_GLOBAL_CTL_CLK_EN CAP_GLOBAL_CTL_BASE + 0x50 +#define CAP_GLOBAL_CTL_CLK_EN_ACCEL_MASK __BITS(25, 20) +#define CAP_GLOBAL_CTL_CLK_EN_AE_MASK __BITS(19, 0) + +/* -------------------------------------------------------------------------- */ +/* AE region */ +#define UPC_MASK 0x1ffff +#define USTORE_SIZE QAT_16K + +#define AE_LOCAL_AE_MASK __BITS(31, 12) +#define AE_LOCAL_CSR_MASK __BITS(9, 0) + +/* AE_LOCAL registers */ +/* Control Store Address Register */ +#define USTORE_ADDRESS 0x000 +#define USTORE_ADDRESS_ECS __BIT(31) + +#define USTORE_ECC_BIT_0 44 +#define USTORE_ECC_BIT_1 45 +#define USTORE_ECC_BIT_2 46 +#define USTORE_ECC_BIT_3 47 +#define USTORE_ECC_BIT_4 48 +#define USTORE_ECC_BIT_5 49 +#define USTORE_ECC_BIT_6 50 + +/* Control Store Data Lower Register */ +#define USTORE_DATA_LOWER 0x004 +/* Control Store Data Upper Register */ +#define USTORE_DATA_UPPER 0x008 +/* Control Store Error Status Register */ +#define USTORE_ERROR_STATUS 0x00c +/* Arithmetic Logic Unit Output Register */ +#define ALU_OUT 0x010 +/* Context Arbiter Control Register */ +#define CTX_ARB_CNTL 0x014 +#define CTX_ARB_CNTL_INIT 0x00000000 +/* Context Enables Register */ +#define CTX_ENABLES 0x018 +#define CTX_ENABLES_INIT 0 +#define CTX_ENABLES_INUSE_CONTEXTS __BIT(31) +#define CTX_ENABLES_CNTL_STORE_PARITY_ERROR __BIT(29) +#define CTX_ENABLES_CNTL_STORE_PARITY_ENABLE __BIT(28) +#define CTX_ENABLES_BREAKPOINT __BIT(27) +#define CTX_ENABLES_PAR_ERR __BIT(25) +#define CTX_ENABLES_NN_MODE __BIT(20) +#define CTX_ENABLES_NN_RING_EMPTY __BIT(18) +#define CTX_ENABLES_LMADDR_1_GLOBAL __BIT(17) +#define CTX_ENABLES_LMADDR_0_GLOBAL __BIT(16) +#define CTX_ENABLES_ENABLE __BITS(15,8) + +#define CTX_ENABLES_IGNORE_W1C_MASK \ + (~(CTX_ENABLES_PAR_ERR | \ + CTX_ENABLES_BREAKPOINT | \ + CTX_ENABLES_CNTL_STORE_PARITY_ERROR)) + +/* cycles from CTX_ENABLE high to CTX entering executing state */ +#define CYCLES_FROM_READY2EXE 8 + +/* Condition Code Enable Register */ +#define CC_ENABLE 0x01c +#define CC_ENABLE_INIT 0x2000 + +/* CSR Context Pointer Register */ +#define CSR_CTX_POINTER 0x020 +#define CSR_CTX_POINTER_CONTEXT __BITS(2,0) +/* Register Error Status Register */ +#define REG_ERROR_STATUS 0x030 +/* Indirect Context Status Register */ +#define CTX_STS_INDIRECT 0x040 +#define CTX_STS_INDIRECT_UPC_INIT 0x00000000 + +/* Active Context Status Register */ +#define ACTIVE_CTX_STATUS 0x044 +#define ACTIVE_CTX_STATUS_ABO __BIT(31) +#define ACTIVE_CTX_STATUS_ACNO __BITS(0, 2) +/* Indirect Context Signal Events Register */ +#define CTX_SIG_EVENTS_INDIRECT 0x048 +#define CTX_SIG_EVENTS_INDIRECT_INIT 0x00000001 +/* Active Context Signal Events Register */ +#define CTX_SIG_EVENTS_ACTIVE 0x04c +/* Indirect Context Wakeup Events Register */ +#define CTX_WAKEUP_EVENTS_INDIRECT 0x050 +#define CTX_WAKEUP_EVENTS_INDIRECT_VOLUNTARY 0x00000001 +#define CTX_WAKEUP_EVENTS_INDIRECT_SLEEP 0x00010000 + +#define CTX_WAKEUP_EVENTS_INDIRECT_INIT 0x00000001 + +/* Active Context Wakeup Events Register */ +#define CTX_WAKEUP_EVENTS_ACTIVE 0x054 +/* Indirect Context Future Count Register */ +#define CTX_FUTURE_COUNT_INDIRECT 0x058 +/* Active Context Future Count Register */ +#define CTX_FUTURE_COUNT_ACTIVE 0x05c +/* Indirect Local Memory Address 0 Register */ +#define LM_ADDR_0_INDIRECT 0x060 +/* Active Local Memory Address 0 Register */ +#define LM_ADDR_0_ACTIVE 0x064 +/* Indirect Local Memory Address 1 Register */ +#define LM_ADDR_1_INDIRECT 0x068 +/* Active Local Memory Address 1 Register */ +#define LM_ADDR_1_ACTIVE 0x06c +/* Byte Index Register */ +#define BYTE_INDEX 0x070 +/* Indirect Local Memory Address 0 Byte Index Register */ +#define INDIRECT_LM_ADDR_0_BYTE_INDEX 0x0e0 +/* Active Local Memory Address 0 Byte Index Register */ +#define ACTIVE_LM_ADDR_0_BYTE_INDEX 0x0e4 +/* Indirect Local Memory Address 1 Byte Index Register */ +#define INDIRECT_LM_ADDR_1_BYTE_INDEX 0x0e8 +/* Active Local Memory Address 1 Byte Index Register */ +#define ACTIVE_LM_ADDR_1_BYTE_INDEX 0x0ec +/* Transfer Index Concatenated with Byte Index Register */ +#define T_INDEX_BYTE_INDEX 0x0f4 +/* Transfer Index Register */ +#define T_INDEX 0x074 +/* Indirect Future Count Signal Signal Register */ +#define FUTURE_COUNT_SIGNAL_INDIRECT 0x078 +/* Active Context Future Count Register */ +#define FUTURE_COUNT_SIGNAL_ACTIVE 0x07c +/* Next Neighbor Put Register */ +#define NN_PUT 0x080 +/* Next Neighbor Get Register */ +#define NN_GET 0x084 +/* Timestamp Low Register */ +#define TIMESTAMP_LOW 0x0c0 +/* Timestamp High Register */ +#define TIMESTAMP_HIGH 0x0c4 +/* Next Neighbor Signal Register */ +#define NEXT_NEIGHBOR_SIGNAL 0x100 +/* Previous Neighbor Signal Register */ +#define PREV_NEIGHBOR_SIGNAL 0x104 +/* Same AccelEngine Signal Register */ +#define SAME_AE_SIGNAL 0x108 +/* Cyclic Redundancy Check Remainder Register */ +#define CRC_REMAINDER 0x140 +/* Profile Count Register */ +#define PROFILE_COUNT 0x144 +/* Pseudorandom Number Register */ +#define PSEUDO_RANDOM_NUMBER 0x148 +/* Signature Enable Register */ +#define SIGNATURE_ENABLE 0x150 +/* Miscellaneous Control Register */ +#define AE_MISC_CONTROL 0x160 +#define AE_MISC_CONTROL_PARITY_ENABLE __BIT(24) +#define AE_MISC_CONTROL_FORCE_BAD_PARITY __BIT(23) +#define AE_MISC_CONTROL_ONE_CTX_RELOAD __BIT(22) +#define AE_MISC_CONTROL_CS_RELOAD __BITS(21, 20) +#define AE_MISC_CONTROL_SHARE_CS __BIT(2) +/* Control Store Address 1 Register */ +#define USTORE_ADDRESS1 0x158 +/* Local CSR Status Register */ +#define LOCAL_CSR_STATUS 0x180 +#define LOCAL_CSR_STATUS_STATUS 0x1 +/* NULL Register */ +#define NULL_CSR 0x3fc + +/* AE_XFER macros */ +#define AE_XFER_AE_MASK __BITS(31, 12) +#define AE_XFER_CSR_MASK __BITS(9, 2) + +#define AEREG_BAD_REGADDR 0xffff /* bad register address */ + +/* -------------------------------------------------------------------------- */ + +#define SSMWDT(i) ((i) * 0x4000 + 0x54) +#define SSMWDTPKE(i) ((i) * 0x4000 + 0x58) +#define INTSTATSSM(i) ((i) * 0x4000 + 0x04) +#define INTSTATSSM_SHANGERR __BIT(13) +#define PPERR(i) ((i) * 0x4000 + 0x08) +#define PPERRID(i) ((i) * 0x4000 + 0x0C) +#define CERRSSMSH(i) ((i) * 0x4000 + 0x10) +#define UERRSSMSH(i) ((i) * 0x4000 + 0x18) +#define UERRSSMSHAD(i) ((i) * 0x4000 + 0x1C) +#define SLICEHANGSTATUS(i) ((i) * 0x4000 + 0x4C) +#define SLICE_HANG_AUTH0_MASK __BIT(0) +#define SLICE_HANG_AUTH1_MASK __BIT(1) +#define SLICE_HANG_CPHR0_MASK __BIT(4) +#define SLICE_HANG_CPHR1_MASK __BIT(5) +#define SLICE_HANG_CMP0_MASK __BIT(8) +#define SLICE_HANG_CMP1_MASK __BIT(9) +#define SLICE_HANG_XLT0_MASK __BIT(12) +#define SLICE_HANG_XLT1_MASK __BIT(13) +#define SLICE_HANG_MMP0_MASK __BIT(16) +#define SLICE_HANG_MMP1_MASK __BIT(17) +#define SLICE_HANG_MMP2_MASK __BIT(18) +#define SLICE_HANG_MMP3_MASK __BIT(19) +#define SLICE_HANG_MMP4_MASK __BIT(20) + +#define SHINTMASKSSM(i) ((i) * 0x4000 + 0x1018) +#define ENABLE_SLICE_HANG 0x000000 +#define MAX_MMP (5) +#define MMP_BASE(i) ((i) * 0x1000 % 0x3800) +#define CERRSSMMMP(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x380) +#define UERRSSMMMP(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x388) +#define UERRSSMMMPAD(i, n) ((i) * 0x4000 + MMP_BASE(n) + 0x38C) + +#define CPP_CFC_ERR_STATUS (0x30000 + 0xC04) +#define CPP_CFC_ERR_PPID (0x30000 + 0xC08) + +#define ERRSOU0 (0x3A000 + 0x00) +#define ERRSOU1 (0x3A000 + 0x04) +#define ERRSOU2 (0x3A000 + 0x08) +#define ERRSOU3 (0x3A000 + 0x0C) +#define ERRSOU4 (0x3A000 + 0xD0) +#define ERRSOU5 (0x3A000 + 0xD8) +#define ERRMSK0 (0x3A000 + 0x10) +#define ERRMSK1 (0x3A000 + 0x14) +#define ERRMSK2 (0x3A000 + 0x18) +#define ERRMSK3 (0x3A000 + 0x1C) +#define ERRMSK4 (0x3A000 + 0xD4) +#define ERRMSK5 (0x3A000 + 0xDC) +#define EMSK3_CPM0_MASK __BIT(2) +#define EMSK3_CPM1_MASK __BIT(3) +#define EMSK5_CPM2_MASK __BIT(16) +#define EMSK5_CPM3_MASK __BIT(17) +#define EMSK5_CPM4_MASK __BIT(18) +#define RICPPINTSTS (0x3A000 + 0x114) +#define RIERRPUSHID (0x3A000 + 0x118) +#define RIERRPULLID (0x3A000 + 0x11C) + +#define TICPPINTSTS (0x3A400 + 0x13C) +#define TIERRPUSHID (0x3A400 + 0x140) +#define TIERRPULLID (0x3A400 + 0x144) +#define SECRAMUERR (0x3AC00 + 0x04) +#define SECRAMUERRAD (0x3AC00 + 0x0C) +#define CPPMEMTGTERR (0x3AC00 + 0x10) +#define ERRPPID (0x3AC00 + 0x14) + +#define ADMINMSGUR 0x3a574 +#define ADMINMSGLR 0x3a578 +#define MAILBOX_BASE 0x20970 +#define MAILBOX_STRIDE 0x1000 +#define ADMINMSG_LEN 32 + +/* -------------------------------------------------------------------------- */ +static const uint8_t mailbox_const_tab[1024] __aligned(1024) = { +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, +0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, +0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, +0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, +0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, +0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, +0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, +0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, +0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, +0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, +0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, +0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, +0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, +0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, +0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, +0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, +0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, +0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, +0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, +0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +/* -------------------------------------------------------------------------- */ +/* Microcode */ + +/* Clear GPR of AE */ +static const uint64_t ae_clear_gprs_inst[] = { + 0x0F0000C0000ull, /* .0 l0000!val = 0 ; immed[l0000!val, 0x0] */ + 0x0F000000380ull, /* .1 l0000!count = 128 ; immed[l0000!count, 0x80] */ + 0x0D805000011ull, /* .2 br!=ctx[0, ctx_init#] */ + 0x0FC082C0300ull, /* .3 local_csr_wr[nn_put, 0] */ + 0x0F0000C0300ull, /* .4 nop */ + 0x0F0000C0300ull, /* .5 nop */ + 0x0F0000C0300ull, /* .6 nop */ + 0x0F0000C0300ull, /* .7 nop */ + 0x0A0643C0000ull, /* .8 init_nn#:alu[*n$index++, --, b, l0000!val] */ + 0x0BAC0000301ull, /* .9 alu[l0000!count, l0000!count, -, 1] */ + 0x0D802000101ull, /* .10 bne[init_nn#] */ + 0x0F0000C0001ull, /* .11 l0000!indx = 0 ; immed[l0000!indx, 0x0] */ + 0x0FC066C0001ull, /* .12 local_csr_wr[active_lm_addr_0, l0000!indx]; + * put indx to lm_addr */ + 0x0F0000C0300ull, /* .13 nop */ + 0x0F0000C0300ull, /* .14 nop */ + 0x0F0000C0300ull, /* .15 nop */ + 0x0F000400300ull, /* .16 l0000!count = 1024 ; immed[l0000!count, 0x400] */ + 0x0A0610C0000ull, /* .17 init_lm#:alu[*l$index0++, --, b, l0000!val] */ + 0x0BAC0000301ull, /* .18 alu[l0000!count, l0000!count, -, 1] */ + 0x0D804400101ull, /* .19 bne[init_lm#] */ + 0x0A0580C0000ull, /* .20 ctx_init#:alu[$l0000!xfers[0], --, b, l0000!val] */ + 0x0A0581C0000ull, /* .21 alu[$l0000!xfers[1], --, b, l0000!val] */ + 0x0A0582C0000ull, /* .22 alu[$l0000!xfers[2], --, b, l0000!val] */ + 0x0A0583C0000ull, /* .23 alu[$l0000!xfers[3], --, b, l0000!val] */ + 0x0A0584C0000ull, /* .24 alu[$l0000!xfers[4], --, b, l0000!val] */ + 0x0A0585C0000ull, /* .25 alu[$l0000!xfers[5], --, b, l0000!val] */ + 0x0A0586C0000ull, /* .26 alu[$l0000!xfers[6], --, b, l0000!val] */ + 0x0A0587C0000ull, /* .27 alu[$l0000!xfers[7], --, b, l0000!val] */ + 0x0A0588C0000ull, /* .28 alu[$l0000!xfers[8], --, b, l0000!val] */ + 0x0A0589C0000ull, /* .29 alu[$l0000!xfers[9], --, b, l0000!val] */ + 0x0A058AC0000ull, /* .30 alu[$l0000!xfers[10], --, b, l0000!val] */ + 0x0A058BC0000ull, /* .31 alu[$l0000!xfers[11], --, b, l0000!val] */ + 0x0A058CC0000ull, /* .32 alu[$l0000!xfers[12], --, b, l0000!val] */ + 0x0A058DC0000ull, /* .33 alu[$l0000!xfers[13], --, b, l0000!val] */ + 0x0A058EC0000ull, /* .34 alu[$l0000!xfers[14], --, b, l0000!val] */ + 0x0A058FC0000ull, /* .35 alu[$l0000!xfers[15], --, b, l0000!val] */ + 0x0A05C0C0000ull, /* .36 alu[$l0000!xfers[16], --, b, l0000!val] */ + 0x0A05C1C0000ull, /* .37 alu[$l0000!xfers[17], --, b, l0000!val] */ + 0x0A05C2C0000ull, /* .38 alu[$l0000!xfers[18], --, b, l0000!val] */ + 0x0A05C3C0000ull, /* .39 alu[$l0000!xfers[19], --, b, l0000!val] */ + 0x0A05C4C0000ull, /* .40 alu[$l0000!xfers[20], --, b, l0000!val] */ + 0x0A05C5C0000ull, /* .41 alu[$l0000!xfers[21], --, b, l0000!val] */ + 0x0A05C6C0000ull, /* .42 alu[$l0000!xfers[22], --, b, l0000!val] */ + 0x0A05C7C0000ull, /* .43 alu[$l0000!xfers[23], --, b, l0000!val] */ + 0x0A05C8C0000ull, /* .44 alu[$l0000!xfers[24], --, b, l0000!val] */ + 0x0A05C9C0000ull, /* .45 alu[$l0000!xfers[25], --, b, l0000!val] */ + 0x0A05CAC0000ull, /* .46 alu[$l0000!xfers[26], --, b, l0000!val] */ + 0x0A05CBC0000ull, /* .47 alu[$l0000!xfers[27], --, b, l0000!val] */ + 0x0A05CCC0000ull, /* .48 alu[$l0000!xfers[28], --, b, l0000!val] */ + 0x0A05CDC0000ull, /* .49 alu[$l0000!xfers[29], --, b, l0000!val] */ + 0x0A05CEC0000ull, /* .50 alu[$l0000!xfers[30], --, b, l0000!val] */ + 0x0A05CFC0000ull, /* .51 alu[$l0000!xfers[31], --, b, l0000!val] */ + 0x0A0400C0000ull, /* .52 alu[l0000!gprega[0], --, b, l0000!val] */ + 0x0B0400C0000ull, /* .53 alu[l0000!gpregb[0], --, b, l0000!val] */ + 0x0A0401C0000ull, /* .54 alu[l0000!gprega[1], --, b, l0000!val] */ + 0x0B0401C0000ull, /* .55 alu[l0000!gpregb[1], --, b, l0000!val] */ + 0x0A0402C0000ull, /* .56 alu[l0000!gprega[2], --, b, l0000!val] */ + 0x0B0402C0000ull, /* .57 alu[l0000!gpregb[2], --, b, l0000!val] */ + 0x0A0403C0000ull, /* .58 alu[l0000!gprega[3], --, b, l0000!val] */ + 0x0B0403C0000ull, /* .59 alu[l0000!gpregb[3], --, b, l0000!val] */ + 0x0A0404C0000ull, /* .60 alu[l0000!gprega[4], --, b, l0000!val] */ + 0x0B0404C0000ull, /* .61 alu[l0000!gpregb[4], --, b, l0000!val] */ + 0x0A0405C0000ull, /* .62 alu[l0000!gprega[5], --, b, l0000!val] */ + 0x0B0405C0000ull, /* .63 alu[l0000!gpregb[5], --, b, l0000!val] */ + 0x0A0406C0000ull, /* .64 alu[l0000!gprega[6], --, b, l0000!val] */ + 0x0B0406C0000ull, /* .65 alu[l0000!gpregb[6], --, b, l0000!val] */ + 0x0A0407C0000ull, /* .66 alu[l0000!gprega[7], --, b, l0000!val] */ + 0x0B0407C0000ull, /* .67 alu[l0000!gpregb[7], --, b, l0000!val] */ + 0x0A0408C0000ull, /* .68 alu[l0000!gprega[8], --, b, l0000!val] */ + 0x0B0408C0000ull, /* .69 alu[l0000!gpregb[8], --, b, l0000!val] */ + 0x0A0409C0000ull, /* .70 alu[l0000!gprega[9], --, b, l0000!val] */ + 0x0B0409C0000ull, /* .71 alu[l0000!gpregb[9], --, b, l0000!val] */ + 0x0A040AC0000ull, /* .72 alu[l0000!gprega[10], --, b, l0000!val] */ + 0x0B040AC0000ull, /* .73 alu[l0000!gpregb[10], --, b, l0000!val] */ + 0x0A040BC0000ull, /* .74 alu[l0000!gprega[11], --, b, l0000!val] */ + 0x0B040BC0000ull, /* .75 alu[l0000!gpregb[11], --, b, l0000!val] */ + 0x0A040CC0000ull, /* .76 alu[l0000!gprega[12], --, b, l0000!val] */ + 0x0B040CC0000ull, /* .77 alu[l0000!gpregb[12], --, b, l0000!val] */ + 0x0A040DC0000ull, /* .78 alu[l0000!gprega[13], --, b, l0000!val] */ + 0x0B040DC0000ull, /* .79 alu[l0000!gpregb[13], --, b, l0000!val] */ + 0x0A040EC0000ull, /* .80 alu[l0000!gprega[14], --, b, l0000!val] */ + 0x0B040EC0000ull, /* .81 alu[l0000!gpregb[14], --, b, l0000!val] */ + 0x0A040FC0000ull, /* .82 alu[l0000!gprega[15], --, b, l0000!val] */ + 0x0B040FC0000ull, /* .83 alu[l0000!gpregb[15], --, b, l0000!val] */ + 0x0D81581C010ull, /* .84 br=ctx[7, exit#] */ + 0x0E000010000ull, /* .85 ctx_arb[kill], any */ + 0x0E000010000ull, /* .86 exit#:ctx_arb[kill], any */ +}; + +static const uint64_t ae_inst_4b[] = { + 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ + 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ + 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ + 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ + 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, + l0000!indx]; put indx to lm_addr */ + 0x0F0000C0300ull, /* .5 nop */ + 0x0F0000C0300ull, /* .6 nop */ + 0x0F0000C0300ull, /* .7 nop */ + 0x0A021000000ull, /* .8 alu[*l$index0++, --, b, l0000!myvalue] */ +}; + +static const uint64_t ae_inst_1b[] = { + 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ + 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ + 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ + 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ + 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, + l0000!indx]; put indx to lm_addr */ + 0x0F0000C0300ull, /* .5 nop */ + 0x0F0000C0300ull, /* .6 nop */ + 0x0F0000C0300ull, /* .7 nop */ + 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ + 0x09080000200ull, /* .9 alu_shf[l0000!myvalue, --, b, + l0000!myvalue, <<24 ] */ + 0x08180280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<8 ] */ + 0x08080280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>8 ] */ + 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ + +}; + +static const uint64_t ae_inst_2b[] = { + 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ + 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ + 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ + 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ + 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, + l0000!indx]; put indx to lm_addr */ + 0x0F0000C0300ull, /* .5 nop */ + 0x0F0000C0300ull, /* .6 nop */ + 0x0F0000C0300ull, /* .7 nop */ + 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ + 0x09100000200ull, /* .9 alu_shf[l0000!myvalue, --, b, + l0000!myvalue, <<16 ] */ + 0x08100280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<16 ] */ + 0x08100280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>16 ] */ + 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ +}; + +static const uint64_t ae_inst_3b[] = { + 0x0F0400C0000ull, /* .0 immed_w0[l0000!indx, 0] */ + 0x0F4400C0000ull, /* .1 immed_w1[l0000!indx, 0] */ + 0x0F040000300ull, /* .2 immed_w0[l0000!myvalue, 0x0] */ + 0x0F440000300ull, /* .3 immed_w1[l0000!myvalue, 0x0] */ + 0x0FC066C0000ull, /* .4 local_csr_wr[active_lm_addr_0, + l0000!indx]; put indx to lm_addr */ + 0x0F0000C0300ull, /* .5 nop */ + 0x0F0000C0300ull, /* .6 nop */ + 0x0F0000C0300ull, /* .7 nop */ + 0x0A000180000ull, /* .8 alu[l0000!val, --, b, *l$index0] */ + 0x09180000200ull, /* .9 alu_shf[l0000!myvalue, --, + b, l0000!myvalue, <<8 ] */ + 0x08080280201ull, /* .10 alu_shf[l0000!val1, --, b, l0000!val, <<24 ] */ + 0x08180280102ull, /* .11 alu_shf[l0000!val1, --, b, l0000!val1 , >>24 ] */ + 0x0BA00100002ull, /* .12 alu[l0000!val2, l0000!val1, or, l0000!myvalue] */ +}; + +/* micro-instr fixup */ +#define INSERT_IMMED_GPRA_CONST(inst, const_val) \ + inst = (inst & 0xFFFF00C03FFull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 10) & 0x0003FC00ull)) +#define INSERT_IMMED_GPRB_CONST(inst, const_val) \ + inst = (inst & 0xFFFF00FFF00ull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 0) & 0x000000FFull)) + +enum aereg_type { + AEREG_NO_DEST, /* no destination */ + AEREG_GPA_REL, /* general-purpose A register under relative mode */ + AEREG_GPA_ABS, /* general-purpose A register under absolute mode */ + AEREG_GPB_REL, /* general-purpose B register under relative mode */ + AEREG_GPB_ABS, /* general-purpose B register under absolute mode */ + AEREG_SR_REL, /* sram register under relative mode */ + AEREG_SR_RD_REL, /* sram read register under relative mode */ + AEREG_SR_WR_REL, /* sram write register under relative mode */ + AEREG_SR_ABS, /* sram register under absolute mode */ + AEREG_SR_RD_ABS, /* sram read register under absolute mode */ + AEREG_SR_WR_ABS, /* sram write register under absolute mode */ + AEREG_SR0_SPILL, /* sram0 spill register */ + AEREG_SR1_SPILL, /* sram1 spill register */ + AEREG_SR2_SPILL, /* sram2 spill register */ + AEREG_SR3_SPILL, /* sram3 spill register */ + AEREG_SR0_MEM_ADDR, /* sram0 memory address register */ + AEREG_SR1_MEM_ADDR, /* sram1 memory address register */ + AEREG_SR2_MEM_ADDR, /* sram2 memory address register */ + AEREG_SR3_MEM_ADDR, /* sram3 memory address register */ + AEREG_DR_REL, /* dram register under relative mode */ + AEREG_DR_RD_REL, /* dram read register under relative mode */ + AEREG_DR_WR_REL, /* dram write register under relative mode */ + AEREG_DR_ABS, /* dram register under absolute mode */ + AEREG_DR_RD_ABS, /* dram read register under absolute mode */ + AEREG_DR_WR_ABS, /* dram write register under absolute mode */ + AEREG_DR_MEM_ADDR, /* dram memory address register */ + AEREG_LMEM, /* local memory */ + AEREG_LMEM0, /* local memory bank0 */ + AEREG_LMEM1, /* local memory bank1 */ + AEREG_LMEM_SPILL, /* local memory spill */ + AEREG_LMEM_ADDR, /* local memory address */ + AEREG_NEIGH_REL, /* next neighbour register under relative mode */ + AEREG_NEIGH_INDX, /* next neighbour register under index mode */ + AEREG_SIG_REL, /* signal register under relative mode */ + AEREG_SIG_INDX, /* signal register under index mode */ + AEREG_SIG_DOUBLE, /* signal register */ + AEREG_SIG_SINGLE, /* signal register */ + AEREG_SCRATCH_MEM_ADDR, /* scratch memory address */ + AEREG_UMEM0, /* ustore memory bank0 */ + AEREG_UMEM1, /* ustore memory bank1 */ + AEREG_UMEM_SPILL, /* ustore memory spill */ + AEREG_UMEM_ADDR, /* ustore memory address */ + AEREG_DR1_MEM_ADDR, /* dram segment1 address */ + AEREG_SR0_IMPORTED, /* sram segment0 imported data */ + AEREG_SR1_IMPORTED, /* sram segment1 imported data */ + AEREG_SR2_IMPORTED, /* sram segment2 imported data */ + AEREG_SR3_IMPORTED, /* sram segment3 imported data */ + AEREG_DR_IMPORTED, /* dram segment0 imported data */ + AEREG_DR1_IMPORTED, /* dram segment1 imported data */ + AEREG_SCRATCH_IMPORTED, /* scratch imported data */ + AEREG_XFER_RD_ABS, /* transfer read register under absolute mode */ + AEREG_XFER_WR_ABS, /* transfer write register under absolute mode */ + AEREG_CONST_VALUE, /* const alue */ + AEREG_ADDR_TAKEN, /* address taken */ + AEREG_OPTIMIZED_AWAY, /* optimized away */ + AEREG_SHRAM_ADDR, /* shared ram0 address */ + AEREG_SHRAM1_ADDR, /* shared ram1 address */ + AEREG_SHRAM2_ADDR, /* shared ram2 address */ + AEREG_SHRAM3_ADDR, /* shared ram3 address */ + AEREG_SHRAM4_ADDR, /* shared ram4 address */ + AEREG_SHRAM5_ADDR, /* shared ram5 address */ + AEREG_ANY = 0xffff /* any register */ +}; +#define AEREG_SR_INDX AEREG_SR_ABS + /* sram transfer register under index mode */ +#define AEREG_DR_INDX AEREG_DR_ABS + /* dram transfer register under index mode */ +#define AEREG_NEIGH_ABS AEREG_NEIGH_INDX + /* next neighbor register under absolute mode */ + + +#define QAT_2K 0x0800 +#define QAT_4K 0x1000 +#define QAT_6K 0x1800 +#define QAT_8K 0x2000 +#define QAT_16K 0x4000 + +#define MOF_OBJ_ID_LEN 8 +#define MOF_FID 0x00666f6d +#define MOF_MIN_VER 0x1 +#define MOF_MAJ_VER 0x0 +#define SYM_OBJS "SYM_OBJS" /* symbol object string */ +#define UOF_OBJS "UOF_OBJS" /* uof object string */ +#define SUOF_OBJS "SUF_OBJS" /* suof object string */ +#define SUOF_IMAG "SUF_IMAG" /* suof chunk ID string */ + +#define UOF_STRT "UOF_STRT" /* string table section ID */ +#define UOF_GTID "UOF_GTID" /* GTID section ID */ +#define UOF_IMAG "UOF_IMAG" /* image section ID */ +#define UOF_IMEM "UOF_IMEM" /* import section ID */ +#define UOF_MSEG "UOF_MSEG" /* memory section ID */ + +#define CRC_POLY 0x1021 +#define CRC_WIDTH 16 +#define CRC_BITMASK(x) (1L << (x)) +#define CRC_WIDTHMASK(width) ((((1L<<(width-1))-1L)<<1)|1L) + +struct mof_file_hdr { + u_int mfh_fid; + u_int mfh_csum; + char mfh_min_ver; + char mfh_maj_ver; + u_short mfh_reserved; + u_short mfh_max_chunks; + u_short mfh_num_chunks; +}; + +struct mof_file_chunk_hdr { + char mfch_id[MOF_OBJ_ID_LEN]; + uint64_t mfch_offset; + uint64_t mfch_size; +}; + +struct mof_uof_hdr { + u_short muh_max_chunks; + u_short muh_num_chunks; + u_int muh_reserved; +}; + +struct mof_uof_chunk_hdr { + char much_id[MOF_OBJ_ID_LEN]; /* should be UOF_IMAG */ + uint64_t much_offset; /* uof image */ + uint64_t much_size; /* uof image size */ + u_int much_name; /* uof name string-table offset */ + u_int much_reserved; +}; + +#define UOF_MAX_NUM_OF_AE 16 /* maximum number of AE */ + +#define UOF_OBJ_ID_LEN 8 /* length of object ID */ +#define UOF_FIELD_POS_SIZE 12 /* field postion size */ +#define MIN_UOF_SIZE 24 /* minimum .uof file size */ +#define UOF_FID 0xc6c2 /* uof magic number */ +#define UOF_MIN_VER 0x11 +#define UOF_MAJ_VER 0x4 + +struct uof_file_hdr { + u_short ufh_id; /* file id and endian indicator */ + u_short ufh_reserved1; /* reserved for future use */ + char ufh_min_ver; /* file format minor version */ + char ufh_maj_ver; /* file format major version */ + u_short ufh_reserved2; /* reserved for future use */ + u_short ufh_max_chunks; /* max chunks in file */ + u_short ufh_num_chunks; /* num of actual chunks */ +}; + +struct uof_file_chunk_hdr { + char ufch_id[UOF_OBJ_ID_LEN]; /* chunk identifier */ + u_int ufch_csum; /* chunk checksum */ + u_int ufch_offset; /* offset of the chunk in the file */ + u_int ufch_size; /* size of the chunk */ +}; + +struct uof_obj_hdr { + u_int uoh_cpu_type; /* CPU type */ + u_short uoh_min_cpu_ver; /* starting CPU version */ + u_short uoh_max_cpu_ver; /* ending CPU version */ + short uoh_max_chunks; /* max chunks in chunk obj */ + short uoh_num_chunks; /* num of actual chunks */ + u_int uoh_reserved1; + u_int uoh_reserved2; +}; + +struct uof_chunk_hdr { + char uch_id[UOF_OBJ_ID_LEN]; + u_int uch_offset; + u_int uch_size; +}; + +struct uof_str_tab { + u_int ust_table_len; /* length of table */ + u_int ust_reserved; /* reserved for future use */ + uint64_t ust_strings; /* pointer to string table. + * NULL terminated strings */ +}; + +#define AE_MODE_RELOAD_CTX_SHARED __BIT(12) +#define AE_MODE_SHARED_USTORE __BIT(11) +#define AE_MODE_LMEM1 __BIT(9) +#define AE_MODE_LMEM0 __BIT(8) +#define AE_MODE_NN_MODE __BITS(7, 4) +#define AE_MODE_CTX_MODE __BITS(3, 0) + +#define AE_MODE_NN_MODE_NEIGH 0 +#define AE_MODE_NN_MODE_SELF 1 +#define AE_MODE_NN_MODE_DONTCARE 0xff + +struct uof_image { + u_int ui_name; /* image name */ + u_int ui_ae_assigned; /* AccelEngines assigned */ + u_int ui_ctx_assigned; /* AccelEngine contexts assigned */ + u_int ui_cpu_type; /* cpu type */ + u_int ui_entry_address; /* entry uaddress */ + u_int ui_fill_pattern[2]; /* uword fill value */ + u_int ui_reloadable_size; /* size of reloadable ustore section */ + + u_char ui_sensitivity; /* + * case sensitivity: 0 = insensitive, + * 1 = sensitive + */ + u_char ui_reserved; /* reserved for future use */ + u_short ui_ae_mode; /* + * unused<15:14>, legacyMode<13>, + * reloadCtxShared<12>, sharedUstore<11>, + * ecc<10>, locMem1<9>, locMem0<8>, + * nnMode<7:4>, ctx<3:0> + */ + + u_short ui_max_ver; /* max cpu ver on which the image can run */ + u_short ui_min_ver; /* min cpu ver on which the image can run */ + + u_short ui_image_attrib; /* image attributes */ + u_short ui_reserved2; /* reserved for future use */ + + u_short ui_num_page_regions; /* number of page regions */ + u_short ui_num_pages; /* number of pages */ + + u_int ui_reg_tab; /* offset to register table */ + u_int ui_init_reg_sym_tab; /* reg/sym init table */ + u_int ui_sbreak_tab; /* offset to sbreak table */ + + u_int ui_app_metadata; /* application meta-data */ + /* ui_npages of code page follows this header */ +}; + +struct uof_obj_table { + u_int uot_nentries; /* number of table entries */ + /* uot_nentries of object follows */ +}; + +struct uof_ae_reg { + u_int uar_name; /* reg name string-table offset */ + u_int uar_vis_name; /* reg visible name string-table offset */ + u_short uar_type; /* reg type */ + u_short uar_addr; /* reg address */ + u_short uar_access_mode; /* uof_RegAccessMode_T: read/write/both/undef */ + u_char uar_visible; /* register visibility */ + u_char uar_reserved1; /* reserved for future use */ + u_short uar_ref_count; /* number of contiguous registers allocated */ + u_short uar_reserved2; /* reserved for future use */ + u_int uar_xoid; /* xfer order ID */ +}; + +enum uof_value_kind { + UNDEF_VAL, /* undefined value */ + CHAR_VAL, /* character value */ + SHORT_VAL, /* short value */ + INT_VAL, /* integer value */ + STR_VAL, /* string value */ + STRTAB_VAL, /* string table value */ + NUM_VAL, /* number value */ + EXPR_VAL /* expression value */ +}; + +enum uof_init_type { + INIT_EXPR, + INIT_REG, + INIT_REG_CTX, + INIT_EXPR_ENDIAN_SWAP +}; + +struct uof_init_reg_sym { + u_int uirs_name; /* symbol name */ + char uirs_init_type; /* 0=expr, 1=register, 2=ctxReg, + * 3=expr_endian_swap */ + char uirs_value_type; /* EXPR_VAL, STRTAB_VAL */ + char uirs_reg_type; /* register type: ae_reg_type */ + u_char uirs_ctx; /* AE context when initType=2 */ + u_int uirs_addr_offset; /* reg address, or sym-value offset */ + u_int uirs_value; /* integer value, or expression */ +}; + +struct uof_sbreak { + u_int us_page_num; /* page number */ + u_int us_virt_uaddr; /* virt uaddress */ + u_char us_sbreak_type; /* sbreak type */ + u_char us_reg_type; /* register type: ae_reg_type */ + u_short us_reserved1; /* reserved for future use */ + u_int us_addr_offset; /* branch target address or offset + * to be used with the reg value to + * calculate the target address */ + u_int us_reg_rddr; /* register address */ +}; +struct uof_code_page { + u_int ucp_page_region; /* page associated region */ + u_int ucp_page_num; /* code-page number */ + u_char ucp_def_page; /* default page indicator */ + u_char ucp_reserved2; /* reserved for future use */ + u_short ucp_reserved1; /* reserved for future use */ + u_int ucp_beg_vaddr; /* starting virtual uaddr */ + u_int ucp_beg_paddr; /* starting physical uaddr */ + u_int ucp_neigh_reg_tab; /* offset to neighbour-reg table */ + u_int ucp_uc_var_tab; /* offset to uC var table */ + u_int ucp_imp_var_tab; /* offset to import var table */ + u_int ucp_imp_expr_tab; /* offset to import expression table */ + u_int ucp_code_area; /* offset to code area */ +}; + +struct uof_code_area { + u_int uca_num_micro_words; /* number of micro words */ + u_int uca_uword_block_tab; /* offset to ublock table */ +}; + +struct uof_uword_block { + u_int uub_start_addr; /* start address */ + u_int uub_num_words; /* number of microwords */ + u_int uub_uword_offset; /* offset to the uwords */ + u_int uub_reserved; /* reserved for future use */ +}; + +struct uof_uword_fixup { + u_int uuf_name; /* offset to string table */ + u_int uuf_uword_address; /* micro word address */ + u_int uuf_expr_value; /* string table offset of expr string, or value */ + u_char uuf_val_type; /* VALUE_UNDEF, VALUE_NUM, VALUE_EXPR */ + u_char uuf_value_attrs; /* bit<0> (Scope: 0=global, 1=local), + * bit<1> (init: 0=no, 1=yes) */ + u_short uuf_reserved1; /* reserved for future use */ + char uuf_field_attrs[UOF_FIELD_POS_SIZE]; + /* field pos, size, and right shift value */ +}; + +struct uof_import_var { + u_int uiv_name; /* import var name string-table offset */ + u_char uiv_value_attrs; /* bit<0> (Scope: 0=global), + * bit<1> (init: 0=no, 1=yes) */ + u_char uiv_reserved1; /* reserved for future use */ + u_short uiv_reserved2; /* reserved for future use */ + uint64_t uiv_value; /* 64-bit imported value */ +}; + +struct uof_mem_val_attr { + u_int umva_byte_offset; /* byte-offset from the allocated memory */ + u_int umva_value; /* memory value */ +}; + +enum uof_mem_region { + SRAM_REGION, /* SRAM region */ + DRAM_REGION, /* DRAM0 region */ + DRAM1_REGION, /* DRAM1 region */ + LMEM_REGION, /* local memory region */ + SCRATCH_REGION, /* SCRATCH region */ + UMEM_REGION, /* micro-store region */ + RAM_REGION, /* RAM region */ + SHRAM_REGION, /* shared memory-0 region */ + SHRAM1_REGION, /* shared memory-1 region */ + SHRAM2_REGION, /* shared memory-2 region */ + SHRAM3_REGION, /* shared memory-3 region */ + SHRAM4_REGION, /* shared memory-4 region */ + SHRAM5_REGION /* shared memory-5 region */ +}; + +#define UOF_SCOPE_GLOBAL 0 +#define UOF_SCOPE_LOCAL 1 + +struct uof_init_mem { + u_int uim_sym_name; /* symbol name */ + char uim_region; /* memory region -- uof_mem_region */ + char uim_scope; /* visibility scope */ + u_short uim_reserved1; /* reserved for future use */ + u_int uim_addr; /* memory address */ + u_int uim_num_bytes; /* number of bytes */ + u_int uim_num_val_attr; /* number of values attributes */ + + /* uim_num_val_attr of uof_mem_val_attr follows this header */ +}; + +struct uof_var_mem_seg { + u_int uvms_sram_base; /* SRAM memory segment base addr */ + u_int uvms_sram_size; /* SRAM segment size bytes */ + u_int uvms_sram_alignment; /* SRAM segment alignment bytes */ + u_int uvms_sdram_base; /* DRAM0 memory segment base addr */ + u_int uvms_sdram_size; /* DRAM0 segment size bytes */ + u_int uvms_sdram_alignment; /* DRAM0 segment alignment bytes */ + u_int uvms_sdram1_base; /* DRAM1 memory segment base addr */ + u_int uvms_sdram1_size; /* DRAM1 segment size bytes */ + u_int uvms_sdram1_alignment; /* DRAM1 segment alignment bytes */ + u_int uvms_scratch_base; /* SCRATCH memory segment base addr */ + u_int uvms_scratch_size; /* SCRATCH segment size bytes */ + u_int uvms_scratch_alignment; /* SCRATCH segment alignment bytes */ +}; + +#define SUOF_OBJ_ID_LEN 8 +#define SUOF_FID 0x53554f46 +#define SUOF_MAJ_VER 0x0 +#define SUOF_MIN_VER 0x1 +#define SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) +#define SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) +#define CSS_FWSK_MODULUS_LEN 256 +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PAD_LEN 252 +#define CSS_FWSK_PUB_LEN (CSS_FWSK_MODULUS_LEN + \ + CSS_FWSK_EXPONENT_LEN + \ + CSS_FWSK_PAD_LEN) +#define CSS_SIGNATURE_LEN 256 +#define CSS_AE_IMG_LEN (sizeof(struct simg_ae_mode) + \ + SIMG_AE_INIT_SEQ_LEN + \ + SIMG_AE_INSTS_LEN) +#define CSS_AE_SIMG_LEN (sizeof(struct css_hdr) + \ + CSS_FWSK_PUB_LEN + \ + CSS_SIGNATURE_LEN + \ + CSS_AE_IMG_LEN) +#define AE_IMG_OFFSET (sizeof(struct css_hdr) + \ + CSS_FWSK_MODULUS_LEN + \ + CSS_FWSK_EXPONENT_LEN + \ + CSS_SIGNATURE_LEN) +#define CSS_MAX_IMAGE_LEN 0x40000 + +struct fw_auth_desc { + u_int fad_img_len; + u_int fad_reserved; + u_int fad_css_hdr_high; + u_int fad_css_hdr_low; + u_int fad_img_high; + u_int fad_img_low; + u_int fad_signature_high; + u_int fad_signature_low; + u_int fad_fwsk_pub_high; + u_int fad_fwsk_pub_low; + u_int fad_img_ae_mode_data_high; + u_int fad_img_ae_mode_data_low; + u_int fad_img_ae_init_data_high; + u_int fad_img_ae_init_data_low; + u_int fad_img_ae_insts_high; + u_int fad_img_ae_insts_low; +}; + +struct auth_chunk { + struct fw_auth_desc ac_fw_auth_desc; + uint64_t ac_chunk_size; + uint64_t ac_chunk_bus_addr; +}; + +enum css_fwtype { + CSS_AE_FIRMWARE = 0, + CSS_MMP_FIRMWARE = 1 +}; + +struct css_hdr { + u_int css_module_type; + u_int css_header_len; + u_int css_header_ver; + u_int css_module_id; + u_int css_module_vendor; + u_int css_date; + u_int css_size; + u_int css_key_size; + u_int css_module_size; + u_int css_exponent_size; + u_int css_fw_type; + u_int css_reserved[21]; +}; + +struct simg_ae_mode { + u_int sam_file_id; + u_short sam_maj_ver; + u_short sam_min_ver; + u_int sam_dev_type; + u_short sam_devmax_ver; + u_short sam_devmin_ver; + u_int sam_ae_mask; + u_int sam_ctx_enables; + char sam_fw_type; + char sam_ctx_mode; + char sam_nn_mode; + char sam_lm0_mode; + char sam_lm1_mode; + char sam_scs_mode; + char sam_lm2_mode; + char sam_lm3_mode; + char sam_tindex_mode; + u_char sam_reserved[7]; + char sam_simg_name[256]; + char sam_appmeta_data[256]; +}; + +struct suof_file_hdr { + u_int sfh_file_id; + u_int sfh_check_sum; + char sfh_min_ver; + char sfh_maj_ver; + char sfh_fw_type; + char sfh_reserved; + u_short sfh_max_chunks; + u_short sfh_num_chunks; +}; + +struct suof_chunk_hdr { + char sch_chunk_id[SUOF_OBJ_ID_LEN]; + uint64_t sch_offset; + uint64_t sch_size; +}; + +struct suof_str_tab { + u_int sst_tab_length; + u_int sst_strings; +}; + +struct suof_obj_hdr { + u_int soh_img_length; + u_int soh_reserved; +}; + +/* -------------------------------------------------------------------------- */ +/* accel */ + +enum fw_slice { + FW_SLICE_NULL = 0, /* NULL slice type */ + FW_SLICE_CIPHER = 1, /* CIPHER slice type */ + FW_SLICE_AUTH = 2, /* AUTH slice type */ + FW_SLICE_DRAM_RD = 3, /* DRAM_RD Logical slice type */ + FW_SLICE_DRAM_WR = 4, /* DRAM_WR Logical slice type */ + FW_SLICE_COMP = 5, /* Compression slice type */ + FW_SLICE_XLAT = 6, /* Translator slice type */ + FW_SLICE_DELIMITER /* End delimiter */ +}; +#define MAX_FW_SLICE FW_SLICE_DELIMITER + +#define QAT_OPTIMAL_ALIGN_SHIFT 6 +#define QAT_OPTIMAL_ALIGN (1 << QAT_OPTIMAL_ALIGN_SHIFT) + +enum hw_auth_algo { + HW_AUTH_ALGO_NULL = 0, /* Null hashing */ + HW_AUTH_ALGO_SHA1 = 1, /* SHA1 hashing */ + HW_AUTH_ALGO_MD5 = 2, /* MD5 hashing */ + HW_AUTH_ALGO_SHA224 = 3, /* SHA-224 hashing */ + HW_AUTH_ALGO_SHA256 = 4, /* SHA-256 hashing */ + HW_AUTH_ALGO_SHA384 = 5, /* SHA-384 hashing */ + HW_AUTH_ALGO_SHA512 = 6, /* SHA-512 hashing */ + HW_AUTH_ALGO_AES_XCBC_MAC = 7, /* AES-XCBC-MAC hashing */ + HW_AUTH_ALGO_AES_CBC_MAC = 8, /* AES-CBC-MAC hashing */ + HW_AUTH_ALGO_AES_F9 = 9, /* AES F9 hashing */ + HW_AUTH_ALGO_GALOIS_128 = 10, /* Galois 128 bit hashing */ + HW_AUTH_ALGO_GALOIS_64 = 11, /* Galois 64 hashing */ + HW_AUTH_ALGO_KASUMI_F9 = 12, /* Kasumi F9 hashing */ + HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, /* UIA2/SNOW_3H F9 hashing */ + HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14, + HW_AUTH_RESERVED_1 = 15, + HW_AUTH_RESERVED_2 = 16, + HW_AUTH_ALGO_SHA3_256 = 17, + HW_AUTH_RESERVED_3 = 18, + HW_AUTH_ALGO_SHA3_512 = 19, + HW_AUTH_ALGO_DELIMITER = 20 +}; + +enum hw_auth_mode { + HW_AUTH_MODE0, + HW_AUTH_MODE1, + HW_AUTH_MODE2, + HW_AUTH_MODE_DELIMITER +}; + +struct hw_auth_config { + uint32_t config; + /* Configuration used for setting up the slice */ + uint32_t reserved; + /* Reserved */ +}; + +#define HW_AUTH_CONFIG_SHA3_ALGO __BITS(22, 23) +#define HW_AUTH_CONFIG_SHA3_PADDING __BIT(16) +#define HW_AUTH_CONFIG_CMPLEN __BITS(14, 8) + /* The length of the digest if the QAT is to the check*/ +#define HW_AUTH_CONFIG_MODE __BITS(7, 4) +#define HW_AUTH_CONFIG_ALGO __BITS(3, 0) + +#define HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \ + __SHIFTIN(mode, HW_AUTH_CONFIG_MODE) | \ + __SHIFTIN(algo, HW_AUTH_CONFIG_ALGO) | \ + __SHIFTIN(cmp_len, HW_AUTH_CONFIG_CMPLEN) + +struct hw_auth_counter { + uint32_t counter; /* Counter value */ + uint32_t reserved; /* Reserved */ +}; + +struct hw_auth_setup { + struct hw_auth_config auth_config; + /* Configuration word for the auth slice */ + struct hw_auth_counter auth_counter; + /* Auth counter value for this request */ +}; + +#define HW_NULL_STATE1_SZ 32 +#define HW_MD5_STATE1_SZ 16 +#define HW_SHA1_STATE1_SZ 20 +#define HW_SHA224_STATE1_SZ 32 +#define HW_SHA256_STATE1_SZ 32 +#define HW_SHA3_256_STATE1_SZ 32 +#define HW_SHA384_STATE1_SZ 64 +#define HW_SHA512_STATE1_SZ 64 +#define HW_SHA3_512_STATE1_SZ 64 +#define HW_SHA3_224_STATE1_SZ 28 +#define HW_SHA3_384_STATE1_SZ 48 +#define HW_AES_XCBC_MAC_STATE1_SZ 16 +#define HW_AES_CBC_MAC_STATE1_SZ 16 +#define HW_AES_F9_STATE1_SZ 32 +#define HW_KASUMI_F9_STATE1_SZ 16 +#define HW_GALOIS_128_STATE1_SZ 16 +#define HW_SNOW_3G_UIA2_STATE1_SZ 8 +#define HW_ZUC_3G_EIA3_STATE1_SZ 8 +#define HW_NULL_STATE2_SZ 32 +#define HW_MD5_STATE2_SZ 16 +#define HW_SHA1_STATE2_SZ 20 +#define HW_SHA224_STATE2_SZ 32 +#define HW_SHA256_STATE2_SZ 32 +#define HW_SHA3_256_STATE2_SZ 0 +#define HW_SHA384_STATE2_SZ 64 +#define HW_SHA512_STATE2_SZ 64 +#define HW_SHA3_512_STATE2_SZ 0 +#define HW_SHA3_224_STATE2_SZ 0 +#define HW_SHA3_384_STATE2_SZ 0 +#define HW_AES_XCBC_MAC_KEY_SZ 16 +#define HW_AES_CBC_MAC_KEY_SZ 16 +#define HW_AES_CCM_CBC_E_CTR0_SZ 16 +#define HW_F9_IK_SZ 16 +#define HW_F9_FK_SZ 16 +#define HW_KASUMI_F9_STATE2_SZ (HW_F9_IK_SZ + HW_F9_FK_SZ) +#define HW_AES_F9_STATE2_SZ HW_KASUMI_F9_STATE2_SZ +#define HW_SNOW_3G_UIA2_STATE2_SZ 24 +#define HW_ZUC_3G_EIA3_STATE2_SZ 32 +#define HW_GALOIS_H_SZ 16 +#define HW_GALOIS_LEN_A_SZ 8 +#define HW_GALOIS_E_CTR0_SZ 16 + +struct hw_auth_sha512 { + struct hw_auth_setup inner_setup; + /* Inner loop configuration word for the slice */ + uint8_t state1[HW_SHA512_STATE1_SZ]; + /* Slice state1 variable */ + struct hw_auth_setup outer_setup; + /* Outer configuration word for the slice */ + uint8_t state2[HW_SHA512_STATE2_SZ]; + /* Slice state2 variable */ +}; + +union hw_auth_algo_blk { + struct hw_auth_sha512 max; + /* This is the largest possible auth setup block size */ +}; + +enum hw_cipher_algo { + HW_CIPHER_ALGO_NULL = 0, /* Null ciphering */ + HW_CIPHER_ALGO_DES = 1, /* DES ciphering */ + HW_CIPHER_ALGO_3DES = 2, /* 3DES ciphering */ + HW_CIPHER_ALGO_AES128 = 3, /* AES-128 ciphering */ + HW_CIPHER_ALGO_AES192 = 4, /* AES-192 ciphering */ + HW_CIPHER_ALGO_AES256 = 5, /* AES-256 ciphering */ + HW_CIPHER_ALGO_ARC4 = 6, /* ARC4 ciphering */ + HW_CIPHER_ALGO_KASUMI = 7, /* Kasumi */ + HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, /* Snow_3G */ + HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, + HW_CIPHER_DELIMITER = 10 /* Delimiter type */ +}; + +enum hw_cipher_mode { + HW_CIPHER_ECB_MODE = 0, /* ECB mode */ + HW_CIPHER_CBC_MODE = 1, /* CBC mode */ + HW_CIPHER_CTR_MODE = 2, /* CTR mode */ + HW_CIPHER_F8_MODE = 3, /* F8 mode */ + HW_CIPHER_XTS_MODE = 6, + HW_CIPHER_MODE_DELIMITER = 7 /* Delimiter type */ +}; + +struct hw_cipher_config { + uint32_t val; /* Cipher slice configuration */ + uint32_t reserved; /* Reserved */ +}; + +#define CIPHER_CONFIG_CONVERT __BIT(9) +#define CIPHER_CONFIG_DIR __BIT(8) +#define CIPHER_CONFIG_MODE __BITS(7, 4) +#define CIPHER_CONFIG_ALGO __BITS(3, 0) +#define HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir) \ + __SHIFTIN(mode, CIPHER_CONFIG_MODE) | \ + __SHIFTIN(algo, CIPHER_CONFIG_ALGO) | \ + __SHIFTIN(convert, CIPHER_CONFIG_CONVERT) | \ + __SHIFTIN(dir, CIPHER_CONFIG_DIR) + +enum hw_cipher_dir { + HW_CIPHER_ENCRYPT = 0, /* encryption is required */ + HW_CIPHER_DECRYPT = 1, /* decryption is required */ +}; + +enum hw_cipher_convert { + HW_CIPHER_NO_CONVERT = 0, /* no key convert is required*/ + HW_CIPHER_KEY_CONVERT = 1, /* key conversion is required*/ +}; + +#define CIPHER_MODE_F8_KEY_SZ_MULT 2 +#define CIPHER_MODE_XTS_KEY_SZ_MULT 2 + +#define HW_DES_BLK_SZ 8 +#define HW_3DES_BLK_SZ 8 +#define HW_NULL_BLK_SZ 8 +#define HW_AES_BLK_SZ 16 +#define HW_KASUMI_BLK_SZ 8 +#define HW_SNOW_3G_BLK_SZ 8 +#define HW_ZUC_3G_BLK_SZ 8 +#define HW_NULL_KEY_SZ 256 +#define HW_DES_KEY_SZ 8 +#define HW_3DES_KEY_SZ 24 +#define HW_AES_128_KEY_SZ 16 +#define HW_AES_192_KEY_SZ 24 +#define HW_AES_256_KEY_SZ 32 +#define HW_AES_128_F8_KEY_SZ (HW_AES_128_KEY_SZ * \ + CIPHER_MODE_F8_KEY_SZ_MULT) +#define HW_AES_192_F8_KEY_SZ (HW_AES_192_KEY_SZ * \ + CIPHER_MODE_F8_KEY_SZ_MULT) +#define HW_AES_256_F8_KEY_SZ (HW_AES_256_KEY_SZ * \ + CIPHER_MODE_F8_KEY_SZ_MULT) +#define HW_AES_128_XTS_KEY_SZ (HW_AES_128_KEY_SZ * \ + CIPHER_MODE_XTS_KEY_SZ_MULT) +#define HW_AES_256_XTS_KEY_SZ (HW_AES_256_KEY_SZ * \ + CIPHER_MODE_XTS_KEY_SZ_MULT) +#define HW_KASUMI_KEY_SZ 16 +#define HW_KASUMI_F8_KEY_SZ (HW_KASUMI_KEY_SZ * \ + CIPHER_MODE_F8_KEY_SZ_MULT) +#define HW_AES_128_XTS_KEY_SZ (HW_AES_128_KEY_SZ * \ + CIPHER_MODE_XTS_KEY_SZ_MULT) +#define HW_AES_256_XTS_KEY_SZ (HW_AES_256_KEY_SZ * \ + CIPHER_MODE_XTS_KEY_SZ_MULT) +#define HW_ARC4_KEY_SZ 256 +#define HW_SNOW_3G_UEA2_KEY_SZ 16 +#define HW_SNOW_3G_UEA2_IV_SZ 16 +#define HW_ZUC_3G_EEA3_KEY_SZ 16 +#define HW_ZUC_3G_EEA3_IV_SZ 16 +#define HW_MODE_F8_NUM_REG_TO_CLEAR 2 + +struct hw_cipher_aes256_f8 { + struct hw_cipher_config cipher_config; + /* Cipher configuration word for the slice set to + * AES-256 and the F8 mode */ + uint8_t key[HW_AES_256_F8_KEY_SZ]; + /* Cipher key */ +}; + +union hw_cipher_algo_blk { + struct hw_cipher_aes256_f8 max; /* AES-256 F8 Cipher */ + /* This is the largest possible cipher setup block size */ +}; + +struct flat_buffer_desc { + uint32_t data_len_in_bytes; + uint32_t reserved; + uint64_t phy_buffer; +}; + +struct buffer_list_desc { + uint64_t resrvd; + uint32_t num_buffers; + uint32_t reserved; +}; + +/* -------------------------------------------------------------------------- */ +/* look aside */ + +enum fw_la_cmd_id { + FW_LA_CMD_CIPHER, /* Cipher Request */ + FW_LA_CMD_AUTH, /* Auth Request */ + FW_LA_CMD_CIPHER_HASH, /* Cipher-Hash Request */ + FW_LA_CMD_HASH_CIPHER, /* Hash-Cipher Request */ + FW_LA_CMD_TRNG_GET_RANDOM, /* TRNG Get Random Request */ + FW_LA_CMD_TRNG_TEST, /* TRNG Test Request */ + FW_LA_CMD_SSL3_KEY_DERIVE, /* SSL3 Key Derivation Request */ + FW_LA_CMD_TLS_V1_1_KEY_DERIVE, /* TLS Key Derivation Request */ + FW_LA_CMD_TLS_V1_2_KEY_DERIVE, /* TLS Key Derivation Request */ + FW_LA_CMD_MGF1, /* MGF1 Request */ + FW_LA_CMD_AUTH_PRE_COMP, /* Auth Pre-Compute Request */ +#if 0 /* incompatible between qat 1.5 and 1.7 */ + FW_LA_CMD_CIPHER_CIPHER, /* Cipher-Cipher Request */ + FW_LA_CMD_HASH_HASH, /* Hash-Hash Request */ + FW_LA_CMD_CIPHER_PRE_COMP, /* Auth Pre-Compute Request */ +#endif + FW_LA_CMD_DELIMITER, /* Delimiter type */ +}; + +#endif Index: head/sys/dev/qat/qatvar.h =================================================================== --- head/sys/dev/qat/qatvar.h +++ head/sys/dev/qat/qatvar.h @@ -0,0 +1,1073 @@ +/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ +/* $NetBSD: qatvar.h,v 1.2 2020/03/14 18:08:39 ad Exp $ */ + +/* + * Copyright (c) 2019 Internet Initiative Japan, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Copyright(c) 2007-2019 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* $FreeBSD$ */ + +#ifndef _DEV_PCI_QATVAR_H_ +#define _DEV_PCI_QATVAR_H_ + +#include +#include + +#include + +#define QAT_NSYMREQ 256 +#define QAT_NSYMCOOKIE ((QAT_NSYMREQ * 2 + 1) * 2) +#define QAT_NASYMREQ 64 +#define QAT_BATCH_SUBMIT_FREE_SPACE 2 + +#define QAT_EV_NAME_SIZE 32 +#define QAT_RING_NAME_SIZE 32 + +#define QAT_MAXSEG 32 /* max segments for sg dma */ +#define QAT_MAXLEN 65535 /* IP_MAXPACKET */ + +#define QAT_HB_INTERVAL 500 /* heartbeat msec */ +#define QAT_SSM_WDT 100 + +enum qat_chip_type { + QAT_CHIP_C2XXX = 0, /* NanoQAT: Atom C2000 */ + QAT_CHIP_C2XXX_IOV, + QAT_CHIP_C3XXX, /* Atom C3000 */ + QAT_CHIP_C3XXX_IOV, + QAT_CHIP_C62X, + QAT_CHIP_C62X_IOV, + QAT_CHIP_D15XX, + QAT_CHIP_D15XX_IOV, + QAT_CHIP_DH895XCC, + QAT_CHIP_DH895XCC_IOV, +}; + +enum qat_sku { + QAT_SKU_UNKNOWN = 0, + QAT_SKU_1, + QAT_SKU_2, + QAT_SKU_3, + QAT_SKU_4, + QAT_SKU_VF, +}; + +enum qat_ae_status { + QAT_AE_ENABLED = 1, + QAT_AE_ACTIVE, + QAT_AE_DISABLED +}; + +#define TIMEOUT_AE_RESET 100 +#define TIMEOUT_AE_CHECK 10000 +#define TIMEOUT_AE_CSR 500 +#define AE_EXEC_CYCLE 20 + +#define QAT_UOF_MAX_PAGE 1 +#define QAT_UOF_MAX_PAGE_REGION 1 + +struct qat_dmamem { + bus_dma_tag_t qdm_dma_tag; + bus_dmamap_t qdm_dma_map; + bus_size_t qdm_dma_size; + bus_dma_segment_t qdm_dma_seg; + void *qdm_dma_vaddr; +}; + +/* Valid internal ring size values */ +#define QAT_RING_SIZE_128 0x01 +#define QAT_RING_SIZE_256 0x02 +#define QAT_RING_SIZE_512 0x03 +#define QAT_RING_SIZE_4K 0x06 +#define QAT_RING_SIZE_16K 0x08 +#define QAT_RING_SIZE_4M 0x10 +#define QAT_MIN_RING_SIZE QAT_RING_SIZE_128 +#define QAT_MAX_RING_SIZE QAT_RING_SIZE_4M +#define QAT_DEFAULT_RING_SIZE QAT_RING_SIZE_16K + +/* Valid internal msg size values */ +#define QAT_MSG_SIZE_32 0x01 +#define QAT_MSG_SIZE_64 0x02 +#define QAT_MSG_SIZE_128 0x04 +#define QAT_MIN_MSG_SIZE QAT_MSG_SIZE_32 +#define QAT_MAX_MSG_SIZE QAT_MSG_SIZE_128 + +/* Size to bytes conversion macros for ring and msg size values */ +#define QAT_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5) +#define QAT_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5) +#define QAT_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7) +#define QAT_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) + +/* Minimum ring bufer size for memory allocation */ +#define QAT_RING_SIZE_BYTES_MIN(SIZE) \ + ((SIZE < QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K)) ? \ + QAT_SIZE_TO_RING_SIZE_IN_BYTES(QAT_RING_SIZE_4K) : SIZE) +#define QAT_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) +#define QAT_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ + SIZE) & ~0x4) +/* Max outstanding requests */ +#define QAT_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \ + ((((1 << (RING_SIZE - 1)) << 3) >> QAT_SIZE_TO_POW(MSG_SIZE)) - 1) + +#define QAT_RING_PATTERN 0x7f + +struct qat_softc; + +typedef int (*qat_cb_t)(struct qat_softc *, void *, void *); + +struct qat_ring { + struct mtx qr_ring_mtx; /* Lock per ring */ + bool qr_need_wakeup; + void *qr_ring_vaddr; + uint32_t * volatile qr_inflight; /* tx/rx shared */ + uint32_t qr_head; + uint32_t qr_tail; + uint8_t qr_msg_size; + uint8_t qr_ring_size; + uint32_t qr_ring; /* ring number in bank */ + uint32_t qr_bank; /* bank number in device */ + uint32_t qr_ring_id; + uint32_t qr_ring_mask; + qat_cb_t qr_cb; + void *qr_cb_arg; + struct qat_dmamem qr_dma; + bus_addr_t qr_ring_paddr; + + const char *qr_name; +}; + +struct qat_bank { + struct qat_softc *qb_sc; /* back pointer to softc */ + uint32_t qb_intr_mask; /* current interrupt mask */ + uint32_t qb_allocated_rings; /* current allocated ring bitfiled */ + uint32_t qb_coalescing_time; /* timer in nano sec, 0: disabled */ +#define COALESCING_TIME_INTERVAL_DEFAULT 10000 +#define COALESCING_TIME_INTERVAL_MIN 500 +#define COALESCING_TIME_INTERVAL_MAX 0xfffff + uint32_t qb_bank; /* bank index */ + struct mtx qb_bank_mtx; + struct resource *qb_ih; + void *qb_ih_cookie; + + struct qat_ring qb_et_rings[MAX_RING_PER_BANK]; + +}; + +struct qat_ap_bank { + uint32_t qab_nf_mask; + uint32_t qab_nf_dest; + uint32_t qab_ne_mask; + uint32_t qab_ne_dest; +}; + +struct qat_ae_page { + struct qat_ae_page *qap_next; + struct qat_uof_page *qap_page; + struct qat_ae_region *qap_region; + u_int qap_flags; +}; + +#define QAT_AE_PAGA_FLAG_WAITING (1 << 0) + +struct qat_ae_region { + struct qat_ae_page *qar_loaded_page; + STAILQ_HEAD(, qat_ae_page) qar_waiting_pages; +}; + +struct qat_ae_slice { + u_int qas_assigned_ctx_mask; + struct qat_ae_region qas_regions[QAT_UOF_MAX_PAGE_REGION]; + struct qat_ae_page qas_pages[QAT_UOF_MAX_PAGE]; + struct qat_ae_page *qas_cur_pages[MAX_AE_CTX]; + struct qat_uof_image *qas_image; +}; + +#define QAT_AE(sc, ae) \ + ((sc)->sc_ae[ae]) + +struct qat_ae { + u_int qae_state; /* AE state */ + u_int qae_ustore_size; /* free micro-store address */ + u_int qae_free_addr; /* free micro-store address */ + u_int qae_free_size; /* free micro-store size */ + u_int qae_live_ctx_mask; /* live context mask */ + u_int qae_ustore_dram_addr; /* mirco-store DRAM address */ + u_int qae_reload_size; /* reloadable code size */ + + /* aefw */ + u_int qae_num_slices; + struct qat_ae_slice qae_slices[MAX_AE_CTX]; + u_int qae_reloc_ustore_dram; /* reloadable ustore-dram address */ + u_int qae_effect_ustore_size; /* effective AE ustore size */ + u_int qae_shareable_ustore; +}; + +struct qat_mof { + void *qmf_sym; /* SYM_OBJS in sc_fw_mof */ + size_t qmf_sym_size; + void *qmf_uof_objs; /* UOF_OBJS in sc_fw_mof */ + size_t qmf_uof_objs_size; + void *qmf_suof_objs; /* SUOF_OBJS in sc_fw_mof */ + size_t qmf_suof_objs_size; +}; + +struct qat_ae_batch_init { + u_int qabi_ae; + u_int qabi_addr; + u_int *qabi_value; + u_int qabi_size; + STAILQ_ENTRY(qat_ae_batch_init) qabi_next; +}; + +STAILQ_HEAD(qat_ae_batch_init_list, qat_ae_batch_init); + +/* overwritten struct uof_uword_block */ +struct qat_uof_uword_block { + u_int quub_start_addr; /* start address */ + u_int quub_num_words; /* number of microwords */ + uint64_t quub_micro_words; /* pointer to the uwords */ +}; + +struct qat_uof_page { + u_int qup_page_num; /* page number */ + u_int qup_def_page; /* default page */ + u_int qup_page_region; /* region of page */ + u_int qup_beg_vaddr; /* begin virtual address */ + u_int qup_beg_paddr; /* begin physical address */ + + u_int qup_num_uc_var; /* num of uC var in array */ + struct uof_uword_fixup *qup_uc_var; + /* array of import variables */ + u_int qup_num_imp_var; /* num of import var in array */ + struct uof_import_var *qup_imp_var; + /* array of import variables */ + u_int qup_num_imp_expr; /* num of import expr in array */ + struct uof_uword_fixup *qup_imp_expr; + /* array of import expressions */ + u_int qup_num_neigh_reg; /* num of neigh-reg in array */ + struct uof_uword_fixup *qup_neigh_reg; + /* array of neigh-reg assignments */ + u_int qup_num_micro_words; /* number of microwords in the seg */ + + u_int qup_num_uw_blocks; /* number of uword blocks */ + struct qat_uof_uword_block *qup_uw_blocks; + /* array of uword blocks */ +}; + +struct qat_uof_image { + struct uof_image *qui_image; /* image pointer */ + struct qat_uof_page qui_pages[QAT_UOF_MAX_PAGE]; + /* array of pages */ + + u_int qui_num_ae_reg; /* num of registers */ + struct uof_ae_reg *qui_ae_reg; /* array of registers */ + + u_int qui_num_init_reg_sym; /* num of reg/sym init values */ + struct uof_init_reg_sym *qui_init_reg_sym; + /* array of reg/sym init values */ + + u_int qui_num_sbreak; /* num of sbreak values */ + struct qui_sbreak *qui_sbreak; /* array of sbreak values */ + + u_int qui_num_uwords_used; + /* highest uword addressreferenced + 1 */ +}; + +struct qat_aefw_uof { + size_t qafu_size; /* uof size */ + struct uof_obj_hdr *qafu_obj_hdr; /* UOF_OBJS */ + + void *qafu_str_tab; + size_t qafu_str_tab_size; + + u_int qafu_num_init_mem; + struct uof_init_mem *qafu_init_mem; + size_t qafu_init_mem_size; + + struct uof_var_mem_seg *qafu_var_mem_seg; + + struct qat_ae_batch_init_list qafu_lm_init[MAX_AE]; + size_t qafu_num_lm_init[MAX_AE]; + size_t qafu_num_lm_init_inst[MAX_AE]; + + u_int qafu_num_imgs; /* number of uof image */ + struct qat_uof_image qafu_imgs[MAX_NUM_AE * MAX_AE_CTX]; + /* uof images */ +}; + +#define QAT_SERVICE_CRYPTO_A (1 << 0) +#define QAT_SERVICE_CRYPTO_B (1 << 1) + +struct qat_admin_rings { + uint32_t qadr_active_aes_per_accel; + uint8_t qadr_srv_mask[MAX_AE_PER_ACCEL]; + + struct qat_dmamem qadr_dma; + struct fw_init_ring_table *qadr_master_ring_tbl; + struct fw_init_ring_table *qadr_cya_ring_tbl; + struct fw_init_ring_table *qadr_cyb_ring_tbl; + + struct qat_ring *qadr_admin_tx; + struct qat_ring *qadr_admin_rx; +}; + +struct qat_accel_init_cb { + int qaic_status; +}; + +struct qat_admin_comms { + struct qat_dmamem qadc_dma; + struct qat_dmamem qadc_const_tbl_dma; + struct qat_dmamem qadc_hb_dma; +}; + +#define QAT_PID_MINOR_REV 0xf +#define QAT_PID_MAJOR_REV (0xf << 4) + +struct qat_suof_image { + char *qsi_simg_buf; + u_long qsi_simg_len; + char *qsi_css_header; + char *qsi_css_key; + char *qsi_css_signature; + char *qsi_css_simg; + u_long qsi_simg_size; + u_int qsi_ae_num; + u_int qsi_ae_mask; + u_int qsi_fw_type; + u_long qsi_simg_name; + u_long qsi_appmeta_data; + struct qat_dmamem qsi_dma; +}; + +struct qat_aefw_suof { + u_int qafs_file_id; + u_int qafs_check_sum; + char qafs_min_ver; + char qafs_maj_ver; + char qafs_fw_type; + char *qafs_suof_buf; + u_int qafs_suof_size; + char *qafs_sym_str; + u_int qafs_sym_size; + u_int qafs_num_simgs; + struct qat_suof_image *qafs_simg; +}; + +enum qat_sym_hash_algorithm { + QAT_SYM_HASH_NONE = 0, + QAT_SYM_HASH_MD5 = 1, + QAT_SYM_HASH_SHA1 = 2, + QAT_SYM_HASH_SHA224 = 3, + QAT_SYM_HASH_SHA256 = 4, + QAT_SYM_HASH_SHA384 = 5, + QAT_SYM_HASH_SHA512 = 6, + QAT_SYM_HASH_AES_XCBC = 7, + QAT_SYM_HASH_AES_CCM = 8, + QAT_SYM_HASH_AES_GCM = 9, + QAT_SYM_HASH_KASUMI_F9 = 10, + QAT_SYM_HASH_SNOW3G_UIA2 = 11, + QAT_SYM_HASH_AES_CMAC = 12, + QAT_SYM_HASH_AES_GMAC = 13, + QAT_SYM_HASH_AES_CBC_MAC = 14, +}; + +#define QAT_HASH_MD5_BLOCK_SIZE 64 +#define QAT_HASH_MD5_DIGEST_SIZE 16 +#define QAT_HASH_MD5_STATE_SIZE 16 +#define QAT_HASH_SHA1_BLOCK_SIZE 64 +#define QAT_HASH_SHA1_DIGEST_SIZE 20 +#define QAT_HASH_SHA1_STATE_SIZE 20 +#define QAT_HASH_SHA224_BLOCK_SIZE 64 +#define QAT_HASH_SHA224_DIGEST_SIZE 28 +#define QAT_HASH_SHA224_STATE_SIZE 32 +#define QAT_HASH_SHA256_BLOCK_SIZE 64 +#define QAT_HASH_SHA256_DIGEST_SIZE 32 +#define QAT_HASH_SHA256_STATE_SIZE 32 +#define QAT_HASH_SHA384_BLOCK_SIZE 128 +#define QAT_HASH_SHA384_DIGEST_SIZE 48 +#define QAT_HASH_SHA384_STATE_SIZE 64 +#define QAT_HASH_SHA512_BLOCK_SIZE 128 +#define QAT_HASH_SHA512_DIGEST_SIZE 64 +#define QAT_HASH_SHA512_STATE_SIZE 64 +#define QAT_HASH_XCBC_PRECOMP_KEY_NUM 3 +#define QAT_HASH_XCBC_MAC_BLOCK_SIZE 16 +#define QAT_HASH_XCBC_MAC_128_DIGEST_SIZE 16 +#define QAT_HASH_CMAC_BLOCK_SIZE 16 +#define QAT_HASH_CMAC_128_DIGEST_SIZE 16 +#define QAT_HASH_AES_CCM_BLOCK_SIZE 16 +#define QAT_HASH_AES_CCM_DIGEST_SIZE 16 +#define QAT_HASH_AES_GCM_BLOCK_SIZE 16 +#define QAT_HASH_AES_GCM_DIGEST_SIZE 16 +#define QAT_HASH_AES_GCM_STATE_SIZE 16 +#define QAT_HASH_KASUMI_F9_BLOCK_SIZE 8 +#define QAT_HASH_KASUMI_F9_DIGEST_SIZE 4 +#define QAT_HASH_SNOW3G_UIA2_BLOCK_SIZE 8 +#define QAT_HASH_SNOW3G_UIA2_DIGEST_SIZE 4 +#define QAT_HASH_AES_CBC_MAC_BLOCK_SIZE 16 +#define QAT_HASH_AES_CBC_MAC_DIGEST_SIZE 16 +#define QAT_HASH_AES_GCM_ICV_SIZE_8 8 +#define QAT_HASH_AES_GCM_ICV_SIZE_12 12 +#define QAT_HASH_AES_GCM_ICV_SIZE_16 16 +#define QAT_HASH_AES_CCM_ICV_SIZE_MIN 4 +#define QAT_HASH_AES_CCM_ICV_SIZE_MAX 16 +#define QAT_HASH_IPAD_BYTE 0x36 +#define QAT_HASH_OPAD_BYTE 0x5c +#define QAT_HASH_IPAD_4_BYTES 0x36363636 +#define QAT_HASH_OPAD_4_BYTES 0x5c5c5c5c +#define QAT_HASH_KASUMI_F9_KEY_MODIFIER_4_BYTES 0xAAAAAAAA + +#define QAT_SYM_XCBC_STATE_SIZE ((QAT_HASH_XCBC_MAC_BLOCK_SIZE) * 3) +#define QAT_SYM_CMAC_STATE_SIZE ((QAT_HASH_CMAC_BLOCK_SIZE) * 3) + +struct qat_sym_hash_alg_info { + uint32_t qshai_digest_len; /* Digest length in bytes */ + uint32_t qshai_block_len; /* Block length in bytes */ + uint32_t qshai_state_size; /* size of above state in bytes */ + const uint8_t *qshai_init_state; /* Initial state */ + + const struct auth_hash *qshai_sah; /* software auth hash */ + uint32_t qshai_state_offset; /* offset to state in *_CTX */ + uint32_t qshai_state_word; +}; + +struct qat_sym_hash_qat_info { + uint32_t qshqi_algo_enc; /* QAT Algorithm encoding */ + uint32_t qshqi_auth_counter; /* Counter value for Auth */ + uint32_t qshqi_state1_len; /* QAT state1 length in bytes */ + uint32_t qshqi_state2_len; /* QAT state2 length in bytes */ +}; + +struct qat_sym_hash_def { + const struct qat_sym_hash_alg_info *qshd_alg; + const struct qat_sym_hash_qat_info *qshd_qat; +}; + +#define QAT_SYM_REQ_PARAMS_SIZE_MAX (24 + 32) +/* Reserve enough space for cipher and authentication request params */ +/* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */ + +#define QAT_SYM_REQ_PARAMS_SIZE_PADDED \ + roundup(QAT_SYM_REQ_PARAMS_SIZE_MAX, QAT_OPTIMAL_ALIGN) +/* Pad out to 64-byte multiple to ensure optimal alignment of next field */ + +#define QAT_SYM_KEY_TLS_PREFIX_SIZE (128) +/* Hash Prefix size in bytes for TLS (128 = MAX = SHA2 (384, 512)*/ + +#define QAT_SYM_KEY_MAX_HASH_STATE_BUFFER \ + (QAT_SYM_KEY_TLS_PREFIX_SIZE * 2) +/* hash state prefix buffer structure that holds the maximum sized secret */ + +#define QAT_SYM_HASH_BUFFER_LEN QAT_HASH_SHA512_STATE_SIZE +/* Buffer length to hold 16 byte MD5 key and 20 byte SHA1 key */ + +#define QAT_GCM_AAD_SIZE_MAX 240 +/* Maximum AAD size */ + +#define QAT_AES_GCM_AAD_ALIGN 16 + +struct qat_sym_bulk_cookie { + uint8_t qsbc_req_params_buf[QAT_SYM_REQ_PARAMS_SIZE_PADDED]; + /* memory block reserved for request params + * NOTE: Field must be correctly aligned in memory for access by QAT + * engine */ + struct qat_crypto *qsbc_crypto; + struct qat_session *qsbc_session; + /* Session context */ + void *qsbc_cb_tag; + /* correlator supplied by the client */ + uint8_t qsbc_msg[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)]; + /* QAT request message */ +} __aligned(QAT_OPTIMAL_ALIGN); + +/* Basis of values are guaranteed in qat_hw*var.h with CTASSERT */ +#define HASH_CONTENT_DESC_SIZE 176 +#define CIPHER_CONTENT_DESC_SIZE 64 + +#define CONTENT_DESC_MAX_SIZE roundup( \ + HASH_CONTENT_DESC_SIZE + CIPHER_CONTENT_DESC_SIZE, \ + QAT_OPTIMAL_ALIGN) + +struct qat_sym_cookie { + union qat_sym_cookie_u { + /* should be 64byte aligned */ + struct qat_sym_bulk_cookie qsc_bulk_cookie; + /* symmetric bulk cookie */ +#ifdef notyet + struct qat_sym_key_cookie qsc_key_cookie; + /* symmetric key cookie */ + struct qat_sym_nrbg_cookie qsc_nrbg_cookie; + /* symmetric NRBG cookie */ +#endif + } u; + + /* should be 64-byte aligned */ + struct buffer_list_desc qsc_buf_list; + struct flat_buffer_desc qsc_flat_bufs[QAT_MAXSEG]; /* should be here */ + + bus_dmamap_t qsc_self_dmamap; /* self DMA mapping and + end of DMA region */ + bus_dma_tag_t qsc_self_dma_tag; + + uint8_t qsc_iv_buf[EALG_MAX_BLOCK_LEN]; + uint8_t qsc_auth_res[QAT_SYM_HASH_BUFFER_LEN]; + uint8_t qsc_gcm_aad[QAT_GCM_AAD_SIZE_MAX]; + uint8_t qsc_content_desc[CONTENT_DESC_MAX_SIZE]; + + bus_dmamap_t qsc_buf_dmamap; /* qsc_flat_bufs DMA mapping */ + bus_dma_tag_t qsc_buf_dma_tag; + void *qsc_buf; + + bus_addr_t qsc_bulk_req_params_buf_paddr; + bus_addr_t qsc_buffer_list_desc_paddr; + bus_addr_t qsc_iv_buf_paddr; + bus_addr_t qsc_auth_res_paddr; + bus_addr_t qsc_gcm_aad_paddr; + bus_addr_t qsc_content_desc_paddr; +}; + +CTASSERT(offsetof(struct qat_sym_cookie, + u.qsc_bulk_cookie.qsbc_req_params_buf) % QAT_OPTIMAL_ALIGN == 0); +CTASSERT(offsetof(struct qat_sym_cookie, qsc_buf_list) % QAT_OPTIMAL_ALIGN == 0); + +#define MAX_CIPHER_SETUP_BLK_SZ \ + (sizeof(struct hw_cipher_config) + \ + 2 * HW_KASUMI_KEY_SZ + 2 * HW_KASUMI_BLK_SZ) +#define MAX_HASH_SETUP_BLK_SZ sizeof(union hw_auth_algo_blk) + +struct qat_crypto_desc { + uint8_t qcd_content_desc[CONTENT_DESC_MAX_SIZE]; /* must be first */ + /* using only for qat 1.5 */ + uint8_t qcd_hash_state_prefix_buf[QAT_GCM_AAD_SIZE_MAX]; + + bus_addr_t qcd_desc_paddr; + bus_addr_t qcd_hash_state_paddr; + + enum fw_slice qcd_slices[MAX_FW_SLICE]; + enum fw_la_cmd_id qcd_cmd_id; + enum hw_cipher_dir qcd_cipher_dir; + + /* content desc info */ + uint8_t qcd_hdr_sz; /* in quad words */ + uint8_t qcd_hw_blk_sz; /* in quad words */ + uint32_t qcd_cipher_offset; + uint32_t qcd_auth_offset; + /* hash info */ + uint8_t qcd_state_storage_sz; /* in quad words */ + uint32_t qcd_gcm_aad_sz_offset1; + uint32_t qcd_gcm_aad_sz_offset2; + /* cipher info */ + uint16_t qcd_cipher_blk_sz; /* in bytes */ + uint16_t qcd_auth_sz; /* in bytes */ + + uint8_t qcd_req_cache[QAT_MSG_SIZE_TO_BYTES(QAT_MAX_MSG_SIZE)]; +} __aligned(QAT_OPTIMAL_ALIGN); + +/* should be aligned to 64bytes */ +struct qat_session { + struct qat_crypto_desc *qs_dec_desc; /* should be at top of struct*/ + /* decrypt or auth then decrypt or auth */ + + struct qat_crypto_desc *qs_enc_desc; + /* encrypt or encrypt then auth */ + + struct qat_dmamem qs_desc_mem; + + enum hw_cipher_algo qs_cipher_algo; + enum hw_cipher_mode qs_cipher_mode; + enum hw_auth_algo qs_auth_algo; + enum hw_auth_mode qs_auth_mode; + + const uint8_t *qs_cipher_key; + int qs_cipher_klen; + const uint8_t *qs_auth_key; + int qs_auth_klen; + int qs_auth_mlen; + + uint32_t qs_status; +#define QAT_SESSION_STATUS_ACTIVE (1 << 0) +#define QAT_SESSION_STATUS_FREEING (1 << 1) + uint32_t qs_inflight; + int qs_aad_length; + bool qs_need_wakeup; + + struct mtx qs_session_mtx; +}; + +struct qat_crypto_bank { + uint16_t qcb_bank; + + struct qat_ring *qcb_sym_tx; + struct qat_ring *qcb_sym_rx; + + struct qat_dmamem qcb_symck_dmamems[QAT_NSYMCOOKIE]; + struct qat_sym_cookie *qcb_symck_free[QAT_NSYMCOOKIE]; + uint32_t qcb_symck_free_count; + + struct mtx qcb_bank_mtx; + + char qcb_ring_names[2][QAT_RING_NAME_SIZE]; /* sym tx,rx */ +}; + +struct qat_crypto { + struct qat_softc *qcy_sc; + uint32_t qcy_bank_mask; + uint16_t qcy_num_banks; + + int32_t qcy_cid; /* OpenCrypto driver ID */ + + struct qat_crypto_bank *qcy_banks; /* array of qat_crypto_bank */ + + uint32_t qcy_session_free_count; + + struct mtx qcy_crypto_mtx; +}; + +struct qat_hw { + int8_t qhw_sram_bar_id; + int8_t qhw_misc_bar_id; + int8_t qhw_etr_bar_id; + + bus_size_t qhw_cap_global_offset; + bus_size_t qhw_ae_offset; + bus_size_t qhw_ae_local_offset; + bus_size_t qhw_etr_bundle_size; + + /* crypto processing callbacks */ + size_t qhw_crypto_opaque_offset; + void (*qhw_crypto_setup_req_params)(struct qat_crypto_bank *, + struct qat_session *, struct qat_crypto_desc const *, + struct qat_sym_cookie *, struct cryptop *); + void (*qhw_crypto_setup_desc)(struct qat_crypto *, struct qat_session *, + struct qat_crypto_desc *); + + uint8_t qhw_num_banks; /* max number of banks */ + uint8_t qhw_num_ap_banks; /* max number of AutoPush banks */ + uint8_t qhw_num_rings_per_bank; /* rings per bank */ + uint8_t qhw_num_accel; /* max number of accelerators */ + uint8_t qhw_num_engines; /* max number of accelerator engines */ + uint8_t qhw_tx_rx_gap; + uint32_t qhw_tx_rings_mask; + uint32_t qhw_clock_per_sec; + bool qhw_fw_auth; + uint32_t qhw_fw_req_size; + uint32_t qhw_fw_resp_size; + + uint8_t qhw_ring_sym_tx; + uint8_t qhw_ring_sym_rx; + uint8_t qhw_ring_asym_tx; + uint8_t qhw_ring_asym_rx; + + /* MSIx */ + uint32_t qhw_msix_ae_vec_gap; /* gap to ae vec from bank */ + + const char *qhw_mof_fwname; + const char *qhw_mmp_fwname; + + uint32_t qhw_prod_type; /* cpu type */ + + /* setup callbacks */ + uint32_t (*qhw_get_accel_mask)(struct qat_softc *); + uint32_t (*qhw_get_ae_mask)(struct qat_softc *); + enum qat_sku (*qhw_get_sku)(struct qat_softc *); + uint32_t (*qhw_get_accel_cap)(struct qat_softc *); + const char *(*qhw_get_fw_uof_name)(struct qat_softc *); + void (*qhw_enable_intr)(struct qat_softc *); + void (*qhw_init_etr_intr)(struct qat_softc *, int); + int (*qhw_init_admin_comms)(struct qat_softc *); + int (*qhw_send_admin_init)(struct qat_softc *); + int (*qhw_init_arb)(struct qat_softc *); + void (*qhw_get_arb_mapping)(struct qat_softc *, const uint32_t **); + void (*qhw_enable_error_correction)(struct qat_softc *); + int (*qhw_check_uncorrectable_error)(struct qat_softc *); + void (*qhw_print_err_registers)(struct qat_softc *); + void (*qhw_disable_error_interrupts)(struct qat_softc *); + int (*qhw_check_slice_hang)(struct qat_softc *); + int (*qhw_set_ssm_wdtimer)(struct qat_softc *); +}; + + +/* sc_flags */ +#define QAT_FLAG_ESRAM_ENABLE_AUTO_INIT (1 << 0) +#define QAT_FLAG_SHRAM_WAIT_READY (1 << 1) + +/* sc_accel_cap */ +#define QAT_ACCEL_CAP_CRYPTO_SYMMETRIC (1 << 0) +#define QAT_ACCEL_CAP_CRYPTO_ASYMMETRIC (1 << 1) +#define QAT_ACCEL_CAP_CIPHER (1 << 2) +#define QAT_ACCEL_CAP_AUTHENTICATION (1 << 3) +#define QAT_ACCEL_CAP_REGEX (1 << 4) +#define QAT_ACCEL_CAP_COMPRESSION (1 << 5) +#define QAT_ACCEL_CAP_LZS_COMPRESSION (1 << 6) +#define QAT_ACCEL_CAP_RANDOM_NUMBER (1 << 7) +#define QAT_ACCEL_CAP_ZUC (1 << 8) +#define QAT_ACCEL_CAP_SHA3 (1 << 9) +#define QAT_ACCEL_CAP_KPT (1 << 10) + +#define QAT_ACCEL_CAP_BITS \ + "\177\020" \ + "b\x0a" "KPT\0" \ + "b\x09" "SHA3\0" \ + "b\x08" "ZUC\0" \ + "b\x07" "RANDOM_NUMBER\0" \ + "b\x06" "LZS_COMPRESSION\0" \ + "b\x05" "COMPRESSION\0" \ + "b\x04" "REGEX\0" \ + "b\x03" "AUTHENTICATION\0" \ + "b\x02" "CIPHER\0" \ + "b\x01" "CRYPTO_ASYMMETRIC\0" \ + "b\x00" "CRYPTO_SYMMETRIC\0" + +#define QAT_HI_PRIO_RING_WEIGHT 0xfc +#define QAT_LO_PRIO_RING_WEIGHT 0xfe +#define QAT_DEFAULT_RING_WEIGHT 0xff +#define QAT_DEFAULT_PVL 0 + +struct firmware; +struct resource; + +struct qat_softc { + device_t sc_dev; + + struct resource *sc_res[MAX_BARS]; + int sc_rid[MAX_BARS]; + bus_space_tag_t sc_csrt[MAX_BARS]; + bus_space_handle_t sc_csrh[MAX_BARS]; + + uint32_t sc_ae_num; + uint32_t sc_ae_mask; + + struct qat_crypto sc_crypto; /* crypto services */ + + struct qat_hw sc_hw; + + uint8_t sc_rev; + enum qat_sku sc_sku; + uint32_t sc_flags; + + uint32_t sc_accel_num; + uint32_t sc_accel_mask; + uint32_t sc_accel_cap; + + struct qat_admin_rings sc_admin_rings; /* use only for qat 1.5 */ + struct qat_admin_comms sc_admin_comms; /* use only for qat 1.7 */ + + /* ETR */ + struct qat_bank *sc_etr_banks; /* array of etr banks */ + struct qat_ap_bank *sc_etr_ap_banks; /* array of etr auto push banks */ + + /* AE */ + struct qat_ae sc_ae[MAX_NUM_AE]; + + /* Interrupt */ + struct resource *sc_ih; /* ae cluster ih */ + void *sc_ih_cookie; /* ae cluster ih cookie */ + + /* Counters */ + counter_u64_t sc_gcm_aad_restarts; + counter_u64_t sc_gcm_aad_updates; + counter_u64_t sc_ring_full_restarts; + + /* Firmware */ + void *sc_fw_mof; /* mof data */ + size_t sc_fw_mof_size; /* mof size */ + struct qat_mof sc_mof; /* mof sections */ + + const char *sc_fw_uof_name; /* uof/suof name in mof */ + + void *sc_fw_uof; /* uof head */ + size_t sc_fw_uof_size; /* uof size */ + struct qat_aefw_uof sc_aefw_uof; /* UOF_OBJS in uof */ + + void *sc_fw_suof; /* suof head */ + size_t sc_fw_suof_size; /* suof size */ + struct qat_aefw_suof sc_aefw_suof; /* suof context */ + + void *sc_fw_mmp; /* mmp data */ + size_t sc_fw_mmp_size; /* mmp size */ +}; + +static inline void +qat_bar_write_4(struct qat_softc *sc, int baroff, bus_size_t offset, + uint32_t value) +{ + + MPASS(baroff >= 0 && baroff < MAX_BARS); + + bus_space_write_4(sc->sc_csrt[baroff], + sc->sc_csrh[baroff], offset, value); +} + +static inline uint32_t +qat_bar_read_4(struct qat_softc *sc, int baroff, bus_size_t offset) +{ + + MPASS(baroff >= 0 && baroff < MAX_BARS); + + return bus_space_read_4(sc->sc_csrt[baroff], + sc->sc_csrh[baroff], offset); +} + +static inline void +qat_misc_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) +{ + + qat_bar_write_4(sc, sc->sc_hw.qhw_misc_bar_id, offset, value); +} + +static inline uint32_t +qat_misc_read_4(struct qat_softc *sc, bus_size_t offset) +{ + + return qat_bar_read_4(sc, sc->sc_hw.qhw_misc_bar_id, offset); +} + +static inline void +qat_misc_read_write_or_4(struct qat_softc *sc, bus_size_t offset, + uint32_t value) +{ + uint32_t reg; + + reg = qat_misc_read_4(sc, offset); + reg |= value; + qat_misc_write_4(sc, offset, reg); +} + +static inline void +qat_misc_read_write_and_4(struct qat_softc *sc, bus_size_t offset, + uint32_t mask) +{ + uint32_t reg; + + reg = qat_misc_read_4(sc, offset); + reg &= mask; + qat_misc_write_4(sc, offset, reg); +} + +static inline void +qat_etr_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) +{ + + qat_bar_write_4(sc, sc->sc_hw.qhw_etr_bar_id, offset, value); +} + +static inline uint32_t +qat_etr_read_4(struct qat_softc *sc, bus_size_t offset) +{ + + return qat_bar_read_4(sc, sc->sc_hw.qhw_etr_bar_id, offset); +} + +static inline void +qat_ae_local_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, + uint32_t value) +{ + + offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) | + (offset & AE_LOCAL_CSR_MASK); + + qat_misc_write_4(sc, sc->sc_hw.qhw_ae_local_offset + offset, + value); +} + +static inline uint32_t +qat_ae_local_read_4(struct qat_softc *sc, u_char ae, bus_size_t offset) +{ + + offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_LOCAL_AE_MASK) | + (offset & AE_LOCAL_CSR_MASK); + + return qat_misc_read_4(sc, sc->sc_hw.qhw_ae_local_offset + offset); +} + +static inline void +qat_ae_xfer_write_4(struct qat_softc *sc, u_char ae, bus_size_t offset, + uint32_t value) +{ + offset = __SHIFTIN(ae & sc->sc_ae_mask, AE_XFER_AE_MASK) | + __SHIFTIN(offset, AE_XFER_CSR_MASK); + + qat_misc_write_4(sc, sc->sc_hw.qhw_ae_offset + offset, value); +} + +static inline void +qat_cap_global_write_4(struct qat_softc *sc, bus_size_t offset, uint32_t value) +{ + + qat_misc_write_4(sc, sc->sc_hw.qhw_cap_global_offset + offset, value); +} + +static inline uint32_t +qat_cap_global_read_4(struct qat_softc *sc, bus_size_t offset) +{ + + return qat_misc_read_4(sc, sc->sc_hw.qhw_cap_global_offset + offset); +} + + +static inline void +qat_etr_bank_write_4(struct qat_softc *sc, int bank, + bus_size_t offset, uint32_t value) +{ + + qat_etr_write_4(sc, sc->sc_hw.qhw_etr_bundle_size * bank + offset, + value); +} + +static inline uint32_t +qat_etr_bank_read_4(struct qat_softc *sc, int bank, + bus_size_t offset) +{ + + return qat_etr_read_4(sc, + sc->sc_hw.qhw_etr_bundle_size * bank + offset); +} + +static inline void +qat_etr_ap_bank_write_4(struct qat_softc *sc, int ap_bank, + bus_size_t offset, uint32_t value) +{ + + qat_etr_write_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset, value); +} + +static inline uint32_t +qat_etr_ap_bank_read_4(struct qat_softc *sc, int ap_bank, + bus_size_t offset) +{ + + return qat_etr_read_4(sc, ETR_AP_BANK_OFFSET * ap_bank + offset); +} + + +static inline void +qat_etr_bank_ring_write_4(struct qat_softc *sc, int bank, int ring, + bus_size_t offset, uint32_t value) +{ + + qat_etr_bank_write_4(sc, bank, (ring << 2) + offset, value); +} + +static inline uint32_t +qat_etr_bank_ring_read_4(struct qat_softc *sc, int bank, int ring, + bus_size_t offset) +{ + + return qat_etr_bank_read_4(sc, bank, (ring << 2) * offset); +} + +static inline void +qat_etr_bank_ring_base_write_8(struct qat_softc *sc, int bank, int ring, + uint64_t value) +{ + uint32_t lo, hi; + + lo = (uint32_t)(value & 0xffffffff); + hi = (uint32_t)((value & 0xffffffff00000000ULL) >> 32); + qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_LBASE, lo); + qat_etr_bank_ring_write_4(sc, bank, ring, ETR_RING_UBASE, hi); +} + +static inline void +qat_arb_ringsrvarben_write_4(struct qat_softc *sc, int index, uint32_t value) +{ + + qat_etr_write_4(sc, ARB_RINGSRVARBEN_OFFSET + + (ARB_REG_SLOT * index), value); +} + +static inline void +qat_arb_sarconfig_write_4(struct qat_softc *sc, int index, uint32_t value) +{ + + qat_etr_write_4(sc, ARB_OFFSET + + (ARB_REG_SIZE * index), value); +} + +static inline void +qat_arb_wrk_2_ser_map_write_4(struct qat_softc *sc, int index, uint32_t value) +{ + + qat_etr_write_4(sc, ARB_OFFSET + ARB_WRK_2_SER_MAP_OFFSET + + (ARB_REG_SIZE * index), value); +} + +void * qat_alloc_mem(size_t); +void qat_free_mem(void *); +void qat_free_dmamem(struct qat_softc *, struct qat_dmamem *); +int qat_alloc_dmamem(struct qat_softc *, struct qat_dmamem *, int, + bus_size_t, bus_size_t); + +int qat_etr_setup_ring(struct qat_softc *, int, uint32_t, uint32_t, + uint32_t, qat_cb_t, void *, const char *, + struct qat_ring **); +int qat_etr_put_msg(struct qat_softc *, struct qat_ring *, + uint32_t *); + +void qat_memcpy_htobe64(void *, const void *, size_t); +void qat_memcpy_htobe32(void *, const void *, size_t); +void qat_memcpy_htobe(void *, const void *, size_t, uint32_t); +void qat_crypto_gmac_precompute(const struct qat_crypto_desc *, + const uint8_t *key, int klen, + const struct qat_sym_hash_def *, uint8_t *); +void qat_crypto_hmac_precompute(const struct qat_crypto_desc *, + const uint8_t *, int, const struct qat_sym_hash_def *, + uint8_t *, uint8_t *); +uint16_t qat_crypto_load_cipher_session(const struct qat_crypto_desc *, + const struct qat_session *); +uint16_t qat_crypto_load_auth_session(const struct qat_crypto_desc *, + const struct qat_session *, + struct qat_sym_hash_def const **); + +#endif Index: head/sys/modules/Makefile =================================================================== --- head/sys/modules/Makefile +++ head/sys/modules/Makefile @@ -300,6 +300,7 @@ pty \ puc \ pwm \ + ${_qat} \ ${_qlxge} \ ${_qlxgb} \ ${_qlxgbe} \ @@ -630,6 +631,7 @@ _ocs_fc= ocs_fc _ossl= ossl _pccard= pccard +_qat= qat .if ${MK_OFED} != "no" || defined(ALL_MODULES) _rdma= rdma .endif Index: head/sys/modules/qat/Makefile =================================================================== --- head/sys/modules/qat/Makefile +++ head/sys/modules/qat/Makefile @@ -0,0 +1,19 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/dev/qat + +KMOD= qat + +SRCS= qat.c \ + qat_ae.c \ + qat_c2xxx.c \ + qat_c3xxx.c \ + qat_c62x.c \ + qat_d15xx.c \ + qat_dh895xcc.c \ + qat_hw15.c \ + qat_hw17.c + +SRCS+= bus_if.h cryptodev_if.h device_if.h pci_if.h + +.include