Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/qat/qat_hw15.c
- This file was added.
/* SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause */ | |||||
/* $NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $ */ | |||||
/* | |||||
* Copyright (c) 2019 Internet Initiative Japan, Inc. | |||||
* All rights reserved. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* 1. Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* 2. Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in the | |||||
* documentation and/or other materials provided with the distribution. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | |||||
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | |||||
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | |||||
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | |||||
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | |||||
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | |||||
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | |||||
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | |||||
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |||||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | |||||
* POSSIBILITY OF SUCH DAMAGE. | |||||
*/ | |||||
/* | |||||
* Copyright(c) 2007-2013 Intel Corporation. All rights reserved. | |||||
* | |||||
* Redistribution and use in source and binary forms, with or without | |||||
* modification, are permitted provided that the following conditions | |||||
* are met: | |||||
* | |||||
* * Redistributions of source code must retain the above copyright | |||||
* notice, this list of conditions and the following disclaimer. | |||||
* * Redistributions in binary form must reproduce the above copyright | |||||
* notice, this list of conditions and the following disclaimer in | |||||
* the documentation and/or other materials provided with the | |||||
* distribution. | |||||
* * Neither the name of Intel Corporation nor the names of its | |||||
* contributors may be used to endorse or promote products derived | |||||
* from this software without specific prior written permission. | |||||
* | |||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
*/ | |||||
#include <sys/cdefs.h> | |||||
__FBSDID("$FreeBSD"); | |||||
#if 0 | |||||
__KERNEL_RCSID(0, "$NetBSD: qat_hw15.c,v 1.1 2019/11/20 09:37:46 hikaru Exp $"); | |||||
#endif | |||||
#include <sys/param.h> | |||||
#include <sys/bus.h> | |||||
#include <sys/kernel.h> | |||||
#include <sys/proc.h> | |||||
#include <sys/systm.h> | |||||
#include <machine/bus.h> | |||||
#include <opencrypto/xform.h> | |||||
#include <dev/pci/pcireg.h> | |||||
#include <dev/pci/pcivar.h> | |||||
#include "qatreg.h" | |||||
#include "qat_hw15reg.h" | |||||
#include "qatvar.h" | |||||
#include "qat_hw15var.h" | |||||
int qat_adm_ring_init_ring_table(struct qat_softc *); | |||||
void qat_adm_ring_build_slice_mask(uint16_t *, uint32_t, uint32_t); | |||||
void qat_adm_ring_build_shram_mask(uint64_t *, uint32_t, uint32_t); | |||||
int qat_adm_ring_build_ring_table(struct qat_softc *, uint32_t); | |||||
int qat_adm_ring_build_init_msg(struct qat_softc *, | |||||
struct fw_init_req *, enum fw_init_cmd_id, uint32_t, | |||||
struct qat_accel_init_cb *); | |||||
int qat_adm_ring_send_init_msg_sync(struct qat_softc *, | |||||
enum fw_init_cmd_id, uint32_t); | |||||
int qat_adm_ring_send_init_msg(struct qat_softc *, | |||||
enum fw_init_cmd_id); | |||||
int qat_adm_ring_intr(struct qat_softc *, void *, void *); | |||||
void | |||||
qat_msg_req_type_populate(struct arch_if_req_hdr *msg, enum arch_if_req type, | |||||
uint32_t rxring) | |||||
{ | |||||
memset(msg, 0, sizeof(struct arch_if_req_hdr)); | |||||
msg->flags = ARCH_IF_FLAGS_VALID_FLAG | | |||||
ARCH_IF_FLAGS_RESP_RING_TYPE_ET | ARCH_IF_FLAGS_RESP_TYPE_S; | |||||
msg->req_type = type; | |||||
msg->resp_pipe_id = rxring; | |||||
} | |||||
void | |||||
qat_msg_cmn_hdr_populate(struct fw_la_bulk_req *msg, bus_addr_t desc_paddr, | |||||
uint8_t hdrsz, uint8_t hwblksz, uint16_t comn_req_flags, uint32_t flow_id) | |||||
{ | |||||
struct fw_comn_req_hdr *hdr = &msg->comn_hdr; | |||||
hdr->comn_req_flags = comn_req_flags; | |||||
hdr->content_desc_params_sz = hwblksz; | |||||
hdr->content_desc_hdr_sz = hdrsz; | |||||
hdr->content_desc_addr = desc_paddr; | |||||
msg->flow_id = flow_id; | |||||
} | |||||
void | |||||
qat_msg_service_cmd_populate(struct fw_la_bulk_req *msg, enum fw_la_cmd_id cmdid, | |||||
uint16_t cmd_flags) | |||||
{ | |||||
msg->comn_la_req.la_cmd_id = cmdid; | |||||
msg->comn_la_req.u.la_flags = cmd_flags; | |||||
} | |||||
void | |||||
qat_msg_cmn_mid_populate(struct fw_comn_req_mid *msg, void *cookie, | |||||
uint64_t src, uint64_t dst) | |||||
{ | |||||
msg->opaque_data = (uint64_t)(uintptr_t)cookie; | |||||
msg->src_data_addr = src; | |||||
if (dst == 0) | |||||
msg->dest_data_addr = src; | |||||
else | |||||
msg->dest_data_addr = dst; | |||||
} | |||||
void | |||||
qat_msg_req_params_populate(struct fw_la_bulk_req *msg, | |||||
bus_addr_t req_params_paddr, uint8_t req_params_sz) | |||||
{ | |||||
msg->req_params_addr = req_params_paddr; | |||||
msg->comn_la_req.u1.req_params_blk_sz = req_params_sz / 8; | |||||
} | |||||
void | |||||
qat_msg_cmn_footer_populate(union fw_comn_req_ftr *msg, uint64_t next_addr) | |||||
{ | |||||
msg->next_request_addr = next_addr; | |||||
} | |||||
void | |||||
qat_msg_params_populate(struct fw_la_bulk_req *msg, | |||||
struct qat_crypto_desc *desc, uint8_t req_params_sz, | |||||
uint16_t service_cmd_flags, uint16_t comn_req_flags) | |||||
{ | |||||
qat_msg_cmn_hdr_populate(msg, desc->qcd_desc_paddr, | |||||
desc->qcd_hdr_sz, desc->qcd_hw_blk_sz, comn_req_flags, 0); | |||||
qat_msg_service_cmd_populate(msg, desc->qcd_cmd_id, service_cmd_flags); | |||||
qat_msg_cmn_mid_populate(&msg->comn_mid, NULL, 0, 0); | |||||
qat_msg_req_params_populate(msg, 0, req_params_sz); | |||||
qat_msg_cmn_footer_populate(&msg->comn_ftr, 0); | |||||
} | |||||
int | |||||
qat_adm_ring_init_ring_table(struct qat_softc *sc) | |||||
{ | |||||
struct qat_admin_rings *qadr = &sc->sc_admin_rings; | |||||
if (sc->sc_ae_num == 1) { | |||||
qadr->qadr_cya_ring_tbl = | |||||
&qadr->qadr_master_ring_tbl[0]; | |||||
qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; | |||||
} else if (sc->sc_ae_num == 2 || | |||||
sc->sc_ae_num == 4) { | |||||
qadr->qadr_cya_ring_tbl = | |||||
&qadr->qadr_master_ring_tbl[0]; | |||||
qadr->qadr_srv_mask[0] = QAT_SERVICE_CRYPTO_A; | |||||
qadr->qadr_cyb_ring_tbl = | |||||
&qadr->qadr_master_ring_tbl[1]; | |||||
qadr->qadr_srv_mask[1] = QAT_SERVICE_CRYPTO_B; | |||||
} | |||||
return 0; | |||||
} | |||||
int | |||||
qat_adm_ring_init(struct qat_softc *sc) | |||||
{ | |||||
struct qat_admin_rings *qadr = &sc->sc_admin_rings; | |||||
int error, i, j; | |||||
error = qat_alloc_dmamem(sc, &qadr->qadr_dma, 1, PAGE_SIZE, PAGE_SIZE); | |||||
if (error) | |||||
return error; | |||||
qadr->qadr_master_ring_tbl = qadr->qadr_dma.qdm_dma_vaddr; | |||||
MPASS(sc->sc_ae_num * | |||||
sizeof(struct fw_init_ring_table) <= PAGE_SIZE); | |||||
/* Initialize the Master Ring Table */ | |||||
for (i = 0; i < sc->sc_ae_num; i++) { | |||||
struct fw_init_ring_table *firt = | |||||
&qadr->qadr_master_ring_tbl[i]; | |||||
for (j = 0; j < INIT_RING_TABLE_SZ; j++) { | |||||
struct fw_init_ring_params *firp = | |||||
&firt->firt_bulk_rings[j]; | |||||
firp->firp_reserved = 0; | |||||
firp->firp_curr_weight = QAT_DEFAULT_RING_WEIGHT; | |||||
firp->firp_init_weight = QAT_DEFAULT_RING_WEIGHT; | |||||
firp->firp_ring_pvl = QAT_DEFAULT_PVL; | |||||
} | |||||
memset(firt->firt_ring_mask, 0, sizeof(firt->firt_ring_mask)); | |||||
} | |||||
error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_TX, | |||||
ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_req_size, | |||||
NULL, NULL, "admin_tx", &qadr->qadr_admin_tx); | |||||
if (error) | |||||
return error; | |||||
error = qat_etr_setup_ring(sc, 0, RING_NUM_ADMIN_RX, | |||||
ADMIN_RING_SIZE, sc->sc_hw.qhw_fw_resp_size, | |||||
qat_adm_ring_intr, qadr, "admin_rx", &qadr->qadr_admin_rx); | |||||
if (error) | |||||
return error; | |||||
/* | |||||
* Finally set up the service indices into the Master Ring Table | |||||
* and convenient ring table pointers for each service enabled. | |||||
* Only the Admin rings are initialized. | |||||
*/ | |||||
error = qat_adm_ring_init_ring_table(sc); | |||||
if (error) | |||||
return error; | |||||
/* | |||||
* Calculate the number of active AEs per QAT | |||||
* needed for Shram partitioning. | |||||
*/ | |||||
for (i = 0; i < sc->sc_ae_num; i++) { | |||||
if (qadr->qadr_srv_mask[i]) | |||||
qadr->qadr_active_aes_per_accel++; | |||||
} | |||||
return 0; | |||||
} | |||||
void | |||||
qat_adm_ring_build_slice_mask(uint16_t *slice_mask, uint32_t srv_mask, | |||||
uint32_t init_shram) | |||||
{ | |||||
uint16_t shram = 0, comn_req = 0; | |||||
if (init_shram) | |||||
shram = COMN_REQ_SHRAM_INIT_REQUIRED; | |||||
if (srv_mask & QAT_SERVICE_CRYPTO_A) | |||||
comn_req |= COMN_REQ_CY0_ONLY(shram); | |||||
if (srv_mask & QAT_SERVICE_CRYPTO_B) | |||||
comn_req |= COMN_REQ_CY1_ONLY(shram); | |||||
*slice_mask = comn_req; | |||||
} | |||||
void | |||||
qat_adm_ring_build_shram_mask(uint64_t *shram_mask, uint32_t active_aes, | |||||
uint32_t ae) | |||||
{ | |||||
*shram_mask = 0; | |||||
if (active_aes == 1) { | |||||
*shram_mask = ~(*shram_mask); | |||||
} else if (active_aes == 2) { | |||||
if (ae == 1) | |||||
*shram_mask = ((~(*shram_mask)) & 0xffffffff); | |||||
else | |||||
*shram_mask = ((~(*shram_mask)) & 0xffffffff00000000ull); | |||||
} else if (active_aes == 3) { | |||||
if (ae == 0) | |||||
*shram_mask = ((~(*shram_mask)) & 0x7fffff); | |||||
else if (ae == 1) | |||||
*shram_mask = ((~(*shram_mask)) & 0x3fffff800000ull); | |||||
else | |||||
*shram_mask = ((~(*shram_mask)) & 0xffffc00000000000ull); | |||||
} else { | |||||
panic("Only three services are supported in current version"); | |||||
} | |||||
} | |||||
int | |||||
qat_adm_ring_build_ring_table(struct qat_softc *sc, uint32_t ae) | |||||
{ | |||||
struct qat_admin_rings *qadr = &sc->sc_admin_rings; | |||||
struct fw_init_ring_table *tbl; | |||||
struct fw_init_ring_params *param; | |||||
uint8_t srv_mask = sc->sc_admin_rings.qadr_srv_mask[ae]; | |||||
if ((srv_mask & QAT_SERVICE_CRYPTO_A)) { | |||||
tbl = qadr->qadr_cya_ring_tbl; | |||||
} else if ((srv_mask & QAT_SERVICE_CRYPTO_B)) { | |||||
tbl = qadr->qadr_cyb_ring_tbl; | |||||
} else { | |||||
device_printf(sc->sc_dev, | |||||
"Invalid execution engine %d\n", ae); | |||||
return EINVAL; | |||||
} | |||||
param = &tbl->firt_bulk_rings[sc->sc_hw.qhw_ring_sym_tx]; | |||||
param->firp_curr_weight = QAT_HI_PRIO_RING_WEIGHT; | |||||
param->firp_init_weight = QAT_HI_PRIO_RING_WEIGHT; | |||||
FW_INIT_RING_MASK_SET(tbl, sc->sc_hw.qhw_ring_sym_tx); | |||||
return 0; | |||||
} | |||||
int | |||||
qat_adm_ring_build_init_msg(struct qat_softc *sc, | |||||
struct fw_init_req *initmsg, enum fw_init_cmd_id cmd, uint32_t ae, | |||||
struct qat_accel_init_cb *cb) | |||||
{ | |||||
struct fw_init_set_ae_info_hdr *aehdr; | |||||
struct fw_init_set_ae_info *aeinfo; | |||||
struct fw_init_set_ring_info_hdr *ringhdr; | |||||
struct fw_init_set_ring_info *ringinfo; | |||||
int init_shram = 0, tgt_id, cluster_id; | |||||
uint32_t srv_mask; | |||||
srv_mask = sc->sc_admin_rings.qadr_srv_mask[ | |||||
ae % sc->sc_ae_num]; | |||||
memset(initmsg, 0, sizeof(struct fw_init_req)); | |||||
qat_msg_req_type_populate(&initmsg->comn_hdr.arch_if, | |||||
ARCH_IF_REQ_QAT_FW_INIT, | |||||
sc->sc_admin_rings.qadr_admin_rx->qr_ring_id); | |||||
qat_msg_cmn_mid_populate(&initmsg->comn_mid, cb, 0, 0); | |||||
switch (cmd) { | |||||
case FW_INIT_CMD_SET_AE_INFO: | |||||
if (ae % sc->sc_ae_num == 0) | |||||
init_shram = 1; | |||||
if (ae >= sc->sc_ae_num) { | |||||
tgt_id = 1; | |||||
cluster_id = 1; | |||||
} else { | |||||
cluster_id = 0; | |||||
if (sc->sc_ae_mask) | |||||
tgt_id = 0; | |||||
else | |||||
tgt_id = 1; | |||||
} | |||||
aehdr = &initmsg->u.set_ae_info; | |||||
aeinfo = &initmsg->u1.set_ae_info; | |||||
aehdr->init_cmd_id = cmd; | |||||
/* XXX that does not support sparse ae_mask */ | |||||
aehdr->init_trgt_id = ae; | |||||
aehdr->init_ring_cluster_id = cluster_id; | |||||
aehdr->init_qat_id = tgt_id; | |||||
qat_adm_ring_build_slice_mask(&aehdr->init_slice_mask, srv_mask, | |||||
init_shram); | |||||
qat_adm_ring_build_shram_mask(&aeinfo->init_shram_mask, | |||||
sc->sc_admin_rings.qadr_active_aes_per_accel, | |||||
ae % sc->sc_ae_num); | |||||
break; | |||||
case FW_INIT_CMD_SET_RING_INFO: | |||||
ringhdr = &initmsg->u.set_ring_info; | |||||
ringinfo = &initmsg->u1.set_ring_info; | |||||
ringhdr->init_cmd_id = cmd; | |||||
/* XXX that does not support sparse ae_mask */ | |||||
ringhdr->init_trgt_id = ae; | |||||
/* XXX */ | |||||
qat_adm_ring_build_ring_table(sc, | |||||
ae % sc->sc_ae_num); | |||||
ringhdr->init_ring_tbl_sz = sizeof(struct fw_init_ring_table); | |||||
ringinfo->init_ring_table_ptr = | |||||
sc->sc_admin_rings.qadr_dma.qdm_dma_seg.ds_addr + | |||||
((ae % sc->sc_ae_num) * | |||||
sizeof(struct fw_init_ring_table)); | |||||
break; | |||||
default: | |||||
return ENOTSUP; | |||||
} | |||||
return 0; | |||||
} | |||||
int | |||||
qat_adm_ring_send_init_msg_sync(struct qat_softc *sc, | |||||
enum fw_init_cmd_id cmd, uint32_t ae) | |||||
{ | |||||
struct fw_init_req initmsg; | |||||
struct qat_accel_init_cb cb; | |||||
int error; | |||||
error = qat_adm_ring_build_init_msg(sc, &initmsg, cmd, ae, &cb); | |||||
if (error) | |||||
return error; | |||||
error = qat_etr_put_msg(sc, sc->sc_admin_rings.qadr_admin_tx, | |||||
(uint32_t *)&initmsg); | |||||
if (error) | |||||
return error; | |||||
error = tsleep(&cb, PZERO, "qat_init", hz * 3 / 2); | |||||
if (error) { | |||||
device_printf(sc->sc_dev, | |||||
"Timed out initialization firmware: %d\n", error); | |||||
return error; | |||||
} | |||||
if (cb.qaic_status) { | |||||
device_printf(sc->sc_dev, "Failed to initialize firmware\n"); | |||||
return EIO; | |||||
} | |||||
return error; | |||||
} | |||||
int | |||||
qat_adm_ring_send_init_msg(struct qat_softc *sc, | |||||
enum fw_init_cmd_id cmd) | |||||
{ | |||||
struct qat_admin_rings *qadr = &sc->sc_admin_rings; | |||||
uint32_t error, ae; | |||||
for (ae = 0; ae < sc->sc_ae_num; ae++) { | |||||
uint8_t srv_mask = qadr->qadr_srv_mask[ae]; | |||||
switch (cmd) { | |||||
case FW_INIT_CMD_SET_AE_INFO: | |||||
case FW_INIT_CMD_SET_RING_INFO: | |||||
if (!srv_mask) | |||||
continue; | |||||
break; | |||||
case FW_INIT_CMD_TRNG_ENABLE: | |||||
case FW_INIT_CMD_TRNG_DISABLE: | |||||
if (!(srv_mask & QAT_SERVICE_CRYPTO_A)) | |||||
continue; | |||||
break; | |||||
default: | |||||
return ENOTSUP; | |||||
} | |||||
error = qat_adm_ring_send_init_msg_sync(sc, cmd, ae); | |||||
if (error) | |||||
return error; | |||||
} | |||||
return 0; | |||||
} | |||||
int | |||||
qat_adm_ring_send_init(struct qat_softc *sc) | |||||
{ | |||||
int error; | |||||
error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_AE_INFO); | |||||
if (error) | |||||
return error; | |||||
error = qat_adm_ring_send_init_msg(sc, FW_INIT_CMD_SET_RING_INFO); | |||||
if (error) | |||||
return error; | |||||
return 0; | |||||
} | |||||
int | |||||
qat_adm_ring_intr(struct qat_softc *sc, void *arg, void *msg) | |||||
{ | |||||
struct arch_if_resp_hdr *resp; | |||||
struct fw_init_resp *init_resp; | |||||
struct qat_accel_init_cb *init_cb; | |||||
int handled = 0; | |||||
resp = (struct arch_if_resp_hdr *)msg; | |||||
switch (resp->resp_type) { | |||||
case ARCH_IF_REQ_QAT_FW_INIT: | |||||
init_resp = (struct fw_init_resp *)msg; | |||||
init_cb = (struct qat_accel_init_cb *) | |||||
(uintptr_t)init_resp->comn_resp.opaque_data; | |||||
init_cb->qaic_status = | |||||
__SHIFTOUT(init_resp->comn_resp.comn_status, | |||||
COMN_RESP_INIT_ADMIN_STATUS); | |||||
wakeup(init_cb); | |||||
break; | |||||
default: | |||||
device_printf(sc->sc_dev, | |||||
"unknown resp type %d\n", resp->resp_type); | |||||
break; | |||||
} | |||||
return handled; | |||||
} | |||||
static inline uint16_t | |||||
qat_hw15_get_comn_req_flags(uint8_t ae) | |||||
{ | |||||
if (ae == 0) { | |||||
return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | | |||||
COMN_REQ_AUTH0_SLICE_REQUIRED | | |||||
COMN_REQ_CIPHER0_SLICE_REQUIRED; | |||||
} else { | |||||
return COMN_REQ_ORD_STRICT | COMN_REQ_PTR_TYPE_SGL | | |||||
COMN_REQ_AUTH1_SLICE_REQUIRED | | |||||
COMN_REQ_CIPHER1_SLICE_REQUIRED; | |||||
} | |||||
} | |||||
static uint32_t | |||||
qat_crypto_setup_cipher_desc(struct qat_crypto_desc *desc, | |||||
struct qat_session *qs, struct fw_cipher_hdr *cipher_hdr, | |||||
uint8_t *hw_blk_ptr, uint32_t hw_blk_offset, enum fw_slice next_slice) | |||||
{ | |||||
struct hw_cipher_config *cipher_config = (struct hw_cipher_config *) | |||||
(hw_blk_ptr + hw_blk_offset); | |||||
uint32_t hw_blk_size; | |||||
const uint8_t *key = qs->qs_cipher_key; | |||||
uint8_t *cipher_key = (uint8_t *)(cipher_config + 1); | |||||
int klen = qs->qs_cipher_klen; | |||||
cipher_config->val = qat_crypto_load_cipher_session(desc, qs); | |||||
cipher_config->reserved = 0; | |||||
cipher_hdr->state_padding_sz = 0; | |||||
cipher_hdr->key_sz = klen / 8; /* bytes to quad words */ | |||||
cipher_hdr->state_sz = desc->qcd_cipher_blk_sz / 8; | |||||
cipher_hdr->next_id = next_slice; | |||||
cipher_hdr->curr_id = FW_SLICE_CIPHER; | |||||
cipher_hdr->offset = hw_blk_offset / 8; | |||||
cipher_hdr->resrvd = 0; | |||||
hw_blk_size = sizeof(struct hw_cipher_config); | |||||
memcpy(cipher_key, key, klen); | |||||
hw_blk_size += klen; | |||||
return hw_blk_size; | |||||
} | |||||
static uint32_t | |||||
qat_crypto_setup_auth_desc(struct qat_crypto_desc *desc, | |||||
struct qat_session *qs, struct fw_auth_hdr *auth_hdr, | |||||
uint8_t *hw_blk_ptr, uint32_t hw_blk_offset, enum fw_slice next_slice) | |||||
{ | |||||
struct qat_sym_hash_def const *hash_def; | |||||
struct hw_auth_setup *auth_setup; | |||||
uint32_t hw_blk_size; | |||||
uint8_t *state1, *state2; | |||||
uint32_t state_size; | |||||
auth_setup = (struct hw_auth_setup *)(hw_blk_ptr + hw_blk_offset); | |||||
auth_setup->auth_config.config = qat_crypto_load_auth_session(desc, qs, | |||||
&hash_def); | |||||
auth_setup->auth_config.reserved = 0; | |||||
/* for HMAC in mode 1 authCounter is the block size | |||||
* else the authCounter is 0. The firmware expects the counter to be | |||||
* big endian */ | |||||
auth_setup->auth_counter.counter = | |||||
htobe32(hash_def->qshd_qat->qshqi_auth_counter); | |||||
auth_setup->auth_counter.reserved = 0; | |||||
auth_hdr->next_id = next_slice; | |||||
auth_hdr->curr_id = FW_SLICE_AUTH; | |||||
auth_hdr->offset = hw_blk_offset / 8; | |||||
auth_hdr->resrvd = 0; | |||||
auth_hdr->hash_flags = FW_AUTH_HDR_FLAG_NO_NESTED; | |||||
auth_hdr->u.inner_prefix_sz = 0; | |||||
auth_hdr->outer_prefix_sz = 0; | |||||
auth_hdr->final_sz = hash_def->qshd_alg->qshai_digest_len; | |||||
auth_hdr->inner_state1_sz = | |||||
roundup(hash_def->qshd_qat->qshqi_state1_len, 8); | |||||
auth_hdr->inner_res_sz = hash_def->qshd_alg->qshai_digest_len; | |||||
auth_hdr->inner_state2_sz = | |||||
roundup(hash_def->qshd_qat->qshqi_state2_len, 8); | |||||
auth_hdr->inner_state2_off = auth_hdr->offset + | |||||
((sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz) / 8); | |||||
hw_blk_size = sizeof(struct hw_auth_setup) + auth_hdr->inner_state1_sz + | |||||
auth_hdr->inner_state2_sz; | |||||
auth_hdr->outer_config_off = 0; | |||||
auth_hdr->outer_state1_sz = 0; | |||||
auth_hdr->outer_res_sz = 0; | |||||
auth_hdr->outer_prefix_off = 0; | |||||
state1 = (uint8_t *)(auth_setup + 1); | |||||
state2 = state1 + auth_hdr->inner_state1_sz; | |||||
state_size = hash_def->qshd_alg->qshai_state_size; | |||||
if (hash_def->qshd_qat->qshqi_algo_enc == HW_AUTH_ALGO_SHA1) { | |||||
uint32_t state1_pad_len = auth_hdr->inner_state1_sz - | |||||
state_size; | |||||
uint32_t state2_pad_len = auth_hdr->inner_state2_sz - | |||||
state_size; | |||||
if (state1_pad_len > 0) | |||||
memset(state1 + state_size, 0, state1_pad_len); | |||||
if (state2_pad_len > 0) | |||||
memset(state2 + state_size, 0, state2_pad_len); | |||||
} | |||||
desc->qcd_state_storage_sz = (sizeof(struct hw_auth_counter) + | |||||
roundup(state_size, 8)) / 8; | |||||
qat_crypto_hmac_precompute(desc, qs, hash_def, state1, state2); | |||||
return hw_blk_size; | |||||
} | |||||
void | |||||
qat_hw15_crypto_setup_desc(struct qat_crypto *qcy, struct qat_session *qs, | |||||
struct qat_crypto_desc *desc) | |||||
{ | |||||
struct fw_cipher_hdr *cipher_hdr; | |||||
struct fw_auth_hdr *auth_hdr; | |||||
struct fw_la_bulk_req *req_cache; | |||||
uint32_t ctrl_blk_size = 0, ctrl_blk_offset = 0, hw_blk_offset = 0; | |||||
int i; | |||||
uint16_t la_cmd_flags = 0; | |||||
uint8_t req_params_sz = 0; | |||||
uint8_t *ctrl_blk_ptr; | |||||
uint8_t *hw_blk_ptr; | |||||
if (qs->qs_cipher_algo != HW_CIPHER_ALGO_NULL) | |||||
ctrl_blk_size += sizeof(struct fw_cipher_hdr); | |||||
if (qs->qs_auth_algo != HW_AUTH_ALGO_NULL) | |||||
ctrl_blk_size += sizeof(struct fw_auth_hdr); | |||||
ctrl_blk_ptr = desc->qcd_content_desc; | |||||
hw_blk_ptr = ctrl_blk_ptr + ctrl_blk_size; | |||||
for (i = 0; i < MAX_FW_SLICE; i++) { | |||||
switch (desc->qcd_slices[i]) { | |||||
case FW_SLICE_CIPHER: | |||||
cipher_hdr = (struct fw_cipher_hdr *)(ctrl_blk_ptr + | |||||
ctrl_blk_offset); | |||||
ctrl_blk_offset += sizeof(struct fw_cipher_hdr); | |||||
hw_blk_offset += qat_crypto_setup_cipher_desc(desc, | |||||
qs, cipher_hdr, hw_blk_ptr, hw_blk_offset, | |||||
desc->qcd_slices[i + 1]); | |||||
req_params_sz += sizeof(struct fw_la_cipher_req_params); | |||||
break; | |||||
case FW_SLICE_AUTH: | |||||
auth_hdr = (struct fw_auth_hdr *)(ctrl_blk_ptr + | |||||
ctrl_blk_offset); | |||||
ctrl_blk_offset += sizeof(struct fw_auth_hdr); | |||||
hw_blk_offset += qat_crypto_setup_auth_desc(desc, | |||||
qs, auth_hdr, hw_blk_ptr, hw_blk_offset, | |||||
desc->qcd_slices[i + 1]); | |||||
req_params_sz += sizeof(struct fw_la_auth_req_params); | |||||
la_cmd_flags |= LA_FLAGS_RET_AUTH_RES; | |||||
/* no digest verify */ | |||||
break; | |||||
case FW_SLICE_DRAM_WR: | |||||
i = MAX_FW_SLICE; /* end of chain */ | |||||
break; | |||||
default: | |||||
MPASS(0); | |||||
break; | |||||
} | |||||
} | |||||
desc->qcd_hdr_sz = ctrl_blk_offset / 8; | |||||
desc->qcd_hw_blk_sz = hw_blk_offset / 8; | |||||
req_cache = (struct fw_la_bulk_req *)desc->qcd_req_cache; | |||||
qat_msg_req_type_populate( | |||||
&req_cache->comn_hdr.arch_if, | |||||
ARCH_IF_REQ_QAT_FW_LA, 0); | |||||
la_cmd_flags |= LA_FLAGS_PROTO_NO; | |||||
qat_msg_params_populate(req_cache, | |||||
desc, req_params_sz, la_cmd_flags, 0); | |||||
bus_dmamap_sync(qs->qs_desc_mem.qdm_dma_tag, | |||||
qs->qs_desc_mem.qdm_dma_map, BUS_DMASYNC_PREWRITE); | |||||
} | |||||
void | |||||
qat_hw15_crypto_setup_req_params(struct qat_crypto_bank *qcb, struct qat_session *qs, | |||||
struct qat_crypto_desc const *desc, struct qat_sym_cookie *qsc, | |||||
struct cryptop *crp) | |||||
{ | |||||
struct qat_sym_bulk_cookie *qsbc; | |||||
struct fw_la_bulk_req *bulk_req; | |||||
struct fw_la_cipher_req_params *cipher_req; | |||||
struct fw_la_auth_req_params *auth_req; | |||||
bus_addr_t digest_paddr; | |||||
uint32_t req_params_offset = 0; | |||||
uint8_t *req_params_ptr; | |||||
enum fw_la_cmd_id cmd_id = desc->qcd_cmd_id; | |||||
enum fw_slice next_slice; | |||||
qsbc = &qsc->u.qsc_bulk_cookie; | |||||
bulk_req = (struct fw_la_bulk_req *)qsbc->qsbc_msg; | |||||
memcpy(bulk_req, &desc->qcd_req_cache, QAT_HW15_SESSION_REQ_CACHE_SIZE); | |||||
bulk_req->comn_hdr.arch_if.resp_pipe_id = qcb->qcb_sym_rx->qr_ring_id; | |||||
bulk_req->comn_hdr.comn_req_flags = | |||||
qat_hw15_get_comn_req_flags(qcb->qcb_bank % 2); | |||||
bulk_req->comn_mid.src_data_addr = qsc->qsc_buffer_list_desc_paddr; | |||||
bulk_req->comn_mid.dest_data_addr = qsc->qsc_buffer_list_desc_paddr; | |||||
bulk_req->req_params_addr = qsc->qsc_bulk_req_params_buf_paddr; | |||||
bulk_req->comn_ftr.next_request_addr = 0; | |||||
bulk_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)qsc; | |||||
digest_paddr = 0; | |||||
if (desc->qcd_auth_sz != 0) { | |||||
bulk_req->comn_la_req.u.la_flags |= LA_FLAGS_DIGEST_IN_BUFFER; | |||||
digest_paddr = qsc->qsc_digest_paddr; | |||||
} | |||||
req_params_ptr = qsbc->qsbc_req_params_buf; | |||||
if (cmd_id != FW_LA_CMD_AUTH) { | |||||
cipher_req = (struct fw_la_cipher_req_params *) | |||||
(req_params_ptr + req_params_offset); | |||||
req_params_offset += sizeof(struct fw_la_cipher_req_params); | |||||
if (cmd_id == FW_LA_CMD_CIPHER || cmd_id == FW_LA_CMD_HASH_CIPHER) | |||||
next_slice = FW_SLICE_DRAM_WR; | |||||
else | |||||
next_slice = FW_SLICE_AUTH; | |||||
cipher_req->resrvd = 0; | |||||
cipher_req->cipher_state_sz = desc->qcd_cipher_blk_sz / 8; | |||||
cipher_req->curr_id = FW_SLICE_CIPHER; | |||||
cipher_req->next_id = next_slice; | |||||
cipher_req->resrvd1 = 0; | |||||
cipher_req->cipher_off = crp->crp_payload_start; | |||||
cipher_req->cipher_len = crp->crp_payload_length; | |||||
cipher_req->state_address = qsc->qsc_iv_buf_paddr; | |||||
} | |||||
if (cmd_id != FW_LA_CMD_CIPHER) { | |||||
auth_req = (struct fw_la_auth_req_params *) | |||||
(req_params_ptr + req_params_offset); | |||||
req_params_offset += sizeof(struct fw_la_auth_req_params); | |||||
if (cmd_id == FW_LA_CMD_HASH_CIPHER) | |||||
next_slice = FW_SLICE_CIPHER; | |||||
else | |||||
next_slice = FW_SLICE_DRAM_WR; | |||||
auth_req->next_id = next_slice; | |||||
auth_req->curr_id = FW_SLICE_AUTH; | |||||
auth_req->auth_res_address = digest_paddr; | |||||
auth_req->auth_res_sz = 16; /* no digest verify */ | |||||
auth_req->auth_len = crp->crp_aad_start; | |||||
auth_req->auth_off = crp->crp_aad_length; | |||||
auth_req->hash_state_sz = 0; | |||||
auth_req->u1.prefix_addr = desc->qcd_hash_state_paddr + | |||||
desc->qcd_state_storage_sz; | |||||
auth_req->u.resrvd = 0; | |||||
} | |||||
} | |||||