diff --git a/sys/arm64/qoriq/caam/caam.c b/sys/arm64/qoriq/caam/caam.c --- a/sys/arm64/qoriq/caam/caam.c +++ b/sys/arm64/qoriq/caam/caam.c @@ -30,7 +30,10 @@ #include #include +#include + #include "caam.h" +#include "caam_crypto.h" #include "caam_debug.h" #include "caam_internal.h" #include "caam_test.h" @@ -450,21 +453,29 @@ goto cleanup_jobrings; } + rv = caam_crypto_attach(dev); + if (rv != 0) { + CAAM_LOG_ERROR("caam_crypto_attach failed rv=(%d)\n", rv); + goto cleanup_trng; + } + #ifdef CAAM_DEBUG /* Add the caam test device for development */ rv = caam_test_attach(dev); if (rv != 0) { CAAM_LOG_ERROR("caam_test_attach failed rv=(%d)\n", rv); - goto cleanup_trng; + goto cleanup_crypto; } #endif return (0); #ifdef CAAM_DEBUG +cleanup_crypto: + caam_crypto_detach(dev); +#endif cleanup_trng: caam_trng_detach(dev); -#endif cleanup_jobrings: caam_jobrings_detach(dev); cleanup_rman: @@ -531,6 +542,8 @@ { struct caam_softc *sc = device_get_softc(dev); + caam_crypto_detach(dev); + /* Remove TRNG drivers */ caam_trng_detach(dev); #ifdef CAAM_DEBUG @@ -681,6 +694,11 @@ DEVMETHOD(bus_activate_resource, caam_activate_resource), DEVMETHOD(bus_release_resource, caam_release_resource), + DEVMETHOD(cryptodev_probesession, caam_crypto_probesession), + DEVMETHOD(cryptodev_newsession, caam_crypto_newsession), + DEVMETHOD(cryptodev_freesession, caam_crypto_freesession), + DEVMETHOD(cryptodev_process, caam_crypto_process), + DEVMETHOD_END }; @@ -688,4 +706,5 @@ simplebus_driver); DRIVER_MODULE(caam, simplebus, caam_driver, 0, 0); +MODULE_DEPEND(caam, crypto, 1, 1, 1); MODULE_VERSION(caam, 1); diff --git a/sys/arm64/qoriq/caam/caam_crypto.h b/sys/arm64/qoriq/caam/caam_crypto.h new file mode 100644 --- /dev/null +++ b/sys/arm64/qoriq/caam/caam_crypto.h @@ -0,0 +1,100 @@ +/* + * Copyright 2024 Alstom Group + * Copyright 2024 Sii Poland + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#ifndef CAAM_CRYPTO_H +#define CAAM_CRYPTO_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "cryptodev_if.h" + +/* + * @brief Register with crypto framework and get assigned crypto id. + * + * @param [in] dev caam device handle + * + * @retval :: 0 is returned on success + * @retval :: errno is returned on internal error + */ +int caam_crypto_attach(device_t dev); + +/* + * @brief Unregister assigned crypto id from crypto framework. + * + * @param [in] dev caam device handle + */ +void caam_crypto_detach(device_t dev); + +/* + * @brief CAAM cryptodev_probesession devmethod. + * + * Informs crypto framework about CAAM crypto supported operations. + * + * @param [in] dev caam device handle + * + * @retval :: int acceleration type CAAM provides for probed algorithm + * @retval :: errno EINVAL on unsuppored operations + */ +int caam_crypto_probesession(device_t dev, + const struct crypto_session_params *csp); + +/* + * @brief CAAM cryptodev_freesession devmethod. + * + * Free a crypto session. + * + * @param [in] dev caam device handle + * @param [in] cses crypto session id + */ +void caam_crypto_freesession(device_t dev, crypto_session_t cses); + +/* + * @brief CAAM cryptodev_newsession devmethod. + * + * Initialize a new crypto session. + * + * @param [in] dev caam device handle + * @param [in] cses crypto session id + * @param [in] csp crypto session parameters + * + * @retval :: 0 is returned on success + * @retval :: errno is returned on internal error + */ +int caam_crypto_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp); + +/* + * @brief CAAM cryptodev_process devmethod. + * + * Start processing a crypto request. + * + * @param [in] dev caam device handle + * @param [in] crp Structure describing complete operation + * @param [in] hint processing hint + * + * @retval :: 0 is returned on success + * @retval :: errno is returned on internal error + */ +int caam_crypto_process(device_t dev, struct cryptop *crp, int hint); + +#endif /* CAAM_CRYPTO_H */ diff --git a/sys/arm64/qoriq/caam/caam_crypto.c b/sys/arm64/qoriq/caam/caam_crypto.c new file mode 100644 --- /dev/null +++ b/sys/arm64/qoriq/caam/caam_crypto.c @@ -0,0 +1,398 @@ +/* + * Copyright 2024 Alstom Group + * Copyright 2024 Sii Poland + * + * SPDX-License-Identifier: BSD-3-Clause + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "caam.h" +#include "caam_crypto.h" +#include "caam_debug.h" +#include "caam_internal.h" +#include "jr/caam_desc_helper.h" +#include "jr/caam_jobdesc.h" +#include "jr/caam_jr.h" +#include "jr/caam_jr_hw.h" + +SDT_PROVIDER_DEFINE(caam_crypto); +SDT_PROBE_DEFINE0(caam_crypto, , func, process__start); +SDT_PROBE_DEFINE0(caam_crypto, , func, process__end); + +MALLOC_DEFINE(M_CAAM_CRYPTO, "caam_crypto", "CAAM crypto device"); + +struct caam_session { + size_t keylen; + size_t data_size; + bool encrypt; + + struct caam_softc *sc; +}; + +struct caam_operation { + struct job_descriptor jobdesc; + size_t keylen; + bool encrypt; + + caam_jr_dma_map_t data; + caam_jr_dma_map_t key; + caam_jr_dma_map_t sgtm; + + uint8_t iv[AES_XTS_IV_LEN]; + struct caam_sgt_entry sgt[MAX_SGT_ENTRIES]; + + struct caam_session *ses; + struct cryptop *crp; + device_t jr_dev; +}; + +MALLOC_DEFINE(M_CAAM, "caam", "caam cryptography buffers"); + +static void +dma_data_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + struct caam_sgt_entry *sg; + struct caam_operation *op; + + op = (struct caam_operation *)arg; + + if (error) { + CAAM_LOG_WARN("dma_data_callback error: %i!\n", error); + return; + } + + if (nseg == 1) { + op->data.bus_addr = segs[0].ds_addr; + return; + } + + op->data.bus_addr = 0; + for (int i = 0; i < nseg; i++) { + sg = &op->sgt[i]; + + sg->ptr_hi = htobe32(PHYS_ADDR_HI(segs[i].ds_addr)); + sg->ptr_lo = htobe32(PHYS_ADDR_LO(segs[i].ds_addr)); + sg->len_f_e = segs[i].ds_len; + sg->offset = 0; + + if (i == nseg - 1) { + sg->len_f_e |= CAAM_SGT_FINAL; + } + sg->len_f_e = htobe32(sg->len_f_e); + } +} + +static void +caam_aes_xts_build_desc(struct caam_operation *op) +{ + uint32_t *iv_ptr = (uint32_t *)op->iv; + uint32_t *desc = op->jobdesc.desc; + + caam_jobdesc_init(desc); + + caam_jobdesc_add_word(desc, DESC_HEADER(0)); + + caam_jobdesc_add_word(desc, LD_KEY_PLAIN(CLASS_1, REG, op->keylen)); + caam_jobdesc_add_ptr(desc, op->key.bus_addr); + + caam_jobdesc_add_word(desc, LD_IMM_OFF(CLASS_1, REG_CTX, 8, 0x28)); + caam_jobdesc_add_word(desc, 0); + caam_jobdesc_add_word(desc, 1 << 15); + + /* NOTE: This will not work on big endian platforms */ + caam_jobdesc_add_word(desc, LD_IMM_OFF(CLASS_1, REG_CTX, 8, 0x20)); + caam_jobdesc_add_word(desc, htobe32(iv_ptr[0])); + caam_jobdesc_add_word(desc, htobe32(iv_ptr[1])); + + caam_jobdesc_add_word(desc, + CIPHER_INIT_FINAL(OP_ALGO(AES), op->encrypt) | ALGO_AAI(AES_XTS)); + + if (op->data.bus_addr != 0) { + caam_jobdesc_add_word(desc, FIFO_LD_EXT(CLASS_1, MSG, LAST_C1)); + caam_jobdesc_add_ptr(desc, op->data.bus_addr); + caam_jobdesc_add_word(desc, op->data.buflen); + + caam_jobdesc_add_word(desc, FIFO_ST_EXT(MSG_DATA)); + caam_jobdesc_add_ptr(desc, op->data.bus_addr); + caam_jobdesc_add_word(desc, op->data.buflen); + } else { + caam_jobdesc_add_word(desc, + FIFO_LD_SGT_EXT(CLASS_1, MSG, LAST_C1)); + caam_jobdesc_add_ptr(desc, op->sgtm.bus_addr); + caam_jobdesc_add_word(desc, op->data.buflen); + + caam_jobdesc_add_word(desc, FIFO_ST_SGT_EXT(MSG_DATA)); + caam_jobdesc_add_ptr(desc, op->sgtm.bus_addr); + caam_jobdesc_add_word(desc, op->data.buflen); + } +} + +int +caam_crypto_probesession(device_t dev, const struct crypto_session_params *csp) +{ + if (csp->csp_flags != 0) + return (EINVAL); + switch (csp->csp_mode) { + case CSP_MODE_CIPHER: + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_XTS: + if (csp->csp_ivlen != AES_XTS_IV_LEN) + return (EINVAL); + switch (csp->csp_cipher_klen * 8) { + case 256: + case 512: + break; + default: + return (EINVAL); + } + break; + default: + return (EINVAL); + } + break; + default: + return (EINVAL); + } + + return (CRYPTODEV_PROBE_HARDWARE); +} + +void +caam_crypto_freesession(device_t dev, crypto_session_t cses) +{ + /* Nothing to do */ +} + +/* + * Generate a new software session. + */ +int +caam_crypto_newsession(device_t dev, crypto_session_t cses, + const struct crypto_session_params *csp) +{ + struct caam_session *ses; + struct caam_softc *sc; + + sc = device_get_softc(dev); + ses = crypto_get_driver_session(cses); + if (ses == NULL) { + CAAM_LOG_DEV_WARN(dev, "Cannot create new session\n"); + return (EINVAL); + } + + ses->sc = sc; + + return (0); +} + +static void +crypto_process_done(uint32_t *desc, uint32_t status, void *arg, device_t dev) +{ +#if defined(CAAM_JR_MODE_ASYNC) + struct caam_operation *op; + bus_dma_tag_t jr_data_tag; + int rv = 0; + + op = (struct caam_operation *)arg; + if (status != 0) + op->crp->crp_etype = EIO; + else + op->crp->crp_etype = 0; + + jr_data_tag = caam_jr_get_data_tag(op->jr_dev); + + rv = caam_jr_dma_unmap(op->jr_dev, &op->key, JR_MAP_NOSYNC); + if (rv != 0) { + CAAM_LOG_ERROR("%s: Failed to destroy key\n", __func__); + } + + bus_dmamap_sync(jr_data_tag, op->data.mmap, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(jr_data_tag, op->data.mmap); + rv = bus_dmamap_destroy(jr_data_tag, op->data.mmap); + if (rv != 0) { + CAAM_LOG_ERROR("%s: Failed to destroy data\n", __func__); + } + + if (op->data.bus_addr == 0) { + rv = caam_jr_dma_unmap(op->jr_dev, &op->sgtm, JR_MAP_NOSYNC); + if (rv != 0) { + CAAM_LOG_ERROR("%s: Failed to destroy sgt\n", __func__); + } + } + + jr_pool_release(&op->ses->sc->jr_pool, op->jr_dev); + + crypto_done(op->crp); + + free(op, M_CAAM_CRYPTO); +#endif /* CAAM_JR_MODE_ASYNC */ +} + +int +caam_crypto_process(device_t dev, struct cryptop *crp, int hint) +{ + const struct crypto_session_params *csp; + struct caam_session *ses; + const void *key_vaddr; + int rv = 0; + struct caam_operation *op; + struct caam_softc *sc; + bus_dma_tag_t jr_data_tag; + + SDT_PROBE0(caam_crypto, , func, process__start); + + sc = device_get_softc(dev); + ses = crypto_get_driver_session(crp->crp_session); + csp = crypto_get_params(crp->crp_session); + ses->keylen = csp->csp_cipher_klen; + ses->data_size = crp->crp_payload_length; + ses->encrypt = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); + + op = malloc_aligned(sizeof(struct caam_operation), PAGE_SIZE, + M_CAAM_CRYPTO, M_NOWAIT); + if (op == NULL) { + CAAM_LOG_DEV_WARN(dev, + "Failed to allocate caam operation structure\n"); + rv = ENOMEM; + goto exit; + } + + op->keylen = ses->keylen; + op->data.buflen = ses->data_size; + op->encrypt = ses->encrypt; + op->ses = ses; + op->crp = crp; + + op->jobdesc.arg = op; + op->jobdesc.callback = crypto_process_done; + + op->jr_dev = jr_pool_acquire(&sc->jr_pool); + if (op->jr_dev == 0) { + CAAM_LOG_DEV_WARN(dev, "Failed to acquire JR\n"); + rv = EAGAIN; + goto operation_cleanup; + } + jr_data_tag = caam_jr_get_data_tag(op->jr_dev); + + crypto_read_iv(crp, op->iv); + if (crp->crp_cipher_key) + key_vaddr = crp->crp_cipher_key; + else + key_vaddr = csp->csp_cipher_key; + + /* Map and sync the key buffer */ + rv = caam_jr_dma_map(op->jr_dev, &op->key, __DECONST(void *, key_vaddr), + ses->keylen, BUS_DMA_COHERENT, BUS_DMA_NOWAIT, JR_MAP_SYNC); + if (rv != 0) { + CAAM_LOG_DEV_WARN(dev, "Failed to create key map\n"); + goto release_jr; + } + + /* Map and sync the data buffer */ + rv = bus_dmamap_create(jr_data_tag, BUS_DMA_COHERENT, &op->data.mmap); + if (rv != 0) { + CAAM_LOG_DEV_WARN(dev, "Failed to create data map\n"); + goto key_map_cleanup; + } + op->data.bus_addr = ULONG_MAX; + rv = bus_dmamap_load_crp_buffer(jr_data_tag, op->data.mmap, + &crp->crp_buf, dma_data_callback, op, BUS_DMA_NOWAIT); + if ((rv != 0) || (op->data.bus_addr == ULONG_MAX)) { + CAAM_LOG_DEV_WARN(dev, "Failed to load data map\n"); + goto data_map_cleanup; + } + bus_dmamap_sync(jr_data_tag, op->data.mmap, BUS_DMASYNC_PREWRITE); + + /* If scatter gather is used map and sync the sgt buffer */ + if (op->data.bus_addr == 0) { + rv = caam_jr_dma_map(op->jr_dev, &op->sgtm, op->sgt, + sizeof(op->sgt), BUS_DMA_COHERENT, BUS_DMA_NOWAIT, + JR_MAP_SYNC); + if (rv != 0) { + CAAM_LOG_DEV_WARN(dev, "Failed to create sgt map\n"); + goto data_load_cleanup; + } + } + + caam_aes_xts_build_desc(op); +#if defined(CAAM_JR_MODE_ASYNC) + rv = caam_run_descriptor_jr_async(dev, &op->jobdesc, op->jr_dev); +#else + rv = caam_run_descriptor_jr_blocking(dev, &op->jobdesc, op->jr_dev); +#endif /* defined(CAAM_JR_MODE_ASYNC) */ + if (rv != 0) { + CAAM_LOG_DEV_WARN(dev, "Failed to run job descriptor\n"); + goto sgt_map_cleanup; + } + +#if defined(CAAM_JR_MODE_ASYNC) + return (0); +#endif /* defined(CAAM_JR_MODE_ASYNC) */ + + /* For S/G paddr should be set to 0 */ +sgt_map_cleanup: + if (op->data.bus_addr == 0) + caam_jr_dma_unmap(op->jr_dev, &op->sgtm, JR_MAP_NOSYNC); +data_load_cleanup: + bus_dmamap_unload(jr_data_tag, op->data.mmap); +data_map_cleanup: + bus_dmamap_destroy(jr_data_tag, op->data.mmap); +key_map_cleanup: + caam_jr_dma_unmap(op->jr_dev, &op->key, JR_MAP_NOSYNC); +release_jr: + jr_pool_release(&sc->jr_pool, op->jr_dev); +operation_cleanup: + free(op, M_CAAM_CRYPTO); +exit: + crp->crp_etype = rv; + crypto_done(crp); + + SDT_PROBE0(caam_crypto, , func, process__end); + + return (0); +} + +int +caam_crypto_attach(device_t dev) +{ + struct caam_softc *sc; + int error = 0; + + sc = device_get_softc(dev); + + sc->crypto_id = crypto_get_driverid(dev, + sizeof(struct caam_session), CRYPTOCAP_F_HARDWARE); + if (sc->crypto_id < 0) { + CAAM_LOG_DEV_WARN(dev, + "Failed to initialize crypto device\n"); + return (ENXIO); + } + + return (error); +} + +void +caam_crypto_detach(device_t dev) +{ + struct caam_softc *sc; + + sc = device_get_softc(dev); + crypto_unregister_all(sc->crypto_id); +} diff --git a/sys/arm64/qoriq/caam/caam_internal.h b/sys/arm64/qoriq/caam/caam_internal.h --- a/sys/arm64/qoriq/caam/caam_internal.h +++ b/sys/arm64/qoriq/caam/caam_internal.h @@ -30,6 +30,8 @@ struct rman mem_rman; struct jr_pool jr_pool; + + int32_t crypto_id; }; #endif /* CAAM_INTERNAL_H */ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -687,6 +687,7 @@ arm64/qoriq/clk/qoriq_clk_pll.c optional clk soc_nxp_ls arm64/qoriq/clk/qoriq_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/caam/caam.c optional caam soc_nxp_ls +arm64/qoriq/caam/caam_crypto.c optional caam soc_nxp_ls arm64/qoriq/caam/caam_trng.c optional caam soc_nxp_ls arm64/qoriq/caam/caam_test.c optional caam caam_debug soc_nxp_ls arm64/qoriq/caam/jr/caam_jr.c optional caam soc_nxp_ls