Differential D45592 Diff 140228 sys/arm64/qoriq/caam/caam_crypto.c

Changeset View

Standalone View

sys/arm64/qoriq/caam/caam_crypto.c

This file was added.

				/*
				* Copyright 2024 Alstom Group
				* Copyright 2024 Sii Poland
				*
				* SPDX-License-Identifier: BSD-3-Clause
				*
				*/

				#include <sys/cdefs.h>
				#include <sys/types.h>
				#include <sys/param.h>
				#include <sys/conf.h>
				#include <sys/kernel.h>
				#include <sys/limits.h>
				#include <sys/mutex.h>
				#include <sys/rwlock.h>
				#include <sys/smp.h>
				#include <sys/uio.h>

				#include <machine/bus.h>
				#include <machine/vfp.h>

				#include <opencrypto/cryptodev.h>
				#include <opencrypto/xform.h>

				#include "caam.h"
				#include "caam_crypto.h"
				#include "caam_debug.h"
				#include "caam_internal.h"
				#include "jr/caam_desc_helper.h"
				#include "jr/caam_jobdesc.h"
				#include "jr/caam_jr.h"
				#include "jr/caam_jr_hw.h"

				SDT_PROVIDER_DEFINE(caam_crypto);
				SDT_PROBE_DEFINE0(caam_crypto, , func, process__start);
				SDT_PROBE_DEFINE0(caam_crypto, , func, process__end);

				MALLOC_DEFINE(M_CAAM_CRYPTO, "caam_crypto", "CAAM crypto device");

				struct caam_session {
				size_t keylen;
				size_t data_size;
				bool encrypt;

				struct caam_softc *sc;
				};

				struct caam_operation {
				struct job_descriptor jobdesc;
				size_t keylen;
				bool encrypt;

				caam_jr_dma_map_t data;
				caam_jr_dma_map_t key;
				caam_jr_dma_map_t sgtm;

				uint8_t iv[AES_XTS_IV_LEN];
				struct caam_sgt_entry sgt[MAX_SGT_ENTRIES];

				struct caam_session *ses;
				struct cryptop *crp;
				device_t jr_dev;
				};

				MALLOC_DEFINE(M_CAAM, "caam", "caam cryptography buffers");

				static void
				dma_data_callback(void arg, bus_dma_segment_t segs, int nseg, int error)
				{
				struct caam_sgt_entry *sg;
				struct caam_operation *op;

				op = (struct caam_operation *)arg;

				if (error) {
				CAAM_LOG_WARN("dma_data_callback error: %i!\n", error);
				return;
				}

				if (nseg == 1) {
				op->data.bus_addr = segs[0].ds_addr;
				return;
				}

				op->data.bus_addr = 0;
				for (int i = 0; i < nseg; i++) {
				sg = &op->sgt[i];

				sg->ptr_hi = htobe32(PHYS_ADDR_HI(segs[i].ds_addr));
				sg->ptr_lo = htobe32(PHYS_ADDR_LO(segs[i].ds_addr));
				sg->len_f_e = segs[i].ds_len;
				sg->offset = 0;

				if (i == nseg - 1) {
				sg->len_f_e \|= CAAM_SGT_FINAL;
				}
				sg->len_f_e = htobe32(sg->len_f_e);
				}
				}

				static void
				caam_aes_xts_build_desc(struct caam_operation *op, bool little_endian)
				{
				uint32_t iv_ptr = (uint32_t )op->iv;
				uint32_t *desc = op->jobdesc.desc;

				caam_jobdesc_init(desc);

				caam_jobdesc_add_word(desc, DESC_HEADER(0));

				caam_jobdesc_add_word(desc, LD_KEY_PLAIN(CLASS_1, REG, op->keylen));
				caam_jobdesc_add_ptr(desc, op->key.bus_addr, little_endian);

				caam_jobdesc_add_word(desc, LD_IMM_OFF(CLASS_1, REG_CTX, 8, 0x28));
				caam_jobdesc_add_word(desc, 0);
				caam_jobdesc_add_word(desc, 1 << 15);

				/* NOTE: This will not work on big endian platforms */
				caam_jobdesc_add_word(desc, LD_IMM_OFF(CLASS_1, REG_CTX, 8, 0x20));
				caam_jobdesc_add_word(desc, htobe32(iv_ptr[0]));
				caam_jobdesc_add_word(desc, htobe32(iv_ptr[1]));

				caam_jobdesc_add_word(desc,
				CIPHER_INIT_FINAL(OP_ALGO(AES), op->encrypt) \| ALGO_AAI(AES_XTS));

				if (op->data.bus_addr != 0) {
				caam_jobdesc_add_word(desc, FIFO_LD_EXT(CLASS_1, MSG, LAST_C1));
				caam_jobdesc_add_ptr(desc, op->data.bus_addr, little_endian);
				caam_jobdesc_add_word(desc, op->data.buflen);

				caam_jobdesc_add_word(desc, FIFO_ST_EXT(MSG_DATA));
				caam_jobdesc_add_ptr(desc, op->data.bus_addr, little_endian);
				caam_jobdesc_add_word(desc, op->data.buflen);
				} else {
				caam_jobdesc_add_word(desc,
				FIFO_LD_SGT_EXT(CLASS_1, MSG, LAST_C1));
				caam_jobdesc_add_ptr(desc, op->sgtm.bus_addr, little_endian);
				caam_jobdesc_add_word(desc, op->data.buflen);

				caam_jobdesc_add_word(desc, FIFO_ST_SGT_EXT(MSG_DATA));
				caam_jobdesc_add_ptr(desc, op->sgtm.bus_addr, little_endian);
				caam_jobdesc_add_word(desc, op->data.buflen);
				}
				}

				int
				caam_crypto_probesession(device_t dev, const struct crypto_session_params *csp)
				{
				if (csp->csp_flags != 0)
				return (EINVAL);
				switch (csp->csp_mode) {
				case CSP_MODE_CIPHER:
				switch (csp->csp_cipher_alg) {
				case CRYPTO_AES_XTS:
				if (csp->csp_ivlen != AES_XTS_IV_LEN)
				return (EINVAL);
				switch (csp->csp_cipher_klen * 8) {
				case 256:
				case 512:
				break;
				default:
				return (EINVAL);
				}
				break;
				default:
				return (EINVAL);
				}
				break;
				default:
				return (EINVAL);
				}

				return (CRYPTODEV_PROBE_HARDWARE);
				}

				void
				caam_crypto_freesession(device_t dev, crypto_session_t cses)
				{
				/* Nothing to do */
				}

				/*
				* Generate a new software session.
				*/
				int
				caam_crypto_newsession(device_t dev, crypto_session_t cses,
				const struct crypto_session_params *csp)
				{
				struct caam_session *ses;
				struct caam_softc *sc;

				sc = device_get_softc(dev);
				ses = crypto_get_driver_session(cses);
				if (ses == NULL) {
				CAAM_LOG_DEV_WARN(dev, "Cannot create new session\n");
				return (EINVAL);
				}

				ses->sc = sc;

				return (0);
				}

				static void
				crypto_process_done(uint32_t desc, uint32_t status, void arg, device_t dev)
				{
				#if defined(CAAM_JR_MODE_ASYNC)
				struct caam_operation *op;
				bus_dma_tag_t jr_data_tag;
				int rv = 0;

				op = (struct caam_operation *)arg;
				if (status != 0)
				op->crp->crp_etype = EIO;
				else
				op->crp->crp_etype = 0;

				jr_data_tag = caam_jr_get_data_tag(op->jr_dev);

				rv = caam_jr_dma_unmap(op->jr_dev, &op->key, JR_MAP_NOSYNC);
				if (rv != 0) {
				CAAM_LOG_ERROR("%s: Failed to destroy key\n", __func__);
				}

				bus_dmamap_sync(jr_data_tag, op->data.mmap, BUS_DMASYNC_POSTREAD);
				bus_dmamap_unload(jr_data_tag, op->data.mmap);
				rv = bus_dmamap_destroy(jr_data_tag, op->data.mmap);
				if (rv != 0) {
				CAAM_LOG_ERROR("%s: Failed to destroy data\n", __func__);
				}

				if (op->data.bus_addr == 0) {
				rv = caam_jr_dma_unmap(op->jr_dev, &op->sgtm, JR_MAP_NOSYNC);
				if (rv != 0) {
				CAAM_LOG_ERROR("%s: Failed to destroy sgt\n", __func__);
				}
				}

				jr_pool_release(&op->ses->sc->jr_pool, op->jr_dev);

				crypto_done(op->crp);

				free(op, M_CAAM_CRYPTO);
				#endif /* CAAM_JR_MODE_ASYNC */
				}

				int
				caam_crypto_process(device_t dev, struct cryptop *crp, int hint)
				{
				const struct crypto_session_params *csp;
				struct caam_session *ses;
				const void *key_vaddr;
				int rv = 0;
				struct caam_operation *op;
				struct caam_softc *sc;
				bus_dma_tag_t jr_data_tag;

				SDT_PROBE0(caam_crypto, , func, process__start);

				sc = device_get_softc(dev);
				ses = crypto_get_driver_session(crp->crp_session);
				csp = crypto_get_params(crp->crp_session);
				ses->keylen = csp->csp_cipher_klen;
				ses->data_size = crp->crp_payload_length;
				ses->encrypt = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);

				op = malloc_aligned(sizeof(struct caam_operation), PAGE_SIZE,
				M_CAAM_CRYPTO, M_NOWAIT);
				if (op == NULL) {
				CAAM_LOG_DEV_WARN(dev,
				"Failed to allocate caam operation structure\n");
				rv = ENOMEM;
				goto exit;
				}

				op->keylen = ses->keylen;
				op->data.buflen = ses->data_size;
				op->encrypt = ses->encrypt;
				op->ses = ses;
				op->crp = crp;

				op->jobdesc.arg = op;
				op->jobdesc.callback = crypto_process_done;

				op->jr_dev = jr_pool_acquire(&sc->jr_pool);
				if (op->jr_dev == 0) {
				CAAM_LOG_DEV_WARN(dev, "Failed to acquire JR\n");
				rv = EAGAIN;
				goto operation_cleanup;
				}
				jr_data_tag = caam_jr_get_data_tag(op->jr_dev);

				crypto_read_iv(crp, op->iv);
				if (crp->crp_cipher_key)
				key_vaddr = crp->crp_cipher_key;
				else
				key_vaddr = csp->csp_cipher_key;

				/* Map and sync the key buffer */
				rv = caam_jr_dma_map(op->jr_dev, &op->key, __DECONST(void *, key_vaddr),
				ses->keylen, BUS_DMA_COHERENT, BUS_DMA_NOWAIT, JR_MAP_SYNC);
				if (rv != 0) {
				CAAM_LOG_DEV_WARN(dev, "Failed to create key map\n");
				goto release_jr;
				}

				/* Map and sync the data buffer */
				rv = bus_dmamap_create(jr_data_tag, BUS_DMA_COHERENT, &op->data.mmap);
				if (rv != 0) {
				CAAM_LOG_DEV_WARN(dev, "Failed to create data map\n");
				goto key_map_cleanup;
				}
				op->data.bus_addr = ULONG_MAX;
				rv = bus_dmamap_load_crp_buffer(jr_data_tag, op->data.mmap,
				&crp->crp_buf, dma_data_callback, op, BUS_DMA_NOWAIT);
				if ((rv != 0) \|\| (op->data.bus_addr == ULONG_MAX)) {
				CAAM_LOG_DEV_WARN(dev, "Failed to load data map\n");
				goto data_map_cleanup;
				}
				bus_dmamap_sync(jr_data_tag, op->data.mmap, BUS_DMASYNC_PREWRITE);

				/* If scatter gather is used map and sync the sgt buffer */
				if (op->data.bus_addr == 0) {
				rv = caam_jr_dma_map(op->jr_dev, &op->sgtm, op->sgt,
				sizeof(op->sgt), BUS_DMA_COHERENT, BUS_DMA_NOWAIT,
				JR_MAP_SYNC);
				if (rv != 0) {
				CAAM_LOG_DEV_WARN(dev, "Failed to create sgt map\n");
				goto data_load_cleanup;
				}
				}

				caam_aes_xts_build_desc(op, sc->little_endian);
				#if defined(CAAM_JR_MODE_ASYNC)
				rv = caam_run_descriptor_jr_async(dev, &op->jobdesc, op->jr_dev);
				#else
				rv = caam_run_descriptor_jr_blocking(dev, &op->jobdesc, op->jr_dev);
				#endif /* defined(CAAM_JR_MODE_ASYNC) */
				if (rv != 0) {
				CAAM_LOG_DEV_WARN(dev, "Failed to run job descriptor\n");
				goto sgt_map_cleanup;
				}

				#if defined(CAAM_JR_MODE_ASYNC)
				return (0);
				#endif /* defined(CAAM_JR_MODE_ASYNC) */

				/* For S/G paddr should be set to 0 */
				sgt_map_cleanup:
				if (op->data.bus_addr == 0)
				caam_jr_dma_unmap(op->jr_dev, &op->sgtm, JR_MAP_NOSYNC);
				data_load_cleanup:
				bus_dmamap_unload(jr_data_tag, op->data.mmap);
				data_map_cleanup:
				bus_dmamap_destroy(jr_data_tag, op->data.mmap);
				key_map_cleanup:
				caam_jr_dma_unmap(op->jr_dev, &op->key, JR_MAP_NOSYNC);
				release_jr:
				jr_pool_release(&sc->jr_pool, op->jr_dev);
				operation_cleanup:
				free(op, M_CAAM_CRYPTO);
				exit:
				crp->crp_etype = rv;
				crypto_done(crp);

				SDT_PROBE0(caam_crypto, , func, process__end);

				return (0);
				}

				int
				caam_crypto_attach(device_t dev)
				{
				struct caam_softc *sc;
				int error = 0;

				sc = device_get_softc(dev);

				sc->crypto_id = crypto_get_driverid(dev,
				sizeof(struct caam_session), CRYPTOCAP_F_HARDWARE);
				if (sc->crypto_id < 0) {
				CAAM_LOG_DEV_WARN(dev,
				"Failed to initialize crypto device\n");
				return (ENXIO);
				}

				return (error);
				}

				void
				caam_crypto_detach(device_t dev)
				{
				struct caam_softc *sc;

				sc = device_get_softc(dev);
				crypto_unregister_all(sc->crypto_id);
				}