Page MenuHomeFreeBSD
Files F171613536

D59581.id.diff
No OneTemporary
Actions

D59581.id.diff
View Options

diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -113,6 +113,8 @@
dev/dpaa/qman_portals.c optional dpaa fdt
dev/dpaa/qman.c optional dpaa
dev/dpaa/qman_fdt.c optional dpaa fdt
+dev/dpaa/sec_dev.c optional dpaa crypto
+dev/dpaa/sec_jr.c optional dpaa crypto
dev/fb/fb.c optional sc
dev/gpio/qoriq_gpio.c optional mpc85xx gpio
dev/hwpmc/hwpmc_e500.c optional hwpmc
@@ -247,7 +249,7 @@
dev/random/darn.c optional powerpc64 !random_loadable | powerpc64le !random_loadable
dev/scc/scc_bfe_macio.c optional scc powermac
dev/sdhci/sdhci_fsl_fdt.c optional mpc85xx sdhci
-dev/sec/sec.c optional sec mpc85xx
+dev/sec/sec.c optional sec mpc85xx powerpc
dev/sound/macio/aoa.c optional snd_davbus | snd_ai2s powermac
dev/sound/macio/davbus.c optional snd_davbus powermac
dev/sound/macio/i2s.c optional snd_ai2s powermac
diff --git a/sys/dev/dpaa/dpaa_common.h b/sys/dev/dpaa/dpaa_common.h
--- a/sys/dev/dpaa/dpaa_common.h
+++ b/sys/dev/dpaa/dpaa_common.h
@@ -24,7 +24,11 @@
uint32_t cmd_stat;
} __packed;
+#define DPAA_FD_FORMAT_SHORT_SBSF 0
+#define DPAA_FD_FORMAT_COMPOUND 1
+#define DPAA_FD_FORMAT_LONG_SBSF 2
#define DPAA_FD_FORMAT_SHORT_MBSF 4
+#define DPAA_FD_FORMAT_LONG_MBSF 6
#define DPAA_FD_RX_STATUS_DCL4C 0x10000000
#define DPAA_FD_RX_STATUS_DME 0x01000000
diff --git a/sys/dev/dpaa/sec_dev.c b/sys/dev/dpaa/sec_dev.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa/sec_dev.c
@@ -0,0 +1,2291 @@
+/*
+ * Copyright (c) 2026 Justin Hibbits
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/callout.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <sys/endian.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include "sec_var.h"
+#include "cryptodev_if.h"
+
+/*
+ * Most of this work is based on the T2080 Security (SEC) Reference Manual.
+ *
+ * The driver uses the Job Ring interface for all jobs. The QI interface can be
+ * added if IPSec, OVPN, or kTLS acceleration is added.
+ */
+
+/* From T2080 Security Reference Manual */
+#define SEC_MAX_SHDESC_WORDS 62
+
+#define SEC_MAX_JR 4 /* T2080 exposes four Job Rings */
+
+/* CCSR register offsets. */
+#define SEC_MCFGR 0x0004
+#define MCFGR_SWRST 0x80000000 /* Software reset */
+#define MCFGR_WDE 0x40000000 /* DECO watchdog enable */
+#define MCFGR_WDF 0x20000000 /* Watchdog fast (test only) */
+#define MCFGR_DMARST 0x10000000 /* DMA reset (with SWRST) */
+#define MCFGR_WRHD 0x08000000 /* Write handoff disable */
+#define MCFGR_DJPC 0x00200000 /* Disable job perf ctrs */
+#define MCFGR_DBPC 0x00100000 /* Disable byte perf ctrs */
+#define MCFGR_PS 0x00010000 /* Large pointers */
+#define MCFGR_ARCACHE_M 0x0000f000 /* AXI read cache attrs */
+#define MCFGR_AWCACHE_M 0x00000f00 /* AXI write cache attrs */
+#define MCFGR_AXIPRI 0x00000008 /* AXI master priority */
+#define MCFGR_LARGE_BURST 0x00000004 /* Enable 256B bursts */
+#define SEC_SCFGR 0x000c
+#define SCFGR_VIRT_EN 0x00008000 /* Virtualization enabled */
+
+#define SEC_RDSTA 0x06c0 /* RNG DRNG Status */
+#define RDSTA_IF0 0x00000001 /* State handle 0 up */
+#define RDSTA_IF1 0x00000002 /* State handle 1 up */
+#define RDSTA_ERRCODE_M 0x000f0000
+#define RDSTA_ERRCODE_S 16
+#define RDSTA_CE 0x00100000 /* Catastrophic error */
+
+/* DECO direct-access registers */
+#define SEC_DECORR 0x009c /* DECO Request Register */
+#define DECORR_DEN0 0x00010000 /* DECO0 enable (RO, bit 16) */
+#define DECORR_RQD0 0x00000001 /* DECO0 request */
+#define SEC_D0LIODNR_MS 0x00a0
+#define SEC_D0LIODNR_LS 0x00a4
+#define SEC_D0JQCR_MS 0x8800 /* JQCR upper: WHL/FOUR/SOB */
+#define DAJQCR_MS_WHL 0x20000000 /* Whole descriptor loaded */
+#define DAJQCR_MS_FOUR 0x10000000 /* >= 4 words in first burst */
+#define DAJQCR_MS_SOB 0x00010000 /* Shared/burst loaded */
+#define DAJQCR_MS_SRC_M 0x00000700 /* Job source */
+#define DAJQCR_MS_SRC_S 8
+#define SEC_D0JQCR_LS 0x8804
+#define SEC_D0DAR_MS 0x8808 /* Descriptor address, upper */
+#define SEC_D0DAR_LS 0x880c
+#define SEC_D0DESB(n) (0x8a00 + (n) * 4) /* n = 0..63 */
+#define SEC_D0DDR 0x8e04 /* Debug status */
+#define DADDR_VALID 0x80000000 /* Job currently running */
+#define DADDR_DECO_STATE_M 0x00f00000 /* Main state machine */
+#define DADDR_DECO_STATE_S 20
+
+/* Fault-address registers. */
+#define SEC_FAR_HI 0x0fc0 /* Fault Address, upper */
+#define SEC_FAR_LO 0x0fc4 /* Fault Address, lower */
+#define SEC_FALR 0x0fc8 /* Fault Address LIODN */
+#define SEC_FADR 0x0fcc /* Fault Address Detail */
+#define FADR_FERR_M 0xc0000000 /* AXI error response */
+#define FADR_FERR_S 30
+#define FADR_FSZ_EXT_M 0x00070000 /* Transfer size high 3 bits */
+#define FADR_FSZ_EXT_S 16
+#define FADR_DTYP 0x00008000 /* 0=message, 1=control */
+#define FADR_JSRC_M 0x00007000 /* Job source */
+#define FADR_JSRC_S 12
+#define FADR_BLKID_M 0x00000f00 /* SEC internal block ID */
+#define FADR_BLKID_S 8
+#define FADR_TYP 0x00000080 /* 0=read, 1=write */
+#define FADR_FSZ_M 0x0000007f /* Transfer size low 7 bits */
+
+#define SEC_RD4(sc, off) bus_read_4((sc)->sc_rres, (off))
+#define SEC_WR4(sc, off, v) bus_write_4((sc)->sc_rres, (off), (v))
+
+/* Descriptor command components */
+/* SEQ commands are intended for network protocols */
+#define CMD_DESC(n) ((n) << 27)
+#define CMD_KEY 0x00 /* Pointer/key follows descriptor */
+#define CMD_SEQ_KEY 0x01
+#define KEY_CLASS_M 0x06000000
+#define KEY_CLASS_1 0x02000000
+#define KEY_CLASS_2 0x04000000
+#define KEY_SGF 0x01000000 /* KEY - Pointer to SGT */
+#define KEY_VLF 0x01000000 /* SK - variable length */
+#define KEY_IMM 0x00800000 /* KEY - Key follows descriptor */
+#define KEY_AIDF 0x00800000 /* SK - Already in Input FIFO */
+#define KEY_ENC 0x00400000 /* Key is encrypted */
+#define KEY_NWB 0x00200000 /* No write back */
+#define KEY_EKT 0x00100000 /* Encrypted Key Type:
+ * 0 - AES-CCB
+ * 1 - AES-CCM
+ */
+#define KEY_KDEST_M 0x00030000 /* Key Destination */
+#define KEY_KDEST_REG 0x00000000 /* Dest is Key register */
+#define KEY_KDEST_PKHA 0x00010000 /* Dest is PKHA E-memory */
+#define KEY_KDEST_AFHA 0x00020000 /* Dest is AFHA S-Box */
+#define KEY_KDEST_MDHA_SPLIT 0x00030000 /* Key is MDHA split key */
+#define KEY_TK 0x00008000 /* Trusted Key */
+#define KEY_LENGTH_M 0x000003ff /* Key length */
+#define CMD_LOAD 0x02
+#define CMD_SEQ_LOAD 0x03
+#define LOAD_CLASS_M 0x06000000
+#define LOAD_CLASS_1 0x02000000
+#define LOAD_CLASS_2 0x04000000
+#define LOAD_CLASS_3 0x06000000
+#define LOAD_SGF 0x01000000 /* LOAD - Pointer to SGT */
+#define LOAD_VLF 0x01000000 /* SL - variable length */
+#define LOAD_IMM 0x00800000 /* LOAD - Data follows descriptor */
+#define LOAD_DST_M 0x007f0000 /* Destination register */
+#define LOAD_DST_S 16
+#define LOAD_KSR 0x00010000 /* Key Size Register (C1/C2) */
+#define LOAD_DSR 0x00020000 /* Data Size Register (C1/C2) */
+#define LOAD_ICVS 0x00030000 /* ICV Size Register (C1/C2) */
+#define LOAD_LSR 0x00040000 /* LIODN Status Register (C3) */
+#define LOAD_DCTRL2 0x00050000 /* DECO Control Register 2(C3) */
+#define LOAD_CCTRL 0x00060000 /* CHA Control Register (C1) */
+#define LOAD_DCTRL 0x00060000 /* DECO Control Register (C3) */
+#define LOAD_ICTRL 0x00070000 /* IRQ Control Register (C0) */
+#define LOAD_DPOVRD 0x00070000 /* DECO Protocol Override (C3) */
+#define LOAD_CLRW 0x00080000 /* Clear Written Register (C0) */
+#define LOAD_MATH0W 0x00080000 /* DECO Math Register 0 (C3) */
+#define LOAD_MATH1W 0x00090000 /* DECO Math Register 1 (C3) */
+#define LOAD_MATH2W 0x000a0000 /* DECO Math Register 2 (C3) */
+#define LOAD_CISEL 0x000a0000 /* CHA Instance Select Reg (C0) */
+#define LOAD_AADSZ 0x000b0000 /* AAD Size Register (C1) */
+#define LOAD_MAT3W 0x000b0000 /* DECO Math Register 3 (C3) */
+#define LOAD_C1VSZ 0x000c0000 /* Class 1 IV SIze Register (C1) */
+#define LOAD_ALTDS1 0x000f0000 /* Alternate Data Size C1 (C1) */
+#define LOAD_PKASZ 0x00100000 /* PKHA A Size Register (C1) */
+#define LOAD_PKBSZ 0x00110000 /* PKHA B Size Register (C1) */
+#define LOAD_PKNSZ 0x00120000 /* PKHA N Size Register (C1) */
+#define LOAD_PKESZ 0x00130000 /* PKHA E Size Register (C1) */
+#define LOAD_CTX 0x00200000 /* Context Register (C1/C2) */
+#define LOAD_KEY 0x00400000 /* Key Register (C1/C2) */
+#define LOAD_DESC_BUF 0x00400000 /* DECO Descriptor Buffer (C3) */
+#define LOAD_NFSL 0x00700000 /* NFIFO and size registers (C0) */
+#define LOAD_NFSM 0x00710000 /* NFIFO and size registers (C0) */
+#define LOAD_NFL 0x00720000 /* NFIFO (C0) */
+#define LOAD_NFM 0x00730000 /* NFIFO (C0) */
+#define LOAD_SL 0x00740000 /* Size register(s) (C0) */
+#define LOAD_SM 0x00750000 /* Size register(s) (C0) */
+#define LOAD_IDFNS 0x00760000 /* Input Data FIFO Nibble Shift (C0) */
+#define LOAD_ODFNS 0x00770000 /* Output Data FIFO Nibble Shift (C0) */
+#define LOAD_AUXDATA 0x00780000 /* Aux Data FIFO (C0) */
+#define LOAD_NFIFO 0x007a0000 /* NFIFO (C0) */
+#define LOAD_IFIFO 0x007c0000 /* Input Data FIFO (C0) */
+#define LOAD_OFIFO 0x007e0000 /* Output Data FIFO (C0) */
+#define LOAD_LENGTH_M 0x000000ff /* Data length (8 bits) */
+#define LOAD_OFFSET_S 8 /* OFFSET field shift (bits 8-15) */
+#define CMD_FIFO_LOAD 0x04
+#define CMD_SEQ_FIFO_LOAD 0x05
+#define CMD_STORE 0x0a
+#define CMD_SEQ_STORE 0x0b
+#define CMD_FIFO_STORE 0x0c
+#define CMD_SEQ_FIFO_STORE 0x0d
+#define CMD_MOVE 0x0e
+#define CMD_MOVE_LEN 0x0f
+#define CMD_OPERATION 0x10
+#define OPTYPE_M 0x07000000
+#define OPTYPE_S 24
+#define OPTYPE_CLASS1_ALG 0x02000000
+#define OPTYPE_CLASS2_ALG 0x04000000
+#define ALG_S 16
+#define CMD_ALGORITHM(m, n) ((m) | ((n) << ALG_S))
+/* Class 1 algorithms */
+#define ALG_AES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x10)
+#define ALG_DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x20)
+#define ALG_3DES CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x21)
+#define ALG_ARC4 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x30)
+#define ALG_RNG CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x50)
+#define ALG_SNOW3G_F8 CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x60)
+#define ALG_KASUMI CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0x70)
+#define ALG_ZUC_ENC CMD_ALGORITHM(OPTYPE_CLASS1_ALG, 0xb0)
+/* Class 2 algorithms */
+#define ALG_MD5 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x40)
+#define ALG_SHA1 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x41)
+#define ALG_SHA224 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x42)
+#define ALG_SHA256 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x43)
+#define ALG_SHA384 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x44)
+#define ALG_SHA512 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x45)
+#define ALG_CRC CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0x90)
+#define ALG_SNOW3G_F9 CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xa0)
+#define ALG_ZUC_AUTH CMD_ALGORITHM(OPTYPE_CLASS2_ALG, 0xc0)
+/* AAI (Additional Algorithm Information) codes. */
+#define AAI_S 4
+/* AES modes */
+#define AAI_AES_CTR (0x00 << AAI_S)
+#define AAI_AES_CBC (0x10 << AAI_S)
+#define AAI_AES_ECB (0x20 << AAI_S)
+#define AAI_AES_CFB (0x30 << AAI_S)
+#define AAI_AES_OFB (0x40 << AAI_S)
+#define AAI_AES_XTS (0x50 << AAI_S)
+#define AAI_AES_CMAC (0x60 << AAI_S)
+#define AAI_AES_XCBC_MAC (0x70 << AAI_S)
+#define AAI_AES_CCM (0x80 << AAI_S)
+#define AAI_AES_GCM (0x90 << AAI_S)
+#define AAI_AES_DK (0x100 << AAI_S) /* Decrypt-key derive */
+/* DES/3DES modes */
+#define AAI_DES_CBC (0x10 << AAI_S)
+#define AAI_DES_ECB (0x20 << AAI_S)
+/* MDHA modes */
+#define AAI_HASH (0x00 << AAI_S)
+#define AAI_HMAC (0x01 << AAI_S)
+#define AAI_HMAC_PRECOMP (0x04 << AAI_S) /* Precomputed IPAD/OPAD */
+/* Algorithm State field (bits 2-3): what phase to run */
+#define AS_S 2
+#define AS_UPDATE (0x0 << AS_S)
+#define AS_INIT (0x1 << AS_S)
+#define AS_FINAL (0x2 << AS_S)
+#define AS_INIT_FINAL (0x3 << AS_S)
+/* RNG-specific: State-Handle field. */
+#define OP_RNG_SH_S 4
+#define OP_RNG_SH(n) ((n) << OP_RNG_SH_S)
+/* Direction / ICV */
+#define OP_ICV 0x00000002
+#define OP_ENC 0x00000001
+
+/* SEQ FIFO LOAD command bits. */
+#define FIFOLD_CLASS_1 0x02000000 /* CLASS = 01b (Class 1) */
+#define FIFOLD_CLASS_2 0x04000000 /* CLASS = 10b (Class 2) */
+#define FIFOLD_CLASS_BOTH 0x06000000 /* CLASS = 11b (snooping) */
+#define FIFOLD_VLF 0x01000000 /* Variable-length flag */
+/*
+ * Input data type: top 3 bits = type,
+ * bottom 3 bits = LC2/LC1/FC1 flags.
+ */
+#define FIFOLD_TYPE_S 16
+#define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_S) /* 010_000 */
+/* Class 1 output fed straight into Class 2, i.e. MAC over ciphertext. */
+#define FIFOLD_TYPE_MSG_C1OUT (0x18 << FIFOLD_TYPE_S) /* 011_000 */
+#define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_S) /* 100_000 */
+#define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_S) /* 110_000 */
+#define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_S) /* 111_000 */
+#define FIFOLD_FC1 (0x01 << FIFOLD_TYPE_S) /* Flush class 1 */
+#define FIFOLD_LC1 (0x02 << FIFOLD_TYPE_S) /* Last for Class 1 */
+#define FIFOLD_LC2 (0x04 << FIFOLD_TYPE_S) /* Last for Class 2 */
+/* Length moves to a 32-bit word after the command. */
+#define FIFO_EXT 0x00400000
+
+/* SEQ FIFO STORE command bits. */
+#define FIFOST_VLF 0x01000000
+#define FIFOST_TYPE_S 16
+#define FIFOST_TYPE_MSG_DATA (0x30 << FIFOST_TYPE_S)
+
+#define CMD_SIGNATURE 0x12
+#define CMD_JUMP 0x14
+#define CMD_MATH 0x15
+#define MATH_FN_ADD (0x0 << 20) /* SRC0 + SRC1 */
+#define MATH_SRC0_SIL (0x8 << 16) /* Sequence In Length */
+#define MATH_SRC1_ZERO (0xF << 12) /* Constant zero */
+#define MATH_DEST_VSIL (0xA << 8) /* Variable SIL */
+#define MATH_DEST_VSOL (0xB << 8) /* Variable SOL */
+#define MATH_LEN_4 0x4
+/* J - Job Descriptor, S - Shared Descriptor */
+#define CMD_DESC_HEADER 0x16
+#define HEADER_EXT 0x04000000 /* Has Extension (J) */
+#define HEADER_RSL 0x02000000 /* Require SEQ LIODN (J) */
+#define HEADER_DNR 0x01000000 /* Do Not Run (J/S) */
+#define HEADER_ONE 0x00800000 /* Must be 1 (J/S) */
+#define HEADER_START_INDEX(n) ((n) << 16) /* Start Index (J/S) */
+#define HEADER_SHR_DESC_L(n) ((n) << 16) /* Shared Desc len (J) */
+/* Bit 16 must be 0 */
+#define HEADER_TDES_M 0x00006000 /* Trusted Descriptor Mask (J) */
+#define HEADER_TDES 0x00004000 /* Trusted Descriptor (J) */
+#define HEADER_TDES_CAND 0x00006000 /* Candidate Trust Desc (J) */
+#define HEADER_SHR 0x00001000 /* Has Shared Descriptor (J) */
+#define HEADER_REO 0x00000800 /* Reverse Execution Order (J) */
+#define HEADER_SHARE_M 0x00000700 /* Share State (J/S) */
+#define HEADER_SHARE_WAIT 0x00000100 /* Wait to share (J/S) */
+#define HEADER_SHARE_SERIAL 0x00000200 /* Serialize (J/S) */
+#define HEADER_SHARE_ALWAYS 0x00000300 /* Always share (stateless) (J/S) */
+#define HEADER_SHARE_DEFER 0x00000400 /* Defer to shared desc (J) */
+#define HEADER_DESCLEN_M 0x0000007f /* Descriptor length */
+#define HEADER_DESCLEN_S 0
+#define HEADER_EXT_FTD 0x00000100 /* Fake Trusted Descriptor */
+#define HEADER_EXT_DSELVALID 0x00000080 /* DECO_SELECT field valid */
+#define HEADER_EXT_DSEL_M 0x0000000f /* DECO Select */
+#define CMD_SHARED_HEADER 0x17
+#define HEADER_RIF 0x02000000 /* Read Input Frame */
+#define HEADER_CIF 0x00002000 /* Clear Input FIFO */
+#define HEADER_SC 0x00001000 /* Save Context */
+#define HEADER_PD 0x00000800 /* Propagate DNR */
+#define CMD_MATHI 0x1d
+#define CMD_SEQ_IN_PTR 0x1e
+#define SEQ_SGF 0x01000000 /* Pointer is SGT (bit 7 NXP) */
+#define SEQ_EXT 0x00400000 /* 32-bit extended length (bit 9 NXP) */
+#define CMD_SEQ_OUT_PTR 0x1f
+
+/* Shared descriptor container. */
+struct sec_context {
+ uint32_t shd[SEC_MAX_SHDESC_WORDS];
+};
+
+
+/*
+ * Session state: one shared descriptor per direction. The shared
+ * descriptor holds just KEY + OPERATION; the per-job JD adds LOAD-IV
+ * and SEQ_IN_PTR / SEQ_OUT_PTR inline.
+ */
+#define SEC_MAX_SPLIT_KEY 128 /* SHA-512 AES-ECB encrypted */
+
+#define SEC_CCM_AAD_MAX 0xfeff
+
+struct sec_session {
+ struct sec_softc *sess_sc;
+ struct sec_context ctx[2]; /* [0]=dec, [1]=enc */
+ uint32_t sdlen[2]; /* words per direction */
+ uint8_t digestlen; /* HMAC output size (0 if none) */
+ uint8_t skeylen; /* HMAC split key size (0 if none) */
+ uint8_t skey[SEC_MAX_SPLIT_KEY];
+};
+
+static device_probe_t sec_probe;
+static device_attach_t sec_attach;
+static device_detach_t sec_detach;
+static cryptodev_probesession_t sec_probe_session;
+static cryptodev_newsession_t sec_new_session;
+static cryptodev_freesession_t sec_free_session;
+static cryptodev_process_t sec_process;
+
+static void sec_intr(void *);
+
+/* Register-level bring-up. Filled in from the SEC reference manual. */
+static int sec_reset(struct sec_softc *);
+static int sec_rng_init(struct sec_softc *);
+
+static struct ofw_compat_data compats[] = {
+ { "fsl,sec-v5.2", 52 },
+ { "fsl,sec-v5.0", 50 },
+ { "fsl,sec-v4.0", 40 },
+ { NULL, 0 }
+};
+
+static device_method_t sec_methods[] = {
+ /* Device methods */
+ DEVMETHOD(device_probe, sec_probe),
+ DEVMETHOD(device_attach, sec_attach),
+ DEVMETHOD(device_detach, sec_detach),
+
+ /* Cryptodev methods */
+ DEVMETHOD(cryptodev_probesession, sec_probe_session),
+ DEVMETHOD(cryptodev_newsession, sec_new_session),
+ DEVMETHOD(cryptodev_freesession, sec_free_session),
+ DEVMETHOD(cryptodev_process, sec_process),
+
+ DEVMETHOD_END
+};
+
+static DEFINE_CLASS_0(sec, sec_driver, sec_methods, sizeof(struct sec_softc));
+DRIVER_MODULE(sec, simplebus, sec_driver, NULL, NULL);
+MODULE_DEPEND(sec, crypto, 1, 1, 1);
+
+MALLOC_DEFINE(M_SEC, "sec", "SEC driver");
+
+static int
+sec_probe(device_t dev)
+{
+ const struct ofw_compat_data *cd;
+
+ cd = ofw_bus_search_compatible(dev, compats);
+ if (cd->ocd_data == 0)
+ return (ENXIO);
+
+ device_set_descf(dev, "Freescale Security Engine v%d.%d",
+ (int)cd->ocd_data / 10, (int)cd->ocd_data % 10);
+
+ return (BUS_PROBE_DEFAULT);
+}
+
+static int
+sec_attach(device_t dev)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ const struct ofw_compat_data *cd;
+
+ sc->sc_dev = dev;
+ sc->sc_cid = -1;
+
+ cd = ofw_bus_search_compatible(dev, compats);
+ sc->sc_version = cd->ocd_data;
+
+ sc->sc_rrid = 0;
+ sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
+ RF_ACTIVE);
+ if (sc->sc_rres == NULL) {
+ device_printf(dev, "could not allocate register resource\n");
+ goto fail;
+ }
+
+ /* TODO: Error IRQ handling. */
+ sc->sc_irid = 0;
+ sc->sc_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irid,
+ RF_ACTIVE | RF_SHAREABLE);
+ if (sc->sc_ires == NULL) {
+ device_printf(dev, "could not allocate error interrupt\n");
+ goto fail;
+ }
+
+ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
+ SEC_MAX_SIZE, SEC_MAX_SEGMENTS, SEC_MAX_SIZE, BUS_DMA_ALLOCNOW,
+ NULL, NULL, &sc->sc_dmatag) != 0) {
+ device_printf(dev, "could not create DMA tag\n");
+ goto fail;
+ }
+
+ if (sec_reset(sc) != 0) {
+ device_printf(dev, "SEC reset failed\n");
+ goto fail;
+ }
+ if (sec_rng_init(sc) != 0) {
+ device_printf(dev, "SEC RNG instantiation failed\n");
+ goto fail;
+ }
+ if (sec_init_rings(sc) == 0) {
+ device_printf(dev, "SEC job ring init failed\n");
+ goto fail;
+ }
+
+ /*
+ * Clear any fault-address latch left over from the bootloader before
+ * enabling the error IRQ. FADR, FAR_HI/LO, and FALR must all be read
+ * before they're all cleared, per the RM.
+ */
+ (void)SEC_RD4(sc, SEC_FADR);
+ (void)SEC_RD4(sc, SEC_FAR_HI);
+ (void)SEC_RD4(sc, SEC_FAR_LO);
+ (void)SEC_RD4(sc, SEC_FALR);
+
+ if (bus_setup_intr(dev, sc->sc_ires, INTR_TYPE_MISC | INTR_MPSAFE,
+ NULL, sec_intr, sc, &sc->sc_icookie) != 0) {
+ device_printf(dev, "could not install error interrupt\n");
+ goto fail;
+ }
+
+ sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
+ CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto fail;
+ }
+
+ return (0);
+
+fail:
+ sec_detach(dev);
+ return (ENXIO);
+}
+
+static int
+sec_detach(device_t dev)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ u_int i;
+
+ if (sc->sc_cid >= 0)
+ crypto_unregister_all(sc->sc_cid);
+
+ /* Silence the rings before halting them. */
+ for (i = 0; i < sc->sc_njr; i++) {
+ struct sec_jr *jr = &sc->sc_jr[i];
+
+ if (jr->jr_icookie != NULL)
+ bus_teardown_intr(dev, jr->jr_ires, jr->jr_icookie);
+ if (jr->jr_ires != NULL)
+ bus_release_resource(dev, SYS_RES_IRQ, jr->jr_irid,
+ jr->jr_ires);
+ sec_jr_teardown(sc, jr);
+ }
+ free(sc->sc_jr, M_DEVBUF);
+
+ if (sc->sc_dmatag != NULL)
+ bus_dma_tag_destroy(sc->sc_dmatag);
+ if (sc->sc_icookie != NULL)
+ bus_teardown_intr(dev, sc->sc_ires, sc->sc_icookie);
+ if (sc->sc_ires != NULL)
+ bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irid,
+ sc->sc_ires);
+ if (sc->sc_rres != NULL)
+ bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
+ sc->sc_rres);
+
+ return (0);
+}
+
+static const char *sec_ferr_str[] = {
+ "OKAY", "reserved", "SLVERR", "DECERR",
+};
+
+static const char *sec_jsrc_str[] = {
+ "JR0", "JR1", "JR2", "JR3", "RTIC", "QI", "rsvd6", "rsvd7",
+};
+
+static void
+sec_intr(void *arg)
+{
+ struct sec_softc *sc = arg;
+ uint32_t fadr, falr;
+ uint64_t far;
+
+ fadr = SEC_RD4(sc, SEC_FADR);
+ if ((fadr & FADR_FERR_M) != 0) {
+ /*
+ * All fault registers are latched by hardware until all are
+ * read, in any order.
+ */
+ far = (uint64_t)SEC_RD4(sc, SEC_FAR_HI) << 32;
+ far |= SEC_RD4(sc, SEC_FAR_LO);
+ falr = SEC_RD4(sc, SEC_FALR);
+
+ device_printf(sc->sc_dev,
+ "bus fault: FADR=%#x FAR=%#jx FALR=%#x "
+ "(%s, %s, src=%s, blkid=%#x, %s, size=%u)\n",
+ fadr, (uintmax_t)far, falr,
+ sec_ferr_str[(fadr & FADR_FERR_M) >> FADR_FERR_S],
+ (fadr & FADR_DTYP) ? "control" : "message",
+ sec_jsrc_str[(fadr & FADR_JSRC_M) >> FADR_JSRC_S],
+ (fadr & FADR_BLKID_M) >> FADR_BLKID_S,
+ (fadr & FADR_TYP) ? "write" : "read",
+ (unsigned)(((fadr & FADR_FSZ_EXT_M) >>
+ (FADR_FSZ_EXT_S - 7)) | (fadr & FADR_FSZ_M)));
+ }
+
+}
+
+/*
+ * Decode a SEC job termination status word.
+ *
+ * Bits 0-3 (MSB) are the "source" of the report; the remaining bits are
+ * source-specific. Zero means clean completion.
+ *
+ * Two cases we care to distinguish:
+ * - CCB (source 2), ERRID field bits 28-31
+ * value 0xA is "ICV check failed" -> EBADMSG.
+ * - DECO (source 4), Error Code bits 24-31
+ * values F0h/F1h/FFh are informational warnings (IPsec TTL,
+ * 3GPP HFN, output-length rollover). The job actually completed,
+ * so map those to success.
+ *
+ * Everything else is logged and reported as EIO. Real per-code
+ * decoding of DECO/QI errors can be layered on as we hit them.
+ */
+#define SEC_STAT_SOURCE(s) (((s) >> 28) & 0xf)
+#define SEC_SRC_NONE 0x0
+#define SEC_SRC_CCB 0x2
+#define SEC_SRC_DECO 0x4
+#define SEC_SRC_QI 0x5
+#define SEC_SRC_JR 0x6
+#define SEC_CCB_ERR_ICV_FAIL 0x0a
+#define SEC_DECO_ERR_WARN_MIN 0xf0
+
+static int
+sec_decode_status(struct sec_softc *sc, uint32_t status)
+{
+ uint32_t source;
+
+ if (status == 0)
+ return (0);
+
+ source = SEC_STAT_SOURCE(status);
+
+ switch (source) {
+ case SEC_SRC_CCB:
+ if ((status & 0xf) == SEC_CCB_ERR_ICV_FAIL)
+ return (EBADMSG);
+ break;
+ case SEC_SRC_DECO:
+ if ((status & 0xff) >= SEC_DECO_ERR_WARN_MIN)
+ return (0);
+ break;
+ }
+
+ device_printf(sc->sc_dev,
+ "job termination status %#x (source %#x)\n", status, source);
+ return (EIO);
+}
+
+/*
+ * Complete one job that SEC has finished processing.
+ */
+void
+sec_complete_one(struct sec_softc *sc, uint64_t desc_pa, uint32_t status)
+{
+ struct sec_job *job;
+ struct cryptop *crp;
+ const struct crypto_session_params *csp;
+ uint8_t expected[SEC_MAX_DIGEST];
+ int dlen;
+
+ job = (struct sec_job *)PHYS_TO_DMAP((vm_paddr_t)desc_pa);
+ crp = job->crp;
+
+ crp->crp_etype = sec_decode_status(sc, status);
+
+ bus_dmamap_sync(sc->sc_dmatag, job->map,
+ BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->sc_dmatag, job->map);
+ bus_dmamap_destroy(sc->sc_dmatag, job->map);
+
+ if (crp->crp_etype == 0) {
+ csp = crypto_get_params(crp->crp_session);
+ dlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen :
+ job->sess->digestlen;
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ dlen, expected);
+ if (timingsafe_bcmp(job->digest, expected,
+ dlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else {
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ }
+ break;
+ case CSP_MODE_ETA:
+ if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0) {
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ break;
+ }
+ crypto_copydata(crp, crp->crp_digest_start, dlen,
+ expected);
+ if (timingsafe_bcmp(job->digest, expected, dlen) != 0)
+ crp->crp_etype = EBADMSG;
+ break;
+ case CSP_MODE_AEAD:
+ if ((crp->crp_op & CRYPTO_OP_ENCRYPT) != 0)
+ crypto_copyback(crp, crp->crp_digest_start,
+ dlen, job->digest);
+ break;
+ }
+ }
+
+ crypto_done(crp);
+ free(job, M_SEC);
+}
+
+static bool
+check_cipher(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 32 ||
+ csp->csp_cipher_klen == 64);
+ default:
+ return (false);
+ }
+}
+
+static bool
+check_aead(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (csp->csp_auth_mlen != 0 &&
+ csp->csp_auth_mlen != AES_GMAC_HASH_LEN)
+ return (false);
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ case CRYPTO_AES_CCM_16:
+ return (csp->csp_cipher_klen == 16 ||
+ csp->csp_cipher_klen == 24 ||
+ csp->csp_cipher_klen == 32);
+ default:
+ return (false);
+ }
+}
+
+/*
+ * Map an opencrypto auth_alg to its SEC selector and digest length.
+ * skeylen is zero for a plain hash, which is what tells the two apart.
+ */
+static bool
+sec_hash_params(int auth_alg, uint32_t *alg, uint8_t *dlen, uint8_t *skeylen)
+{
+
+ switch (auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ *alg = ALG_SHA1; *dlen = 20; *skeylen = 40; return (true);
+ case CRYPTO_SHA2_224_HMAC:
+ *alg = ALG_SHA224; *dlen = 28; *skeylen = 64; return (true);
+ case CRYPTO_SHA2_256_HMAC:
+ *alg = ALG_SHA256; *dlen = 32; *skeylen = 64; return (true);
+ case CRYPTO_SHA2_384_HMAC:
+ *alg = ALG_SHA384; *dlen = 48; *skeylen = 128; return (true);
+ case CRYPTO_SHA2_512_HMAC:
+ *alg = ALG_SHA512; *dlen = 64; *skeylen = 128; return (true);
+ case CRYPTO_SHA1:
+ *alg = ALG_SHA1; *dlen = 20; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_224:
+ *alg = ALG_SHA224; *dlen = 28; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_256:
+ *alg = ALG_SHA256; *dlen = 32; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_384:
+ *alg = ALG_SHA384; *dlen = 48; *skeylen = 0; return (true);
+ case CRYPTO_SHA2_512:
+ *alg = ALG_SHA512; *dlen = 64; *skeylen = 0; return (true);
+ }
+ return (false);
+}
+
+static bool
+check_digest(const struct crypto_session_params *csp)
+{
+ uint32_t alg;
+ uint8_t dlen, skeylen;
+
+ /* GMAC is AESA rather than MDHA, so it has its own constraints. */
+ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (false);
+ if (csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+ return (false);
+ return (csp->csp_auth_klen == 16 ||
+ csp->csp_auth_klen == 24 ||
+ csp->csp_auth_klen == 32);
+ }
+
+ if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
+ return (false);
+ /* Keyed variants require a key; plain hashes must not carry one. */
+ if ((skeylen != 0) != (csp->csp_auth_klen != 0))
+ return (false);
+ return (csp->csp_auth_mlen <= dlen);
+}
+
+static bool
+check_eta(const struct crypto_session_params *csp)
+{
+
+ /*
+ * ESN appends four bytes from crp_esn to the MAC input, which the
+ * descriptor has no way to splice in, so refuse rather than
+ * authenticate the wrong span.
+ */
+ if ((csp->csp_flags & CSP_F_ESN) != 0)
+ return (false);
+ /*
+ * XTS carries its tweak in the class 1 context and pairs with no
+ * MAC; its shared descriptor is shaped differently.
+ */
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ return (false);
+ /* The MAC half has to be keyed; a bare hash authenticates nothing. */
+ if (csp->csp_auth_klen == 0)
+ return (false);
+ return (check_cipher(csp) && check_digest(csp));
+}
+
+/*
+ * Software split-key generator: computes the HMAC ipad/opad hash-state
+ * halves in software and packs them big-endian for SEC's Class 2 KEY
+ * register.
+ *
+ * Runs the CPU through one SHA block per pad (two total). Much cheaper than
+ * the round trip through the job ring for setup.
+ */
+static void
+sec_pack_state32(uint8_t *dst, const uint32_t *src, unsigned int nbytes)
+{
+ unsigned int i;
+
+ for (i = 0; i < nbytes; i += 4)
+ be32enc(dst + i, src[i / 4]);
+}
+
+static void
+sec_pack_state64(uint8_t *dst, const uint64_t *src, unsigned int nbytes)
+{
+ unsigned int i;
+
+ for (i = 0; i < nbytes; i += 8)
+ be64enc(dst + i, src[i / 8]);
+}
+
+static void
+sec_sw_gen_split_key(const struct crypto_session_params *csp,
+ uint8_t *out, size_t out_len)
+{
+ union authctx ictx, octx;
+ const struct auth_hash *axf;
+ uint8_t half;
+
+ axf = crypto_auth_hash(csp);
+ hmac_init_ipad(axf, csp->csp_auth_key, csp->csp_auth_klen, &ictx);
+ hmac_init_opad(axf, csp->csp_auth_key, csp->csp_auth_klen, &octx);
+
+ KASSERT(out_len % 2 == 0, ("split key len must be even"));
+ half = out_len / 2;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ sec_pack_state32(out, ictx.sha1ctx.h.b32, half);
+ sec_pack_state32(out + half, octx.sha1ctx.h.b32, half);
+ break;
+ case CRYPTO_SHA2_224_HMAC:
+ sec_pack_state32(out, ictx.sha224ctx.state, half);
+ sec_pack_state32(out + half, octx.sha224ctx.state, half);
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ sec_pack_state32(out, ictx.sha256ctx.state, half);
+ sec_pack_state32(out + half, octx.sha256ctx.state, half);
+ break;
+ case CRYPTO_SHA2_384_HMAC:
+ sec_pack_state64(out, ictx.sha384ctx.state, half);
+ sec_pack_state64(out + half, octx.sha384ctx.state, half);
+ break;
+ case CRYPTO_SHA2_512_HMAC:
+ sec_pack_state64(out, ictx.sha512ctx.state, half);
+ sec_pack_state64(out + half, octx.sha512ctx.state, half);
+ break;
+ }
+
+ explicit_bzero(&ictx, sizeof(ictx));
+ explicit_bzero(&octx, sizeof(octx));
+}
+
+/*
+ * Descriptor builder. Word 0 is the HEADER and is filled in last, since its
+ * length field is only known once the body has been emitted.
+ */
+struct sec_desc_builder {
+ uint32_t *desc;
+ unsigned int idx; /* next word to write */
+ unsigned int max;
+ int err;
+};
+
+static inline void
+sec_desc_init(struct sec_desc_builder *b, uint32_t *desc, unsigned int max)
+{
+
+ b->desc = desc;
+ b->idx = 1; /* reserve word 0 for the HEADER */
+ b->max = max;
+ b->err = 0;
+}
+
+static inline void
+sec_desc_word(struct sec_desc_builder *b, uint32_t w)
+{
+
+ if (b->err != 0)
+ return;
+ if (b->idx >= b->max) {
+ b->err = ENOSPC;
+ return;
+ }
+ b->desc[b->idx++] = w;
+}
+
+/* Emit a KEY command with the key inline after it. */
+static inline void
+sec_desc_key_imm(struct sec_desc_builder *b, uint32_t class,
+ const void *key, unsigned int keylen)
+{
+ unsigned int nwords = howmany(keylen, sizeof(uint32_t));
+
+ if (b->err != 0)
+ return;
+ if (b->idx + 1 + nwords > b->max) {
+ b->err = ENOSPC;
+ return;
+ }
+ b->desc[b->idx++] = CMD_DESC(CMD_KEY) | class | KEY_IMM |
+ (keylen & KEY_LENGTH_M);
+ memcpy(&b->desc[b->idx], key, keylen);
+ b->idx += nwords;
+}
+
+static int
+sec_desc_finalize_shared(struct sec_desc_builder *b, uint32_t flags,
+ uint32_t *sdlenp)
+{
+
+ if (b->err != 0)
+ return (b->err);
+ if (b->idx > SEC_MAX_SHDESC_WORDS)
+ return (ENOSPC);
+ b->desc[0] = CMD_DESC(CMD_SHARED_HEADER) | HEADER_ONE |
+ (flags & (HEADER_SHARE_M | HEADER_SC)) |
+ (b->idx & HEADER_DESCLEN_M);
+ *sdlenp = b->idx;
+ return (0);
+}
+
+static int
+sec_desc_finalize_job(struct sec_desc_builder *b, uint32_t word,
+ uint32_t *dlenp)
+{
+
+ if (b->err != 0)
+ return (b->err);
+ if (b->idx > SEC_MAX_DESC_WORDS)
+ return (ENOSPC);
+ b->desc[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE |
+ word | (b->idx & HEADER_DESCLEN_M);
+ *dlenp = b->idx;
+ return (0);
+}
+
+
+/*
+ * Job descriptor builder conveniences.
+ */
+
+static inline void
+sec_jd_ptr(struct sec_desc_builder *b, vm_paddr_t pa)
+{
+ sec_desc_word(b, (uint32_t)(pa >> 32));
+ sec_desc_word(b, (uint32_t)pa);
+}
+
+/* Build a SEQ_IN/SEQ_OUT descriptor command. */
+static inline void
+sec_jd_seq(struct sec_desc_builder *b, bool inout, uint32_t flags,
+ vm_paddr_t ptr, uint32_t len)
+{
+ sec_desc_word(b,
+ CMD_DESC(inout ? CMD_SEQ_OUT_PTR : CMD_SEQ_IN_PTR) | flags);
+ sec_jd_ptr(b, ptr);
+ sec_desc_word(b, len);
+}
+
+static inline void
+sec_jd_load(struct sec_desc_builder *b, bool seq, uint32_t class,
+ uint32_t flags, uint32_t dst, uint32_t off, uint32_t len, vm_paddr_t ptr)
+{
+ uint32_t cmd = seq ? CMD_SEQ_LOAD : CMD_LOAD;
+
+ sec_desc_word(b, CMD_DESC(cmd) | class | flags | dst |
+ (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M));
+ if (!seq)
+ sec_jd_ptr(b, ptr);
+}
+
+static inline void
+sec_jd_store(struct sec_desc_builder *b, bool seq, uint32_t class, uint32_t src,
+ uint32_t off, uint32_t len, vm_paddr_t ptr)
+{
+ uint32_t cmd = seq ? CMD_SEQ_STORE : CMD_STORE;
+
+ sec_desc_word(b, CMD_DESC(cmd) | class | src |
+ (off << LOAD_OFFSET_S) | (len & LOAD_LENGTH_M));
+ if (!seq)
+ sec_jd_ptr(b, ptr);
+}
+
+static inline void
+sec_jd_fifo(struct sec_desc_builder *b, uint32_t cmd, uint32_t len)
+{
+
+ if (len > 0xffff) {
+ sec_desc_word(b, cmd | FIFO_EXT);
+ sec_desc_word(b, len);
+ } else {
+ sec_desc_word(b, cmd | len);
+ }
+}
+
+/*
+ * AES-XTS Class 1 context layout (byte offsets into the CTX register).
+ * The 16-byte tweak is split either side of the sector-size field.
+ */
+#define SEC_XTS_CTX_TWEAK_LO 0x20
+#define SEC_XTS_CTX_SECTOR 0x28
+#define SEC_XTS_CTX_TWEAK_HI 0x30
+
+/*
+ * Sector size tells the hardware how often to re-derive the tweak.
+ * opencrypto's XTS runs one continuous tweak over the whole request, so
+ * this only needs to exceed any payload we accept; sec_jd_build_cipher
+ * rejects requests that would cross the boundary.
+ */
+#define SEC_XTS_SECTOR_SIZE 0x8000
+
+/*
+ * Build the CCM context block and formatted-AAD length prefix.
+ *
+ * The hardware wants B0 in context dwords 0-1 and the initial counter
+ * CTR0 in dwords 2-3, with dwords 4-6 zeroed because AS is
+ * INITIALIZE/FINALIZE. Both blocks are laid out per RFC 3610: with a
+ * nonce of n bytes, the length field occupies the trailing L = 15 - n
+ * bytes and the flags byte carries L-1 plus, for B0, the encoded tag
+ * size and an AAD-present flag.
+ *
+ * The AAD itself is prefixed with its length and then zero-padded to a
+ * 16-byte boundary by the hardware, which pads AAD and IV FIFO loads
+ * when the flush-class-1 bit is set.
+ */
+static int
+sec_ccm_prep(struct sec_job *job, const struct crypto_session_params *csp)
+{
+ uint8_t *b0 = job->ccm_ctx;
+ uint8_t *ctr0 = job->ccm_ctx + 16;
+ uint32_t aadlen = job->crp->crp_aad_length;
+ uint64_t paylen = job->crp->crp_payload_length;
+ u_int i, lfield = 15 - csp->csp_ivlen;
+
+ if (aadlen > SEC_CCM_AAD_MAX)
+ return (EOPNOTSUPP);
+ /*
+ * B0 carries the payload length in its trailing lfield bytes, so
+ * the nonce is what really caps the payload: a 13-byte nonce
+ * leaves two bytes and stops at 64 KB, while the usual 12-byte one
+ * leaves three and reaches 16 MB.
+ */
+ if (lfield < sizeof(paylen) && paylen >= (uint64_t)1 << (8 * lfield))
+ return (EOPNOTSUPP);
+
+ memset(job->ccm_ctx, 0, sizeof(job->ccm_ctx));
+
+ b0[0] = (aadlen > 0 ? 0x40 : 0x00) |
+ (((job->sess->digestlen - 2) / 2) << 3) | (lfield - 1);
+ memcpy(b0 + 1, job->iv, csp->csp_ivlen);
+ for (i = 0; i < lfield; i++)
+ b0[15 - i] = (paylen >> (8 * i)) & 0xff;
+
+ ctr0[0] = lfield - 1;
+ memcpy(ctr0 + 1, job->iv, csp->csp_ivlen);
+
+ be16enc(job->ccm_alen, aadlen);
+ return (0);
+}
+
+/*
+ * Bytes of IV the input sequence carries. XTS is the odd one out:
+ * opencrypto's IV is just the 8-byte block number, but the hardware
+ * loads both halves of the 16-byte tweak from the sequence.
+ */
+static uint32_t
+sec_cipher_ivlen(const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ return (AES_BLOCK_LEN);
+ return (csp->csp_ivlen);
+}
+
+/*
+ * Expand opencrypto's 8-byte XTS IV in place into the 16-byte tweak the
+ * hardware expects. The IV holds a block number in host order
+ * (xform_aes_xts.c:aes_xts_reinit) and the tweak is that number's
+ * little-endian encoding followed by zeroes.
+ */
+static void
+sec_xts_tweak(uint8_t *iv)
+{
+ uint64_t blocknum;
+
+ memcpy(&blocknum, iv, sizeof(blocknum));
+ le64enc(iv, blocknum);
+ memset(iv + sizeof(blocknum), 0, AES_BLOCK_LEN - sizeof(blocknum));
+}
+
+static uint32_t
+sec_cipher_ctx_offset(uint32_t cipher_alg)
+{
+ switch (cipher_alg) {
+ case CRYPTO_AES_ICM:
+ return (16);
+ };
+
+ return (0);
+}
+
+/* Per-mode shared-descriptor builders. */
+/*
+ * Cipher shared descriptor has the following format:
+ * [0] - Header
+ * [1..klen] - KEY descriptor + key
+ * [XTS:..5] -- XTS specific
+ * [0..2] - LOAD XTS context
+ * [3..4] - LOAD XTS tweak
+ * [!XTS:1] -- Load IV into Context register
+ * [] - Operation
+ * [] - MATH - Move SIL register to VSIL for FIFO IN
+ * [] - MATH - Move SOL register to VSOL for FIFO OUT
+ * [] - FIFO LOAD
+ * [] - FIFO STORE
+ */
+static int
+sec_shd_build_cipher(struct sec_session *sess,
+ const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
+{
+ struct sec_desc_builder b;
+ uint32_t flags, op;
+ uint32_t ctx_offset;
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ op = CMD_DESC(CMD_OPERATION) | ALG_AES |
+ AAI_AES_CBC | AS_INIT_FINAL;
+ break;
+ case CRYPTO_AES_ICM:
+ op = CMD_DESC(CMD_OPERATION) | ALG_AES |
+ AAI_AES_CTR | AS_INIT_FINAL;
+ break;
+ case CRYPTO_AES_XTS:
+ op = CMD_DESC(CMD_OPERATION) | ALG_AES |
+ AAI_AES_XTS | AS_INIT_FINAL;
+ break;
+ default:
+ return (EOPNOTSUPP);
+ }
+
+ ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg);
+
+ if (enc)
+ op |= OP_ENC;
+
+ sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
+
+ if (csp->csp_cipher_klen > 0)
+ sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS) {
+ sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_CTX,
+ SEC_XTS_CTX_SECTOR, 8, SEC_XTS_SECTOR_SIZE);
+
+ sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX,
+ SEC_XTS_CTX_TWEAK_LO, 8, 0);
+ sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX,
+ SEC_XTS_CTX_TWEAK_HI, 8, 0);
+ } else {
+ sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset,
+ csp->csp_ivlen, 0);
+ }
+
+ sec_desc_word(&b, op);
+
+ /*
+ * Copy SIL into VSIL and VSOL so the following VLF-flagged FIFO
+ * commands know how many bytes to move. VLF reads the VS*L
+ * registers, so we need to get the values from the SEQ registers
+ * the SEQ IN/OUT PTR descriptors populate.
+ */
+ sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
+ MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4);
+ sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
+ MATH_SRC1_ZERO | MATH_DEST_VSOL | MATH_LEN_4);
+
+ sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC1);
+
+ sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) | FIFOST_VLF |
+ FIFOST_TYPE_MSG_DATA);
+
+ /* XTS keeps its tweak in the context, so the CCB has to save it. */
+ flags = HEADER_SHARE_SERIAL;
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ flags |= HEADER_SC;
+
+ return (sec_desc_finalize_shared(&b, flags, sdlenp));
+}
+
+/*
+ * Digest shared descriptor. The split key is computed in software at
+ * session setup, so MDHA is told it is precomputed and skips the
+ * ipad/opad expansion.
+ */
+static int
+sec_shd_build_digest(struct sec_session *sess,
+ const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
+{
+ struct sec_desc_builder b;
+ uint32_t alg;
+ uint8_t dlen, skeylen;
+
+ /*
+ * GMAC runs on AESA, not MDHA: the shared descriptor is just the
+ * class 1 key, and the JD drives it as GCM with no message.
+ */
+ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
+ sess->digestlen = csp->csp_auth_mlen != 0 ?
+ csp->csp_auth_mlen : AES_GMAC_HASH_LEN;
+ sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
+ sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL,
+ sdlenp));
+ }
+
+ if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
+ return (EOPNOTSUPP);
+ sess->digestlen = dlen;
+
+ sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
+
+ /*
+ * A plain hash takes no key at all; the keyed variants load the
+ * precomputed ipad || opad blob as an MDHA split key, which is
+ * what AAI_HMAC_PRECOMP tells MDHA to expect.
+ */
+ if (skeylen != 0) {
+ unsigned int nwords = howmany(skeylen, 4);
+
+ b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 |
+ KEY_KDEST_MDHA_SPLIT | KEY_IMM |
+ (skeylen & KEY_LENGTH_M);
+ memcpy(&b.desc[b.idx], sess->skey, skeylen);
+ if (skeylen % 4 != 0)
+ memset((uint8_t *)&b.desc[b.idx] + skeylen, 0,
+ nwords * 4 - skeylen);
+ b.idx += nwords;
+ }
+
+ sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg |
+ (skeylen != 0 ? AAI_HMAC_PRECOMP : AAI_HASH) | AS_INIT_FINAL);
+
+ /* VLF FIFO_LOAD needs VSIL, which SEQ_IN_PTR doesn't populate. */
+ sec_desc_word(&b, CMD_DESC(CMD_MATH) | MATH_FN_ADD | MATH_SRC0_SIL |
+ MATH_SRC1_ZERO | MATH_DEST_VSIL | MATH_LEN_4);
+
+ sec_desc_word(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_2 |
+ FIFOLD_VLF | FIFOLD_TYPE_MSG | FIFOLD_LC2);
+
+ /*
+ * Drain the completed hash from the Class 2 CCB Context register.
+ * SEQ_STORE with class 2 + SRC=CTX (0x20) blocks until MDHA is
+ * done.
+ */
+ sec_jd_store(&b, true, LOAD_CLASS_2, LOAD_CTX, 0, dlen, 0);
+
+ return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
+}
+
+/*
+ * AEAD shared descriptor. AAD and payload lengths vary per job, so
+ * everything but the key lives in the JD. Execution order is not
+ * reversed here: the key has to be loaded before the JD drives data.
+ *
+ * When ICV is set the ENC bit must be clear, which is the only
+ * difference between the two direction slots.
+ */
+static int
+sec_shd_build_aead(struct sec_session *sess,
+ const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
+{
+ struct sec_desc_builder b;
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ break;
+ default:
+ return (EOPNOTSUPP);
+ }
+ (void)enc;
+
+ /* Cache tag length once (both directions share). */
+ sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : 16;
+
+ /*
+ * SHD holds just the AES key.
+ */
+ sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
+ if (csp->csp_cipher_klen > 0)
+ sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+ return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
+}
+
+/*
+ * Shared descriptor for encrypt-then-auth: both keys and both mode
+ * registers, nothing else.
+ */
+static int
+sec_shd_build_eta(struct sec_session *sess,
+ const struct crypto_session_params *csp, int enc, uint32_t *sdlenp)
+{
+ struct sec_desc_builder b;
+ uint32_t alg, op;
+ uint8_t dlen, skeylen;
+
+ if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
+ return (EOPNOTSUPP);
+ sess->digestlen = csp->csp_auth_mlen != 0 ? csp->csp_auth_mlen : dlen;
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CBC |
+ AS_INIT_FINAL;
+ break;
+ case CRYPTO_AES_ICM:
+ op = CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CTR |
+ AS_INIT_FINAL;
+ break;
+ default:
+ return (EOPNOTSUPP);
+ }
+ if (enc)
+ op |= OP_ENC;
+
+ sec_desc_init(&b, sess->ctx[enc].shd, SEC_MAX_SHDESC_WORDS);
+
+ sec_desc_key_imm(&b, KEY_CLASS_1, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+
+ /* Class 2 takes the precomputed ipad/opad blob, as for plain HMAC. */
+ {
+ unsigned int nwords = howmany(skeylen, 4);
+
+ b.desc[b.idx++] = CMD_DESC(CMD_KEY) | KEY_CLASS_2 |
+ KEY_KDEST_MDHA_SPLIT | KEY_IMM | (skeylen & KEY_LENGTH_M);
+ memcpy(&b.desc[b.idx], sess->skey, skeylen);
+ if (skeylen % 4 != 0)
+ memset((uint8_t *)&b.desc[b.idx] + skeylen, 0,
+ nwords * 4 - skeylen);
+ b.idx += nwords;
+ }
+
+ sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | alg | AAI_HMAC_PRECOMP |
+ AS_INIT_FINAL);
+ sec_desc_word(&b, op);
+
+ return (sec_desc_finalize_shared(&b, HEADER_SHARE_SERIAL, sdlenp));
+}
+
+static int
+sec_probe_session(device_t dev, const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ if (!check_cipher(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_DIGEST:
+ if (!check_digest(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ if (!check_aead(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!check_eta(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+sec_new_session(device_t dev, crypto_session_t session,
+ const struct crypto_session_params *csp)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ struct sec_session *sess;
+ uint32_t sdlen;
+ int enc, error;
+
+ sess = crypto_get_driver_session(session);
+ sess->sess_sc = sc;
+
+ if ((csp->csp_mode == CSP_MODE_DIGEST ||
+ csp->csp_mode == CSP_MODE_ETA) && csp->csp_auth_klen > 0 &&
+ csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) {
+ uint32_t alg;
+ uint8_t dlen, skeylen;
+
+ if (!sec_hash_params(csp->csp_auth_alg, &alg, &dlen, &skeylen))
+ return (EOPNOTSUPP);
+ (void)alg;
+ sec_sw_gen_split_key(csp, sess->skey, skeylen);
+ sess->skeylen = skeylen;
+ }
+
+ for (enc = 0; enc <= 1; enc++) {
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ error = sec_shd_build_cipher(sess, csp, enc, &sdlen);
+ break;
+ case CSP_MODE_DIGEST:
+ error = sec_shd_build_digest(sess, csp, enc, &sdlen);
+ break;
+ case CSP_MODE_AEAD:
+ error = sec_shd_build_aead(sess, csp, enc, &sdlen);
+ break;
+ case CSP_MODE_ETA:
+ error = sec_shd_build_eta(sess, csp, enc, &sdlen);
+ break;
+ default:
+ return (EINVAL);
+ }
+ if (error != 0)
+ return (error);
+ sess->sdlen[enc] = sdlen;
+ }
+ return (0);
+}
+
+static void
+sec_free_session(device_t dev, crypto_session_t session)
+{
+ /* Nothing to do here. */
+}
+
+static void
+sec_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+ struct sec_job *job = arg;
+
+ if (error != 0) {
+ job->nsegs = 0;
+ return;
+ }
+ KASSERT(nsegs <= SEC_MAX_SEGMENTS,
+ ("SEC job segment overflow: %d > %d", nsegs, SEC_MAX_SEGMENTS));
+ memcpy(job->segs, segs, nsegs * sizeof(segs[0]));
+ job->nsegs = nsegs;
+}
+
+/*
+ * Append the segments covering [start, start + len) of the mapped buffer.
+ * Returns the next free index, or -1 if the table would overflow or the
+ * range runs past the mapping.
+ */
+static int
+dpaa_sgte_append(struct sec_job *job, struct dpaa_sgte *sgt, int i, int max,
+ uint32_t start, uint32_t len)
+{
+ int s;
+
+ for (s = 0; s < job->nsegs && len > 0; s++) {
+ bus_addr_t addr = job->segs[s].ds_addr;
+ bus_size_t seglen = job->segs[s].ds_len;
+ uint32_t take;
+
+ if (start >= seglen) {
+ start -= seglen;
+ continue;
+ }
+ addr += start;
+ seglen -= start;
+ start = 0;
+
+ take = seglen > len ? len : seglen;
+ len -= take;
+
+ if (i >= max)
+ return (-1);
+ sgt[i].addr = addr;
+ sgt[i].extension = 0;
+ sgt[i].final = 0;
+ sgt[i].length = take;
+ sgt[i].bpid = 0;
+ sgt[i].offset = 0;
+ i++;
+ }
+ if (len != 0)
+ return (-1);
+ return (i);
+}
+
+/*
+ * Populate the SGTs from the DMA-loaded segment list. Entry order is
+ * what the descriptor's SEQ commands consume, so it is fixed per mode:
+ * CIPHER:
+ * in_sgt[0] = job->iv
+ * in_sgt[1..n] = payload segments
+ * out_sgt[0..n-1] = payload segments (in-place)
+ * DIGEST (HMAC, no IV):
+ * in_sgt[0..n-1] = payload segments
+ * out_sgt unused (JD points SEQ_OUT_PTR directly at job->digest).
+ * AEAD and ETA:
+ * in_sgt[0] = job->iv, or job->ccm_ctx for CCM
+ * in_sgt[1..A] = AAD (crp_aad, or crp_buf at crp_aad_start)
+ * in_sgt[A+1..N] = payload segments
+ * in_sgt[N+1..] = (decrypt only) received tag from crp_buf
+ * out_sgt[0..] = payload segments (in-place); the tag goes to
+ * job->digest via a separate STORE.
+ * The final SGT entry in each populated table gets F=1.
+ */
+static int
+sec_job_build_sgts(struct sec_job *job, const struct crypto_session_params *csp)
+{
+ struct cryptop *crp = job->crp;
+ uint32_t skip = crp->crp_payload_start;
+ uint32_t left = crp->crp_payload_length;
+ int i, out_i;
+ int iv_slot = csp->csp_ivlen > 0 ? 1 : 0;
+
+ if (csp->csp_mode == CSP_MODE_AEAD ||
+ csp->csp_mode == CSP_MODE_ETA) {
+ const int inmax = 1 + SEC_MAX_SEGMENTS;
+ bool encrypt = (crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
+ bool ccm = csp->csp_mode == CSP_MODE_AEAD &&
+ csp->csp_cipher_alg == CRYPTO_AES_CCM_16;
+ int in_i = 0, npay, pay_i;
+
+ /* IV, or for CCM the B0 || CTR0 context block. */
+ if (ccm) {
+ job->in_sgt[in_i].addr =
+ pmap_kextract((vm_offset_t)job->ccm_ctx);
+ job->in_sgt[in_i].length = SEC_CCM_CTX_LEN;
+ } else {
+ job->in_sgt[in_i].addr =
+ pmap_kextract((vm_offset_t)job->iv);
+ job->in_sgt[in_i].length = csp->csp_ivlen;
+ }
+ job->in_sgt[in_i].extension = 0;
+ job->in_sgt[in_i].final = 0;
+ job->in_sgt[in_i].bpid = 0;
+ job->in_sgt[in_i].offset = 0;
+ in_i++;
+
+ if (crp->crp_aad_length > 0) {
+ /* CCM feeds the AAD length ahead of the AAD. */
+ if (ccm) {
+ job->in_sgt[in_i].addr = pmap_kextract(
+ (vm_offset_t)job->ccm_alen);
+ job->in_sgt[in_i].extension = 0;
+ job->in_sgt[in_i].final = 0;
+ job->in_sgt[in_i].length =
+ sizeof(job->ccm_alen);
+ job->in_sgt[in_i].bpid = 0;
+ job->in_sgt[in_i].offset = 0;
+ in_i++;
+ }
+ if (crp->crp_aad != NULL) {
+ /*
+ * A dedicated AAD buffer is not part of the
+ * crp mapping; it is small enough that one
+ * entry always covers it.
+ */
+ job->in_sgt[in_i].addr = pmap_kextract(
+ (vm_offset_t)crp->crp_aad);
+ job->in_sgt[in_i].extension = 0;
+ job->in_sgt[in_i].final = 0;
+ job->in_sgt[in_i].length = crp->crp_aad_length;
+ job->in_sgt[in_i].bpid = 0;
+ job->in_sgt[in_i].offset = 0;
+ in_i++;
+ } else {
+ in_i = dpaa_sgte_append(job, job->in_sgt, in_i,
+ inmax, crp->crp_aad_start,
+ crp->crp_aad_length);
+ if (in_i < 0)
+ return (E2BIG);
+ }
+ }
+
+ pay_i = in_i;
+ in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax, skip,
+ left);
+ if (in_i < 0)
+ return (E2BIG);
+ npay = in_i - pay_i;
+ if (npay == 0)
+ return (EINVAL);
+
+ /*
+ * AEAD decrypt hands the received tag to the CHA for its
+ * own compare; ETA drains the MAC to job->digest instead
+ * and compares in software, so it needs no entry here.
+ */
+ if (!encrypt && csp->csp_mode == CSP_MODE_AEAD) {
+ in_i = dpaa_sgte_append(job, job->in_sgt, in_i, inmax,
+ crp->crp_digest_start, job->sess->digestlen);
+ if (in_i < 0)
+ return (E2BIG);
+ }
+ job->in_sgt[in_i - 1].final = 1;
+
+ /* Output mirrors the payload segments, in place. */
+ memcpy(job->out_sgt, &job->in_sgt[pay_i],
+ npay * sizeof(job->out_sgt[0]));
+ job->out_sgt[npay - 1].final = 1;
+ return (0);
+ }
+
+ if (iv_slot) {
+ job->in_sgt[0].addr = pmap_kextract((vm_offset_t)job->iv);
+ job->in_sgt[0].extension = 0;
+ job->in_sgt[0].final = 0;
+ job->in_sgt[0].length = sec_cipher_ivlen(csp);
+ job->in_sgt[0].bpid = 0;
+ job->in_sgt[0].offset = 0;
+ }
+
+ out_i = 0;
+ for (i = 0; i < job->nsegs && left > 0; i++) {
+ bus_addr_t addr = job->segs[i].ds_addr;
+ bus_size_t len = job->segs[i].ds_len;
+ uint32_t take;
+
+ if (skip >= len) {
+ skip -= len;
+ continue;
+ }
+ addr += skip;
+ len -= skip;
+ skip = 0;
+
+ take = (len > left) ? left : len;
+ left -= take;
+
+ if (out_i >= SEC_MAX_SEGMENTS)
+ return (E2BIG);
+
+ job->in_sgt[iv_slot + out_i].addr = addr;
+ job->in_sgt[iv_slot + out_i].extension = 0;
+ job->in_sgt[iv_slot + out_i].final = 0;
+ job->in_sgt[iv_slot + out_i].length = take;
+ job->in_sgt[iv_slot + out_i].bpid = 0;
+ job->in_sgt[iv_slot + out_i].offset = 0;
+
+ if (iv_slot)
+ job->out_sgt[out_i] = job->in_sgt[iv_slot + out_i];
+ out_i++;
+ }
+ if (left != 0)
+ return (EINVAL);
+ if (out_i == 0)
+ return (EINVAL);
+
+ job->in_sgt[iv_slot + out_i - 1].final = 1;
+ if (iv_slot)
+ job->out_sgt[out_i - 1].final = 1;
+ return (0);
+}
+
+/*
+ * JD for the cipher modes. The shared descriptor runs the pipeline, so
+ * the JD only points at the SGTs.
+ *
+ * Both sequences use SGF and EXT unconditionally. Always using a table
+ * keeps the builder from caring how many segments there are, and the
+ * 16-bit length in the command word is too small for the payloads geli
+ * and kTLS hand down.
+ */
+static int
+sec_jd_build_cipher(struct sec_job *job,
+ const struct crypto_session_params *csp)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ int enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op);
+ uint32_t sdlen = sess->sdlen[enc];
+ uint32_t desclen;
+ uint32_t in_len = sec_cipher_ivlen(csp) + job->crp->crp_payload_length;
+ vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa;
+
+ /*
+ * The hardware restarts the tweak every SEC_XTS_SECTOR_SIZE bytes;
+ * opencrypto expects one continuous tweak, so anything that would
+ * cross the boundary has to go back to software.
+ */
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
+ job->crp->crp_payload_length > SEC_XTS_SECTOR_SIZE)
+ return (EOPNOTSUPP);
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa,
+ job->crp->crp_payload_length);
+
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
+
+ /*
+ * HEADER_REO (Reverse Execution Order) makes SEC run the JD
+ * commands FIRST (SEQ_OUT_PTR / SEQ_IN_PTR set up the input and
+ * output sequences), then fall into the shared descriptor. The
+ * shared descriptor's SEQ_LOAD / SEQ_FIFO_LOAD / SEQ_FIFO_STORE
+ * commands depend on those sequences being programmed. Without
+ * this bit the shared desc runs first and SEQ_LOAD hits an
+ * uninitialized input sequence, and DECO reports an invalid
+ * sequence command (error 0x10).
+ */
+ return (sec_desc_finalize_job(&b, HEADER_SHR |
+ HEADER_REO | HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_DEFER,
+ &desclen));
+}
+
+/*
+ * JD for the digest modes. There is no IV to prepend and the output is
+ * a small fixed buffer, so SEQ_OUT_PTR addresses it directly.
+ */
+static int
+sec_jd_build_digest(struct sec_job *job)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ uint32_t sdlen = sess->sdlen[0];
+ uint32_t desclen;
+ vm_paddr_t shd_pa, in_sgt_pa, digest_pa;
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ digest_pa = pmap_kextract((vm_offset_t)job->digest);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ sec_jd_seq(&b, true, SEQ_EXT, digest_pa, sess->digestlen);
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa,
+ job->crp->crp_payload_length);
+
+ return (sec_desc_finalize_job(&b,
+ HEADER_SHR | HEADER_REO | HEADER_SHR_DESC_L(sdlen) |
+ HEADER_SHARE_DEFER, &desclen));
+}
+
+/*
+ * JD for AEAD (AES-GCM).
+ *
+ * The shared descriptor holds only the class 1 key and runs first, so
+ * the job descriptor sets up both sequences and drives all of the data.
+ * The data size counts the IV and AAD rounded up to 16 bytes even though
+ * the FIFO loads supply them unpadded; SEC pads them internally.
+ *
+ * [0] - Header
+ * [1..2] - Shared descriptor pointer
+ * [3..6] - SEQ OUT PTR - ciphertext only, the tag leaves via STORE
+ * [7..10] - SEQ IN PTR - iv + aad + payload, and the tag when decrypting
+ * [11] - Operation
+ * [12] - LOAD Class 1 Data Size, which starts processing
+ * [13] - FIFO LOAD IV
+ * [14] - FIFO LOAD AAD
+ * [15] - FIFO STORE ciphertext
+ * [16] - FIFO LOAD message
+ * [encrypt:17..19] - STORE the computed tag to job->digest
+ * [decrypt:17] - FIFO LOAD received ICV
+ */
+static int
+sec_jd_build_aead(struct sec_job *job)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ const struct crypto_session_params *csp;
+ uint32_t sdlen = sess->sdlen[0];
+ uint32_t desclen;
+ vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa;
+ uint32_t ivlen, aadlen, paylen, taglen;
+ uint32_t padded_iv, padded_aad, dsr_val;
+ uint32_t in_len, out_len;
+ vm_paddr_t digest_pa;
+ int enc;
+
+ csp = crypto_get_params(job->crp->crp_session);
+ enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
+ ivlen = csp->csp_ivlen;
+ aadlen = job->crp->crp_aad_length;
+ paylen = job->crp->crp_payload_length;
+ taglen = sess->digestlen;
+
+ padded_iv = roundup(ivlen, 16);
+ padded_aad = roundup(aadlen, 16);
+ dsr_val = padded_iv + padded_aad + paylen;
+ in_len = ivlen + aadlen + paylen + (enc ? 0 : taglen);
+ /* Output sequence is ciphertext only; tag goes via direct STORE. */
+ out_len = paylen;
+ digest_pa = pmap_kextract((vm_offset_t)job->digest);
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, out_len);
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
+
+ /*
+ * Writing the data size starts processing, so OPERATION has to arm
+ * the CHA in GCM mode before the DSR load below.
+ */
+ sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM |
+ AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV));
+
+ sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
+ (uint64_t)dsr_val << 32);
+
+ /* IV: FC1 so SEC pads to 16 without ending class 1 input. */
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen);
+ /*
+ * Always emit an AAD FIFO_LOAD (even with length 0) so SEC gets
+ * an explicit "AAD phase done" signal via FC1.
+ */
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_AAD | FIFOLD_FC1, aadlen);
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
+ FIFOST_TYPE_MSG_DATA, paylen);
+
+ /* MSG: LC1 for encrypt (last class-1 input), FC1 for decrypt. */
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen);
+ if (enc)
+ sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0, taglen,
+ digest_pa);
+ else
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
+ FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen);
+
+ /* SHR=1, NO REO, so the shd (KEY only) runs first, then the JD. */
+ return (sec_desc_finalize_job(&b, HEADER_SHR |
+ HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
+}
+
+/*
+ * JD for AES-CCM.
+ */
+static int
+sec_jd_build_ccm(struct sec_job *job)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ uint32_t sdlen = sess->sdlen[0];
+ uint32_t desclen, aadlen, paylen, taglen, in_len;
+ vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa;
+ int enc;
+
+ enc = (job->crp->crp_op & CRYPTO_OP_ENCRYPT) != 0;
+ aadlen = job->crp->crp_aad_length;
+ paylen = job->crp->crp_payload_length;
+ taglen = sess->digestlen;
+
+ in_len = SEC_CCM_CTX_LEN + paylen + (enc ? 0 : taglen);
+ if (aadlen > 0)
+ in_len += sizeof(job->ccm_alen) + aadlen;
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
+ digest_pa = pmap_kextract((vm_offset_t)job->digest);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen);
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
+
+ /* B0 || CTR0 || zeroed result dwords, from the head of the input. */
+ sec_desc_word(&b, CMD_DESC(CMD_SEQ_LOAD) | LOAD_CLASS_1 | LOAD_CTX |
+ (SEC_CCM_CTX_LEN & LOAD_LENGTH_M));
+
+ sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_CCM |
+ AS_INIT_FINAL | (enc ? OP_ENC : OP_ICV));
+
+ /* Writing the data size starts the operation. */
+ sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
+ (uint64_t)paylen << 32);
+
+ /* Length-prefixed AAD; the hardware pads it out to 16 bytes. */
+ if (aadlen > 0)
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
+ FIFOLD_CLASS_1 | FIFOLD_TYPE_AAD | FIFOLD_FC1,
+ sizeof(job->ccm_alen) + aadlen);
+
+ /* Arm the drain before the message, as for GCM. */
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
+ FIFOST_TYPE_MSG_DATA, paylen);
+
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_MSG | (enc ? FIFOLD_LC1 : FIFOLD_FC1), paylen);
+
+ if (enc)
+ sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 32, taglen,
+ digest_pa);
+ else
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
+ FIFOLD_CLASS_1 | FIFOLD_TYPE_ICV | FIFOLD_LC1, taglen);
+
+ return (sec_desc_finalize_job(&b, HEADER_SHR |
+ HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
+}
+
+/*
+ * JD for encrypt-then-auth.
+ */
+static int
+sec_jd_build_eta(struct sec_job *job)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ const struct crypto_session_params *csp;
+ uint32_t sdlen, desclen, aadlen, paylen, ivlen, in_len;
+ uint32_t ctx_offset;
+ vm_paddr_t shd_pa, in_sgt_pa, out_sgt_pa, digest_pa;
+ int enc;
+
+ csp = crypto_get_params(job->crp->crp_session);
+ enc = CRYPTO_OP_IS_ENCRYPT(job->crp->crp_op);
+ sdlen = sess->sdlen[enc];
+ ivlen = csp->csp_ivlen;
+ aadlen = job->crp->crp_aad_length;
+ paylen = job->crp->crp_payload_length;
+
+ in_len = ivlen + aadlen + paylen;
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[enc].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ out_sgt_pa = pmap_kextract((vm_offset_t)job->out_sgt);
+ digest_pa = pmap_kextract((vm_offset_t)job->digest);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ sec_jd_seq(&b, true, SEQ_SGF | SEQ_EXT, out_sgt_pa, paylen);
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
+
+ ctx_offset = sec_cipher_ctx_offset(csp->csp_cipher_alg);
+
+ /* IV into the class 1 context; also drops SIL by ivlen. */
+ sec_jd_load(&b, true, LOAD_CLASS_1, 0, LOAD_CTX, ctx_offset,
+ ivlen, 0);
+
+ /* AAD is authenticated only, so class 2 alone. */
+ if (aadlen > 0)
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
+ FIFOLD_CLASS_2 | FIFOLD_TYPE_MSG, aadlen);
+
+ /* Arm the ciphertext drain before feeding the message. */
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_STORE) |
+ FIFOST_TYPE_MSG_DATA, paylen);
+
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) |
+ FIFOLD_CLASS_BOTH | FIFOLD_LC1 | FIFOLD_LC2 |
+ (enc ? FIFOLD_TYPE_MSG_C1OUT : FIFOLD_TYPE_MSG), paylen);
+
+ /* Drain the MAC to job->digest; the caller compares or copies back. */
+ sec_jd_store(&b, false, LOAD_CLASS_2, LOAD_CTX, 0,
+ sess->digestlen, digest_pa);
+
+ return (sec_desc_finalize_job(&b, HEADER_SHR |
+ HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
+}
+
+/*
+ * JD for AES-GMAC: GCM with nothing to encrypt.
+ */
+static int
+sec_jd_build_gmac(struct sec_job *job)
+{
+ struct sec_desc_builder b;
+ struct sec_session *sess = job->sess;
+ const struct crypto_session_params *csp;
+ uint32_t sdlen = sess->sdlen[0];
+ uint32_t desclen, ivlen, datalen, dsr_val, in_len;
+ vm_paddr_t shd_pa, in_sgt_pa, digest_pa;
+
+ csp = crypto_get_params(job->crp->crp_session);
+ ivlen = csp->csp_ivlen;
+ datalen = job->crp->crp_payload_length;
+
+ /*
+ * The digest-mode SGT maps the payload only, so AAD has nowhere
+ * to come from.
+ */
+ if (job->crp->crp_aad_length != 0)
+ return (EOPNOTSUPP);
+
+ dsr_val = roundup(ivlen, 16) + roundup(datalen, 16);
+ in_len = ivlen + datalen;
+
+ shd_pa = pmap_kextract((vm_offset_t)sess->ctx[0].shd);
+ in_sgt_pa = pmap_kextract((vm_offset_t)job->in_sgt);
+ digest_pa = pmap_kextract((vm_offset_t)job->digest);
+
+ sec_desc_init(&b, job->jd, SEC_MAX_DESC_WORDS);
+ sec_jd_ptr(&b, shd_pa);
+
+ /* No output sequence: the tag leaves through an inline STORE. */
+ sec_jd_seq(&b, false, SEQ_SGF | SEQ_EXT, in_sgt_pa, in_len);
+
+ sec_desc_word(&b, CMD_DESC(CMD_OPERATION) | ALG_AES | AAI_AES_GCM |
+ AS_INIT_FINAL | OP_ENC);
+
+ sec_jd_load(&b, false, LOAD_CLASS_1, LOAD_IMM, LOAD_DSR, 0, 8,
+ (uint64_t)dsr_val << 32);
+
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_IV | FIFOLD_FC1, ivlen);
+ sec_jd_fifo(&b, CMD_DESC(CMD_SEQ_FIFO_LOAD) | FIFOLD_CLASS_1 |
+ FIFOLD_TYPE_AAD | FIFOLD_LC1, datalen);
+
+ sec_jd_store(&b, false, LOAD_CLASS_1, LOAD_CTX, 0,
+ sess->digestlen, digest_pa);
+
+ return (sec_desc_finalize_job(&b, HEADER_SHR |
+ HEADER_SHR_DESC_L(sdlen) | HEADER_SHARE_SERIAL, &desclen));
+}
+
+static int
+sec_process(device_t dev, struct cryptop *crp, int hint)
+{
+ struct sec_softc *sc = device_get_softc(dev);
+ struct sec_session *sess = crypto_get_driver_session(crp->crp_session);
+ const struct crypto_session_params *csp;
+ struct sec_job *job;
+ struct sec_jr *jr;
+ int error;
+
+ job = malloc(sizeof(*job), M_SEC, M_NOWAIT | M_ZERO);
+ if (job == NULL) {
+ crp->crp_etype = ENOMEM;
+ crypto_done(crp);
+ return (0);
+ }
+ job->crp = crp;
+ job->sess = sess;
+
+ error = bus_dmamap_create(sc->sc_dmatag, 0, &job->map);
+ if (error != 0)
+ goto fail_free;
+
+ error = bus_dmamap_load_crp(sc->sc_dmatag, job->map, crp,
+ sec_load_cb, job, BUS_DMA_NOWAIT);
+ if (error != 0 || job->nsegs == 0) {
+ if (error == 0)
+ error = EIO;
+ goto fail_destroy;
+ }
+
+ if (crp->crp_payload_length == 0) {
+ error = EINVAL;
+ goto fail_unload;
+ }
+
+ csp = crypto_get_params(crp->crp_session);
+ if (csp->csp_ivlen > 0)
+ crypto_read_iv(crp, job->iv);
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ sec_xts_tweak(job->iv);
+ if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
+ error = sec_ccm_prep(job, csp);
+ if (error != 0)
+ goto fail_unload;
+ }
+
+ error = sec_job_build_sgts(job, csp);
+ if (error != 0)
+ goto fail_unload;
+
+ bus_dmamap_sync(sc->sc_dmatag, job->map,
+ BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC)
+ error = sec_jd_build_gmac(job);
+ else
+ error = sec_jd_build_digest(job);
+ break;
+ case CSP_MODE_AEAD:
+ if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
+ error = sec_jd_build_ccm(job);
+ else
+ error = sec_jd_build_aead(job);
+ break;
+ case CSP_MODE_ETA:
+ error = sec_jd_build_eta(job);
+ break;
+ default:
+ error = sec_jd_build_cipher(job, csp);
+ break;
+ }
+ if (error != 0)
+ goto fail_unload;
+
+
+ /*
+ * Hand the job to a ring and return; sec_jr_intr() completes it.
+ */
+ jr = &sc->sc_jr[curcpu % sc->sc_njr];
+ sec_jr_submit_job(sc, jr, job);
+ return (0);
+
+fail_unload:
+ bus_dmamap_sync(sc->sc_dmatag, job->map,
+ BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->sc_dmatag, job->map);
+fail_destroy:
+ bus_dmamap_destroy(sc->sc_dmatag, job->map);
+fail_free:
+ free(job, M_SEC);
+ /* ERESTART means opencrypto retries this crp, so don't complete it. */
+ if (error == ERESTART)
+ return (ERESTART);
+ crp->crp_etype = error;
+ crypto_done(crp);
+ return (0);
+}
+
+static int
+sec_reset(struct sec_softc *sc)
+{
+ uint32_t mcfgr;
+ int i;
+
+ /*
+ * Preserve cache-attribute fields (AWCACHE/ARCACHE) and burst
+ * settings across the reset. The MCFGR write overwrites those
+ * along with SWRST.
+ */
+ mcfgr = SEC_RD4(sc, SEC_MCFGR);
+ SEC_WR4(sc, SEC_MCFGR, mcfgr | MCFGR_SWRST);
+
+ /* Poll SWRST for self-clear. */
+ for (i = 0; i < 10000; i++) {
+ if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) == 0)
+ break;
+ DELAY(10);
+ }
+ if ((SEC_RD4(sc, SEC_MCFGR) & MCFGR_SWRST) != 0) {
+ device_printf(sc->sc_dev, "MCFGR.SWRST did not clear\n");
+ return (EIO);
+ }
+
+ /*
+ * Post-reset configuration: 40-bit pointers, DECO watchdog on,
+ * large bursts. Preserve whatever cache attributes the bootloader
+ * left in place.
+ */
+ mcfgr &= (MCFGR_ARCACHE_M | MCFGR_AWCACHE_M);
+ mcfgr |= MCFGR_PS | MCFGR_WDE | MCFGR_LARGE_BURST;
+ SEC_WR4(sc, SEC_MCFGR, mcfgr);
+
+ return (0);
+}
+
+/*
+ * Instantiate one RNG state handle via DECO0 direct access.
+ */
+static int
+sec_deco_rng_init(struct sec_softc *sc, int sh)
+{
+ uint32_t jd[2];
+ uint32_t reg, decorr, scfgr;
+ int i;
+
+ jd[0] = CMD_DESC(CMD_DESC_HEADER) | HEADER_ONE |
+ (2 & HEADER_DESCLEN_M);
+ jd[1] = CMD_DESC(CMD_OPERATION) | ALG_RNG | AS_INIT | OP_RNG_SH(sh);
+
+ /* Request DECO0 and wait for the grant (DEN0=1). */
+ SEC_WR4(sc, SEC_DECORR, DECORR_RQD0);
+ decorr = SEC_RD4(sc, SEC_DECORR);
+ for (i = 0; i < 10000; i++) {
+ decorr = SEC_RD4(sc, SEC_DECORR);
+ if ((decorr & DECORR_DEN0) != 0)
+ break;
+ DELAY(10);
+ }
+ if ((decorr & DECORR_DEN0) == 0) {
+ scfgr = SEC_RD4(sc, SEC_SCFGR);
+ device_printf(sc->sc_dev,
+ "DECO0 acquire timeout (DECORR=%#x SCFGR=%#x%s)\n",
+ decorr, scfgr,
+ (scfgr & SCFGR_VIRT_EN) ? " VIRT_EN" : "");
+ SEC_WR4(sc, SEC_DECORR, 0);
+ return (ETIMEDOUT);
+ }
+
+ SEC_WR4(sc, SEC_D0DESB(0), jd[0]);
+ SEC_WR4(sc, SEC_D0DESB(1), jd[1]);
+
+ SEC_WR4(sc, SEC_D0JQCR_MS, DAJQCR_MS_WHL);
+
+ /* Wait for job completion */
+ reg = 0;
+ for (i = 0; i < 100000; i++) {
+ reg = SEC_RD4(sc, SEC_D0DDR);
+ if ((reg & DADDR_VALID) == 0)
+ break;
+ DELAY(10);
+ }
+
+ /* Release DECO0 either way. */
+ SEC_WR4(sc, SEC_DECORR, 0);
+
+ if ((reg & DADDR_VALID) != 0) {
+ device_printf(sc->sc_dev,
+ "RNG SH%d instantiate timeout (D0DDR=%#x)\n", sh, reg);
+ return (ETIMEDOUT);
+ }
+ if (((reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S) != 0) {
+ device_printf(sc->sc_dev,
+ "RNG SH%d instantiate error (D0DDR=%#x, DECO_STATE=%u)\n",
+ sh, reg,
+ (reg & DADDR_DECO_STATE_M) >> DADDR_DECO_STATE_S);
+ return (EIO);
+ }
+ return (0);
+}
+
+static int
+sec_rng_init(struct sec_softc *sc)
+{
+ uint32_t rdsta;
+ int error, sh;
+
+ /*
+ * SEC v4/v5 requires the DRNG state handles to be instantiated
+ * before any class-1 (AES/DES/RNG) job will execute. This is typically
+ * done by the bootloader, but finish what it didn't.
+ */
+ rdsta = SEC_RD4(sc, SEC_RDSTA);
+
+ if ((rdsta & RDSTA_CE) != 0) {
+ device_printf(sc->sc_dev,
+ "RNG catastrophic error (RDSTA=%#x, ERRCODE=%u)\n",
+ rdsta, (rdsta & RDSTA_ERRCODE_M) >> RDSTA_ERRCODE_S);
+ return (EIO);
+ }
+
+ /* Instantiate anything the bootloader didn't. */
+ for (sh = 0; sh <= 1; sh++) {
+ uint32_t bit = (sh == 0) ? RDSTA_IF0 : RDSTA_IF1;
+
+ if ((rdsta & bit) != 0)
+ continue;
+ error = sec_deco_rng_init(sc, sh);
+ if (error != 0)
+ return (error);
+ }
+
+ /* Verify the handles are now up. */
+ rdsta = SEC_RD4(sc, SEC_RDSTA);
+ if ((rdsta & (RDSTA_IF0 | RDSTA_IF1)) !=
+ (RDSTA_IF0 | RDSTA_IF1)) {
+ device_printf(sc->sc_dev,
+ "RNG instantiation left RDSTA=%#x\n", rdsta);
+ return (EIO);
+ }
+ return (0);
+}
diff --git a/sys/dev/dpaa/sec_jr.c b/sys/dev/dpaa/sec_jr.c
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa/sec_jr.c
@@ -0,0 +1,400 @@
+/*
+ * Copyright 2026 Justin Hibbits
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+#include "sec_var.h"
+
+/*
+ * A job that doesn't complete in 5 seconds (should take microseconds or less)
+ * is considered a failure.
+ */
+#define SEC_JOB_TIMEOUT (5 * hz)
+
+/*
+ * Job Ring register offsets, relative to the JR's base within SEC's
+ * CCSR window.
+ */
+#define JR_IRBAR_MS 0x00 /* Input ring base, upper (64-bit reg) */
+#define JR_IRBAR_LS 0x04 /* Input ring base, lower */
+#define JR_IRSR 0x0c /* Input ring size (ring entries) */
+#define JR_IRSAR 0x14 /* Input ring slots available (add-to) */
+#define JR_IRJAR 0x1c /* Input ring jobs added (bump on enqueue) */
+#define JR_ORBAR_MS 0x20 /* Output ring base, upper */
+#define JR_ORBAR_LS 0x24 /* Output ring base, lower */
+#define JR_ORSR 0x2c /* Output ring size */
+#define JR_ORJRR 0x34 /* Output ring jobs removed */
+#define JR_ORSFR 0x3c /* Output ring slots full */
+#define JR_JRSTAR 0x44 /* Output status (per-job termination) */
+#define JR_JRINTR 0x4c /* Interrupt status (W1C) */
+#define JRINTR_JRI 0x00000001 /* JR interrupt asserted */
+#define JRINTR_JRE 0x00000002 /* JR error */
+/*
+ * HALT tracks a flush requested through JRCR: 01b while SEC
+ * is still draining, 10b once every job has reached the output ring.
+ * Writing the field's high bit clears it and lets the ring run again.
+ */
+#define JRINTR_HALT_M 0x0000000c
+#define JRINTR_HALT_ONGOING 0x00000004
+#define JRINTR_HALT_DONE 0x00000008
+#define JR_JRCFGR_MS 0x50 /* Configuration, upper */
+#define JR_JRCFGR_LS 0x54 /* Configuration, lower */
+#define JRCFGR_LS_IMSK 0x00000001 /* Mask interrupts (1=masked) */
+#define JRCFGR_LS_ICEN 0x00000002 /* Interrupt coalescing enable */
+#define JR_JRCR 0x6c /* Command: flush/reset */
+#define JRCR_RESET 0x00000001 /* Flush, or reset if halted */
+#define JR_IRRIR 0x5c /* Input ring read index (RO) */
+#define JR_ORWIR 0x64 /* Output ring write index (RO) */
+
+#define JR_RING_SIZE 16 /* power of 2, small for scaffolding */
+#define JR_RING_MASK (JR_RING_SIZE - 1)
+
+#define JR_RD4(sec, jr, off) bus_read_4(sec->sc_rres, jr->jr_off + off)
+#define JR_WR4(sec, jr, off, v) \
+ bus_write_4(sec->sc_rres, jr->jr_off + off, v)
+
+static void sec_jr_intr(void *arg);
+
+#define FOREACH_JOB_RING(node) \
+ for (phandle_t child = OF_child(node); child != 0; \
+ child = OF_peer(child)) \
+ if ((ofw_bus_node_is_compatible(child, \
+ "fsl,sec-v5.0-job-ring") || \
+ ofw_bus_node_is_compatible(child, \
+ "fsl,sec-v4.0-job-ring")) && \
+ ofw_bus_node_status_okay(child) && \
+ OF_getproplen(child, "reg") == 2 * sizeof(pcell_t))
+/*
+ * Job Ring helpers.
+ */
+
+static int
+sec_jr_count(struct sec_softc *sc)
+{
+ phandle_t node = ofw_bus_get_node(sc->sc_dev);
+ int n = 0;
+
+ FOREACH_JOB_RING(node)
+ n++;
+
+ return (n);
+}
+
+static void
+sec_jr_dma_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{
+ vm_paddr_t *pa = arg;
+ *pa = error == 0 && nsegs == 1 ? segs[0].ds_addr : 0;
+}
+
+static int
+sec_jr_irq_setup(struct sec_softc *sc, struct sec_jr *jr, u_int idx)
+{
+ device_t dev = sc->sc_dev;
+ struct resource_list *rl;
+ phandle_t iparent;
+ pcell_t *cells;
+ int ncells, irqnum;
+
+ if (jr->jr_node == 0)
+ return (ENXIO);
+
+ if (ofw_bus_intr_by_rid(dev, jr->jr_node, 0, &iparent, &ncells,
+ &cells) != 0)
+ return (ENXIO);
+ irqnum = ofw_bus_map_intr(dev, iparent, ncells, cells);
+ OF_prop_free(cells);
+ if (irqnum <= 0)
+ return (ENXIO);
+
+ rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
+ jr->jr_irid = 1 + idx; /* rid 0 is the SEC top-level error IRQ */
+ resource_list_add(rl, SYS_RES_IRQ, jr->jr_irid, irqnum, irqnum, 1);
+
+ jr->jr_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+ &jr->jr_irid, RF_ACTIVE);
+ if (jr->jr_ires == NULL)
+ return (ENXIO);
+
+ if (bus_setup_intr(dev, jr->jr_ires, INTR_TYPE_MISC | INTR_MPSAFE,
+ NULL, sec_jr_intr, jr, &jr->jr_icookie) != 0)
+ return (ENXIO);
+
+ /*
+ * Enable JR interrupts (IMSK=0, ICEN=0 = fire on every completion).
+ * Reset default is already IMSK=0, but be explicit.
+ */
+ JR_WR4(sc, jr, JR_JRCFGR_LS, 0);
+
+ return (0);
+}
+
+/*
+ * Watchdog for a wedged ring. Nothing else reclaims a job that never
+ * reaches the output ring, so its caller would wait forever.
+ *
+ * Writing JRCR[RESET] while RESET reads 0 flushes the ring: jobs already
+ * in the holding tanks or DECOs are terminated onto the output ring with
+ * an error status, and the ordinary completion path reclaims them. So
+ * this only starts the flush and later clears HALT. Jobs merely stalled
+ * in the input ring resume from there.
+ */
+static void
+sec_jr_watchdog(void *arg)
+{
+ struct sec_jr *jr = arg;
+ struct sec_softc *sc = jr->jr_sc;
+ struct sec_job *job;
+ uint32_t intr;
+
+ if (jr->jr_dying)
+ return;
+
+ if (jr->jr_flushing) {
+ intr = JR_RD4(sc, jr, JR_JRINTR);
+ if ((intr & JRINTR_HALT_M) == JRINTR_HALT_DONE) {
+ JR_WR4(sc, jr, JR_JRINTR, JRINTR_HALT_DONE);
+ jr->jr_flushing = false;
+ device_printf(sc->sc_dev,
+ "job ring at %#x resumed after flush\n",
+ jr->jr_off);
+ }
+ } else if ((job = TAILQ_FIRST(&jr->jr_active)) != NULL &&
+ (int)(ticks - job->job_deadline) >= 0) {
+ device_printf(sc->sc_dev,
+ "job ring at %#x stalled with %u job%s outstanding, "
+ "flushing\n", jr->jr_off, jr->jr_inflight,
+ jr->jr_inflight != 1 ? "s" : "");
+ JR_WR4(sc, jr, JR_JRCR, JRCR_RESET);
+ jr->jr_flushing = true;
+ }
+
+ callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr);
+}
+
+/* A per-job non-zero status arrives in the OR entry, not through JRE. */
+static void
+sec_jr_intr(void *arg)
+{
+ struct sec_jr *jr = arg;
+ struct sec_softc *sc = jr->jr_sc;
+ struct sec_or_entry done[JR_RING_SIZE];
+ uint32_t i, intr, n, tail;
+ int blocked;
+
+ mtx_lock(&jr->jr_lock);
+
+ intr = JR_RD4(sc, jr, JR_JRINTR);
+ if ((intr & (JRINTR_JRI | JRINTR_JRE)) == 0) {
+ mtx_unlock(&jr->jr_lock);
+ return;
+ }
+ if ((intr & JRINTR_JRE) != 0)
+ device_printf(sc->sc_dev, "JR error, JRINTR=%#x\n", intr);
+
+ /*
+ * Acknowledge before draining. A job completing between the ORSFR
+ * read and the W1C would otherwise have its interrupt cleared along
+ * with the one being serviced, and would sit there with nothing
+ * left to raise it again. Acknowledging first costs at worst a
+ * spurious interrupt that finds the ring empty.
+ */
+ JR_WR4(sc, jr, JR_JRINTR, intr & (JRINTR_JRI | JRINTR_JRE));
+
+ /*
+ * Completion has to run with jr_lock dropped, since crypto_done()
+ * can dispatch the next request straight back into sec_process().
+ */
+ n = JR_RD4(sc, jr, JR_ORSFR);
+ if (n > JR_RING_SIZE)
+ n = JR_RING_SIZE;
+ for (i = 0; i < n; i++) {
+ tail = (jr->jr_or_tail + i) & JR_RING_MASK;
+ done[i] = jr->jr_or[tail];
+ TAILQ_REMOVE(&jr->jr_active, (struct sec_job *)
+ PHYS_TO_DMAP((vm_paddr_t)done[i].desc_addr), job_link);
+ }
+ if (n != 0) {
+ /* Finish reading the entries before freeing their slots. */
+ atomic_thread_fence_rel();
+ jr->jr_or_tail += n;
+ JR_WR4(sc, jr, JR_ORJRR, n);
+ jr->jr_inflight -= n;
+ }
+
+ blocked = 0;
+ if (jr->jr_blocked != 0 && jr->jr_inflight < JR_RING_SIZE) {
+ blocked = jr->jr_blocked;
+ jr->jr_blocked = 0;
+ }
+
+ mtx_unlock(&jr->jr_lock);
+
+ if (blocked != 0)
+ crypto_unblock(sc->sc_cid, blocked);
+
+ for (i = 0; i < n; i++)
+ sec_complete_one(sc, done[i].desc_addr, done[i].status);
+}
+
+static int
+sec_jr_init(struct sec_softc *sc, struct sec_jr *jr)
+{
+ void *ring_va;
+ size_t ir_bytes = JR_RING_SIZE * sizeof(uint64_t);
+ size_t or_bytes = JR_RING_SIZE * sizeof(struct sec_or_entry);
+ size_t total = ir_bytes + or_bytes;
+
+ mtx_init(&jr->jr_lock, device_get_nameunit(sc->sc_dev), NULL, MTX_DEF);
+ TAILQ_INIT(&jr->jr_active);
+ callout_init_mtx(&jr->jr_wdog, &jr->jr_lock, 0);
+
+ if (bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 64, 0,
+ BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
+ total, 1, total, BUS_DMA_ALLOCNOW, NULL, NULL,
+ &jr->jr_ring_tag) != 0)
+ return (ENOMEM);
+ if (bus_dmamem_alloc(jr->jr_ring_tag, &ring_va,
+ BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT,
+ &jr->jr_map) != 0)
+ return (ENOMEM);
+
+ jr->jr_ir = ring_va;
+ jr->jr_or = (struct sec_or_entry *)((uint8_t *)ring_va + ir_bytes);
+
+ if (bus_dmamap_load(jr->jr_ring_tag, jr->jr_map, ring_va,
+ total, sec_jr_dma_cb, &jr->jr_ir_pa,
+ BUS_DMA_NOWAIT) != 0 || jr->jr_ir_pa == 0)
+ return (ENOMEM);
+ jr->jr_or_pa = jr->jr_ir_pa + ir_bytes;
+
+ JR_WR4(sc, jr, JR_IRBAR_MS, (uint32_t)(jr->jr_ir_pa >> 32));
+ JR_WR4(sc, jr, JR_IRBAR_LS, (uint32_t)jr->jr_ir_pa);
+ JR_WR4(sc, jr, JR_IRSR, JR_RING_SIZE);
+
+ JR_WR4(sc, jr, JR_ORBAR_MS, (uint32_t)(jr->jr_or_pa >> 32));
+ JR_WR4(sc, jr, JR_ORBAR_LS, (uint32_t)jr->jr_or_pa);
+ JR_WR4(sc, jr, JR_ORSR, JR_RING_SIZE);
+
+ /* Enable the ring by writing IRSAR = ring size (all slots free). */
+ JR_WR4(sc, jr, JR_IRSAR, JR_RING_SIZE);
+
+ mtx_lock(&jr->jr_lock);
+ callout_reset(&jr->jr_wdog, hz, sec_jr_watchdog, jr);
+ mtx_unlock(&jr->jr_lock);
+
+ return (0);
+}
+
+int
+sec_init_rings(struct sec_softc *sc)
+{
+ uint32_t reg[2];
+ int err, i, njrs;
+
+ njrs = sec_jr_count(sc);
+ if (njrs == 0)
+ return (0);
+
+ sc->sc_jr = mallocarray(njrs, sizeof(struct sec_jr), M_DEVBUF,
+ M_WAITOK | M_ZERO);
+
+ i = 0;
+ FOREACH_JOB_RING(ofw_bus_get_node(sc->sc_dev)) {
+ struct sec_jr *jr = &sc->sc_jr[i];
+
+ OF_getencprop(child, "reg", reg, sizeof(reg));
+ jr->jr_sc = sc;
+ jr->jr_node = child;
+ /* Offset within SEC's CCSR window. */
+ jr->jr_off = reg[0];
+
+ err = sec_jr_init(sc, jr);
+ if (err != 0)
+ goto fail;
+
+ err = sec_jr_irq_setup(sc, jr, i);
+ if (err != 0) {
+ device_printf(sc->sc_dev,
+ "could not install JR%u interrupt\n", i);
+ goto fail;
+ }
+ i++;
+ }
+
+ sc->sc_njr = njrs;
+
+ return (njrs);
+
+fail:
+ /*
+ * Teardown copes with a partly built ring, so running it over the
+ * whole array also cleans up the one that failed.
+ */
+ for (i = 0; i < njrs; i++)
+ sec_jr_teardown(sc, &sc->sc_jr[i]);
+ free(sc->sc_jr, M_DEVBUF);
+ sc->sc_jr = NULL;
+
+ return (0);
+}
+
+void
+sec_jr_teardown(struct sec_softc *sc, struct sec_jr *jr)
+{
+
+ if (mtx_initialized(&jr->jr_lock)) {
+ mtx_lock(&jr->jr_lock);
+ jr->jr_dying = true;
+ callout_stop(&jr->jr_wdog);
+ mtx_unlock(&jr->jr_lock);
+ callout_drain(&jr->jr_wdog);
+ }
+
+ if (jr->jr_ring_tag != NULL) {
+ /* Halt the JR by writing 0 to input ring size. */
+ if (sc->sc_rres != NULL)
+ JR_WR4(sc, jr, JR_IRSR, 0);
+
+ if (jr->jr_ir != NULL) {
+ bus_dmamap_unload(jr->jr_ring_tag, jr->jr_map);
+ bus_dmamem_free(jr->jr_ring_tag, jr->jr_ir,
+ jr->jr_map);
+ }
+ bus_dma_tag_destroy(jr->jr_ring_tag);
+ }
+
+ /* sec_jr_init() can fail after taking the lock but before the tag. */
+ if (mtx_initialized(&jr->jr_lock))
+ mtx_destroy(&jr->jr_lock);
+}
+
+int
+sec_jr_submit_job(struct sec_softc *sc, struct sec_jr *jr, struct sec_job *job)
+{
+ vm_paddr_t job_pa;
+ int slot;
+
+ job_pa = pmap_kextract((vm_offset_t)job);
+ mtx_lock(&jr->jr_lock);
+
+ if (jr->jr_inflight >= JR_RING_SIZE) {
+ jr->jr_blocked = CRYPTO_SYMQ;
+ mtx_unlock(&jr->jr_lock);
+ return (ERESTART);
+ }
+
+ slot = jr->jr_ir_head & JR_RING_MASK;
+ jr->jr_ir[slot] = (uint64_t)job_pa;
+ jr->jr_ir_head++;
+ jr->jr_inflight++;
+ job->job_deadline = ticks + SEC_JOB_TIMEOUT;
+ TAILQ_INSERT_TAIL(&jr->jr_active, job, job_link);
+ /* The ring entry must be visible before the doorbell. */
+ atomic_thread_fence_rel();
+ JR_WR4(sc, jr, JR_IRJAR, 1);
+
+ mtx_unlock(&jr->jr_lock);
+
+ return (0);
+}
diff --git a/sys/dev/dpaa/sec_var.h b/sys/dev/dpaa/sec_var.h
new file mode 100644
--- /dev/null
+++ b/sys/dev/dpaa/sec_var.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright 2026 Justin Hibbits
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef DPAA_SEC_VAR_H
+#define DPAA_SEC_VAR_H
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/callout.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <sys/endian.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
+
+#include "dpaa_common.h"
+
+#include "cryptodev_if.h"
+
+/* A job descriptor can have up to 64 words */
+#define SEC_MAX_DESC_WORDS 64
+
+/* Arbitrary limit */
+#define SEC_MAX_SEGMENTS 64
+#define SEC_MAX_SIZE ((SEC_MAX_SEGMENTS - 1) * PAGE_SIZE)
+
+#define SEC_MAX_DIGEST 64 /* SHA-512 */
+#define SEC_CCM_CTX_LEN 56
+
+/*
+ * SEC's output ring entry: 8-byte descriptor phys addr
+ * echoed from the input ring, followed by a 4-byte job termination
+ * status word. Entries are packed.
+ */
+struct sec_or_entry {
+ uint64_t desc_addr;
+ uint32_t status;
+} __packed;
+
+_Static_assert(sizeof(struct sec_or_entry) == 12, "OR entry size");
+
+struct sec_job;
+struct sec_session;
+
+/*
+ * One Job Ring. Rings are independent all the way down: separate
+ * register block, separate completion interrupt, separate lock.
+ * Spreading jobs across them keeps submitters off each other's locks.
+ * The rings themselves live in DMA-safe memory from bus_dmamem_alloc.
+ */
+struct sec_jr {
+ struct sec_softc *jr_sc; /* for the interrupt handler */
+ phandle_t jr_node; /* FDT node for this JR */
+ uint32_t jr_off; /* JR base within sc_rres */
+
+ bus_dma_tag_t jr_ring_tag;
+ bus_dmamap_t jr_map; /* covers both rings */
+ uint64_t *jr_ir; /* input, JR_RING_SIZE * 8B */
+ vm_paddr_t jr_ir_pa;
+ struct sec_or_entry *jr_or; /* output ring */
+ vm_paddr_t jr_or_pa;
+ uint32_t jr_ir_head; /* next slot driver writes */
+ uint32_t jr_or_tail; /* next slot driver reads */
+
+ /* Completion IRQ, separate from the SEC top-level error IRQ. */
+ struct resource *jr_ires;
+ int jr_irid;
+ void *jr_icookie;
+
+ /*
+ * Jobs handed to the ring and not yet seen on the output ring.
+ * jr_lock covers this and both ring indices, and is held only for
+ * the ring manipulation itself, never across a job. When the ring
+ * fills, jr_blocked records that opencrypto needs a
+ * crypto_unblock() once slots free up.
+ */
+ uint32_t jr_inflight;
+ int jr_blocked;
+
+ /*
+ * Outstanding jobs in submission order, so the watchdog only has
+ * to look at the head to find the oldest. jr_flushing marks the
+ * window between asking SEC to flush the ring and clearing HALT.
+ */
+ TAILQ_HEAD(, sec_job) jr_active;
+ struct callout jr_wdog;
+ bool jr_flushing;
+ bool jr_dying;
+
+ struct mtx jr_lock;
+};
+
+struct sec_softc {
+ device_t sc_dev;
+ struct resource *sc_rres; /* CCSR MMIO for SEC */
+ int sc_rrid;
+ struct resource *sc_ires; /* SEC error IRQ */
+ int sc_irid;
+ void *sc_icookie;
+ bus_dma_tag_t sc_dmatag; /* for crypto payloads */
+ int32_t sc_cid; /* opencrypto driver id */
+ int sc_version;
+
+ struct sec_jr *sc_jr; /* Job rings */
+ u_int sc_njr;
+};
+
+/*
+ * Per-request state. The job descriptor at the head becomes the
+ * address we push into the input ring. SEC echoes that same address
+ * in the output ring, and PHYS_TO_DMAP() gives us the struct back.
+ */
+
+struct sec_job {
+ uint32_t jd[SEC_MAX_DESC_WORDS];
+ struct dpaa_sgte in_sgt[1 + SEC_MAX_SEGMENTS];
+ struct dpaa_sgte out_sgt[SEC_MAX_SEGMENTS];
+ /* Driver-private (SEC does not touch anything below). */
+ struct cryptop *crp;
+ struct sec_session *sess;
+ bus_dmamap_t map;
+ TAILQ_ENTRY(sec_job) job_link;
+ int job_deadline; /* ticks */
+ /* Also holds the expanded 16-byte XTS tweak; see sec_xts_tweak(). */
+ uint8_t iv[AES_BLOCK_LEN];
+ uint8_t ccm_ctx[SEC_CCM_CTX_LEN];
+ uint8_t ccm_alen[2];
+ uint8_t digest[SEC_MAX_DIGEST];
+ int nsegs;
+ bus_dma_segment_t segs[SEC_MAX_SEGMENTS];
+};
+
+int sec_init_rings(struct sec_softc *sc);
+int sec_destroy_rings(struct sec_softc *sc);
+int sec_jr_submit_job(struct sec_softc *sc, struct sec_jr *jr,
+ struct sec_job *job);
+void sec_jr_teardown(struct sec_softc *sc, struct sec_jr *jr);
+void sec_complete_one(struct sec_softc *sc, uint64_t desc_pa, uint32_t status);
+
+#endif

File Metadata

Mime Type
text/plain
Expires
Sun, Sep 13, 4:28 AM (5 h, 7 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
38801857
Default Alt Text
D59581.id.diff (88 KB)

Event Timeline