Page MenuHomeFreeBSD
Files F137463695

D23677.id68298.diff
No OneTemporary
Actions

D23677.id68298.diff
View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
Index: ObsoleteFiles.inc
===================================================================
--- ObsoleteFiles.inc
+++ ObsoleteFiles.inc
@@ -36,6 +36,10 @@
# xargs -n1 | sort | uniq -d;
# done
+# 2020xxxx: OCF refactoring
+OLD_FILES+=usr/share/man/man9/crypto_find_driver.9
+OLD_FILES+=usr/share/man/man9/crypto_register.9
+OLD_FILES+=usr/share/man/man9/crypto_unregister.9
# 20200206: Remove elf2aout
OLD_FILES+=usr/bin/elf2aout
OLD_FILES+=usr/share/man/man1/elf2aout.1.gz
Index: notes
===================================================================
--- /dev/null
+++ notes
@@ -0,0 +1,239 @@
+The goal of this branch is to simplify the data structures used to describe
+session and crypto op parameters.
+
+- replace linked/list of cryptoini for session init with a structure
+ - explicit IV length?
+- replace linked/list of cryptodesc with members in struct cryptop
+ - maybe require data to always be laid out as
+ AAD || IV (if not sideband) || DATA || TAG/MAC
+ (had to scrap this due to IPsec AH and permit arbitrary IV and
+ digest offsets)
+ - generate IV if needed in common code, not drivers
+- add flag for hash verify rather than hash compute
+- change copyto/copyback to operate on the crp instead of the fields
+- permit separate in/out buffers?
+
+- need to ensure crypto_getreq() zeroes the crp (it does)
+- change for AEAD:
+ - previously with AES+HMAC crda covered all of data to auth, now
+ need to hash AAD region + payload. If IV needs to be hashed,
+ AAD should cover IV.
+
+- Notes on IVs:
+ - CRYPTO_F_IV_GENERATE (encrypt only)
+ - generate full IV and copy to crp_iv_start before encryption
+ - CRYPTO_F_IV_SEPARATE (both, cannot use with GENERATE)
+ - IV is in crp_iv, crp_iv_start should be ignored
+ - neither flag
+ - full IV is in data stream at crp_iv_start
+
+- AES-GCM now uses CIPHER_MODE_AEAD which ignores most of csp_auth_*
+ except for csp_auth_mlen
+
+- Eventually retire 3 separate GMAC algs and repurpose NIST_GMAC as a single
+ MAC?
+
+- Maybe retire driver HIDs with 'struct cryptocap' pointers instead? We
+ could just use a TAILQ instead of an array
+
+- Separate session operation to rekey instead of implicit rekeying?
+ Implicit klen is not really great for auth, for example
+
+- XXX: Need to think about how to define GMAC in the new world order. I
+ think it should be CSP_MODE_DIGEST, but with csp_cipher_ivlen set?
+
+- Yes, GMAC is CSP_MODE_DIGEST with payload area describing data to hash.
+ It uses csp_ivlen.
+
+- Maybe change crp_cipher_key to not change session key but only the
+ specific operation?
+
+- Need to figure out what to do about accelerated software vs plain
+ software
+ - maybe need "priority" and a "session probe/test" instead of just
+ "newsession"
+
+- Add a reference count on sessions
+ - crypto_getop would take a session pointer and set crp_session along
+ with bumping refcount now
+
+- Kill the UIO usage in cryptodev.c, it's silly (done)
+
+- GCM IV length is kind of busted, perhaps need IV length as part of crp
+ instead of csp?
+
+- Testing
+ + cryptocheck
+ + soft
+ + ccr
+ - ipsec
+ - auth
+ - hmac-md5
+ - hmac-sha1
+ - keyed-md5
+ - keyed-sha1
+ - null
+ - hmac-sha2-256
+ - hmac-sha2-384
+ - hmac-sha2-512
+ - hmac-ripemd160
+ - aes-xcbc-mac
+ - tcp-md5
+ - encrypt
+ - des-cbc
+ - 3desc-cbc
+ - null
+ - blowfish-cbc
+ - cast128-cbc
+ - des-deriv
+ - des-32iv
+ - rijndael-cbc
+ - aes-ctr
+ - aes-gcm16
+ - camellia-cbc
+
+ - 3des-hmac-md5
+ + soft
+ - aes-cbc-hmac-sha
+ + soft
+ - aes-cbc
+ + soft
+ - aes-ctr-hmac-sha2-512
+ + soft
+ - aes-ctr
+ + soft
+ - aes-gcm-256
+ + soft
+ - aes-gcm
+ + soft
+ - blowfish
+ + soft
+ - camellia
+ + soft
+ - cast128
+ + soft
+ - des-32iv
+ + soft
+ - des-deriv
+ + soft
+ - hmac-sha1
+ + soft
+ - hmac-sha2-256
+ + soft
+ - hmac-sha2-384
+ + soft
+ - hmac-sha2-512
+ + soft
+ - ipcomp
+ - doesn't work with stock FreeBSD
+ - null-ipcomp
+ - doesn't work with stock FreeBSD
+ - null
+ + soft
+ - rijndael-hmac-sha2-256
+ + soft
+ - rijndael
+ + soft
+ - tcp-md5
+ - needs kernel with TCP_SIGNATURE
+
+- Tested with cryptocheck (re-check)
+ - cryptosoft
+ - aesni
+ - blake2
+ - ccr
+
+- Needs testing:
+ - ccp
+ - aesni with sha
+ - hifn
+ - kgssapi_krb5
+ - ubsec
+ - padlock
+ - safe
+ - GELI
+ - armv8_crypto (aarch64)
+ - glxsb (i386)
+ - sec (ppc)
+ - cesa (armv7)
+ - cryptocteon (mips64)
+ - nlmsec (mips64)
+
+- Benchmarks
+ - aes-gcm setup
+ - frodo
+ - kldload ipsec
+ - ifconfig cc0 inet 172.16.0.1/24 alias
+ - setkey -f /mnt/jhb/work/git/netperf/IPSEC/Configs/source-aes-gcm-256.conf
+ - netperf -H 172.16.0.2 -cC
+ - sam
+ - kldload ipsec
+ - ifconfig cc0 inet 172.16.0.2/24 alias
+ - setkey -f /mnt/jhb/work/git/netperf/IPSEC/Configs/dut-aes-gcm-256.conf
+ - netserver
+ - clean svn (r348883)
+ - no ipsec 44000 18% 25%
+ - null 5500 28% 28%
+ - aes-gcm
+ - cryptosoft 550 26% 17%
+ - aesni 3000 28% 26%
+ - ccr 4300 45% 45%
+ - ocf_rework (r348883)
+ - no ipsec 44000 18% 25%
+ - null 5800 28% 28%
+ - aes-gcm
+ - cryptosoft 580 26% 17%
+ - aesni 3000 28% 26%
+ - ccr 4300 45% 45%
+
+ - ktls using aes-gcm
+ - frodo
+ - ktls.sh
+ - kldload ktls_ocf
+ - service nginx start
+ - sam
+ - normal ktls client
+ - clean svn (r354096)
+ - cryptosoft 2.9 Gbps 76-80% CPU
+ - aesni 27 Gbps 63-66% CPU
+ - ccr 30 Gbps 46-44% CPU
+ - ocf_rework (r354096)
+ - cryptosoft 2.9 Gbps 100% CPU
+ - aesni 27 Gbps 63-66% CPU
+ - ccr 30 Gbps 40-44% CPU
+
+- steps to convert a driver
+ - add probesession
+ - cri -> csp
+ - crd -> crp fields
+ - axe crypto_register
+
++ rename CRYPTO_BUF_IOV -> CRYPTO_BUF_UIO
++ make key pointers const in csp and crp
+ (keep drivers from XOR'ing keys with IPAD/OPAD)
++ kill klen in crp (assume they match session)
+ + allow NULL keys during setup but require klen to be
+ set during probesession / newsession, also have to require a key
+ when processing
++ change klen to bytes?
+- sysctl node to list crypto drivers
+ - update python test to use this
+- update places for new sysctl names
+
++ csp_lookup_hash() returns auth_hash *?
++ csp_lookup_cipher() returns enc_xform *?
+
++ pull checks into crypto_newsession
+ + checks on mlen
+ + csp_auth_klen 0 vs !0
+
++ kill CSP_IV_VALID and update drivers to all bail for flags != 0 to
+ permit adding new flags in the future
+- update crypto.9
+
++ bus_dmamap_load_crp()
+ + hifn
+ + sec (ppc)
+ + safe
+ + cesa (armv7)
+ + ubsec
Index: share/man/man4/crypto.4
===================================================================
--- share/man/man4/crypto.4
+++ share/man/man4/crypto.4
@@ -156,7 +156,7 @@
The symmetric-key operation mode provides a context-based API
to traditional symmetric-key encryption (or privacy) algorithms,
or to keyed and unkeyed one-way hash (HMAC and MAC) algorithms.
-The symmetric-key mode also permits fused operation,
+The symmetric-key mode also permits encrypt-then-authenticate fused operation,
where the hardware performs both a privacy algorithm and an integrity-check
algorithm in a single pass over the data: either a fused
encrypt/HMAC-generate operation, or a fused HMAC-verify/decrypt operation.
@@ -314,16 +314,14 @@
.Fa cr_op-\*[Gt]iv
supply the addresses of the input buffer, output buffer,
one-way hash, and initialization vector, respectively.
-If a session is using both a privacy algorithm and a hash algorithm,
-the request will generate a hash of the input buffer before
-generating the output buffer by default.
-If the
-.Dv COP_F_CIPHER_FIRST
-flag is included in the
-.Fa cr_op-\*[Gt]flags
-field,
-then the request will generate a hash of the output buffer after
-executing the privacy algorithm.
+.Pp
+If a session is using either fused encrypt-then-authenticate or
+an AEAD algorithm,
+decryption operations require the associated hash as an input.
+If the hash is incorrect, the
+operation will fail with
+.Dv EBADMSG
+and the output buffer will remain unchanged.
.It Dv CIOCCRYPTAEAD Fa struct crypt_aead *cr_aead
.Bd -literal
struct crypt_aead {
Index: share/man/man7/crypto.7
===================================================================
--- share/man/man7/crypto.7
+++ share/man/man7/crypto.7
@@ -68,19 +68,13 @@
.El
.Pp
This algorithm implements Galois/Counter Mode.
-This is the cipher part of an AEAD
+This cipher uses AEAD
.Pq Authenticated Encryption with Associated Data
mode.
-This requires use of the use of a proper authentication mode, one of
-.Dv CRYPTO_AES_128_NIST_GMAC ,
-.Dv CRYPTO_AES_192_NIST_GMAC
-or
-.Dv CRYPTO_AES_256_NIST_GMAC ,
-that corresponds with the number of bits in the key that you are using.
.Pp
-The associated data (if any) must be provided by the authentication mode op.
-The authentication tag will be read/written from/to the offset crd_inject
-specified in the descriptor for the authentication mode.
+The authentication tag will be read/written from/to the offset
+.Va crp_digest_start
+specified in the request.
.Pp
Note: You must provide an IV on every call.
.It Dv CRYPTO_AES_ICM
@@ -118,22 +112,6 @@
NOTE: The ciphertext stealing part is not implemented which is why this cipher
is listed as having a block size of 16 instead of 1.
.El
-.Pp
-Authentication algorithms:
-.Bl -tag -width ".Dv CRYPTO_AES_256_NIST_GMAC"
-.It CRYPTO_AES_128_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
-.It CRYPTO_AES_192_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
-.It CRYPTO_AES_256_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
-.El
.Sh SEE ALSO
.Xr crypto 4 ,
.Xr crypto 9
Index: share/man/man9/Makefile
===================================================================
--- share/man/man9/Makefile
+++ share/man/man9/Makefile
@@ -71,6 +71,10 @@
cr_seeothergids.9 \
cr_seeotheruids.9 \
crypto.9 \
+ crypto_asym.9 \
+ crypto_driver.9 \
+ crypto_request.9 \
+ crypto_session.9 \
CTASSERT.9 \
DB_COMMAND.9 \
DECLARE_GEOM_CLASS.9 \
@@ -889,20 +893,33 @@
cpuset.9 CPU_COPY_STORE_REL.9
MLINKS+=critical_enter.9 critical.9 \
critical_enter.9 critical_exit.9
-MLINKS+=crypto.9 crypto_dispatch.9 \
- crypto.9 crypto_done.9 \
- crypto.9 crypto_freereq.9 \
- crypto.9 crypto_freesession.9 \
- crypto.9 crypto_get_driverid.9 \
- crypto.9 crypto_getreq.9 \
- crypto.9 crypto_kdispatch.9 \
- crypto.9 crypto_kdone.9 \
- crypto.9 crypto_kregister.9 \
- crypto.9 crypto_newsession.9 \
- crypto.9 crypto_register.9 \
- crypto.9 crypto_unblock.9 \
- crypto.9 crypto_unregister.9 \
- crypto.9 crypto_unregister_all.9
+MLINKS+=crypto_asym.9 crypto_kdispatch.9 \
+ crypto_asym.9 crypto_kdone.9 \
+ crypto_asym.9 crypto_kregister.9 \
+ crypto_asym.9 CRYPTODEV_KPROCESS.9
+MLINKS+=crypto_driver.9 crypto_apply.9 \
+ crypto_driver.9 crypto_contiguous_segment.9 \
+ crypto_driver.9 crypto_copyback.9 \
+ crypto_driver.9 crypto_copydata.9 \
+ crypto_driver.9 crypto_done.9 \
+ crypto_driver.9 crypto_get_driverid.9 \
+ crypto_driver.9 crypto_get_driver_session.9 \
+ crypto_driver.9 crypto_unblock.9 \
+ crypto_driver.9 crypto_unregister_all.9 \
+ crypto_driver.9 CRYPTODEV_FREESESSION.9 \
+ crypto_driver.9 CRYPTODEV_NEWSESSION.9 \
+ crypto_driver.9 CRYPTODEV_PROBESESSION.9 \
+ crypto_driver.9 CRYPTODEV_PROCESS.9 \
+ crypto_driver.9 hmac_init_ipad.9 \
+ crypto_driver.9 hmac_init_opad.9
+MLINKS+=crypto_request.9 crypto_dispatch.9 \
+ crypto_request.9 crypto_freereq.9 \
+ crypto_request.9 crypto_getreq.9
+MLINKS+=crypto_session.9 crypto_auth_hash.9 \
+ crypto_session.9 crypto_cipher.9 \
+ crypto_session.9 crypto_get_params.9 \
+ crypto_session.9 crypto_newsession.9 \
+ crypto_session.9 crypto_freesession.9
MLINKS+=DB_COMMAND.9 DB_SHOW_ALL_COMMAND.9 \
DB_COMMAND.9 DB_SHOW_COMMAND.9
MLINKS+=DECLARE_MODULE.9 DECLARE_MODULE_TIED.9
Index: share/man/man9/bus_dma.9
===================================================================
--- share/man/man9/bus_dma.9
+++ share/man/man9/bus_dma.9
@@ -68,6 +68,7 @@
.Nm bus_dmamap_load ,
.Nm bus_dmamap_load_bio ,
.Nm bus_dmamap_load_ccb ,
+.Nm bus_dmamap_load_crp ,
.Nm bus_dmamap_load_mbuf ,
.Nm bus_dmamap_load_mbuf_sg ,
.Nm bus_dmamap_load_uio ,
@@ -118,6 +119,10 @@
"union ccb *ccb" "bus_dmamap_callback_t *callback" "void *callback_arg" \
"int flags"
.Ft int
+.Fn bus_dmamap_load_crp "bus_dma_tag_t dmat" "bus_dmamap_t map" \
+"struct crypto *crp" "bus_dmamap_callback_t *callback" "void *callback_arg" \
+"int flags"
+.Ft int
.Fn bus_dmamap_load_mbuf "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct mbuf *mbuf" "bus_dmamap_callback2_t *callback" "void *callback_arg" \
"int flags"
@@ -387,9 +392,10 @@
.Vt bus_dmamap_t
via
.Fn bus_dmamap_load ,
-.Fn bus_dmamap_load_bio
+.Fn bus_dmamap_load_bio ,
+.Fn bus_dmamap_load_ccb ,
or
-.Fn bus_dmamap_load_ccb .
+.Fn bus_dmamap_load_crp .
Callbacks are of the format:
.Bl -tag -width indent
.It Ft void
@@ -879,6 +885,15 @@
.It
XPT_SCSI_IO
.El
+.It Fn bus_dmamap_load_crp "dmat" "map" "crp" "callback" "callback_arg" "flags"
+This is a variation of
+.Fn bus_dmamap_load
+which maps buffers pointed to by
+.Fa crp
+for DMA transfers.
+The
+.Dv BUS_DMA_NOWAIT
+flag is implied, thus no callback deferral will happen.
.It Fn bus_dmamap_load_mbuf "dmat" "map" "mbuf" "callback2" "callback_arg" \
"flags"
This is a variation of
Index: share/man/man9/crypto.9
===================================================================
--- share/man/man9/crypto.9
+++ share/man/man9/crypto.9
@@ -25,120 +25,50 @@
.Nd API for cryptographic services in the kernel
.Sh SYNOPSIS
.In opencrypto/cryptodev.h
-.Ft int32_t
-.Fn crypto_get_driverid "device_t dev" "size_t session_size" "int flags"
-.Ft int
-.Fn crypto_register "uint32_t driverid" "int alg" "uint16_t maxoplen" "uint32_t flags"
-.Ft int
-.Fn crypto_kregister "uint32_t driverid" "int kalg" "uint32_t flags"
-.Ft int
-.Fn crypto_unregister "uint32_t driverid" "int alg"
-.Ft int
-.Fn crypto_unregister_all "uint32_t driverid"
-.Ft void
-.Fn crypto_done "struct cryptop *crp"
-.Ft void
-.Fn crypto_kdone "struct cryptkop *krp"
-.Ft int
-.Fn crypto_find_driver "const char *match"
-.Ft int
-.Fn crypto_newsession "crypto_session_t *cses" "struct cryptoini *cri" "int crid"
-.Ft int
-.Fn crypto_freesession "crypto_session_t cses"
-.Ft int
-.Fn crypto_dispatch "struct cryptop *crp"
-.Ft int
-.Fn crypto_kdispatch "struct cryptkop *krp"
-.Ft int
-.Fn crypto_unblock "uint32_t driverid" "int what"
-.Ft "struct cryptop *"
-.Fn crypto_getreq "int num"
-.Ft void
-.Fn crypto_freereq "struct cryptop *crp"
-.Bd -literal
-#define CRYPTO_SYMQ 0x1
-#define CRYPTO_ASYMQ 0x2
-
-#define EALG_MAX_BLOCK_LEN 16
-
-struct cryptoini {
- int cri_alg;
- int cri_klen;
- int cri_mlen;
- caddr_t cri_key;
- uint8_t cri_iv[EALG_MAX_BLOCK_LEN];
- struct cryptoini *cri_next;
-};
-
-struct cryptodesc {
- int crd_skip;
- int crd_len;
- int crd_inject;
- int crd_flags;
- struct cryptoini CRD_INI;
-#define crd_iv CRD_INI.cri_iv
-#define crd_key CRD_INI.cri_key
-#define crd_alg CRD_INI.cri_alg
-#define crd_klen CRD_INI.cri_klen
- struct cryptodesc *crd_next;
-};
-
-struct cryptop {
- TAILQ_ENTRY(cryptop) crp_next;
- crypto_session_t crp_session;
- int crp_ilen;
- int crp_olen;
- int crp_etype;
- int crp_flags;
- caddr_t crp_buf;
- caddr_t crp_opaque;
- struct cryptodesc *crp_desc;
- int (*crp_callback) (struct cryptop *);
- caddr_t crp_mac;
-};
-
-struct crparam {
- caddr_t crp_p;
- u_int crp_nbits;
-};
-
-#define CRK_MAXPARAM 8
-
-struct cryptkop {
- TAILQ_ENTRY(cryptkop) krp_next;
- u_int krp_op; /* ie. CRK_MOD_EXP or other */
- u_int krp_status; /* return status */
- u_short krp_iparams; /* # of input parameters */
- u_short krp_oparams; /* # of output parameters */
- uint32_t krp_hid;
- struct crparam krp_param[CRK_MAXPARAM];
- int (*krp_callback)(struct cryptkop *);
-};
-.Ed
.Sh DESCRIPTION
.Nm
-is a framework for drivers of cryptographic hardware to register with
-the kernel so
-.Dq consumers
-(other kernel subsystems, and
-users through the
+is a framework for in-kernel cryptography.
+It permits in-kernel consumers to encrypt and decrypt data
+and also enables userland applications to use cryptographic hardware
+through the
.Pa /dev/crypto
-device) are able to make use of it.
-Drivers register with the framework the algorithms they support,
-and provide entry points (functions) the framework may call to
-establish, use, and tear down sessions.
-Sessions are used to cache cryptographic information in a particular driver
-(or associated hardware), so initialization is not needed with every request.
-Consumers of cryptographic services pass a set of
-descriptors that instruct the framework (and the drivers registered
-with it) of the operations that should be applied on the data (more
-than one cryptographic operation can be requested).
+device.
.Pp
-Keying operations are supported as well.
-Unlike the symmetric operators described above,
-these sessionless commands perform mathematical operations using
-input and output parameters.
+.Nm
+supports two modes of operation:
+one mode for symmetric-keyed cryptographic requests and digest,
+and a second mode for asymmetric-key requests and modular arithmetic.
+.Ss Symmetric-Key Mode
+Symmetric-key operations include encryption and decryption operations
+using block and stream ciphers as well as computation and verification
+of message authentication codes (MACs).
+In this mode,
+consumers allocate sessions to describe a transform as discussed in
+.Xr crypto_session 9 .
+Consumers then allocate request objects to describe each transformation
+such as encrypting a network packet or decrypting a disk sector.
+Requests are described in
+.Xr crypto_request 9 .
.Pp
+Device drivers are responsible for processing requests submitted by
+consumers.
+.Xr crypto_driver 9
+describes the interfaces drivers use to register with the framework,
+helper routines the framework provides to faciliate request processing,
+and the interfaces drivers are required to provide.
+.Ss Asymmetric-Key Mode
+Assymteric-key operations do not use sessions.
+Instead,
+these operations perform individual mathematical operations using a set
+of input and output parameters.
+These operations are described in
+.Xr crypto_asym 9 .
+Drivers that support asymmetric operations use additional interfaces
+described in
+.Xr crypto_asym 9
+in addition to the base interfaces described in
+.Xr crypto_driver 9 .
+.Ss Callbacks
Since the consumers may not be associated with a process, drivers may
not
.Xr sleep 9 .
@@ -148,88 +78,38 @@
callback is specified by the consumer on a per-request basis).
The callback is invoked by the framework whether the request was
successfully completed or not.
-An error indication is provided in the latter case.
-A specific error code,
+Errors are reported to the callback function.
+.Pp
+Session initialization does not use callbacks and returns errors
+synchronously.
+.Ss Session Migration
+For symmetric-key operations,
+a specific error code,
.Er EAGAIN ,
is used to indicate that a session handle has changed and that the
request may be re-submitted immediately with the new session.
-Errors are only returned to the invoking function if not
-enough information to call the callback is available (meaning, there
-was a fatal error in verifying the arguments).
-For session initialization and teardown no callback mechanism is used.
+The consumer should update its saved copy of the session handle
+to the value of
+.Fa crp_session
+so that future requests use the new session.
+.Ss Supported Algorithms
+More details on some algorithms may be found in
+.Xr crypto 7 .
+These algorithms are used for symmetric-mode operations.
+Asymmetric-mode operations support operations described in
+.Xr crypto_asym 9 .
.Pp
-The
-.Fn crypto_find_driver
-returns the driver id of the device whose name matches
-.Fa match .
-.Fa match
-can either be the exact name of a device including the unit
-or the driver name without a unit.
-In the latter case,
-the id of the first device with the matching driver name is returned.
-If no matching device is found,
-the value -1 is returned.
+The following authentication algorithms are supported:
.Pp
-The
-.Fn crypto_newsession
-routine is called by consumers of cryptographic services (such as the
-.Xr ipsec 4
-stack) that wish to establish a new session with the framework.
-The
-.Fa cri
-argument points to a
-.Vt cryptoini
-structure containing all the necessary information for
-the driver to establish the session.
-The
-.Fa crid
-argument is either a specific driver id or a bitmask of flags.
-The flags are
-.Dv CRYPTOCAP_F_HARDWARE ,
-to select hardware devices,
-or
-.Dv CRYPTOCAP_F_SOFTWARE ,
-to select software devices.
-If both are specified, hardware devices are preferred over software
-devices.
-On success, the opaque session handle of the new session will be stored in
-.Fa *cses .
-The
-.Vt cryptoini
-structure pointed to by
-.Fa cri
-contains these fields:
-.Bl -tag -width ".Va cri_next"
-.It Va cri_alg
-An algorithm identifier.
-Currently supported algorithms are:
-.Pp
-.Bl -tag -width ".Dv CRYPTO_RIPEMD160_HMAC" -compact
-.It Dv CRYPTO_AES_128_NIST_GMAC
-.It Dv CRYPTO_AES_192_NIST_GMAC
-.It Dv CRYPTO_AES_256_NIST_GMAC
-.It Dv CRYPTO_AES_CBC
-.It Dv CRYPTO_AES_CCM_16
+.Bl -tag -offset indent -width CRYPTO_AES_CCM_CBC_MAC -compact
.It Dv CRYPTO_AES_CCM_CBC_MAC
-.It Dv CRYPTO_AES_ICM
-.It Dv CRYPTO_AES_NIST_GCM_16
.It Dv CRYPTO_AES_NIST_GMAC
-.It Dv CRYPTO_AES_XTS
-.It Dv CRYPTO_ARC4
.It Dv CRYPTO_BLAKE2B
.It Dv CRYPTO_BLAKE2S
-.It Dv CRYPTO_BLF_CBC
-.It Dv CRYPTO_CAMELLIA_CBC
-.It Dv CRYPTO_CAST_CBC
-.It Dv CRYPTO_CHACHA20
-.It Dv CRYPTO_DEFLATE_COMP
-.It Dv CRYPTO_DES_CBC
-.It Dv CRYPTO_3DES_CBC
.It Dv CRYPTO_MD5
.It Dv CRYPTO_MD5_HMAC
.It Dv CRYPTO_MD5_KPDK
.It Dv CRYPTO_NULL_HMAC
-.It Dv CRYPTO_NULL_CBC
.It Dv CRYPTO_POLY1305
.It Dv CRYPTO_RIPEMD160
.It Dv CRYPTO_RIPEMD160_HMAC
@@ -244,488 +124,38 @@
.It Dv CRYPTO_SHA2_384_HMAC
.It Dv CRYPTO_SHA2_512
.It Dv CRYPTO_SHA2_512_HMAC
+.El
+.Pp
+The following encryption algorithms are supported:
+.Pp
+.Bl -tag -offset indent -width CRYPTO_CAMELLIA_CBC -compact
+.It Dv CRYPTO_AES_CBC
+.It Dv CRYPTO_AES_ICM
+.It Dv CRYPTO_AES_XTS
+.It Dv CRYPTO_ARC4
+.It Dv CRYPTO_BLF_CBC
+.It Dv CRYPTO_CAMELLIA_CBC
+.It Dv CRYPTO_CAST_CBC
+.It Dv CRYPTO_CHACHA20
+.It Dv CRYPTO_DES_CBC
+.It Dv CRYPTO_3DES_CBC
+.It Dv CRYPTO_NULL_CBC
.It Dv CRYPTO_SKIPJACK_CBC
.El
-.It Va cri_klen
-For variable-size key algorithms, the length of the key in bits.
-.It Va cri_mlen
-If non-zero, truncate the calculated hash to this many bytes.
-.It Va cri_key
-The key to be used.
-.It Va cri_iv
-An explicit initialization vector if it does not prefix
-the data.
-This field is ignored during initialization
-.Pq Nm crypto_newsession .
-If no IV is explicitly passed (see below on details), a random IV is used
-by the device driver processing the request.
-.It Va cri_next
-Pointer to another
-.Vt cryptoini
-structure.
-This is used to establish dual-algorithm sessions, such as combining a
-cipher with a MAC.
-.El
-.Pp
-The
-.Vt cryptoini
-structure and its contents will not be modified or referenced by the
-framework or any cryptographic drivers.
-The memory associated with
-.Fa cri
-can be released once
-.Fn crypto_newsession
-returns.
-.Pp
-.Fn crypto_freesession
-is called with the session handle returned by
-.Fn crypto_newsession
-to free the session.
.Pp
-.Fn crypto_dispatch
-is called to process a request.
-The various fields in the
-.Vt cryptop
-structure are:
-.Bl -tag -width ".Va crp_callback"
-.It Va crp_session
-The session handle.
-.It Va crp_ilen
-The total length in bytes of the buffer to be processed.
-.It Va crp_olen
-On return, contains the total length of the result.
-For symmetric crypto operations, this will be the same as the input length.
-This will be used if the framework needs to allocate a new
-buffer for the result (or for re-formatting the input).
-.It Va crp_callback
-Callback routine invoked when a request is completed via
-.Fn crypto_done .
-The callback routine should inspect the
-.Va crp_etype
-to determine if the request was successfully completed.
-.It Va crp_etype
-The error type, if any errors were encountered, or zero if
-the request was successfully processed.
-If the
-.Er EAGAIN
-error code is returned, the session handle has changed (and has been recorded
-in the
-.Va crp_session
-field).
-The consumer should record the new session handle and use it in all subsequent
-requests.
-In this case, the request may be re-submitted immediately.
-This mechanism is used by the framework to perform
-session migration (move a session from one driver to another, because
-of availability, performance, or other considerations).
+The following authenticated encryption with additional data (AEAD)
+algorithms are supported:
.Pp
-This field is only valid in the context of the callback routine specified by
-.Va crp_callback .
-Errors are returned to the invoker of
-.Fn crypto_process
-only when enough information is not present to call the callback
-routine (i.e., if the pointer passed is
-.Dv NULL
-or if no callback routine was specified).
-.It Va crp_flags
-A bitmask of flags associated with this request.
-Currently defined flags are:
-.Bl -tag -width ".Dv CRYPTO_F_CBIFSYNC"
-.It Dv CRYPTO_F_IMBUF
-The buffer is an mbuf chain pointed to by
-.Va crp_mbuf .
-.It Dv CRYPTO_F_IOV
-The buffer is a
-.Vt uio
-structure pointed to by
-.Va crp_uio .
-.It Dv CRYPTO_F_BATCH
-Batch operation if possible.
-.It Dv CRYPTO_F_CBIMM
-Do callback immediately instead of doing it from a dedicated kernel thread.
-.It Dv CRYPTO_F_DONE
-Operation completed.
-.It Dv CRYPTO_F_CBIFSYNC
-Do callback immediately if operation is synchronous (that the driver
-specified the
-.Dv CRYPTOCAP_F_SYNC
-flag).
-.It Dv CRYPTO_F_ASYNC
-Try to do the crypto operation in a pool of workers
-if the operation is synchronous (that is, if the driver specified the
-.Dv CRYPTOCAP_F_SYNC
-flag).
-It aims to speed up processing by dispatching crypto operations
-on different processors.
-.It Dv CRYPTO_F_ASYNC_KEEPORDER
-Dispatch callbacks in the same order they are posted.
-Only relevant if the
-.Dv CRYPTO_F_ASYNC
-flag is set and if the operation is synchronous.
-.El
-.It Va crp_buf
-Data buffer unless
-.Dv CRYPTO_F_IMBUF
-or
-.Dv CRYPTO_F_IOV
-is set in
-.Va crp_flags .
-The length in bytes is set in
-.Va crp_ilen .
-.It Va crp_mbuf
-Data buffer mbuf chain when
-.Dv CRYPTO_F_IMBUF
-is set in
-.Va crp_flags .
-.It Va crp_uio
-.Vt struct uio
-data buffer when
-.Dv CRYPTO_F_IOV
-is set in
-.Va crp_flags .
-.It Va crp_opaque
-Cookie passed through the crypto framework untouched.
-It is
-intended for the invoking application's use.
-.It Va crp_desc
-A linked list of descriptors.
-Each descriptor provides
-information about what type of cryptographic operation should be done
-on the input buffer.
-The various fields are:
-.Bl -tag -width ".Va crd_inject"
-.It Va crd_iv
-When the flag
-.Dv CRD_F_IV_EXPLICIT
-is set, this field contains the IV.
-.It Va crd_key
-When the
-.Dv CRD_F_KEY_EXPLICIT
-flag is set, the
-.Va crd_key
-points to a buffer with encryption or authentication key.
-.It Va crd_alg
-An algorithm to use.
-Must be the same as the one given at newsession time.
-.It Va crd_klen
-The
-.Va crd_key
-key length.
-.It Va crd_skip
-The offset in the input buffer where processing should start.
-.It Va crd_len
-How many bytes, after
-.Va crd_skip ,
-should be processed.
-.It Va crd_inject
-The
-.Va crd_inject
-field specifies an offset in bytes from the beginning of the buffer.
-For encryption algorithms, this may be where the IV will be inserted
-when encrypting or where the IV may be found for
-decryption (subject to
-.Va crd_flags ) .
-For MAC algorithms, this is where the result of the keyed hash will be
-inserted.
-.It Va crd_flags
-The following flags are defined:
-.Bl -tag -width 3n
-.It Dv CRD_F_ENCRYPT
-For encryption algorithms, this bit is set when encryption is required
-(when not set, decryption is performed).
-.It Dv CRD_F_IV_PRESENT
-.\" This flag name has nothing to do w/ it's behavior, fix the name.
-For encryption, if this bit is not set the IV used to encrypt the packet
-will be written at the location pointed to by
-.Va crd_inject .
-The IV length is assumed to be equal to the blocksize of the
-encryption algorithm.
-For encryption, if this bit is set, nothing is done.
-For decryption, this flag has no meaning.
-Applications that do special
-.Dq "IV cooking" ,
-such as the half-IV mode in
-.Xr ipsec 4 ,
-can use this flag to indicate that the IV should not be written on the packet.
-This flag is typically used in conjunction with the
-.Dv CRD_F_IV_EXPLICIT
-flag.
-.It Dv CRD_F_IV_EXPLICIT
-This bit is set when the IV is explicitly
-provided by the consumer in the
-.Va crd_iv
-field.
-Otherwise, for encryption operations the IV is provided for by
-the driver used to perform the operation, whereas for decryption
-operations the offset of the IV is provided by the
-.Va crd_inject
-field.
-This flag is typically used when the IV is calculated
-.Dq "on the fly"
-by the consumer, and does not precede the data.
-.It Dv CRD_F_KEY_EXPLICIT
-For encryption and authentication (MAC) algorithms, this bit is set when the key
-is explicitly provided by the consumer in the
-.Va crd_key
-field for the given operation.
-Otherwise, the key is taken at newsession time from the
-.Va cri_key
-field.
-As calculating the key schedule may take a while, it is recommended that often
-used keys are given their own session.
-.It Dv CRD_F_COMP
-For compression algorithms, this bit is set when compression is required (when
-not set, decompression is performed).
-.El
-.It Va CRD_INI
-This
-.Vt cryptoini
-structure will not be modified by the framework or the device drivers.
-Since this information accompanies every cryptographic
-operation request, drivers may re-initialize state on-demand
-(typically an expensive operation).
-Furthermore, the cryptographic
-framework may re-route requests as a result of full queues or hardware
-failure, as described above.
-.It Va crd_next
-Point to the next descriptor.
-Linked operations are useful in protocols such as
-.Xr ipsec 4 ,
-where multiple cryptographic transforms may be applied on the same
-block of data.
+.Bl -tag -offset indent -width CRYPTO_AES_NIST_GCM_16 -compact
+.It Dv CRYPTO_AES_CCM_16
+.It Dv CRYPTO_AES_NIST_GCM_16
.El
-.El
-.Pp
-.Fn crypto_getreq
-allocates a
-.Vt cryptop
-structure with a linked list of
-.Fa num
-.Vt cryptodesc
-structures.
.Pp
-.Fn crypto_freereq
-deallocates a structure
-.Vt cryptop
-and any
-.Vt cryptodesc
-structures linked to it.
-Note that it is the responsibility of the
-callback routine to do the necessary cleanups associated with the
-opaque field in the
-.Vt cryptop
-structure.
+The following compression algorithms are supported:
.Pp
-.Fn crypto_kdispatch
-is called to perform a keying operation.
-The various fields in the
-.Vt cryptkop
-structure are:
-.Bl -tag -width ".Va krp_callback"
-.It Va krp_op
-Operation code, such as
-.Dv CRK_MOD_EXP .
-.It Va krp_status
-Return code.
-This
-.Va errno Ns -style
-variable indicates whether lower level reasons
-for operation failure.
-.It Va krp_iparams
-Number of input parameters to the specified operation.
-Note that each operation has a (typically hardwired) number of such parameters.
-.It Va krp_oparams
-Number of output parameters from the specified operation.
-Note that each operation has a (typically hardwired) number of such parameters.
-.It Va krp_kvp
-An array of kernel memory blocks containing the parameters.
-.It Va krp_hid
-Identifier specifying which low-level driver is being used.
-.It Va krp_callback
-Callback called on completion of a keying operation.
+.Bl -tag -offset indent -width CRYPTO_DEFLATE_COMP -compact
+.It Dv CRYPTO_DEFLATE_COMP
.El
-.Sh DRIVER-SIDE API
-The
-.Fn crypto_get_driverid ,
-.Fn crypto_get_driver_session ,
-.Fn crypto_register ,
-.Fn crypto_kregister ,
-.Fn crypto_unregister ,
-.Fn crypto_unblock ,
-and
-.Fn crypto_done
-routines are used by drivers that provide support for cryptographic
-primitives to register and unregister with the kernel crypto services
-framework.
-.Pp
-Drivers must first use the
-.Fn crypto_get_driverid
-function to acquire a driver identifier, specifying the
-.Fa flags
-as an argument.
-One of
-.Dv CRYPTOCAP_F_SOFTWARE
-or
-.Dv CRYPTOCAP_F_HARDWARE
-must be specified.
-The
-.Dv CRYPTOCAP_F_SYNC
-may also be specified, and should be specified if the driver does all of
-it's operations synchronously.
-Drivers must pass the size of their session structure as the second argument.
-An appropriately sized memory will be allocated by the framework, zeroed, and
-passed to the driver's
-.Fn newsession
-method.
-.Pp
-For each algorithm the driver supports, it must then call
-.Fn crypto_register .
-The first two arguments are the driver and algorithm identifiers.
-The next two arguments specify the largest possible operator length (in bits,
-important for public key operations) and flags for this algorithm.
-.Pp
-.Fn crypto_unregister
-is called by drivers that wish to withdraw support for an algorithm.
-The two arguments are the driver and algorithm identifiers, respectively.
-Typically, drivers for
-PCMCIA
-crypto cards that are being ejected will invoke this routine for all
-algorithms supported by the card.
-.Fn crypto_unregister_all
-will unregister all algorithms registered by a driver
-and the driver will be disabled (no new sessions will be allocated on
-that driver, and any existing sessions will be migrated to other
-drivers).
-The same will be done if all algorithms associated with a driver are
-unregistered one by one.
-After a call to
-.Fn crypto_unregister_all
-there will be no threads in either the newsession or freesession function
-of the driver.
-.Pp
-The calling convention for the driver-supplied routines are:
-.Pp
-.Bl -item -compact
-.It
-.Ft int
-.Fn \*[lp]*newsession\*[rp] "device_t" "crypto_session_t" "struct cryptoini *" ;
-.It
-.Ft void
-.Fn \*[lp]*freesession\*[rp] "device_t" "crypto_session_t" ;
-.It
-.Ft int
-.Fn \*[lp]*process\*[rp] "device_t" "struct cryptop *" "int" ;
-.It
-.Ft int
-.Fn \*[lp]*kprocess\*[rp] "device_t" "struct cryptkop *" "int" ;
-.El
-.Pp
-On invocation, the first argument to
-all routines is the
-.Fa device_t
-that was provided to
-.Fn crypto_get_driverid .
-The second argument to
-.Fn newsession
-is the opaque session handle for the new session.
-The third argument is identical to that of
-.Fn crypto_newsession .
-.Pp
-Drivers obtain a pointer to their session memory by invoking
-.Fn crypto_get_driver_session
-on the opaque
-.Vt crypto_session_t
-handle.
-.Pp
-The
-.Fn freesession
-routine takes as arguments the opaque data value and the session handle.
-It should clear any context associated with the session (clear hardware
-registers, memory, etc.).
-If no resources need to be released other than the contents of session memory,
-the method is optional.
-The
-.Nm
-framework will zero and release the allocated session memory (after running the
-.Fn freesession
-method, if one exists).
-.Pp
-The
-.Fn process
-routine is invoked with a request to perform crypto processing.
-This routine must not block or sleep, but should queue the request and return
-immediately or process the request to completion.
-In case of an unrecoverable error, the error indication must be placed in the
-.Va crp_etype
-field of the
-.Vt cryptop
-structure.
-When the request is completed, or an error is detected, the
-.Fn process
-routine must invoke
-.Fn crypto_done .
-Session migration may be performed, as mentioned previously.
-.Pp
-In case of a temporary resource exhaustion, the
-.Fn process
-routine may return
-.Er ERESTART
-in which case the crypto services will requeue the request, mark the driver
-as
-.Dq blocked ,
-and stop submitting requests for processing.
-The driver is then responsible for notifying the crypto services
-when it is again able to process requests through the
-.Fn crypto_unblock
-routine.
-This simple flow control mechanism should only be used for short-lived
-resource exhaustion as it causes operations to be queued in the crypto
-layer.
-Doing so is preferable to returning an error in such cases as
-it can cause network protocols to degrade performance by treating the
-failure much like a lost packet.
-.Pp
-The
-.Fn kprocess
-routine is invoked with a request to perform crypto key processing.
-This routine must not block, but should queue the request and return
-immediately.
-Upon processing the request, the callback routine should be invoked.
-In case of an unrecoverable error, the error indication must be placed in the
-.Va krp_status
-field of the
-.Vt cryptkop
-structure.
-When the request is completed, or an error is detected, the
-.Fn kprocess
-routine should invoked
-.Fn crypto_kdone .
-.Sh RETURN VALUES
-.Fn crypto_register ,
-.Fn crypto_kregister ,
-.Fn crypto_unregister ,
-.Fn crypto_newsession ,
-.Fn crypto_freesession ,
-and
-.Fn crypto_unblock
-return 0 on success, or an error code on failure.
-.Fn crypto_get_driverid
-returns a non-negative value on error, and \-1 on failure.
-.Fn crypto_getreq
-returns a pointer to a
-.Vt cryptop
-structure and
-.Dv NULL
-on failure.
-.Fn crypto_dispatch
-returns
-.Er EINVAL
-if its argument or the callback function was
-.Dv NULL ,
-and 0 otherwise.
-The callback is provided with an error code in case of failure, in the
-.Va crp_etype
-field.
.Sh FILES
.Bl -tag -width ".Pa sys/opencrypto/crypto.c"
.It Pa sys/opencrypto/crypto.c
@@ -735,7 +165,10 @@
.Xr crypto 4 ,
.Xr ipsec 4 ,
.Xr crypto 7 ,
-.Xr malloc 9 ,
+.Xr crypto_asym 9 ,
+.Xr crypto_driver 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9 ,
.Xr sleep 9
.Sh HISTORY
The cryptographic framework first appeared in
@@ -743,14 +176,6 @@
and was written by
.An Angelos D. Keromytis Aq Mt angelos@openbsd.org .
.Sh BUGS
-The framework currently assumes that all the algorithms in a
-.Fn crypto_newsession
-operation must be available by the same driver.
-If that is not the case, session initialization will fail.
-.Pp
-The framework also needs a mechanism for determining which driver is
+The framework needs a mechanism for determining which driver is
best for a specific set of algorithms associated with a session.
Some type of benchmarking is in order here.
-.Pp
-Multiple instances of the same algorithm in the same session are not
-supported.
Index: share/man/man9/crypto_asym.9
===================================================================
--- /dev/null
+++ share/man/man9/crypto_asym.9
@@ -0,0 +1,178 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\" this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\" contributors may be used to endorse or promote products derived from
+.\" this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd January 31, 2020
+.Dt CRYPTO_ASYM 9
+.Os
+.Sh NAME
+.Nm crypto_asym
+.Nd asymmetric cryptographic operations
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fn crypto_kdispatch "struct cryptkop *krp"
+.Ft void
+.Fn crypto_kdone "struct cryptkop *krp"
+.Ft int
+.Fn crypto_kregister "uint32_t driverid" "int kalg" "uint32_t flags"
+.Ft int
+.Fn CRYPTODEV_KPROCESS "device_t dev" "struct cryptop *krp" "int flags"
+.Sh DESCRIPTION
+The in-kernel cryptographic kernel framework supports asymmetric
+requests (keying requests) in addition to symmetric operations.
+There are currently no in-kernel users of these requests,
+but applications can make requests of hardware drivers via the
+.Pa /dev/crypto
+device .
+.Pp
+Some APIs are shared with the framework's symmetric request support.
+This manual describes the APIs and data structures unique to
+asymmetric requests.
+.Pp
+.Ss Request Objects
+A request is described by a
+.Vt struct cryptkop
+containing the following fields:
+.Bl -tag -width "krp_callback"
+.It Fa krp_op
+Operation to perform.
+Available operations include
+.Dv CRK_MOD_EXP ,
+.Dv CRK_MOD_EXP_CRT ,
+.Dv CRK_DSA_SIGN ,
+.Dv CRK_DSA_VERIFY ,
+and
+.Dv CRK_DH_COMPUTE_KEY .
+.It Fa krp_status
+Error status.
+Either zero on success,
+or an error if an operation fails.
+Set by drivers prior to completing a request via
+.Fn crypto_kdone .
+.It Fa krp_iparams
+Count of input parameters.
+.It Fa krp_oparams
+Count of output parameters.
+.It Fa krp_crid
+Requested device.
+.It Fa krp_hid
+Device used to complete the request.
+.It Fa krp_param
+Array of parameters.
+The array contains the input parameters first followed by the output
+parameters.
+Each parameter is stored as a bignum.
+Each bignum is described by a
+.Vt struct crparam
+containing the following fields:
+.Bl -tag -width "crp_nbits"
+.It Fa crp_p
+Pointer to array of packed bytes.
+.It Fa crp_nbits
+Size of bignum in bits.
+.El
+.It Fa krp_callback
+Callback function.
+This must point to a callback function of type
+.Vt void (*)(struct cryptkop *) .
+The callback function should inspect
+.Fa krp_status
+to determine the status of the completed operation.
+.El
+.Pp
+New requests should be initialized to zero before setting fields to
+appropriate values.
+Once the request has been populated,
+it should be passed to
+.Fn crypto_kdispatch .
+.Pp
+.Fn crypto_kdispatch
+will choose a device driver to perform the operation described by
+.Fa krp
+and invoke that driver's
+.Fn CRYPTO_KPROCESS
+method.
+.Ss Driver API
+Drivers register support for asymmetric operations by calling
+.Fn crypto_kregister
+for each supported algorithm.
+.Fa driverid
+should be the value returned by an earlier call to
+.Fn crypto_get_driverid .
+.Fa kalg
+should list one of the operations that can be set in
+.Fa krp_op .
+.Fa flags
+is a bitmask of zero or more of the following values:
+.Bl -tag -width "CRYPTO_ALG_FLAG_RNG_ENABLE"
+.It Dv CRYPTO_ALG_FLAG_RNG_ENABLE
+Device has a hardware RNG for DH/DSA.
+.It Dv CRYPTO_ALG_FLAG_DSA_SHA
+Device can compute a SHA digest of a message.
+.El
+.Pp
+Drivers unregister with the framework via
+.Fn crypto_unregister_all .
+.Pp
+Similar to
+.Fn CRYPTO_PROCESS ,
+.Fn CRYPTO_KPROCESS
+should complete the request or schedule it for asynchronous
+completion.
+If this method is not able to complete a request due to insufficient
+resources,
+it can defer the request (and future asymmetric requests) by returning
+.Dv ERESTART .
+Once resources are available,
+the driver should invoke
+.Fn crypto_unblock
+with
+.Dv CRYPTO_ASYMQ
+to resume processing of asymmetric requests.
+.Pp
+Once a request is completed,
+the driver should set
+.Fa krp_status
+and then call
+.Fn crypto_kdone .
+.Sh RETURN VALUES
+.Fn crypto_kdispatch ,
+.Fn crypto_kregister ,
+and
+.Fn CRYPTODEV_KPROCESS
+return zero on success or an error on failure.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_driver 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9
Index: share/man/man9/crypto_driver.9
===================================================================
--- /dev/null
+++ share/man/man9/crypto_driver.9
@@ -0,0 +1,392 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\" this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\" contributors may be used to endorse or promote products derived from
+.\" this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd January 31, 2020
+.Dt CRYPTO_DRIVER 9
+.Os
+.Sh NAME
+.Nm crypto_driver
+.Nd interface for symmetric cryptographic drivers
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fo crypto_apply
+.Fa "struct cryptop *crp"
+.Fa "int off"
+.Fa "int len"
+.Fa "int (*f)(void *, void *, u_int)"
+.Fa "void *arg"
+.Fc
+.Ft void *
+.Fo crypto_contiguous_subsegment
+.Fa "struct cryptop *crp"
+.Fa "size_t skip"
+.Fa "size_t len"
+.Fc
+.Ft void
+.Fn crypto_copyback "struct cryptop *crp" "int off" "int size" "const void *src"
+.Ft void
+.Fn crypto_copydata "struct cryptop *crp" "int off" "int size" "void *dst"
+.Ft void
+.Fn crypto_done "struct cryptop *crp"
+.Ft int32_t
+.Fn crypto_get_driverid "device_t dev" "size_t session_size" "int flags"
+.Ft void *
+.Fn crypto_get_driver_session "crypto_session_t crypto_session"
+.Ft int
+.Fn crypto_unblock "uint32_t driverid" "int what"
+.Ft int
+.Fn crypto_unregister_all "uint32_t driverid"
+.Ft int
+.Fn CRYPTODEV_FREESESSION "device_t dev" "crypto_session_t crypto_session"
+.Ft int
+.Fo CRYPTODEV_NEWSESSION
+.Fa "device_t dev"
+.Fa "crypto_session_t crypto_session"
+.Fa "const struct crypto_session_params *csp"
+.Fc
+.Ft int
+.Fo CRYPTODEV_PROBESESSION
+.Fa "device_t dev"
+.Fa "const struct crypto_session_params *csp"
+.Fc
+.Ft int
+.Fn CRYPTODEV_PROCESS "device_t dev" "struct cryptop *crp" "int flags"
+.Ft void
+.Fo hmac_init_ipad
+.Fa "struct auth_hash *axf"
+.Fa "const char *key"
+.Fa "int klen"
+.Fa "void *auth_ctx"
+.Fc
+.Ft void
+.Fo hmac_init_opad
+.Fa "struct auth_hash *axf"
+.Fa "const char *key"
+.Fa "int klen"
+.Fa "void *auth_ctx"
+.Fc
+.Sh DESCRIPTION
+Symmetric cryptographic drivers process cryptographic requests
+submitted to sessions associated with the driver.
+.Pp
+Cryptographic drivers call
+.Fn crypto_get_driverid
+to register with the cryptographic framework.
+.Fa dev
+is the device used to service requests.
+The
+.Fn CRYPTODEV
+methods are defined in the method table for the device driver attached to
+.Fa dev .
+.Fa session_size
+specifies the size of a driver-specific per-session structure allocated by
+the cryptographic framework.
+.Fa flags
+is a bitmask of properties about the driver.
+Exactly one of
+.Dv CRYPTOCAP_F_SOFTWARE
+or
+.Dv CRYPTOCAP_F_HARDWARE
+must be specified.
+.Dv CRYPTOCAP_F_SOFTWARE
+should be used for drivers which process requests using host CPUs.
+.Dv CRYPTOCAP_F_HARDWARE
+should be used for drivers which process requests on separate co-processors.
+.Dv CRYPTOCAP_F_SYNC
+should be set for drivers which process requests synchronously in
+.Fn CRYPTODEV_PROCESS .
+.Fn crypto_get_driverid
+returns an opaque driver id.
+.Pp
+.Fn crypto_unregister_all
+unregisters a driver from the cryptographic framework.
+If there are any pending operations or open sessions,
+this function will sleep.
+.Fa driverid
+is the value returned by an earlier call to
+.Fn crypto_get_driverid .
+.Pp
+When a new session is created by
+.Fn crypto_newsession ,
+.Fn CRYPTODEV_PROBESESSION
+is invoked by the cryptographic framework on each active driver to
+determine the best driver to use for the session.
+This method should inspect the session parameters in
+.Fa csp .
+If a driver does not support requests described by
+.Fa csp ,
+this method should return an error value.
+If the driver does support requests described by
+.Fa csp ,
+it should return a negative value.
+The framework prefers drivers with the largest negative value,
+similar to
+.Xr DEVICE_PROBE 9 .
+The following values are defined for non-error return values from this
+method:
+.Bl -tag -width "CRYPTODEV_PROBE_ACCEL_SOFTWARE"
+.It Dv CRYPTODEV_PROBE_HARDWARE
+The driver processes requests via a co-processor.
+.It Dv CRYPTODEV_PROBE_ACCEL_SOFTWARE
+The driver processes requests on the host CPU using optimized instructions
+such as AES-NI.
+.It Dv CRYPTODEV_PROBE_SOFTWARE
+The driver processes requests on the host CPU.
+.El
+.Pp
+This method should not sleep.
+.Pp
+Once the framework has chosen a driver for a session,
+the framework invokes the
+.Fn CRYPTODEV_NEWSESSION
+method to initialize driver-specific session state.
+Prior to calling this method,
+the framework allocates a per-session driver-specific data structure.
+This structure is initialized with zeroes,
+and its size is set by the
+.Fa session_size
+passed to
+.Fn crypto_get_driverid .
+This method can retrieve a pointer to this data structure by passing
+.Fa crypto_session
+to
+.Fn crypto_get_driver_session .
+Session parameters are described in
+.Fa csp .
+.Pp
+This method should not sleep.
+.Pp
+.Fn CRYPTODEV_FREESESSION
+is invoked to release any driver-specific state when a session is
+destroyed.
+The per-session driver-specific data structure is explicitly zeroed
+and freed by the framework after this method returns.
+If a driver requires no additional tear-down steps, it can leave
+this method undefined.
+.Pp
+This method should not sleep.
+.Pp
+.Fn CRYPTODEV_PROCESS
+is invoked for each request submitted to an active session.
+This method can either complete a request synchronously or
+schedule it to be completed asynchronously,
+but it must not sleep.
+.Pp
+If this method is not able to complete a request due to insufficient
+resources such as a full command queue,
+it can defer the request by returning
+.Dv ERESTART .
+The request will be queued by the framework and retried once the
+driver releases pending requests via
+.Fn crypto_unblock .
+Any requests submitted to sessions belonging to the driver will also
+be queued until
+.Fn crypto_unblock
+is called.
+.Pp
+If a driver encounters errors while processing a request,
+it should report them via the
+.Fa crp_etype
+field of
+.Fa crp
+rather than returning an error directly.
+.Pp
+.Fa flags
+may be set to
+.Dv CRYPTO_HINT_MORE
+if there are additional requests queued for this driver.
+The driver can use this as a hint to batch completion interrupts.
+Note that these additional requests may be from different sessions.
+.Pp
+.Fn crypto_get_driver_session
+returns a pointer to the driver-specific per-session data structure
+for the session
+.Fa crypto_session .
+This function can be used in the
+.Fn CRYPTODEV_NEWSESSION ,
+.Fn CRYPTODEV_PROCESS ,
+and
+.Fn CRYPTODEV_FREESESSION
+callbacks.
+.Pp
+.Fn crypto_copydata
+copies
+.Fa size
+bytes out of the data buffer for
+.Fa crp
+into a local buffer pointed to by
+.Fa dst .
+The bytes are read starting at an offset of
+.Fa off
+bytes in the request's data buffer.
+.Pp
+.Fn crypto_copyback
+copies
+.Fa size
+bytes from the local buffer pointed to by
+.Fa src
+into the data buffer for
+.Fa crp .
+The bytes are written starting at an offset of
+.Fa off
+bytes in the request's data buffer.
+.Pp
+A driver calls
+.Fn crypto_done
+to mark the request
+.Fa crp
+as completed.
+Any errors should be set in
+.Fa crp_etype
+prior to calling this function.
+.Pp
+If a driver defers a request by returning
+.Dv ERESTART
+from
+.Dv CRYPTO_PROCESS ,
+the framework will queue all requests for the driver until the driver calls
+.Fn crypto_unblock
+to indicate that the temporary resource shortage has been relieved.
+For example,
+if a driver returns
+.Dv ERESTART
+due to a full command ring,
+it would invoke
+.Fn crypto_unblock
+from a command completion interrupt that makes a command ring entry available.
+.Fa driverid
+is the value returned by
+.Fn crypto_get_driverid .
+.Fa what
+indicates which types of requests the driver is able to handle again:
+.Bl -tag -width "CRYPTO_ASYMQ"
+.It Dv CRYPTO_SYMQ
+indicates that the driver is able to handle symmetric requests passed to
+.Fn CRYPTODEV_PROCESS .
+.It Dv CRYPTO_ASYMQ
+indicates that the driver is able to handle asymmetric requests passed to
+.Fn CRYPTODEV_KPROCESS .
+.El
+.Pp
+.Fn crypto_apply
+is a helper routine that can be used to invoke a caller-supplied function
+to a region of the data buffer for
+.Fa crp .
+The function
+.Fa f
+is called one or more times.
+For each invocation,
+the first argument to
+.Fa f
+is the value of
+.Fa arg passed to
+.Fn crypto_apply .
+The second and third arguments to
+.Fa f
+are a pointer and length to a segment of the buffer mapped into the kernel.
+The function is called enough times to cover the
+.Fa len
+bytes of the data buffer which starts at an offset
+.Fa off .
+If any invocation of
+.Fa f
+returns a non-zero value,
+.Fn crypto_apply
+immediately returns that value without invoking
+.Fa f
+on any remaining segments of the region,
+otherwise
+.Fn crypto_apply
+returns the value from the final call to
+.Fa f .
+.Pp
+.Fn crypto_contiguous_subsegment
+attempts to locate a single, virtually-contiguous segment of the data buffer
+for
+.Fa crp .
+The segment must be
+.Fa len
+bytes long and start at an offset of
+.Fa skip
+bytes.
+If a segment is found,
+a pointer to the start of the segment is returned.
+Otherwise,
+.Dv NULL
+is returned.
+.Pp
+.Fn hmac_init_ipad
+prepares an authentication context to generate the inner hash of an HMAC.
+.Fa axf
+is a software implementation of an authentication algorithm such as the
+value returned by
+.Fn crypto_auth_hash .
+.Fa key
+is a pointer to a HMAC key of
+.Fa klen
+bytes.
+.Fa auth_ctx
+points to a valid authentication context for the desired algorithm.
+The function initializes the context with the supplied key.
+.Pp
+.Fn hmac_init_opad
+is similar to
+.Fn hmac_init_ipad
+except that it prepares an authentication context to generate the
+outer hash of an HMAC.
+.Sh RETURN VALUES
+.Fn crypto_apply
+returns the return value from the caller-supplied callback function.
+.Pp
+.Fn crypto_contiguous_subsegment
+returns a pointer to a contiguous segment or
+.Dv NULL .
+.Pp
+.Fn crypto_get_driverid
+returns a driver identifier on success or -1 on error.
+.Pp
+.Fn crypto_unblock ,
+.Fn crypto_unregister_all ,
+.Fn CRYPTODEV_FREESESSION ,
+.Fn CRYPTODEV_NEWSESSION ,
+and
+.Fn CRYPTODEV_PROCESS
+return zero on success or an error on failure.
+.Pp
+.Fn CRYPTODEV_PROBESESSION
+returns a negative value on success or an error on failure.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9
Index: share/man/man9/crypto_request.9
===================================================================
--- /dev/null
+++ share/man/man9/crypto_request.9
@@ -0,0 +1,419 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\" this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\" contributors may be used to endorse or promote products derived from
+.\" this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd January 14, 2020
+.Dt CRYPTO_REQUEST 9
+.Os
+.Sh NAME
+.Nm crypto_request
+.Nd symmetric cryptographic operations
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fn crypto_dispatch "struct cryptop *crp"
+.Ft void
+.Fn crypto_freereq "struct cryptop *crp"
+.Ft "struct cryptop *"
+.Fn crypto_getreq "crypto_session_t cses" "int how"
+.Sh DESCRIPTION
+Each symmetric cryptographic operation in the kernel is described by
+an instance of
+.Vt struct cryptop
+and is associated with an active session.
+.Pp
+New requests are allocated by
+.Fn crypto_getreq .
+.Fa cses
+is a reference to an active session.
+.Fa how
+is passed to
+.Xr malloc 9
+and should be set to either
+.Dv M_NOWAIT
+or
+.Dv M_WAITOK .
+The caller should then set fields in the returned structure to describe
+request-specific parameters.
+Unused fields should be left as-is.
+.Pp
+.Fn crypto_dispatch
+passes a crypto request to the driver attached to the request's session.
+If there are errors in the request's fields, this function may return
+an error to the caller.
+If errors are encountered while servicing the request, they will instead
+be reported to the request's callback function
+.Pq Fa crp_callback
+via
+.Fa crp_etype .
+.Pp
+Note that a request's callback function may be invoked before
+.Fn crypto_dispatch
+returns.
+.Pp
+Once a request has signaled completion by invoking its callback function,
+it should be feed via
+.Fn crypto_freereq .
+.Pp
+Cryptographic operations include several fields to describe the request.
+.Ss Buffer Types
+Requests are associated with a single data buffer that is modified in place.
+The type of the data buffer and the buffer itself are described by the
+following fields:
+.Bl -tag -width crp_buf_type
+.It Fa crp_buf_type
+The type of the data buffer.
+The following types are supported:
+.Bl -tag -width CRYPTO_BUF_CONTIG
+.It Dv CRYPTO_BUF_CONTIG
+An array of bytes mapped into the kernel's address space.
+.It Dv CRYPTO_BUF_UIO
+A scatter/gather list of kernel buffers as described in
+.Xr uio 9 .
+.It Dv CRYPTO_BUF_MBUF
+A network memory buffer as described in
+.Xr mbuf 9 .
+.El
+.It Fa crp_buf
+A pointer to the start of a
+.Dv CRYPTO_BUF_CONTIG
+data buffer.
+.It Fa crp_ilen
+The length of a
+.Dv CRYPTO_BUF_CONTIG
+data buffer
+.It Fa crp_mbuf
+A pointer to a
+.Vt struct mbuf
+for
+.Dv CRYPTO_BUF_MBUF .
+.It Fa crp_uio
+A pointer to a
+.Vt struct uio
+for
+.Dv CRYPTO_BUF_UIO .
+.It Fa crp_olen
+Used with compression and decompression requests to describe the updated
+length of the payload region in the data buffer.
+.Pp
+If a compression request increases the size of the payload,
+then the data buffer is unmodified, the request completes successfully,
+and
+.Fa crp_olen
+is set to the size the compressed data would have used.
+Callers can compare this to the payload region length to determine if
+the compressed data was discarded.
+.El
+.Ss Request Regions
+Each request describes one or more regions in the data buffer using.
+Each region is described by an offset relative to the start of the
+data buffer and a length.
+The length of some regions is the same for all requests belonging to
+a session.
+Those lengths are set in the session parameters of the associated
+session.
+All requests must define a payload region.
+Other regions are only required for specific session modes.
+The following regions are defined:
+.Bl -column "Payload" "crp_payload_start" "crp_payload_length"
+.It Sy Region Ta Sy Start Ta Sy Length Ta Sy Description
+.It AAD Ta Fa crp_aad_start Ta Fa crp_aad_length Ta
+Additional Authenticated Data
+.It IV Ta Fa crp_iv_start Ta Fa csp_ivlen Ta
+Embedded IV or nonce
+.It Payload Ta Fa crp_payload_start Ta Fa crp_payload_length Ta
+Data to encrypt, decrypt, compress, or decompress
+.It Digest Ta Fa crp_digest_start Ta Fa csp_auth_mlen Ta
+Authentication digest, hash, or tag
+.El
+.Pp
+Requests are permitted to operate on only a subset of the data buffer.
+For example,
+requests from IPsec operate on network packets that include headers not
+used as either additional authentication data (AAD) or payload data.
+.Ss Request Operations
+All requests must specify the type of operation to perform in
+.Fa crp_op .
+Available operations depend on the session's mode.
+.Pp
+Compression requests support the following operations:
+.Bl -tag -width CRYPTO_OP_DECOMPRESS
+.It Dv CRYPTO_OP_COMPRESS
+Compress the data in the payload region of the data buffer.
+.It Dv CRYPTO_OP_DECOMPRESS
+Decompress the data in the payload region of the data buffer.
+.El
+.Pp
+Cipher requests support the following operations:
+.Bl -tag -width CRYPTO_OP_DECRYPT
+.It Dv CRYPTO_OP_ENCRYPT
+Encrypt the data in the payload region of the data buffer.
+.It Dv CRYPTO_OP_DECRYPT
+Decrypt the data in the payload region of the data buffer.
+.El
+.Pp
+Digest requests support the following operations:
+.Bl -tag -width CRYPTO_OP_COMPUTE_DIGEST
+.It Dv CRYPTO_OP_COMPUTE_DIGEST
+Calculate a digest over the payload region of the data buffer
+and store the result in the digest region.
+.It Dv CRYPTO_OP_VERIFY_DIGEST
+Calculate a digest over the payload region of the data buffer.
+Compare the calculated digest to the existing digest from the digest region.
+If the digests match,
+complete the request successfully.
+If the digests do not match,
+fail the request with
+.Er EBADMSG .
+.El
+.Pp
+AEAD and Encrypt-then-Authenticate requests support the following
+operations:
+.Bl -tag -width CRYPTO_OP
+.It Dv CRYPTO_OP_ENCRYPT | Dv CRYPTO_OP_COMPUTE_DIGEST
+Encrypt the data in the payload region of the data buffer.
+Calculate a digest over the AAD and payload regions and store the
+result in the data buffer.
+.It Dv CRYPTO_OP_DECRYPT | Dv CRYPTO_OP_VERIFY_DIGEST
+Calculate a digest over the AAD and payload regions of the data buffer.
+Compare the calculated digest to the existing digest from the digest region.
+If the digests match,
+decrypt the payload region.
+If the digests do not match,
+fail the request with
+.Er EBADMSG .
+.El
+.Ss Request IV and/or Nonce
+Some cryptographic operations require an IV or nonce as an input.
+An IV may be stored either in the IV region of the data buffer or in
+.Fa crp_iv .
+By default,
+the IV is assumed to be stored in the IV region.
+If the IV is stored in
+.Fa crp_iv ,
+.Dv CRYPTO_F_IV_SEPARATE
+should be set in
+.Fa crp_flags
+and
+.Fa crp_digest_start
+should be left as zero.
+.Pp
+An encryption request using an IV stored in the IV region may set
+.Dv CRYPTO_F_IV_GENERATE
+in
+.Fa crp_flags
+to request that the driver generate a random IV.
+Note that
+.Dv CRYPTO_F_IV_GENERATE
+cannot be used with decryption operations or in combination with
+.Dv CRYPTO_F_IV_SEPARATE .
+.Pp
+Requests that store part, but not all, of the IV in the data buffer should
+store the partial IV in the data buffer and pass the full IV separately in
+.Fa crp_iv .
+.Ss Request and Callback Scheduling
+The crypto framework provides multiple methods of scheduling the dispatch
+of requests to drivers along with the processing of driver callbacks.
+Requests use flags in
+.Fa crp_flags
+to select the desired scheduling methods.
+.Pp
+.Fn crypto_dispatch
+can pass the request to the session's driver via three different methods:
+.Bl -enum
+.It
+The request is queued to a taskqueue backed by a pool of worker threads.
+By default the pool is sized to provide one thread for each CPU.
+Worker threads dequeue requests and pass them to the driver
+asynchronously.
+.It
+The request is passed to the driver synchronously in the context of the
+thread invoking
+.Fn crypto_dispatch .
+.It
+The request is queued to a queue of pending requests.
+A single worker thread dequeues requests and passes them to the driver
+asynchronously.
+.El
+.Pp
+To select the first method (taskqueue backed by multiple threads),
+requests should set
+.Dv CRYPTO_F_ASYNC .
+To always use the third method (queue to single worker thread),
+requests should set
+.Dv CRYPTO_F_BATCH .
+If both flags are set,
+.Dv CRYPTO_F_ASYNC
+takes precedence.
+If neither flag is set,
+.Fn crypto_dispatch
+will first attempt the second method (invoke driver synchronously).
+If the driver is blocked,
+the request will be queued using the third method.
+One caveat is that the first method is only used for requests using software
+drivers which use host CPUs to process requests.
+Requests whose session is associated with a hardware driver will ignore
+.Dv CRYPTO_F_ASYNC
+and only use
+.Dv CRYPTO_F_BATCH
+to determine how requests should be scheduled.
+.Pp
+In addition to bypassing synchronous dispatch in
+.Fn crypto_dispatch ,
+.Dv CRYPTO_F_BATCH
+requests additional changes aimed at optimizing batches of requests to
+the same driver.
+When the worker thread processes a request with
+.Dv CRYPTO_F_BATCH ,
+it will search the pending request queue for any other requests for the same
+driver,
+including requests from different sessions.
+If any other requests are present,
+.Dv CRYPTO_HINT_MORE
+is passed to the driver's process method.
+Drivers may use this to batch completion interrupts.
+.Pp
+Callback function scheduling is simpler than request scheduling.
+Callbacks can either be invoked synchronously from
+.Fn crypto_done ,
+or they can be queued to a pool of worker threads.
+This pool of worker threads is also sized to provide one worker thread
+for each CPU by default.
+Note that a callback function invoked synchronously from
+.Fn crypto_done
+must follow the same restrictions placed on threaded interrupt handlers.
+.Pp
+By default,
+callbacks are invoked asynchronously by a worker thread.
+If
+.Dv CRYPTO_F_CBIMM
+is set,
+the callback is always invoked synchronously from
+.Fn crypto_done .
+If
+.Dv CRYPTO_F_CBIFSYNC
+is set,
+the callback is invoked synchronously if the request was processed by a
+software driver or asynchronously if the request was processed by a
+hardware driver.
+.Pp
+If a request was scheduled to the taskqueue via
+.Dv CRYPTO_F_ASYNC ,
+callbacks are always invoked asynchronously ignoring
+.Dv CRYPTO_F_CBIMM
+and
+.Dv CRYPTO_F_CBIFSYNC .
+In this case,
+.Dv CRYPTO_F_ASYNC_KEEPORDER
+may be set to ensure that callbacks for requests on a given session are
+invoked in the same order that requests were queued to the session via
+.Fn crypto_dispatch .
+This flag is used by IPsec to ensure that decrypted network packets are
+passed up the network stack in roughly the same order they were received.
+.Pp
+.Ss Other Request Fields
+In addition to the fields and flags enumerated above,
+.Vt struct cryptop
+includes the following:
+.Bl -tag -width crp_payload_length
+.It Fa crp_session
+A reference to the active session.
+This is set when the request is created by
+.Fn crypto_getreq
+and should not be modified.
+Drivers can use this to fetch driver-specific session state or
+session parameters.
+.It Fa crp_etype
+Error status.
+Either zero on success, or an error if a request fails.
+Set by drivers prior to completing a request via
+.Fn crypto_done .
+.It Fa crp_flags
+A bitmask of flags.
+The following flags are available in addition to flags discussed previously:
+.Bl -tag -width CRYPTO_F_DONE
+.It Dv CRYPTO_F_DONE
+Set by
+.Fa crypto_done
+before calling
+.Fa crp_callback .
+This flag is not very useful and will likely be removed in the future.
+It can only be safely checked from the callback routine at which point
+it is always set.
+.El
+.It Fa crp_cipher_key
+Pointer to a request-specific encryption key.
+If this value is not set,
+the request uses the session encryption key.
+.It Fa crp_auth_key
+Pointer to a request-specific authentication key.
+If this value is not set,
+the request uses the session authentication key.
+.It Fa crp_opaque
+An opaque pointer.
+This pointer permits users of the cryptographic framework to store
+information about a request to be used in the callback.
+.It Fa crp_callback
+Callback function.
+This must point to a callback function of type
+.Vt void (*)(struct cryptop *) .
+The callback function should inspect
+.Fa crp_etype
+to determine the status of the completed operation.
+It should also arrange for the request to be freed via
+.Fn crypto_freereq .
+.El
+.Sh RETURN VALUES
+.Fn crypto_dispatch
+returns an error if the request contained invalid fields,
+or zero if the request was valid.
+.Fn crypto_getreq
+returns a pointer to a new request structure on success,
+or
+.Dv NULL
+on failure.
+.Dv NULL
+can only be returned if
+.Dv M_NOWAIT
+was passed in
+.Fa how .
+.Sh SEE ALSO
+.Xr ipsec 4 ,
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_session 9 ,
+.Xr mbuf 9
+.Xr uio 9
+.Sh BUGS
+Not all drivers properly handle mixing session and per-request keys
+within a single session.
+Consumers should either use a single key for a session specified in
+the session parameters or always use per-request keys.
Index: share/man/man9/crypto_session.9
===================================================================
--- /dev/null
+++ share/man/man9/crypto_session.9
@@ -0,0 +1,245 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\" this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\" contributors may be used to endorse or promote products derived from
+.\" this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd January 13, 2020
+.Dt CRYPTO_SESSION 9
+.Os
+.Sh NAME
+.Nm crypto_session
+.Nd state used for symmetric cryptographic services
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft struct auth_hash *
+.Fn crypto_auth_hash "const struct crypto_session_params *csp"
+.Ft struct enc_xform *
+.Fn crypto_cipher "const struct crypto_session_params *csp"
+.Ft const struct crypto_session_params *
+.Fn crypto_get_params "crypto_session_t cses"
+.Ft int
+.Fo crypto_newsession
+.Fa "crypto_session_t *cses"
+.Fa "const struct crypto_session_params *csp"
+.Fa "int crid"
+.Fc
+.Ft int
+.Fn crypto_freesession "crypto_session_t cses"
+.Sh DESCRIPTION
+Symmetric cryptographic operations in the kernel are associated with
+cryptographic sessions.
+Sessions hold state shared across multiple requests.
+Active sessions are associated with a single cryptographic driver.
+.Pp
+The
+.Vt crypto_session_t
+type represents an opaque reference to an active session.
+Session objects are allocated and managed by the cryptographic
+framework.
+.Pp
+New sessions are created by
+.Fn crypto_newsession .
+.Fa csp
+describes various parameters associated with the new session such as
+the algorithms to use and any session-wide keys.
+.Fa crid
+can be used to request either a specific cryptographic driver or
+classes of drivers.
+For the latter case,
+.Fa crid
+should be set to a mask of the following values:
+.Bl -tag -width "CRYPTOCAP_F_HARDWARE"
+.It Dv CRYPTOCAP_F_HARDWARE
+Request hardware drivers.
+Hardware drivers do not use the host CPU to perform operations.
+Typically, a separate co-processor performs the operations asynchronously.
+.It Dv CRYPTOCAP_F_SOFTWARE
+Request software drivers.
+Software drivers use the host CPU to perform operations.
+The kernel includes a simple, yet portable implementation of each supported
+algorithm in the
+.Xr cryptosoft 4
+driver.
+Additional software drivers may also be available on architectures which
+provide instructions designed to accelerate cryptographic operations.
+.El
+.Pp
+If both hardware and software drivers are requested,
+hardware drivers are preferred over software drivers.
+Accelerated software drivers are preferred over the baseline software driver.
+If multiple hardware drivers are available,
+the framework will distribute sessions across these drivers in a round-robin
+fashion.
+.Pp
+On success,
+.Fn crypto_newsession
+saves a reference to the newly created session in
+.Fa cses .
+.Pp
+.Fn crypto_freesession
+is used to free the resources associated with the session
+.Fa cses .
+.Pp
+.Fn crypto_auth_hash
+returns a structure describing the baseline software implementation of an
+authentication algorithm requested by
+.Fa csp .
+If
+.Fa csp
+does not specify an authentication algorithm,
+or requests an invalid algorithm,
+.Dv NULL
+is returned.
+.Pp
+.Fn crypto_cipher
+returns a structure describing the baseline software implementation of an
+encryption algorithm requested by
+.Fa csp .
+If
+.Fa csp
+does not specify an encryption algorithm,
+or requests an invalid algorithm,
+.Dv NULL
+is returned.
+.Pp
+.Fn crypto_get_params
+returns a pointer to the session parameters used by
+.Fa cses .
+.Ss Session Parameters
+Session parameters are used to describe the cryptographic operations
+performed by cryptographic requests.
+Parameters are stored in an instance of
+.Vt struct crypto_session_params .
+When initializing parameters to pass to
+.Fn crypto_newsession ,
+the entire structure should first be zeroed.
+Needed fields should then be set leaving unused fields as zero.
+This structure contains the following fields:
+.Bl -tag -width csp_cipher_klen
+.It Fa csp_mode
+Type of operation to perform.
+This field must be set to one of the following:
+.Bl -tag -width CSP_MODE_COMPRESS
+.It Dv CSP_MODE_COMPRESS
+Compress or decompress request payload.
+.Pp
+The compression algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_CIPHER
+Encrypt or decrypt request payload.
+.Pp
+The encryption algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_DIGEST
+Compute or verify a digest, or hash, of request payload.
+.Pp
+The authentication algorithm is specified in
+.Fa csp_auth_alg .
+.It Dv CSP_MODE_AEAD
+Authenticated encryption with additional data.
+Decryption operations require the digest, or tag,
+and fail if it does not match.
+.Pp
+The AEAD algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_ETA
+Encrypt-then-Authenticate.
+In this mode, encryption operations encrypt the payload and then
+compute an authentication digest over the request additional authentication
+data followed by the encrypted payload.
+Decryption operations fail without decrypting the data if the provided digest
+does not match.
+.Pp
+The encryption algorithm is specified in
+.Fa csp_cipher_alg
+and the authentication algorithm is specified in
+.Fa csp_auth_alg .
+.El
+.It Fa csp_flags
+Currently, no additional flags are defined and this field should be set to
+zero.
+.It Fa csp_ivlen
+If either the cipher or authentication algorithms require an explicit
+initialization vector (IV) or nonce,
+this specifies the length in bytes.
+All requests for a session use the same IV length.
+.It Fa csp_cipher_alg
+Encryption or compression algorithm.
+.It Fa csp_cipher_klen
+Length of encryption or decryption key in bytes.
+All requests for a session use the same key length.
+.It Fa csp_cipher_key
+Pointer to encryption or decryption key.
+If all requests for a session use request-specific keys,
+this field should be left as
+.Dv NULL .
+This pointer and associated key must remain valid for the duration of the
+crypto session.
+.It Fa csp_auth_alg
+Authentication algorithm.
+.It Fa csp_auth_klen
+Length of authentication key in bytes.
+If the authentication algorithm does not use a key,
+this field should be left as zero.
+.It Fa csp_auth_key
+Pointer to the authentication key.
+If all requests for a session use request-specific keys,
+this field should be left as
+.Dv NULL .
+This pointer and associated key must remain valid for the duration of the
+crypto session.
+.It Fa csp_auth_mlen
+The length in bytes of the digest.
+If zero, the full length of the digest is used.
+If non-zero, the first
+.Fa csp_auth_mlen
+bytes of the digest are used.
+.El
+.Sh RETURN VALUES
+.Fn crypto_newsession
+returns a non-zero value if an error occurs or zero on success.
+.Pp
+.Fn crypto_auth_hash
+and
+.Fn crypto_cipher
+return
+.Dv NULL
+if the request is valid or a pointer to a structure on success.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_request 9
+.Sh BUGS
+The current implementation of
+.Nm crypto_freesession
+does not provide a way for the caller to know that there are no other
+references to the keys stored in the session's associated parameters.
+This function should probably sleep until any in-flight cryptographic
+operations associated with the session are completed.
Index: sys/crypto/aesni/aesni.h
===================================================================
--- sys/crypto/aesni/aesni.h
+++ sys/crypto/aesni/aesni.h
@@ -56,16 +56,16 @@
uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16);
uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16);
- /* Same as the SHA256 Blocksize. */
- uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
- int algo;
int rounds;
/* uint8_t *ses_ictx; */
/* uint8_t *ses_octx; */
- /* int ses_mlen; */
int used;
- int auth_algo;
int mlen;
+ int hash_len;
+ void (*hash_init)(void *);
+ void (*hash_update)(void *, const void *, unsigned);
+ void (*hash_finalize)(void *, void *);
+ bool hmac;
};
/*
@@ -120,7 +120,7 @@
const unsigned char *addt, const unsigned char *ivec,
const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
const unsigned char *key, int nr);
-int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
- int keylen);
+void aesni_cipher_setup_common(struct aesni_session *ses,
+ const struct crypto_session_params *csp, const uint8_t *key, int keylen);
#endif /* _AESNI_H_ */
Index: sys/crypto/aesni/aesni.c
===================================================================
--- sys/crypto/aesni/aesni.c
+++ sys/crypto/aesni/aesni.c
@@ -88,16 +88,13 @@
(ctx) = NULL; \
} while (0)
-static int aesni_newsession(device_t, crypto_session_t cses,
- struct cryptoini *cri);
static int aesni_cipher_setup(struct aesni_session *ses,
- struct cryptoini *encini, struct cryptoini *authini);
-static int aesni_cipher_process(struct aesni_session *ses,
- struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
-static int aesni_cipher_crypt(struct aesni_session *ses,
- struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
-static int aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
- struct cryptop *crp);
+ const struct crypto_session_params *csp);
+static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp);
+static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp);
+static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp);
MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
@@ -170,7 +167,7 @@
sc = device_get_softc(dev);
sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
- CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+ CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM);
@@ -187,25 +184,6 @@
}
detect_cpu_features(&sc->has_aes, &sc->has_sha);
- if (sc->has_aes) {
- crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
- }
- if (sc->has_sha) {
- crypto_register(sc->cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_224, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_224_HMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_256, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- }
return (0);
}
@@ -223,115 +201,125 @@
return (0);
}
+static bool
+aesni_auth_supported(struct aesni_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if (!sc->has_sha)
+ return (false);
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ break;
+ default:
+ return (false);
+ }
+
+ return (true);
+}
+
+static bool
+aesni_cipher_supported(struct aesni_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if (!sc->has_aes)
+ return (false);
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ return (sc->has_aes);
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ return (sc->has_aes);
+ default:
+ return (false);
+ }
+}
+
static int
-aesni_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+aesni_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct aesni_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!aesni_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!aesni_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (csp->csp_auth_mlen != 0 &&
+ csp->csp_auth_mlen != GMAC_DIGEST_LEN)
+ return (EINVAL);
+ if (csp->csp_ivlen != AES_GCM_IV_LEN ||
+ !sc->has_aes)
+ return (EINVAL);
+ break;
+ case CRYPTO_AES_CCM_16:
+ if (csp->csp_auth_mlen != 0 &&
+ csp->csp_auth_mlen != AES_CBC_MAC_HASH_LEN)
+ return (EINVAL);
+ if (csp->csp_ivlen != AES_CCM_IV_LEN ||
+ !sc->has_aes)
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ case CSP_MODE_ETA:
+ if (!aesni_auth_supported(sc, csp) ||
+ !aesni_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
+
+static int
+aesni_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
struct aesni_softc *sc;
struct aesni_session *ses;
- struct cryptoini *encini, *authini;
- bool gcm_hash, gcm;
- bool cbc_hash, ccm;
int error;
- KASSERT(cses != NULL, ("EDOOFUS"));
- if (cri == NULL) {
- CRYPTDEB("no cri");
- return (EINVAL);
- }
-
sc = device_get_softc(dev);
ses = crypto_get_driver_session(cses);
- authini = NULL;
- encini = NULL;
- gcm = false;
- gcm_hash = false;
- ccm = cbc_hash = false;
-
- for (; cri != NULL; cri = cri->cri_next) {
- switch (cri->cri_alg) {
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_CCM_16:
- if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) {
- gcm = true;
- } else if (cri->cri_alg == CRYPTO_AES_CCM_16) {
- ccm = true;
- }
- /* FALLTHROUGH */
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_XTS:
- if (!sc->has_aes)
- goto unhandled;
- if (encini != NULL) {
- CRYPTDEB("encini already set");
- return (EINVAL);
- }
- encini = cri;
- break;
- case CRYPTO_AES_CCM_CBC_MAC:
- cbc_hash = true;
- authini = cri;
- break;
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- /*
- * nothing to do here, maybe in the future cache some
- * values for GHASH
- */
- if (authini != NULL) {
- CRYPTDEB("authini already set");
- return (EINVAL);
- }
- gcm_hash = true;
- authini = cri;
- break;
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_256_HMAC:
- if (!sc->has_sha)
- goto unhandled;
- if (authini != NULL) {
- CRYPTDEB("authini already set");
- return (EINVAL);
- }
- authini = cri;
- break;
- default:
-unhandled:
- CRYPTDEB("unhandled algorithm");
- return (EINVAL);
- }
- }
- if (encini == NULL && authini == NULL) {
- CRYPTDEB("no cipher");
- return (EINVAL);
- }
- /*
- * GMAC algorithms are only supported with simultaneous GCM. Likewise
- * GCM is not supported without GMAC.
- */
- if (gcm_hash != gcm) {
- CRYPTDEB("gcm_hash != gcm");
- return (EINVAL);
- }
-
- if (cbc_hash != ccm) {
- CRYPTDEB("cbc_hash != ccm");
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_CIPHER:
+ case CSP_MODE_AEAD:
+ case CSP_MODE_ETA:
+ break;
+ default:
return (EINVAL);
}
-
- if (encini != NULL)
- ses->algo = encini->cri_alg;
- if (authini != NULL)
- ses->auth_algo = authini->cri_alg;
-
- error = aesni_cipher_setup(ses, encini, authini);
+ error = aesni_cipher_setup(ses, csp);
if (error != 0) {
CRYPTDEB("setup failed");
return (error);
@@ -344,108 +332,31 @@
aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
{
struct aesni_session *ses;
- struct cryptodesc *crd, *enccrd, *authcrd;
- int error, needauth;
-
- ses = NULL;
- error = 0;
- enccrd = NULL;
- authcrd = NULL;
- needauth = 0;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL ||
- crp->crp_session == NULL) {
- error = EINVAL;
- goto out;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_CCM_16:
- needauth = 1;
- /* FALLTHROUGH */
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_XTS:
- if (enccrd != NULL) {
- error = EINVAL;
- goto out;
- }
- enccrd = crd;
- break;
-
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- case CRYPTO_AES_CCM_CBC_MAC:
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_256_HMAC:
- if (authcrd != NULL) {
- error = EINVAL;
- goto out;
- }
- authcrd = crd;
- break;
-
- default:
- error = EINVAL;
- goto out;
- }
- }
-
- if ((enccrd == NULL && authcrd == NULL) ||
- (needauth && authcrd == NULL)) {
- error = EINVAL;
- goto out;
- }
-
- /* CBC & XTS can only handle full blocks for now */
- if (enccrd != NULL && (enccrd->crd_alg == CRYPTO_AES_CBC ||
- enccrd->crd_alg == CRYPTO_AES_XTS) &&
- (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
- error = EINVAL;
- goto out;
- }
+ int error;
ses = crypto_get_driver_session(crp->crp_session);
- KASSERT(ses != NULL, ("EDOOFUS"));
- error = aesni_cipher_process(ses, enccrd, authcrd, crp);
- if (error != 0)
- goto out;
+ error = aesni_cipher_process(ses, crp);
-out:
crp->crp_etype = error;
crypto_done(crp);
- return (error);
+ return (0);
}
static uint8_t *
-aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
- bool *allocated)
+aesni_cipher_alloc(struct cryptop *crp, int start, int length, bool *allocated)
{
uint8_t *addr;
- addr = crypto_contiguous_subsegment(crp->crp_flags,
- crp->crp_buf, enccrd->crd_skip, enccrd->crd_len);
+ addr = crypto_contiguous_subsegment(crp, start, length);
if (addr != NULL) {
*allocated = false;
return (addr);
}
- addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
+ addr = malloc(length, M_AESNI, M_NOWAIT);
if (addr != NULL) {
*allocated = true;
- crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, addr);
+ crypto_copydata(crp, start, length, addr);
} else
*allocated = false;
return (addr);
@@ -457,6 +368,7 @@
DEVMETHOD(device_attach, aesni_attach),
DEVMETHOD(device_detach, aesni_detach),
+ DEVMETHOD(cryptodev_probesession, aesni_probesession),
DEVMETHOD(cryptodev_newsession, aesni_newsession),
DEVMETHOD(cryptodev_process, aesni_process),
@@ -474,63 +386,7 @@
MODULE_VERSION(aesni, 1);
MODULE_DEPEND(aesni, crypto, 1, 1, 1);
-static int
-aesni_authprepare(struct aesni_session *ses, int klen, const void *cri_key)
-{
- int keylen;
-
- if (klen % 8 != 0)
- return (EINVAL);
- keylen = klen / 8;
- if (keylen > sizeof(ses->hmac_key))
- return (EINVAL);
- if (ses->auth_algo == CRYPTO_SHA1 && keylen > 0)
- return (EINVAL);
- memcpy(ses->hmac_key, cri_key, keylen);
- return (0);
-}
-
-static int
-aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini,
- struct cryptoini *authini)
-{
- struct fpu_kern_ctx *ctx;
- int kt, ctxidx, error;
-
- switch (ses->auth_algo) {
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_256_HMAC:
- error = aesni_authprepare(ses, authini->cri_klen,
- authini->cri_key);
- if (error != 0)
- return (error);
- ses->mlen = authini->cri_mlen;
- }
-
- kt = is_fpu_kern_thread(0) || (encini == NULL);
- if (!kt) {
- ACQUIRE_CTX(ctxidx, ctx);
- fpu_kern_enter(curthread, ctx,
- FPU_KERN_NORMAL | FPU_KERN_KTHR);
- }
-
- error = 0;
- if (encini != NULL)
- error = aesni_cipher_setup_common(ses, encini->cri_key,
- encini->cri_klen);
-
- if (!kt) {
- fpu_kern_leave(curthread, ctx);
- RELEASE_CTX(ctxidx, ctx);
- }
- return (error);
-}
-
-static int
+static void
intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
{
struct sha1_ctxt *ctx = vctx;
@@ -563,7 +419,6 @@
intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
off += copysiz;
}
- return (0);
}
static void
@@ -578,7 +433,7 @@
sha1_result(ctx, digest);
}
-static int
+static void
intel_sha256_update(void *vctx, const void *vdata, u_int len)
{
SHA256_CTX *ctx = vctx;
@@ -599,7 +454,7 @@
/* Handle the case where we don't need to perform any transforms */
if (len < 64 - r) {
memcpy(&ctx->buf[r], src, len);
- return (0);
+ return;
}
/* Finish the current block */
@@ -618,7 +473,6 @@
/* Copy left over data into buffer */
memcpy(ctx->buf, src, len);
- return (0);
}
static void
@@ -645,42 +499,145 @@
SHA256_Final(digest, ctx);
}
-/*
- * Compute the HASH( (key ^ xorbyte) || buf )
- */
-static void
-hmac_internal(void *ctx, uint32_t *res,
- int (*update)(void *, const void *, u_int),
- void (*finalize)(void *, void *), uint8_t *key, uint8_t xorbyte,
- const void *buf, size_t off, size_t buflen, int crpflags)
+static int
+aesni_authprepare(struct aesni_session *ses, int klen)
+{
+
+ if (klen > SHA1_BLOCK_LEN)
+ return (EINVAL);
+ if ((ses->hmac && klen == 0) || (!ses->hmac && klen != 0))
+ return (EINVAL);
+ return (0);
+}
+
+static int
+aesni_cipherprepare(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_ICM:
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ case CRYPTO_AES_CBC:
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ CRYPTDEB("invalid CBC/ICM/GCM key length");
+ return (EINVAL);
+ }
+ break;
+ case CRYPTO_AES_XTS:
+ switch (csp->csp_cipher_klen * 8) {
+ case 256:
+ case 512:
+ break;
+ default:
+ CRYPTDEB("invalid XTS key length");
+ return (EINVAL);
+ }
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (0);
+}
+
+static int
+aesni_cipher_setup(struct aesni_session *ses,
+ const struct crypto_session_params *csp)
{
- size_t i;
+ struct fpu_kern_ctx *ctx;
+ int kt, ctxidx, error;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ ses->hmac = true;
+ /* FALLTHROUGH */
+ case CRYPTO_SHA1:
+ ses->hash_len = SHA1_HASH_LEN;
+ ses->hash_init = SHA1_Init_fn;
+ ses->hash_update = intel_sha1_update;
+ ses->hash_finalize = SHA1_Finalize_fn;
+ break;
+ case CRYPTO_SHA2_224_HMAC:
+ ses->hmac = true;
+ /* FALLTHROUGH */
+ case CRYPTO_SHA2_224:
+ ses->hash_len = SHA2_224_HASH_LEN;
+ ses->hash_init = SHA224_Init_fn;
+ ses->hash_update = intel_sha256_update;
+ ses->hash_finalize = SHA224_Finalize_fn;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ ses->hmac = true;
+ /* FALLTHROUGH */
+ case CRYPTO_SHA2_256:
+ ses->hash_len = SHA2_256_HASH_LEN;
+ ses->hash_init = SHA256_Init_fn;
+ ses->hash_update = intel_sha256_update;
+ ses->hash_finalize = SHA256_Finalize_fn;
+ break;
+ }
+
+ if (ses->hash_len != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->mlen = ses->hash_len;
+ else
+ ses->mlen = csp->csp_auth_mlen;
+
+ error = aesni_authprepare(ses, csp->csp_auth_klen);
+ if (error != 0)
+ return (error);
+ }
+
+ error = aesni_cipherprepare(csp);
+ if (error != 0)
+ return (error);
+
+ kt = is_fpu_kern_thread(0) || (csp->csp_cipher_alg == 0);
+ if (!kt) {
+ ACQUIRE_CTX(ctxidx, ctx);
+ fpu_kern_enter(curthread, ctx,
+ FPU_KERN_NORMAL | FPU_KERN_KTHR);
+ }
- for (i = 0; i < 64; i++)
- key[i] ^= xorbyte;
- update(ctx, key, 64);
- for (i = 0; i < 64; i++)
- key[i] ^= xorbyte;
+ error = 0;
+ if (csp->csp_cipher_key != NULL)
+ aesni_cipher_setup_common(ses, csp, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
- crypto_apply(crpflags, __DECONST(void *, buf), off, buflen,
- __DECONST(int (*)(void *, void *, u_int), update), ctx);
- finalize(res, ctx);
+ if (!kt) {
+ fpu_kern_leave(curthread, ctx);
+ RELEASE_CTX(ctxidx, ctx);
+ }
+ return (error);
}
static int
-aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
- struct cryptodesc *authcrd, struct cryptop *crp)
+aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp)
{
+ const struct crypto_session_params *csp;
struct fpu_kern_ctx *ctx;
int error, ctxidx;
bool kt;
- if (enccrd != NULL) {
- if ((enccrd->crd_alg == CRYPTO_AES_ICM ||
- enccrd->crd_alg == CRYPTO_AES_CCM_16 ||
- enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) &&
- (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
+ csp = crypto_get_params(crp->crp_session);
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_ICM:
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
return (EINVAL);
+ break;
+ case CRYPTO_AES_CBC:
+ case CRYPTO_AES_XTS:
+ /* CBC & XTS can only handle full blocks for now */
+ if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0)
+ return (EINVAL);
+ break;
}
ctx = NULL;
@@ -694,28 +651,21 @@
}
/* Do work */
- if (enccrd != NULL && authcrd != NULL) {
- /* Perform the first operation */
- if (crp->crp_desc == enccrd)
- error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
- else
- error = aesni_cipher_mac(ses, authcrd, crp);
- if (error != 0)
- goto out;
- /* Perform the second operation */
- if (crp->crp_desc == enccrd)
- error = aesni_cipher_mac(ses, authcrd, crp);
- else
- error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
- } else if (enccrd != NULL)
- error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = aesni_cipher_crypt(ses, crp, csp);
+ if (error == 0)
+ error = aesni_cipher_mac(ses, crp, csp);
+ } else {
+ error = aesni_cipher_mac(ses, crp, csp);
+ if (error == 0)
+ error = aesni_cipher_crypt(ses, crp, csp);
+ }
+ } else if (csp->csp_mode == CSP_MODE_DIGEST)
+ error = aesni_cipher_mac(ses, crp, csp);
else
- error = aesni_cipher_mac(ses, authcrd, crp);
+ error = aesni_cipher_crypt(ses, crp, csp);
- if (error != 0)
- goto out;
-
-out:
if (!kt) {
fpu_kern_leave(curthread, ctx);
RELEASE_CTX(ctxidx, ctx);
@@ -724,28 +674,24 @@
}
static int
-aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd,
- struct cryptodesc *authcrd, struct cryptop *crp)
+aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp)
{
uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf;
- int error, ivlen;
+ int error;
bool encflag, allocated, authallocated;
- KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 &&
- ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL,
- ("AES_NIST_GCM_16/AES_CCM_16 must include MAC descriptor"));
-
- ivlen = 0;
- authbuf = NULL;
-
- buf = aesni_cipher_alloc(enccrd, crp, &allocated);
+ buf = aesni_cipher_alloc(crp, crp->crp_payload_start,
+ crp->crp_payload_length, &allocated);
if (buf == NULL)
return (ENOMEM);
authallocated = false;
- if (ses->algo == CRYPTO_AES_NIST_GCM_16 ||
- ses->algo == CRYPTO_AES_CCM_16) {
- authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
+ authbuf = NULL;
+ if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 ||
+ csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
+ authbuf = aesni_cipher_alloc(crp, crp->crp_aad_start,
+ crp->crp_aad_length, &authallocated);
if (authbuf == NULL) {
error = ENOMEM;
goto out;
@@ -753,221 +699,161 @@
}
error = 0;
- encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
- if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
- error = aesni_cipher_setup_common(ses, enccrd->crd_key,
- enccrd->crd_klen);
- if (error != 0)
- goto out;
- }
-
- switch (enccrd->crd_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- ivlen = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_XTS:
- ivlen = 8;
- break;
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_CCM_16:
- ivlen = 12; /* should support arbitarily larger */
- break;
- }
+ encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
+ if (crp->crp_cipher_key != NULL)
+ aesni_cipher_setup_common(ses, csp, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
/* Setup iv */
- if (encflag) {
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, iv, ivlen);
- else
- arc4rand(iv, ivlen, 0);
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivlen, iv);
- } else {
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, iv, ivlen);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivlen, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
- switch (ses->algo) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_CBC:
if (encflag)
aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
- enccrd->crd_len, buf, buf, iv);
+ crp->crp_payload_length, buf, buf, iv);
else
aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
- enccrd->crd_len, buf, iv);
+ crp->crp_payload_length, buf, iv);
break;
case CRYPTO_AES_ICM:
/* encryption & decryption are the same */
aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
- enccrd->crd_len, buf, buf, iv);
+ crp->crp_payload_length, buf, buf, iv);
break;
case CRYPTO_AES_XTS:
if (encflag)
aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
- ses->xts_schedule, enccrd->crd_len, buf, buf,
- iv);
+ ses->xts_schedule, crp->crp_payload_length, buf,
+ buf, iv);
else
aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
- ses->xts_schedule, enccrd->crd_len, buf, buf,
- iv);
+ ses->xts_schedule, crp->crp_payload_length, buf,
+ buf, iv);
break;
case CRYPTO_AES_NIST_GCM_16:
- if (!encflag)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, sizeof(tag), tag);
- else
- bzero(tag, sizeof tag);
-
if (encflag) {
+ memset(tag, 0, sizeof(tag));
AES_GCM_encrypt(buf, buf, authbuf, iv, tag,
- enccrd->crd_len, authcrd->crd_len, ivlen,
- ses->enc_schedule, ses->rounds);
-
- if (authcrd != NULL)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, sizeof(tag), tag);
+ crp->crp_payload_length, crp->crp_aad_length,
+ csp->csp_ivlen, ses->enc_schedule, ses->rounds);
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
+ tag);
} else {
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
+ tag);
if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag,
- enccrd->crd_len, authcrd->crd_len, ivlen,
- ses->enc_schedule, ses->rounds))
+ crp->crp_payload_length, crp->crp_aad_length,
+ csp->csp_ivlen, ses->enc_schedule, ses->rounds))
error = EBADMSG;
}
break;
case CRYPTO_AES_CCM_16:
- if (!encflag)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, sizeof(tag), tag);
- else
- bzero(tag, sizeof tag);
if (encflag) {
+ memset(tag, 0, sizeof(tag));
AES_CCM_encrypt(buf, buf, authbuf, iv, tag,
- enccrd->crd_len, authcrd->crd_len, ivlen,
- ses->enc_schedule, ses->rounds);
- if (authcrd != NULL)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- authcrd->crd_inject, sizeof(tag), tag);
+ crp->crp_payload_length, crp->crp_aad_length,
+ csp->csp_ivlen, ses->enc_schedule, ses->rounds);
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
+ tag);
} else {
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
+ tag);
if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag,
- enccrd->crd_len, authcrd->crd_len, ivlen,
- ses->enc_schedule, ses->rounds))
+ crp->crp_payload_length, crp->crp_aad_length,
+ csp->csp_ivlen, ses->enc_schedule, ses->rounds))
error = EBADMSG;
}
break;
}
if (allocated && error == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, buf);
+ crypto_copyback(crp, crp->crp_payload_start,
+ crp->crp_payload_length, buf);
out:
if (allocated) {
- explicit_bzero(buf, enccrd->crd_len);
+ explicit_bzero(buf, crp->crp_payload_length);
free(buf, M_AESNI);
}
if (authallocated) {
- explicit_bzero(authbuf, authcrd->crd_len);
+ explicit_bzero(authbuf, crp->crp_aad_length);
free(authbuf, M_AESNI);
}
return (error);
}
static int
-aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
- struct cryptop *crp)
+aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp)
{
union {
struct SHA256Context sha2 __aligned(16);
struct sha1_ctxt sha1 __aligned(16);
} sctx;
+ uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)];
- int hashlen, error;
- void *ctx;
- void (*InitFn)(void *);
- int (*UpdateFn)(void *, const void *, unsigned);
- void (*FinalizeFn)(void *, void *);
+ uint32_t res2[SHA2_256_HASH_LEN / sizeof(uint32_t)];
+ const uint8_t *key;
+ int i, keylen;
- bool hmac;
+ if (crp->crp_auth_key != NULL)
+ key = crp->crp_auth_key;
+ else
+ key = csp->csp_auth_key;
+ keylen = csp->csp_auth_klen;
- if ((crd->crd_flags & ~CRD_F_KEY_EXPLICIT) != 0) {
- CRYPTDEB("%s: Unsupported MAC flags: 0x%x", __func__,
- (crd->crd_flags & ~CRD_F_KEY_EXPLICIT));
- return (EINVAL);
- }
- if ((crd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
- error = aesni_authprepare(ses, crd->crd_klen, crd->crd_key);
- if (error != 0)
- return (error);
- }
-
- hmac = false;
- switch (ses->auth_algo) {
- case CRYPTO_SHA1_HMAC:
- hmac = true;
- /* FALLTHROUGH */
- case CRYPTO_SHA1:
- hashlen = SHA1_HASH_LEN;
- InitFn = SHA1_Init_fn;
- UpdateFn = intel_sha1_update;
- FinalizeFn = SHA1_Finalize_fn;
- ctx = &sctx.sha1;
- break;
-
- case CRYPTO_SHA2_256_HMAC:
- hmac = true;
- /* FALLTHROUGH */
- case CRYPTO_SHA2_256:
- hashlen = SHA2_256_HASH_LEN;
- InitFn = SHA256_Init_fn;
- UpdateFn = intel_sha256_update;
- FinalizeFn = SHA256_Finalize_fn;
- ctx = &sctx.sha2;
- break;
-
- case CRYPTO_SHA2_224_HMAC:
- hmac = true;
- /* FALLTHROUGH */
- case CRYPTO_SHA2_224:
- hashlen = SHA2_224_HASH_LEN;
- InitFn = SHA224_Init_fn;
- UpdateFn = intel_sha256_update;
- FinalizeFn = SHA224_Finalize_fn;
- ctx = &sctx.sha2;
- break;
- default:
- /*
- * AES-GMAC authentication is verified while processing the
- * enccrd
- */
- return (0);
- }
-
- if (hmac) {
+ if (ses->hmac) {
/* Inner hash: (K ^ IPAD) || data */
- InitFn(ctx);
- hmac_internal(ctx, res, UpdateFn, FinalizeFn, ses->hmac_key,
- 0x36, crp->crp_buf, crd->crd_skip, crd->crd_len,
- crp->crp_flags);
+ ses->hash_init(&sctx);
+ for (i = 0; i < keylen; i++)
+ hmac_key[i] = key[i] ^ HMAC_IPAD_VAL;
+ for (i = keylen; i < sizeof(hmac_key); i++)
+ hmac_key[i] = 0 ^ HMAC_IPAD_VAL;
+ ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
+
+ crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+ __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+ &sctx);
+ crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length,
+ __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+ &sctx);
+ ses->hash_finalize(res, &sctx);
+
/* Outer hash: (K ^ OPAD) || inner hash */
- InitFn(ctx);
- hmac_internal(ctx, res, UpdateFn, FinalizeFn, ses->hmac_key,
- 0x5C, res, 0, hashlen, 0);
+ ses->hash_init(&sctx);
+ for (i = 0; i < keylen; i++)
+ hmac_key[i] = key[i] ^ HMAC_OPAD_VAL;
+ for (i = keylen; i < sizeof(hmac_key); i++)
+ hmac_key[i] = 0 ^ HMAC_OPAD_VAL;
+ ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
+ ses->hash_update(&sctx, res, ses->hash_len);
+ ses->hash_finalize(res, &sctx);
} else {
- InitFn(ctx);
- crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, __DECONST(int (*)(void *, void *, u_int),
- UpdateFn), ctx);
- FinalizeFn(res, ctx);
+ ses->hash_init(&sctx);
+
+ crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+ __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+ &sctx);
+ crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length,
+ __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+ &sctx);
+
+ ses->hash_finalize(res, &sctx);
}
- if (ses->mlen != 0 && ses->mlen < hashlen)
- hashlen = ses->mlen;
-
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
- (void *)res);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
+ if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
return (0);
}
Index: sys/crypto/aesni/aesni_wrap.c
===================================================================
--- sys/crypto/aesni/aesni_wrap.c
+++ sys/crypto/aesni/aesni_wrap.c
@@ -435,51 +435,37 @@
iv, 0);
}
-int
-aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
- int keylen)
+void
+aesni_cipher_setup_common(struct aesni_session *ses,
+ const struct crypto_session_params *csp, const uint8_t *key, int keylen)
{
int decsched;
decsched = 1;
- switch (ses->algo) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_ICM:
case CRYPTO_AES_NIST_GCM_16:
case CRYPTO_AES_CCM_16:
decsched = 0;
- /* FALLTHROUGH */
- case CRYPTO_AES_CBC:
- switch (keylen) {
- case 128:
- ses->rounds = AES128_ROUNDS;
- break;
- case 192:
- ses->rounds = AES192_ROUNDS;
- break;
- case 256:
- ses->rounds = AES256_ROUNDS;
- break;
- default:
- CRYPTDEB("invalid CBC/ICM/GCM key length");
- return (EINVAL);
- }
break;
- case CRYPTO_AES_XTS:
- switch (keylen) {
- case 256:
- ses->rounds = AES128_ROUNDS;
- break;
- case 512:
- ses->rounds = AES256_ROUNDS;
- break;
- default:
- CRYPTDEB("invalid XTS key length");
- return (EINVAL);
- }
+ }
+
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ keylen /= 2;
+
+ switch (keylen * 8) {
+ case 128:
+ ses->rounds = AES128_ROUNDS;
+ break;
+ case 192:
+ ses->rounds = AES192_ROUNDS;
+ break;
+ case 256:
+ ses->rounds = AES256_ROUNDS;
break;
default:
- return (EINVAL);
+ panic("shouldn't happen");
}
aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
@@ -487,9 +473,7 @@
aesni_set_deckey(ses->enc_schedule, ses->dec_schedule,
ses->rounds);
- if (ses->algo == CRYPTO_AES_XTS)
- aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ aesni_set_enckey(key + keylen, ses->xts_schedule,
ses->rounds);
-
- return (0);
}
Index: sys/crypto/armv8/armv8_crypto.c
===================================================================
--- sys/crypto/armv8/armv8_crypto.c
+++ sys/crypto/armv8/armv8_crypto.c
@@ -85,7 +85,7 @@
} while (0)
static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
- struct cryptodesc *, struct cryptop *);
+ struct cryptop *);
MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
@@ -131,7 +131,7 @@
sc->dieing = 0;
sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
- CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+ CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM);
@@ -149,8 +149,6 @@
mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
}
- crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
-
return (0);
}
@@ -185,83 +183,74 @@
}
static int
+armv8_crypto_probesession(device_t dev,
+ const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (EINVAL);
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ default:
+ return (EINVAL);
+ }
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
+
+static void
armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
- struct cryptoini *encini)
+ const struct crypto_session_params *csp)
{
int i;
- switch (ses->algo) {
- case CRYPTO_AES_CBC:
- switch (encini->cri_klen) {
- case 128:
- ses->rounds = AES128_ROUNDS;
- break;
- case 192:
- ses->rounds = AES192_ROUNDS;
- break;
- case 256:
- ses->rounds = AES256_ROUNDS;
- break;
- default:
- CRYPTDEB("invalid CBC/ICM/GCM key length");
- return (EINVAL);
- }
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ ses->rounds = AES128_ROUNDS;
+ break;
+ case 192:
+ ses->rounds = AES192_ROUNDS;
+ break;
+ case 256:
+ ses->rounds = AES256_ROUNDS;
break;
default:
- return (EINVAL);
+ panic("invalid CBC key length");
}
- rijndaelKeySetupEnc(ses->enc_schedule, encini->cri_key,
- encini->cri_klen);
- rijndaelKeySetupDec(ses->dec_schedule, encini->cri_key,
- encini->cri_klen);
+ rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key,
+ csp->csp_cipher_klen * 8);
+ rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key,
+ csp->csp_cipher_klen * 8);
for (i = 0; i < nitems(ses->enc_schedule); i++) {
ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
}
-
- return (0);
}
static int
armv8_crypto_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri)
+ const struct crypto_session_params *csp)
{
struct armv8_crypto_softc *sc;
struct armv8_crypto_session *ses;
- struct cryptoini *encini;
- int error;
-
- if (cri == NULL) {
- CRYPTDEB("no cri");
- return (EINVAL);
- }
sc = device_get_softc(dev);
- if (sc->dieing)
- return (EINVAL);
-
- ses = NULL;
- encini = NULL;
- for (; cri != NULL; cri = cri->cri_next) {
- switch (cri->cri_alg) {
- case CRYPTO_AES_CBC:
- if (encini != NULL) {
- CRYPTDEB("encini already set");
- return (EINVAL);
- }
- encini = cri;
- break;
- default:
- CRYPTDEB("unhandled algorithm");
- return (EINVAL);
- }
- }
- if (encini == NULL) {
- CRYPTDEB("no cipher");
- return (EINVAL);
- }
-
rw_wlock(&sc->lock);
if (sc->dieing) {
rw_wunlock(&sc->lock);
@@ -269,15 +258,7 @@
}
ses = crypto_get_driver_session(cses);
- ses->algo = encini->cri_alg;
-
- error = armv8_crypto_cipher_setup(ses, encini);
- if (error != 0) {
- CRYPTDEB("setup failed");
- rw_wunlock(&sc->lock);
- return (error);
- }
-
+ armv8_crypto_cipher_setup(ses, csp);
rw_wunlock(&sc->lock);
return (0);
}
@@ -285,50 +266,17 @@
static int
armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
{
- struct cryptodesc *crd, *enccrd;
struct armv8_crypto_session *ses;
int error;
- error = 0;
- enccrd = NULL;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
- error = EINVAL;
- goto out;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- if (enccrd != NULL) {
- error = EINVAL;
- goto out;
- }
- enccrd = crd;
- break;
- default:
- error = EINVAL;
- goto out;
- }
- }
-
- if (enccrd == NULL) {
- error = EINVAL;
- goto out;
- }
-
/* We can only handle full blocks for now */
- if ((enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+ if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
error = EINVAL;
goto out;
}
ses = crypto_get_driver_session(crp->crp_session);
- error = armv8_crypto_cipher_process(ses, enccrd, crp);
+ error = armv8_crypto_cipher_process(ses, crp);
out:
crp->crp_etype = error;
@@ -337,37 +285,21 @@
}
static uint8_t *
-armv8_crypto_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
- int *allocated)
+armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated)
{
- struct mbuf *m;
- struct uio *uio;
- struct iovec *iov;
uint8_t *addr;
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- m = (struct mbuf *)crp->crp_buf;
- if (m->m_next != NULL)
- goto alloc;
- addr = mtod(m, uint8_t *);
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- uio = (struct uio *)crp->crp_buf;
- if (uio->uio_iovcnt != 1)
- goto alloc;
- iov = uio->uio_iov;
- addr = (uint8_t *)iov->iov_base;
- } else
- addr = (uint8_t *)crp->crp_buf;
- *allocated = 0;
- addr += enccrd->crd_skip;
- return (addr);
-
-alloc:
- addr = malloc(enccrd->crd_len, M_ARMV8_CRYPTO, M_NOWAIT);
+ addr = crypto_contiguous_subsegment(crp, crp->crp_payload_start,
+ crp->crp_payload_length);
+ if (addr != NULL) {
+ *allocated = 0;
+ return (addr);
+ }
+ addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
if (addr != NULL) {
*allocated = 1;
- crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, addr);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, addr);
} else
*allocated = 0;
return (addr);
@@ -375,18 +307,20 @@
static int
armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
- struct cryptodesc *enccrd, struct cryptop *crp)
+ struct cryptop *crp)
{
+ const struct crypto_session_params *csp;
struct fpu_kern_ctx *ctx;
uint8_t *buf;
uint8_t iv[AES_BLOCK_LEN];
int allocated, i;
- int encflag, ivlen;
+ int encflag;
int kt;
- encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+ csp = crypto_get_params(crp->crp_session);
+ encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
- buf = armv8_crypto_cipher_alloc(enccrd, crp, &allocated);
+ buf = armv8_crypto_cipher_alloc(crp, &allocated);
if (buf == NULL)
return (ENOMEM);
@@ -397,56 +331,41 @@
FPU_KERN_NORMAL | FPU_KERN_KTHR);
}
- if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
- panic("CRD_F_KEY_EXPLICIT");
- }
-
- switch (enccrd->crd_alg) {
- case CRYPTO_AES_CBC:
- ivlen = AES_BLOCK_LEN;
- break;
+ if (crp->crp_cipher_key != NULL) {
+ panic("armv8: new cipher key");
}
/* Setup iv */
- if (encflag) {
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, iv, ivlen);
- else
- arc4rand(iv, ivlen, 0);
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivlen, iv);
- } else {
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, iv, ivlen);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivlen, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
/* Do work */
- switch (ses->algo) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_CBC:
if (encflag)
armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule,
- enccrd->crd_len, buf, buf, iv);
+ crp->crp_payload_length, buf, buf, iv);
else
armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
- enccrd->crd_len, buf, iv);
+ crp->crp_payload_length, buf, iv);
break;
}
if (allocated)
- crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, buf);
+ crypto_copyback(crp, crp->crp_payload_start,
+ crp->crp_payload_length, buf);
if (!kt) {
fpu_kern_leave(curthread, ctx);
RELEASE_CTX(i, ctx);
}
if (allocated) {
- bzero(buf, enccrd->crd_len);
+ bzero(buf, crp->crp_payload_length);
free(buf, M_ARMV8_CRYPTO);
}
return (0);
@@ -458,6 +377,7 @@
DEVMETHOD(device_attach, armv8_crypto_attach),
DEVMETHOD(device_detach, armv8_crypto_detach),
+ DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
DEVMETHOD(cryptodev_process, armv8_crypto_process),
Index: sys/crypto/blake2/blake2_cryptodev.c
===================================================================
--- sys/crypto/blake2/blake2_cryptodev.c
+++ sys/crypto/blake2/blake2_cryptodev.c
@@ -50,10 +50,7 @@
#endif
struct blake2_session {
- int algo;
- size_t klen;
size_t mlen;
- uint8_t key[BLAKE2B_KEYBYTES];
};
CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
@@ -79,10 +76,8 @@
(ctx) = NULL; \
} while (0)
-static int blake2_newsession(device_t, crypto_session_t cses,
- struct cryptoini *cri);
static int blake2_cipher_setup(struct blake2_session *ses,
- struct cryptoini *authini);
+ const struct crypto_session_params *csp);
static int blake2_cipher_process(struct blake2_session *ses,
struct cryptop *crp);
@@ -134,7 +129,7 @@
sc->dying = false;
sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
- CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+ CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
if (sc->cid < 0) {
device_printf(dev, "Could not get crypto driver id.\n");
return (ENOMEM);
@@ -152,8 +147,6 @@
rw_init(&sc->lock, "blake2_lock");
- crypto_register(sc->cid, CRYPTO_BLAKE2B, 0, 0);
- crypto_register(sc->cid, CRYPTO_BLAKE2S, 0, 0);
return (0);
}
@@ -177,52 +170,47 @@
}
static int
-blake2_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+blake2_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct blake2_softc *sc;
- struct blake2_session *ses;
- struct cryptoini *authini;
- int error;
- if (cri == NULL) {
- CRYPTDEB("no cri");
+ if (csp->csp_flags != 0)
return (EINVAL);
- }
-
- sc = device_get_softc(dev);
-
- authini = NULL;
- for (; cri != NULL; cri = cri->cri_next) {
- switch (cri->cri_alg) {
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ switch (csp->csp_auth_alg) {
case CRYPTO_BLAKE2B:
case CRYPTO_BLAKE2S:
- if (authini != NULL) {
- CRYPTDEB("authini already set");
- return (EINVAL);
- }
- authini = cri;
break;
default:
- CRYPTDEB("unhandled algorithm");
return (EINVAL);
}
- }
- if (authini == NULL) {
- CRYPTDEB("no cipher");
+ break;
+ default:
return (EINVAL);
}
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
- rw_wlock(&sc->lock);
- if (sc->dying) {
- rw_wunlock(&sc->lock);
- return (EINVAL);
- }
- rw_wunlock(&sc->lock);
+static int
+blake2_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct blake2_softc *sc;
+ struct blake2_session *ses;
+ int error;
+
+ sc = device_get_softc(dev);
ses = crypto_get_driver_session(cses);
- ses->algo = authini->cri_alg;
- error = blake2_cipher_setup(ses, authini);
+ rw_rlock(&sc->lock);
+ if (sc->dying) {
+ rw_runlock(&sc->lock);
+ return (EINVAL);
+ }
+ rw_runlock(&sc->lock);
+
+ error = blake2_cipher_setup(ses, csp);
if (error != 0) {
CRYPTDEB("setup failed");
return (error);
@@ -235,48 +223,14 @@
blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
{
struct blake2_session *ses;
- struct cryptodesc *crd, *authcrd;
int error;
- ses = NULL;
- error = 0;
- authcrd = NULL;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
- error = EINVAL;
- goto out;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_BLAKE2B:
- case CRYPTO_BLAKE2S:
- if (authcrd != NULL) {
- error = EINVAL;
- goto out;
- }
- authcrd = crd;
- break;
-
- default:
- error = EINVAL;
- goto out;
- }
- }
-
ses = crypto_get_driver_session(crp->crp_session);
error = blake2_cipher_process(ses, crp);
- if (error != 0)
- goto out;
-out:
crp->crp_etype = error;
crypto_done(crp);
- return (error);
+ return (0);
}
static device_method_t blake2_methods[] = {
@@ -285,6 +239,7 @@
DEVMETHOD(device_attach, blake2_attach),
DEVMETHOD(device_detach, blake2_detach),
+ DEVMETHOD(cryptodev_probesession, blake2_probesession),
DEVMETHOD(cryptodev_newsession, blake2_newsession),
DEVMETHOD(cryptodev_process, blake2_process),
@@ -302,37 +257,48 @@
MODULE_VERSION(blake2, 1);
MODULE_DEPEND(blake2, crypto, 1, 1, 1);
+static bool
+blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
+{
+
+ if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
+ return (klen <= BLAKE2S_KEYBYTES);
+ else
+ return (klen <= BLAKE2B_KEYBYTES);
+}
+
static int
-blake2_cipher_setup(struct blake2_session *ses, struct cryptoini *authini)
+blake2_cipher_setup(struct blake2_session *ses,
+ const struct crypto_session_params *csp)
{
- int keylen;
+ int hashlen;
CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
- if (authini->cri_mlen < 0)
+ if (!blake2_check_klen(csp, csp->csp_auth_klen))
return (EINVAL);
- switch (ses->algo) {
+ if (csp->csp_auth_mlen < 0)
+ return (EINVAL);
+
+ switch (csp->csp_auth_alg) {
case CRYPTO_BLAKE2S:
- if (authini->cri_mlen != 0 &&
- authini->cri_mlen > BLAKE2S_OUTBYTES)
- return (EINVAL);
- /* FALLTHROUGH */
+ hashlen = BLAKE2S_OUTBYTES;
+ break;
case CRYPTO_BLAKE2B:
- if (authini->cri_mlen != 0 &&
- authini->cri_mlen > BLAKE2B_OUTBYTES)
- return (EINVAL);
-
- if (authini->cri_klen % 8 != 0)
- return (EINVAL);
- keylen = authini->cri_klen / 8;
- if (keylen > sizeof(ses->key) ||
- (ses->algo == CRYPTO_BLAKE2S && keylen > BLAKE2S_KEYBYTES))
- return (EINVAL);
- ses->klen = keylen;
- memcpy(ses->key, authini->cri_key, keylen);
- ses->mlen = authini->cri_mlen;
+ hashlen = BLAKE2B_OUTBYTES;
+ break;
+ default:
+ return (EINVAL);
}
+
+ if (csp->csp_auth_mlen > hashlen)
+ return (EINVAL);
+
+ if (csp->csp_auth_mlen == 0)
+ ses->mlen = hashlen;
+ else
+ ses->mlen = csp->csp_auth_mlen;
return (0);
}
@@ -365,15 +331,15 @@
blake2b_state sb;
blake2s_state ss;
} bctx;
- char res[BLAKE2B_OUTBYTES];
+ char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
+ const struct crypto_session_params *csp;
struct fpu_kern_ctx *ctx;
+ const void *key;
int ctxidx;
bool kt;
- struct cryptodesc *crd;
int error, rc;
- size_t hashlen;
+ unsigned klen;
- crd = crp->crp_desc;
ctx = NULL;
ctxidx = 0;
error = EINVAL;
@@ -385,47 +351,42 @@
FPU_KERN_NORMAL | FPU_KERN_KTHR);
}
- if (crd->crd_flags != 0)
- goto out;
-
- switch (ses->algo) {
+ csp = crypto_get_params(crp->crp_session);
+ if (crp->crp_auth_key != NULL)
+ key = crp->crp_auth_key;
+ else
+ key = csp->csp_auth_key;
+ klen = csp->csp_auth_klen;
+ switch (csp->csp_auth_alg) {
case CRYPTO_BLAKE2B:
- if (ses->mlen != 0)
- hashlen = ses->mlen;
+ if (klen > 0)
+ rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
else
- hashlen = BLAKE2B_OUTBYTES;
- if (ses->klen > 0)
- rc = blake2b_init_key(&bctx.sb, hashlen, ses->key, ses->klen);
- else
- rc = blake2b_init(&bctx.sb, hashlen);
+ rc = blake2b_init(&bctx.sb, ses->mlen);
if (rc != 0)
goto out;
- error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, blake2b_applicator, &bctx.sb);
+ error = crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length, blake2b_applicator, &bctx.sb);
if (error != 0)
goto out;
- rc = blake2b_final(&bctx.sb, res, hashlen);
+ rc = blake2b_final(&bctx.sb, res, ses->mlen);
if (rc != 0) {
error = EINVAL;
goto out;
}
break;
case CRYPTO_BLAKE2S:
- if (ses->mlen != 0)
- hashlen = ses->mlen;
+ if (klen > 0)
+ rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
else
- hashlen = BLAKE2S_OUTBYTES;
- if (ses->klen > 0)
- rc = blake2s_init_key(&bctx.ss, hashlen, ses->key, ses->klen);
- else
- rc = blake2s_init(&bctx.ss, hashlen);
+ rc = blake2s_init(&bctx.ss, ses->mlen);
if (rc != 0)
goto out;
- error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, blake2s_applicator, &bctx.ss);
+ error = crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length, blake2s_applicator, &bctx.ss);
if (error != 0)
goto out;
- rc = blake2s_final(&bctx.ss, res, hashlen);
+ rc = blake2s_final(&bctx.ss, res, ses->mlen);
if (rc != 0) {
error = EINVAL;
goto out;
@@ -435,8 +396,12 @@
panic("unreachable");
}
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
- (void *)res);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
+ if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
out:
if (!kt) {
Index: sys/crypto/ccp/ccp.h
===================================================================
--- sys/crypto/ccp/ccp.h
+++ sys/crypto/ccp/ccp.h
@@ -58,14 +58,18 @@
SHA2_256, SHA2_384, SHA2_512
};
+/*
+ * XXX: The hmac.res, gmac.final_block, and blkcipher.iv fields are
+ * used by individual requests meaning that sessions cannot have more
+ * than a single request in flight at a time.
+ */
struct ccp_session_hmac {
struct auth_hash *auth_hash;
int hash_len;
- unsigned int partial_digest_len;
unsigned int auth_mode;
- unsigned int mk_size;
char ipad[CCP_HASH_MAX_BLOCK_SIZE];
char opad[CCP_HASH_MAX_BLOCK_SIZE];
+ char res[CCP_HASH_MAX_BLOCK_SIZE];
};
struct ccp_session_gmac {
@@ -77,14 +81,12 @@
unsigned cipher_mode;
unsigned cipher_type;
unsigned key_len;
- unsigned iv_len;
char enckey[CCP_AES_MAX_KEY_LEN];
char iv[CCP_MAX_CRYPTO_IV_LEN];
};
struct ccp_session {
- bool active : 1;
- bool cipher_first : 1;
+ bool active;
int pending;
enum { HMAC, BLKCIPHER, AUTHENC, GCM } mode;
unsigned queue;
@@ -217,12 +219,11 @@
* Internal hardware crypt-op submission routines.
*/
int ccp_authenc(struct ccp_queue *sc, struct ccp_session *s,
- struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde)
- __must_check;
+ struct cryptop *crp) __must_check;
int ccp_blkcipher(struct ccp_queue *sc, struct ccp_session *s,
struct cryptop *crp) __must_check;
-int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde) __must_check;
+int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp)
+ __must_check;
int ccp_hmac(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp)
__must_check;
Index: sys/crypto/ccp/ccp.c
===================================================================
--- sys/crypto/ccp/ccp.c
+++ sys/crypto/ccp/ccp.c
@@ -96,22 +96,28 @@
int error;
sglist_reset(sg);
- if (crp->crp_flags & CRYPTO_F_IMBUF)
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
error = sglist_append_mbuf(sg, crp->crp_mbuf);
- else if (crp->crp_flags & CRYPTO_F_IOV)
+ break;
+ case CRYPTO_BUF_UIO:
error = sglist_append_uio(sg, crp->crp_uio);
- else
+ break;
+ case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+ break;
+ default:
+ error = EINVAL;
+ }
return (error);
}
/*
* Handle a GCM request with an empty payload by performing the
- * operation in software. Derived from swcr_authenc().
+ * operation in software.
*/
static void
-ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp)
{
struct aes_gmac_ctx gmac_ctx;
char block[GMAC_BLOCK_LEN];
@@ -123,21 +129,11 @@
* This assumes a 12-byte IV from the crp. See longer comment
* above in ccp_gcm() for more details.
*/
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- arc4rand(iv, 12, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+ crp->crp_etype = EINVAL;
+ goto out;
}
+ memcpy(iv, crp->crp_iv, 12);
*(uint32_t *)&iv[12] = htobe32(1);
/* Initialize the MAC. */
@@ -146,34 +142,34 @@
AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));
/* MAC the AAD. */
- for (i = 0; i < crda->crd_len; i += sizeof(block)) {
- len = imin(crda->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+ len = imin(crp->crp_aad_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_aad_start + i, len, block);
bzero(block + len, sizeof(block) - len);
AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
}
/* Length block. */
bzero(block, sizeof(block));
- ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
+ ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
AES_GMAC_Final(digest, &gmac_ctx);
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest), digest);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+ digest);
crp->crp_etype = 0;
} else {
char digest2[GMAC_DIGEST_LEN];
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest2), digest2);
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+ digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
crp->crp_etype = 0;
else
crp->crp_etype = EBADMSG;
}
+out:
crypto_done(crp);
}
@@ -259,22 +255,6 @@
random_source_register(&random_ccp);
}
- if ((sc->hw_features & VERSION_CAP_AES) != 0) {
- crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
- }
- if ((sc->hw_features & VERSION_CAP_SHA) != 0) {
- crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
- }
-
return (0);
}
@@ -304,8 +284,7 @@
}
static void
-ccp_init_hmac_digest(struct ccp_session *s, int cri_alg, char *key,
- int klen)
+ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
{
union authctx auth_ctx;
struct auth_hash *axf;
@@ -316,7 +295,6 @@
* the key as the key instead.
*/
axf = s->hmac.auth_hash;
- klen /= 8;
if (klen > axf->blocksize) {
axf->Init(&auth_ctx);
axf->Update(&auth_ctx, key, klen);
@@ -335,26 +313,26 @@
}
}
-static int
+static bool
ccp_aes_check_keylen(int alg, int klen)
{
- switch (klen) {
+ switch (klen * 8) {
case 128:
case 192:
if (alg == CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
case 256:
break;
case 512:
if (alg != CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
default:
- return (EINVAL);
+ return (false);
}
- return (0);
+ return (true);
}
static void
@@ -363,9 +341,9 @@
unsigned kbits;
if (alg == CRYPTO_AES_XTS)
- kbits = klen / 2;
+ kbits = (klen / 2) * 8;
else
- kbits = klen;
+ kbits = klen * 8;
switch (kbits) {
case 128:
@@ -381,123 +359,154 @@
panic("should not get here");
}
- s->blkcipher.key_len = klen / 8;
+ s->blkcipher.key_len = klen;
memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
}
+static bool
+ccp_auth_supported(struct ccp_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if ((sc->hw_features & VERSION_CAP_SHA) == 0)
+ return (false);
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ if (csp->csp_auth_key == NULL)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+ccp_cipher_supported(struct ccp_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if ((sc->hw_features & VERSION_CAP_AES) == 0)
+ return (false);
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (ccp_aes_check_keylen(csp->csp_cipher_alg,
+ csp->csp_cipher_klen));
+}
+
static int
-ccp_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ccp_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct ccp_softc *sc;
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ sc = device_get_softc(dev);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!ccp_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!ccp_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (EINVAL);
+ if (csp->csp_auth_mlen < 0 ||
+ csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+ return (EINVAL);
+ if ((sc->hw_features & VERSION_CAP_AES) == 0)
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ case CSP_MODE_ETA:
+ if (!ccp_auth_supported(sc, csp) ||
+ !ccp_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+ccp_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
struct ccp_softc *sc;
struct ccp_session *s;
struct auth_hash *auth_hash;
- struct cryptoini *c, *hash, *cipher;
enum ccp_aes_mode cipher_mode;
- unsigned auth_mode, iv_len;
- unsigned partial_digest_len;
+ unsigned auth_mode;
unsigned q;
- int error;
- bool gcm_hash;
- if (cri == NULL)
- return (EINVAL);
-
- s = crypto_get_driver_session(cses);
-
- gcm_hash = false;
- cipher = NULL;
- hash = NULL;
- auth_hash = NULL;
/* XXX reconcile auth_mode with use by ccp_sha */
- auth_mode = 0;
- cipher_mode = CCP_AES_MODE_ECB;
- iv_len = 0;
- partial_digest_len = 0;
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- if (hash)
- return (EINVAL);
- hash = c;
- switch (c->cri_alg) {
- case CRYPTO_SHA1_HMAC:
- auth_hash = &auth_hash_hmac_sha1;
- auth_mode = SHA1;
- partial_digest_len = SHA1_HASH_LEN;
- break;
- case CRYPTO_SHA2_256_HMAC:
- auth_hash = &auth_hash_hmac_sha2_256;
- auth_mode = SHA2_256;
- partial_digest_len = SHA2_256_HASH_LEN;
- break;
- case CRYPTO_SHA2_384_HMAC:
- auth_hash = &auth_hash_hmac_sha2_384;
- auth_mode = SHA2_384;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_SHA2_512_HMAC:
- auth_hash = &auth_hash_hmac_sha2_512;
- auth_mode = SHA2_512;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- gcm_hash = true;
-#if 0
- auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
-#endif
- break;
- }
- break;
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_XTS:
- if (cipher)
- return (EINVAL);
- cipher = c;
- switch (c->cri_alg) {
- case CRYPTO_AES_CBC:
- cipher_mode = CCP_AES_MODE_CBC;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_ICM:
- cipher_mode = CCP_AES_MODE_CTR;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_NIST_GCM_16:
- cipher_mode = CCP_AES_MODE_GCTR;
- iv_len = AES_GCM_IV_LEN;
- break;
- case CRYPTO_AES_XTS:
- cipher_mode = CCP_AES_MODE_XTS;
- iv_len = AES_BLOCK_LEN;
- break;
- }
- if (c->cri_key != NULL) {
- error = ccp_aes_check_keylen(c->cri_alg,
- c->cri_klen);
- if (error != 0)
- return (error);
- }
- break;
- default:
- return (EINVAL);
- }
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1_HMAC:
+ auth_hash = &auth_hash_hmac_sha1;
+ auth_mode = SHA1;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_256;
+ auth_mode = SHA2_256;
+ break;
+ case CRYPTO_SHA2_384_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_384;
+ auth_mode = SHA2_384;
+ break;
+ case CRYPTO_SHA2_512_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_512;
+ auth_mode = SHA2_512;
+ break;
+ default:
+ auth_hash = NULL;
+ auth_mode = 0;
+ break;
+ }
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ cipher_mode = CCP_AES_MODE_CBC;
+ break;
+ case CRYPTO_AES_ICM:
+ cipher_mode = CCP_AES_MODE_CTR;
+ break;
+ case CRYPTO_AES_NIST_GCM_16:
+ cipher_mode = CCP_AES_MODE_GCTR;
+ break;
+ case CRYPTO_AES_XTS:
+ cipher_mode = CCP_AES_MODE_XTS;
+ break;
+ default:
+ cipher_mode = CCP_AES_MODE_ECB;
+ break;
}
- if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR))
- return (EINVAL);
- if (hash == NULL && cipher == NULL)
- return (EINVAL);
- if (hash != NULL && hash->cri_key == NULL)
- return (EINVAL);
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
@@ -506,6 +515,8 @@
return (ENXIO);
}
+ s = crypto_get_driver_session(cses);
+
/* Just grab the first usable queue for now. */
for (q = 0; q < nitems(sc->queues); q++)
if ((sc->valid_queues & (1 << q)) != 0)
@@ -516,38 +527,40 @@
}
s->queue = q;
- if (gcm_hash)
+ switch (csp->csp_mode) {
+ case CSP_MODE_AEAD:
s->mode = GCM;
- else if (hash != NULL && cipher != NULL)
+ break;
+ case CSP_MODE_ETA:
s->mode = AUTHENC;
- else if (hash != NULL)
+ break;
+ case CSP_MODE_DIGEST:
s->mode = HMAC;
- else {
- MPASS(cipher != NULL);
+ break;
+ case CSP_MODE_CIPHER:
s->mode = BLKCIPHER;
+ break;
}
- if (gcm_hash) {
- if (hash->cri_mlen == 0)
+
+ if (s->mode == GCM) {
+ if (csp->csp_auth_mlen == 0)
s->gmac.hash_len = AES_GMAC_HASH_LEN;
else
- s->gmac.hash_len = hash->cri_mlen;
- } else if (hash != NULL) {
+ s->gmac.hash_len = csp->csp_auth_mlen;
+ } else if (auth_hash != NULL) {
s->hmac.auth_hash = auth_hash;
s->hmac.auth_mode = auth_mode;
- s->hmac.partial_digest_len = partial_digest_len;
- if (hash->cri_mlen == 0)
+ if (csp->csp_auth_mlen == 0)
s->hmac.hash_len = auth_hash->hashsize;
else
- s->hmac.hash_len = hash->cri_mlen;
- ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
- hash->cri_klen);
+ s->hmac.hash_len = csp->csp_auth_mlen;
+ ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
}
- if (cipher != NULL) {
+ if (cipher_mode != CCP_AES_MODE_ECB) {
s->blkcipher.cipher_mode = cipher_mode;
- s->blkcipher.iv_len = iv_len;
- if (cipher->cri_key != NULL)
- ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
- cipher->cri_klen);
+ if (csp->csp_cipher_key != NULL)
+ ccp_aes_setkey(s, csp->csp_cipher_alg,
+ csp->csp_cipher_key, csp->csp_cipher_klen);
}
s->active = true;
@@ -573,19 +586,17 @@
static int
ccp_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct ccp_softc *sc;
struct ccp_queue *qp;
struct ccp_session *s;
- struct cryptodesc *crd, *crda, *crde;
int error;
bool qpheld;
qpheld = false;
qp = NULL;
- if (crp == NULL)
- return (EINVAL);
- crd = crp->crp_desc;
+ csp = crypto_get_params(crp->crp_session);
s = crypto_get_driver_session(crp->crp_session);
sc = device_get_softc(dev);
mtx_lock(&sc->lock);
@@ -600,89 +611,47 @@
if (error != 0)
goto out;
+ if (crp->crp_auth_key != NULL) {
+ KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
+ ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
+ }
+ if (crp->crp_cipher_key != NULL)
+ ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
+
switch (s->mode) {
case HMAC:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
- ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
- crd->crd_klen);
+ if (s->pending != 0) {
+ error = EAGAIN;
+ break;
+ }
error = ccp_hmac(qp, s, crp);
break;
case BLKCIPHER:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccp_aes_check_keylen(crd->crd_alg,
- crd->crd_klen);
- if (error != 0)
- break;
- ccp_aes_setkey(s, crd->crd_alg, crd->crd_key,
- crd->crd_klen);
+ if (s->pending != 0) {
+ error = EAGAIN;
+ break;
}
error = ccp_blkcipher(qp, s, crp);
break;
case AUTHENC:
- error = 0;
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_XTS:
- /* Only encrypt-then-authenticate supported. */
- crde = crd;
- crda = crd->crd_next;
- if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
- error = EINVAL;
- break;
- }
- s->cipher_first = true;
+ if (s->pending != 0) {
+ error = EAGAIN;
break;
- default:
- crda = crd;
- crde = crd->crd_next;
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- error = EINVAL;
- break;
- }
- s->cipher_first = false;
- break;
- }
- if (error != 0)
- break;
- if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
- ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
- crda->crd_klen);
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccp_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error != 0)
- break;
- ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
}
- error = ccp_authenc(qp, s, crp, crda, crde);
+ error = ccp_authenc(qp, s, crp);
break;
case GCM:
- error = 0;
- if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
- crde = crd;
- crda = crd->crd_next;
- s->cipher_first = true;
- } else {
- crda = crd;
- crde = crd->crd_next;
- s->cipher_first = false;
- }
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccp_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error != 0)
- break;
- ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
- }
- if (crde->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
mtx_unlock(&qp->cq_lock);
- ccp_gcm_soft(s, crp, crda, crde);
+ ccp_gcm_soft(s, crp);
return (0);
}
- error = ccp_gcm(qp, s, crp, crda, crde);
+ if (s->pending != 0) {
+ error = EAGAIN;
+ break;
+ }
+ error = ccp_gcm(qp, s, crp);
break;
}
@@ -716,6 +685,7 @@
DEVMETHOD(device_attach, ccp_attach),
DEVMETHOD(device_detach, ccp_detach),
+ DEVMETHOD(cryptodev_probesession, ccp_probesession),
DEVMETHOD(cryptodev_newsession, ccp_newsession),
DEVMETHOD(cryptodev_freesession, ccp_freesession),
DEVMETHOD(cryptodev_process, ccp_process),
Index: sys/crypto/ccp/ccp_hardware.c
===================================================================
--- sys/crypto/ccp/ccp_hardware.c
+++ sys/crypto/ccp/ccp_hardware.c
@@ -894,7 +894,7 @@
remain = len;
for (i = 0; i < sgl->sg_nseg && remain != 0; i++) {
seg = &sgl->sg_segs[i];
- /* crd_len is int, so 32-bit min() is ok. */
+ /* crp lengths are int, so 32-bit min() is ok. */
nb = min(remain, seg->ss_len);
if (tolsb)
@@ -1115,7 +1115,7 @@
lsbaddr = ccp_queue_lsb_address(qp, LSB_ENTRY_SHA);
for (i = 0; i < sgl_dst->sg_nseg; i++) {
seg = &sgl_dst->sg_segs[i];
- /* crd_len is int, so 32-bit min() is ok. */
+ /* crp lengths are int, so 32-bit min() is ok. */
nb = min(remaining, seg->ss_len);
error = ccp_passthrough(qp, seg->ss_paddr, CCP_MEMTYPE_SYSTEM,
@@ -1201,7 +1201,7 @@
static void
ccp_do_hmac_done(struct ccp_queue *qp, struct ccp_session *s,
- struct cryptop *crp, struct cryptodesc *crd, int error)
+ struct cryptop *crp, int error)
{
char ihash[SHA2_512_HASH_LEN /* max hash len */];
union authctx auth_ctx;
@@ -1219,21 +1219,26 @@
/* Do remaining outer hash over small inner hash in software */
axf->Init(&auth_ctx);
axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize);
- ccp_sha_copy_result(ihash, s->hmac.ipad, s->hmac.auth_mode);
+ ccp_sha_copy_result(ihash, s->hmac.res, s->hmac.auth_mode);
#if 0
INSECURE_DEBUG(dev, "%s sha intermediate=%64D\n", __func__,
(u_char *)ihash, " ");
#endif
axf->Update(&auth_ctx, ihash, axf->hashsize);
- axf->Final(s->hmac.ipad, &auth_ctx);
+ axf->Final(s->hmac.res, &auth_ctx);
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- s->hmac.hash_len, s->hmac.ipad);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
+ ihash);
+ if (timingsafe_bcmp(s->hmac.res, ihash, s->hmac.hash_len) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
+ s->hmac.res);
/* Avoid leaking key material */
explicit_bzero(&auth_ctx, sizeof(auth_ctx));
- explicit_bzero(s->hmac.ipad, sizeof(s->hmac.ipad));
- explicit_bzero(s->hmac.opad, sizeof(s->hmac.opad));
+ explicit_bzero(s->hmac.res, sizeof(s->hmac.res));
out:
crypto_done(crp);
@@ -1243,17 +1248,15 @@
ccp_hmac_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
int error)
{
- struct cryptodesc *crd;
struct cryptop *crp;
crp = vcrp;
- crd = crp->crp_desc;
- ccp_do_hmac_done(qp, s, crp, crd, error);
+ ccp_do_hmac_done(qp, s, crp, error);
}
static int __must_check
ccp_do_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crd, const struct ccp_completion_ctx *cctx)
+ const struct ccp_completion_ctx *cctx)
{
device_t dev;
struct auth_hash *axf;
@@ -1271,15 +1274,21 @@
error = sglist_append(qp->cq_sg_ulptx, s->hmac.ipad, axf->blocksize);
if (error != 0)
return (error);
+ if (crp->crp_aad_length != 0) {
+ error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
+ crp->crp_aad_start, crp->crp_aad_length);
+ if (error != 0)
+ return (error);
+ }
error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error != 0) {
DPRINTF(dev, "%s: sglist too short\n", __func__);
return (error);
}
- /* Populate SGL for output -- just reuse hmac.ipad buffer. */
+ /* Populate SGL for output -- use hmac.res buffer. */
sglist_reset(qp->cq_sg_dst);
- error = sglist_append(qp->cq_sg_dst, s->hmac.ipad,
+ error = sglist_append(qp->cq_sg_dst, s->hmac.res,
roundup2(axf->hashsize, LSB_ENTRY_SIZE));
if (error != 0)
return (error);
@@ -1297,15 +1306,12 @@
ccp_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
{
struct ccp_completion_ctx ctx;
- struct cryptodesc *crd;
-
- crd = crp->crp_desc;
ctx.callback_fn = ccp_hmac_done;
ctx.callback_arg = crp;
ctx.session = s;
- return (ccp_do_hmac(qp, s, crp, crd, &ctx));
+ return (ccp_do_hmac(qp, s, crp, &ctx));
}
static void
@@ -1328,7 +1334,7 @@
{
struct cryptop *crp;
- explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
+ explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
crp = vcrp;
@@ -1342,57 +1348,39 @@
}
static void
-ccp_collect_iv(struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crd)
-{
+ccp_collect_iv(struct cryptop *crp, const struct crypto_session_params *csp,
+ char *iv)
+{
- if (crd->crd_flags & CRD_F_ENCRYPT) {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(s->blkcipher.iv, crd->crd_iv,
- s->blkcipher.iv_len);
- else
- arc4rand(s->blkcipher.iv, s->blkcipher.iv_len, 0);
- if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len,
- s->blkcipher.iv);
- } else {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(s->blkcipher.iv, crd->crd_iv,
- s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len,
- s->blkcipher.iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
/*
* If the input IV is 12 bytes, append an explicit counter of 1.
*/
- if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16 &&
- s->blkcipher.iv_len == 12) {
- *(uint32_t *)&s->blkcipher.iv[12] = htobe32(1);
- s->blkcipher.iv_len = AES_BLOCK_LEN;
- }
+ if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 &&
+ csp->csp_ivlen == 12)
+ *(uint32_t *)&iv[12] = htobe32(1);
- if (crd->crd_alg == CRYPTO_AES_XTS && s->blkcipher.iv_len != AES_BLOCK_LEN) {
- DPRINTF(NULL, "got ivlen != 16: %u\n", s->blkcipher.iv_len);
- if (s->blkcipher.iv_len < AES_BLOCK_LEN)
- memset(&s->blkcipher.iv[s->blkcipher.iv_len], 0,
- AES_BLOCK_LEN - s->blkcipher.iv_len);
- s->blkcipher.iv_len = AES_BLOCK_LEN;
- }
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
+ csp->csp_ivlen < AES_BLOCK_LEN)
+ memset(&iv[csp->csp_ivlen], 0, AES_BLOCK_LEN - csp->csp_ivlen);
/* Reverse order of IV material for HW */
- INSECURE_DEBUG(NULL, "%s: IV: %16D len: %u\n", __func__,
- s->blkcipher.iv, " ", s->blkcipher.iv_len);
+ INSECURE_DEBUG(NULL, "%s: IV: %16D len: %u\n", __func__, iv, " ",
+ csp->csp_ivlen);
/*
* For unknown reasons, XTS mode expects the IV in the reverse byte
* order to every other AES mode.
*/
- if (crd->crd_alg != CRYPTO_AES_XTS)
- ccp_byteswap(s->blkcipher.iv, s->blkcipher.iv_len);
+ if (csp->csp_cipher_alg != CRYPTO_AES_XTS)
+ ccp_byteswap(iv, AES_BLOCK_LEN);
}
static int __must_check
@@ -1413,8 +1401,7 @@
static int __must_check
ccp_do_xts(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crd, enum ccp_cipher_dir dir,
- const struct ccp_completion_ctx *cctx)
+ enum ccp_cipher_dir dir, const struct ccp_completion_ctx *cctx)
{
struct ccp_desc *desc;
device_t dev;
@@ -1426,7 +1413,8 @@
dev = qp->cq_softc->dev;
for (i = 0; i < nitems(ccp_xts_unitsize_map); i++)
- if (ccp_xts_unitsize_map[i].cxu_size == crd->crd_len) {
+ if (ccp_xts_unitsize_map[i].cxu_size ==
+ crp->crp_payload_length) {
usize = ccp_xts_unitsize_map[i].cxu_id;
break;
}
@@ -1483,25 +1471,26 @@
static int __must_check
ccp_do_blkcipher(struct ccp_queue *qp, struct ccp_session *s,
- struct cryptop *crp, struct cryptodesc *crd,
- const struct ccp_completion_ctx *cctx)
+ struct cryptop *crp, const struct ccp_completion_ctx *cctx)
{
+ const struct crypto_session_params *csp;
struct ccp_desc *desc;
char *keydata;
device_t dev;
enum ccp_cipher_dir dir;
- int error;
+ int error, iv_len;
size_t keydata_len;
unsigned i, j;
dev = qp->cq_softc->dev;
- if (s->blkcipher.key_len == 0 || crd->crd_len == 0) {
+ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) {
DPRINTF(dev, "%s: empty\n", __func__);
return (EINVAL);
}
- if ((crd->crd_len % AES_BLOCK_LEN) != 0) {
- DPRINTF(dev, "%s: len modulo: %d\n", __func__, crd->crd_len);
+ if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
+ DPRINTF(dev, "%s: len modulo: %d\n", __func__,
+ crp->crp_payload_length);
return (EINVAL);
}
@@ -1518,16 +1507,20 @@
}
/* Gather IV/nonce data */
- ccp_collect_iv(s, crp, crd);
+ csp = crypto_get_params(crp->crp_session);
+ ccp_collect_iv(crp, csp, s->blkcipher.iv);
+ iv_len = csp->csp_ivlen;
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ iv_len = AES_BLOCK_LEN;
- if ((crd->crd_flags & CRD_F_ENCRYPT) != 0)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
dir = CCP_CIPHER_DIR_ENCRYPT;
else
dir = CCP_CIPHER_DIR_DECRYPT;
/* Set up passthrough op(s) to copy IV into LSB */
error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
- s->blkcipher.iv, s->blkcipher.iv_len);
+ s->blkcipher.iv, iv_len);
if (error != 0)
return (error);
@@ -1538,15 +1531,16 @@
keydata_len = 0;
keydata = NULL;
- switch (crd->crd_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_XTS:
for (j = 0; j < nitems(ccp_xts_unitsize_map); j++)
- if (ccp_xts_unitsize_map[j].cxu_size == crd->crd_len)
+ if (ccp_xts_unitsize_map[j].cxu_size ==
+ crp->crp_payload_length)
break;
/* Input buffer must be a supported UnitSize */
if (j >= nitems(ccp_xts_unitsize_map)) {
device_printf(dev, "%s: rejected block size: %u\n",
- __func__, crd->crd_len);
+ __func__, crp->crp_payload_length);
return (EOPNOTSUPP);
}
/* FALLTHROUGH */
@@ -1559,14 +1553,14 @@
INSECURE_DEBUG(dev, "%s: KEY(%zu): %16D\n", __func__, keydata_len,
keydata, " ");
- if (crd->crd_alg == CRYPTO_AES_XTS)
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
INSECURE_DEBUG(dev, "%s: KEY(XTS): %64D\n", __func__, keydata, " ");
/* Reverse order of key material for HW */
ccp_byteswap(keydata, keydata_len);
/* Store key material into LSB to avoid page boundaries */
- if (crd->crd_alg == CRYPTO_AES_XTS) {
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS) {
/*
* XTS mode uses 2 256-bit vectors for the primary key and the
* tweak key. For 128-bit keys, the vectors are zero-padded.
@@ -1610,7 +1604,7 @@
*/
sglist_reset(qp->cq_sg_ulptx);
error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error != 0)
return (error);
@@ -1622,8 +1616,8 @@
if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg)
return (EAGAIN);
- if (crd->crd_alg == CRYPTO_AES_XTS)
- return (ccp_do_xts(qp, s, crp, crd, dir, cctx));
+ if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+ return (ccp_do_xts(qp, s, crp, dir, cctx));
for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
struct sglist_seg *seg;
@@ -1646,7 +1640,7 @@
desc->aes.encrypt = dir;
desc->aes.mode = s->blkcipher.cipher_mode;
desc->aes.type = s->blkcipher.cipher_type;
- if (crd->crd_alg == CRYPTO_AES_ICM)
+ if (csp->csp_cipher_alg == CRYPTO_AES_ICM)
/*
* Size of CTR value in bits, - 1. ICM mode uses all
* 128 bits as counter.
@@ -1683,38 +1677,29 @@
ccp_blkcipher(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
{
struct ccp_completion_ctx ctx;
- struct cryptodesc *crd;
-
- crd = crp->crp_desc;
ctx.callback_fn = ccp_blkcipher_done;
ctx.session = s;
ctx.callback_arg = crp;
- return (ccp_do_blkcipher(qp, s, crp, crd, &ctx));
+ return (ccp_do_blkcipher(qp, s, crp, &ctx));
}
static void
ccp_authenc_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
int error)
{
- struct cryptodesc *crda;
struct cryptop *crp;
- explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
+ explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
crp = vcrp;
- if (s->cipher_first)
- crda = crp->crp_desc->crd_next;
- else
- crda = crp->crp_desc;
- ccp_do_hmac_done(qp, s, crp, crda, error);
+ ccp_do_hmac_done(qp, s, crp, error);
}
int __must_check
-ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
{
struct ccp_completion_ctx ctx;
int error;
@@ -1724,18 +1709,18 @@
ctx.callback_arg = crp;
/* Perform first operation */
- if (s->cipher_first)
- error = ccp_do_blkcipher(qp, s, crp, crde, NULL);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ error = ccp_do_blkcipher(qp, s, crp, NULL);
else
- error = ccp_do_hmac(qp, s, crp, crda, NULL);
+ error = ccp_do_hmac(qp, s, crp, NULL);
if (error != 0)
return (error);
/* Perform second operation */
- if (s->cipher_first)
- error = ccp_do_hmac(qp, s, crp, crda, &ctx);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ error = ccp_do_hmac(qp, s, crp, &ctx);
else
- error = ccp_do_blkcipher(qp, s, crp, crde, &ctx);
+ error = ccp_do_blkcipher(qp, s, crp, &ctx);
return (error);
}
@@ -1852,17 +1837,9 @@
int error)
{
char tag[GMAC_DIGEST_LEN];
- struct cryptodesc *crde, *crda;
struct cryptop *crp;
crp = vcrp;
- if (s->cipher_first) {
- crde = crp->crp_desc;
- crda = crp->crp_desc->crd_next;
- } else {
- crde = crp->crp_desc->crd_next;
- crda = crp->crp_desc;
- }
s->pending--;
@@ -1872,27 +1849,26 @@
}
/* Encrypt is done. Decrypt needs to verify tag. */
- if ((crde->crd_flags & CRD_F_ENCRYPT) != 0)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
goto out;
/* Copy in message tag. */
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(tag), tag);
+ crypto_copydata(crp, crp->crp_digest_start, s->gmac.hash_len, tag);
/* Verify tag against computed GMAC */
if (timingsafe_bcmp(tag, s->gmac.final_block, s->gmac.hash_len) != 0)
crp->crp_etype = EBADMSG;
out:
- explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
- explicit_bzero(&s->gmac, sizeof(s->gmac));
+ explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
+ explicit_bzero(&s->gmac.final_block, sizeof(s->gmac.final_block));
crypto_done(crp);
}
int __must_check
-ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
{
+ const struct crypto_session_params *csp;
struct ccp_completion_ctx ctx;
enum ccp_cipher_dir dir;
device_t dev;
@@ -1902,16 +1878,9 @@
if (s->blkcipher.key_len == 0)
return (EINVAL);
- /*
- * AAD is only permitted before the cipher/plain text, not
- * after.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
- return (EINVAL);
-
dev = qp->cq_softc->dev;
- if ((crde->crd_flags & CRD_F_ENCRYPT) != 0)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
dir = CCP_CIPHER_DIR_ENCRYPT;
else
dir = CCP_CIPHER_DIR_DECRYPT;
@@ -1920,14 +1889,15 @@
memset(s->blkcipher.iv, 0, sizeof(s->blkcipher.iv));
/* Gather IV data */
- ccp_collect_iv(s, crp, crde);
+ csp = crypto_get_params(crp->crp_session);
+ ccp_collect_iv(crp, csp, s->blkcipher.iv);
/* Reverse order of key material for HW */
ccp_byteswap(s->blkcipher.enckey, s->blkcipher.key_len);
/* Prepare input buffer of concatenated lengths for final GHASH */
- be64enc(s->gmac.final_block, (uint64_t)crda->crd_len * 8);
- be64enc(&s->gmac.final_block[8], (uint64_t)crde->crd_len * 8);
+ be64enc(s->gmac.final_block, (uint64_t)crp->crp_aad_length * 8);
+ be64enc(&s->gmac.final_block[8], (uint64_t)crp->crp_payload_length * 8);
/* Send IV + initial zero GHASH, key data, and lengths buffer to LSB */
error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
@@ -1945,10 +1915,10 @@
return (error);
/* First step - compute GHASH over AAD */
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
sglist_reset(qp->cq_sg_ulptx);
error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
- crda->crd_skip, crda->crd_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error != 0)
return (error);
@@ -1970,7 +1940,7 @@
/* Feed data piece by piece into GCTR */
sglist_reset(qp->cq_sg_ulptx);
error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error != 0)
return (error);
@@ -1996,7 +1966,7 @@
seg = &qp->cq_sg_ulptx->sg_segs[i];
error = ccp_do_gctr(qp, s, dir, seg,
- (i == 0 && crda->crd_len == 0),
+ (i == 0 && crp->crp_aad_length == 0),
i == (qp->cq_sg_ulptx->sg_nseg - 1));
if (error != 0)
return (error);
@@ -2004,7 +1974,7 @@
/* Send just initial IV (not GHASH!) to LSB again */
error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
- s->blkcipher.iv, s->blkcipher.iv_len);
+ s->blkcipher.iv, AES_BLOCK_LEN);
if (error != 0)
return (error);
@@ -2021,7 +1991,7 @@
sglist_reset(qp->cq_sg_ulptx);
if (dir == CCP_CIPHER_DIR_ENCRYPT)
error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
- crda->crd_inject, s->gmac.hash_len);
+ crp->crp_digest_start, s->gmac.hash_len);
else
/*
* For decrypting, copy the computed tag out to our session
Index: sys/crypto/via/padlock.h
===================================================================
--- sys/crypto/via/padlock.h
+++ sys/crypto/via/padlock.h
@@ -68,7 +68,6 @@
union padlock_cw ses_cw __aligned(16);
uint32_t ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */
uint32_t ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16); /* 128 bit aligned */
- uint8_t ses_iv[16] __aligned(16); /* 128 bit aligned */
struct auth_hash *ses_axf;
uint8_t *ses_ictx;
uint8_t *ses_octx;
@@ -79,13 +78,14 @@
#define PADLOCK_ALIGN(p) (void *)(roundup2((uintptr_t)(p), 16))
int padlock_cipher_setup(struct padlock_session *ses,
- struct cryptoini *encini);
+ const struct crypto_session_params *csp);
int padlock_cipher_process(struct padlock_session *ses,
- struct cryptodesc *enccrd, struct cryptop *crp);
+ struct cryptop *crp, const struct crypto_session_params *csp);
+bool padlock_hash_check(const struct crypto_session_params *csp);
int padlock_hash_setup(struct padlock_session *ses,
- struct cryptoini *macini);
+ const struct crypto_session_params *csp);
int padlock_hash_process(struct padlock_session *ses,
- struct cryptodesc *maccrd, struct cryptop *crp);
+ struct cryptop *crp, const struct crypto_session_params *csp);
void padlock_hash_free(struct padlock_session *ses);
#endif /* !_PADLOCK_H_ */
Index: sys/crypto/via/padlock.c
===================================================================
--- sys/crypto/via/padlock.c
+++ sys/crypto/via/padlock.c
@@ -60,7 +60,9 @@
int32_t sc_cid;
};
-static int padlock_newsession(device_t, crypto_session_t cses, struct cryptoini *cri);
+static int padlock_probesession(device_t, const struct crypto_session_params *);
+static int padlock_newsession(device_t, crypto_session_t cses,
+ const struct crypto_session_params *);
static void padlock_freesession(device_t, crypto_session_t cses);
static void padlock_freesession_one(struct padlock_softc *sc,
struct padlock_session *ses);
@@ -123,13 +125,6 @@
return (ENOMEM);
}
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
return (0);
}
@@ -143,63 +138,65 @@
}
static int
-padlock_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+padlock_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct padlock_softc *sc = device_get_softc(dev);
- struct padlock_session *ses = NULL;
- struct cryptoini *encini, *macini;
- struct thread *td;
- int error;
- if (cri == NULL)
+ if (csp->csp_flags != 0)
return (EINVAL);
- encini = macini = NULL;
- for (; cri != NULL; cri = cri->cri_next) {
- switch (cri->cri_alg) {
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- if (macini != NULL)
- return (EINVAL);
- macini = cri;
- break;
- case CRYPTO_AES_CBC:
- if (encini != NULL)
- return (EINVAL);
- encini = cri;
- break;
- default:
- return (EINVAL);
- }
- }
-
/*
* We only support HMAC algorithms to be able to work with
* ipsec(4), so if we are asked only for authentication without
- * encryption, don't pretend we can accellerate it.
+ * encryption, don't pretend we can accelerate it.
+ *
+ * XXX: For CPUs with SHA instructions we should probably
+ * permit CSP_MODE_DIGEST so that those can be tested.
*/
- if (encini == NULL)
+ switch (csp->csp_mode) {
+ case CSP_MODE_ETA:
+ if (!padlock_hash_check(csp))
+ return (EINVAL);
+ /* FALLTHROUGH */
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ default:
return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
+
+static int
+padlock_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct padlock_softc *sc = device_get_softc(dev);
+ struct padlock_session *ses = NULL;
+ struct thread *td;
+ int error;
ses = crypto_get_driver_session(cses);
ses->ses_fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
- error = padlock_cipher_setup(ses, encini);
+ error = padlock_cipher_setup(ses, csp);
if (error != 0) {
padlock_freesession_one(sc, ses);
return (error);
}
- if (macini != NULL) {
+ if (csp->csp_mode == CSP_MODE_ETA) {
td = curthread;
fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL |
FPU_KERN_KTHR);
- error = padlock_hash_setup(ses, macini);
+ error = padlock_hash_setup(ses, csp);
fpu_kern_leave(td, ses->ses_fpu_ctx);
if (error != 0) {
padlock_freesession_one(sc, ses);
@@ -231,68 +228,34 @@
static int
padlock_process(device_t dev, struct cryptop *crp, int hint __unused)
{
- struct padlock_session *ses = NULL;
- struct cryptodesc *crd, *enccrd, *maccrd;
- int error = 0;
+ const struct crypto_session_params *csp;
+ struct padlock_session *ses;
+ int error;
- enccrd = maccrd = NULL;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
- error = EINVAL;
- goto out;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- if (maccrd != NULL) {
- error = EINVAL;
- goto out;
- }
- maccrd = crd;
- break;
- case CRYPTO_AES_CBC:
- if (enccrd != NULL) {
- error = EINVAL;
- goto out;
- }
- enccrd = crd;
- break;
- default:
- return (EINVAL);
- }
- }
- if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+ if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
error = EINVAL;
goto out;
}
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
- /* Perform data authentication if requested before encryption. */
- if (maccrd != NULL && maccrd->crd_next == enccrd) {
- error = padlock_hash_process(ses, maccrd, crp);
+ /* Perform data authentication if requested before decryption. */
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = padlock_hash_process(ses, crp, csp);
if (error != 0)
goto out;
}
- error = padlock_cipher_process(ses, enccrd, crp);
+ error = padlock_cipher_process(ses, crp, csp);
if (error != 0)
goto out;
/* Perform data authentication if requested after encryption. */
- if (maccrd != NULL && enccrd->crd_next == maccrd) {
- error = padlock_hash_process(ses, maccrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = padlock_hash_process(ses, crp, csp);
if (error != 0)
goto out;
}
@@ -320,6 +283,7 @@
DEVMETHOD(device_attach, padlock_attach),
DEVMETHOD(device_detach, padlock_detach),
+ DEVMETHOD(cryptodev_probesession, padlock_probesession),
DEVMETHOD(cryptodev_newsession, padlock_newsession),
DEVMETHOD(cryptodev_freesession,padlock_freesession),
DEVMETHOD(cryptodev_process, padlock_process),
Index: sys/crypto/via/padlock_cipher.c
===================================================================
--- sys/crypto/via/padlock_cipher.c
+++ sys/crypto/via/padlock_cipher.c
@@ -98,7 +98,7 @@
}
static void
-padlock_cipher_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+padlock_cipher_key_setup(struct padlock_session *ses, const void *key, int klen)
{
union padlock_cw *cw;
int i;
@@ -106,8 +106,8 @@
cw = &ses->ses_cw;
if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) {
/* Build expanded keys for both directions */
- rijndaelKeySetupEnc(ses->ses_ekey, key, klen);
- rijndaelKeySetupDec(ses->ses_dkey, key, klen);
+ rijndaelKeySetupEnc(ses->ses_ekey, key, klen * 8);
+ rijndaelKeySetupDec(ses->ses_dkey, key, klen * 8);
for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) {
ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]);
ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]);
@@ -119,12 +119,13 @@
}
int
-padlock_cipher_setup(struct padlock_session *ses, struct cryptoini *encini)
+padlock_cipher_setup(struct padlock_session *ses,
+ const struct crypto_session_params *csp)
{
union padlock_cw *cw;
- if (encini->cri_klen != 128 && encini->cri_klen != 192 &&
- encini->cri_klen != 256) {
+ if (csp->csp_cipher_klen != 16 && csp->csp_cipher_klen != 25 &&
+ csp->csp_cipher_klen != 32) {
return (EINVAL);
}
@@ -133,7 +134,7 @@
cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES;
cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW;
cw->cw_intermediate = 0;
- switch (encini->cri_klen) {
+ switch (csp->csp_cipher_klen * 8) {
case 128:
cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128;
cw->cw_key_size = PADLOCK_KEY_SIZE_128;
@@ -151,12 +152,10 @@
cw->cw_key_size = PADLOCK_KEY_SIZE_256;
break;
}
- if (encini->cri_key != NULL) {
- padlock_cipher_key_setup(ses, encini->cri_key,
- encini->cri_klen);
+ if (csp->csp_cipher_key != NULL) {
+ padlock_cipher_key_setup(ses, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
}
-
- arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
return (0);
}
@@ -166,56 +165,60 @@
* If it isn't, new buffer is allocated.
*/
static u_char *
-padlock_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
- int *allocated)
+padlock_cipher_alloc(struct cryptop *crp, int *allocated)
{
u_char *addr;
- if (crp->crp_flags & CRYPTO_F_IMBUF)
- goto alloc;
- else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- struct uio *uio;
- struct iovec *iov;
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ break;
+ case CRYPTO_BUF_UIO: {
+ struct uio *uio;
+ struct iovec *iov;
- uio = (struct uio *)crp->crp_buf;
- if (uio->uio_iovcnt != 1)
- goto alloc;
- iov = uio->uio_iov;
- addr = (u_char *)iov->iov_base + enccrd->crd_skip;
- } else {
- addr = (u_char *)crp->crp_buf;
- }
+ uio = crp->crp_uio;
+ if (uio->uio_iovcnt != 1)
+ break;
+ iov = uio->uio_iov;
+ addr = (u_char *)iov->iov_base + crp->crp_payload_start;
if (((uintptr_t)addr & 0xf) != 0) /* 16 bytes aligned? */
- goto alloc;
+ break;
*allocated = 0;
return (addr);
}
-alloc:
+ case CRYPTO_BUF_CONTIG:
+ addr = (u_char *)crp->crp_buf + crp->crp_payload_start;
+ if (((uintptr_t)addr & 0xf) != 0) /* 16 bytes aligned? */
+ break;
+ *allocated = 0;
+ return (addr);
+ }
+
*allocated = 1;
- addr = malloc(enccrd->crd_len + 16, M_PADLOCK, M_NOWAIT);
+ addr = malloc(crp->crp_payload_length + 16, M_PADLOCK, M_NOWAIT);
return (addr);
}
int
-padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd,
- struct cryptop *crp)
+padlock_cipher_process(struct padlock_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp)
{
union padlock_cw *cw;
struct thread *td;
u_char *buf, *abuf;
uint32_t *key;
+ uint8_t iv[AES_BLOCK_LEN] __aligned(16);
int allocated;
- buf = padlock_cipher_alloc(enccrd, crp, &allocated);
+ buf = padlock_cipher_alloc(crp, &allocated);
if (buf == NULL)
return (ENOMEM);
/* Buffer has to be 16 bytes aligned. */
abuf = PADLOCK_ALIGN(buf);
- if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
- padlock_cipher_key_setup(ses, enccrd->crd_key,
- enccrd->crd_klen);
+ if (crp->crp_cipher_key != NULL) {
+ padlock_cipher_key_setup(ses, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
cw = &ses->ses_cw;
@@ -223,52 +226,39 @@
cw->cw_filler1 = 0;
cw->cw_filler2 = 0;
cw->cw_filler3 = 0;
- if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
+
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, AES_BLOCK_LEN, 0);
+ crypto_copyback(crp, crp->crp_iv_start, AES_BLOCK_LEN, iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, AES_BLOCK_LEN);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, AES_BLOCK_LEN, iv);
+
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT;
key = ses->ses_ekey;
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
- }
} else {
cw->cw_direction = PADLOCK_DIRECTION_DECRYPT;
key = ses->ses_dkey;
- if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
- bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
- else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
- }
}
if (allocated) {
- crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, abuf);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, abuf);
}
td = curthread;
fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR);
- padlock_cbc(abuf, abuf, enccrd->crd_len / AES_BLOCK_LEN, key, cw,
- ses->ses_iv);
+ padlock_cbc(abuf, abuf, crp->crp_payload_length / AES_BLOCK_LEN, key,
+ cw, iv);
fpu_kern_leave(td, ses->ses_fpu_ctx);
if (allocated) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
- enccrd->crd_len, abuf);
- }
+ crypto_copyback(crp, crp->crp_payload_start,
+ crp->crp_payload_length, abuf);
- /* copy out last block for use as next session IV */
- if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
- AES_BLOCK_LEN, ses->ses_iv);
- }
-
- if (allocated) {
- bzero(buf, enccrd->crd_len + 16);
+ explicit_bzero(buf, crp->crp_payload_length + 16);
free(buf, M_PADLOCK);
}
return (0);
Index: sys/crypto/via/padlock_hash.c
===================================================================
--- sys/crypto/via/padlock_hash.c
+++ sys/crypto/via/padlock_hash.c
@@ -44,7 +44,6 @@
#include <machine/pcb.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
#include <opencrypto/xform.h>
#include <crypto/via/padlock.h>
@@ -249,12 +248,11 @@
}
static void
-padlock_hash_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+padlock_hash_key_setup(struct padlock_session *ses, const uint8_t *key,
+ int klen)
{
struct auth_hash *axf;
- int i;
- klen /= 8;
axf = ses->ses_axf;
/*
@@ -265,32 +263,17 @@
padlock_free_ctx(axf, ses->ses_ictx);
padlock_free_ctx(axf, ses->ses_octx);
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
-
- axf->Init(ses->ses_ictx);
- axf->Update(ses->ses_ictx, key, klen);
- axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- axf->Init(ses->ses_octx);
- axf->Update(ses->ses_octx, key, klen);
- axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ hmac_init_ipad(axf, key, klen, ses->ses_ictx);
+ hmac_init_opad(axf, key, klen, ses->ses_octx);
}
/*
* Compute keyed-hash authenticator.
*/
static int
-padlock_authcompute(struct padlock_session *ses, struct cryptodesc *crd,
- caddr_t buf, int flags)
+padlock_authcompute(struct padlock_session *ses, struct cryptop *crp)
{
- u_char hash[HASH_MAX_LEN];
+ u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN];
struct auth_hash *axf;
union authctx ctx;
int error;
@@ -298,7 +281,14 @@
axf = ses->ses_axf;
padlock_copy_ctx(axf, ses->ses_ictx, &ctx);
- error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+ error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+ (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+ if (error != 0) {
+ padlock_free_ctx(axf, &ctx);
+ return (error);
+ }
+ error = crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length,
(int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
if (error != 0) {
padlock_free_ctx(axf, &ctx);
@@ -310,48 +300,75 @@
axf->Update(&ctx, hash, axf->hashsize);
axf->Final(hash, &ctx);
- /* Inject the authentication data */
- crypto_copyback(flags, buf, crd->crd_inject,
- ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash2);
+ if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash);
return (0);
}
-int
-padlock_hash_setup(struct padlock_session *ses, struct cryptoini *macini)
+/* Find software structure which describes HMAC algorithm. */
+static struct auth_hash *
+padlock_hash_lookup(int alg)
{
+ struct auth_hash *axf;
- ses->ses_mlen = macini->cri_mlen;
-
- /* Find software structure which describes HMAC algorithm. */
- switch (macini->cri_alg) {
+ switch (alg) {
case CRYPTO_NULL_HMAC:
- ses->ses_axf = &auth_hash_null;
+ axf = &auth_hash_null;
break;
case CRYPTO_MD5_HMAC:
- ses->ses_axf = &auth_hash_hmac_md5;
+ axf = &auth_hash_hmac_md5;
break;
case CRYPTO_SHA1_HMAC:
if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
- ses->ses_axf = &padlock_hmac_sha1;
+ axf = &padlock_hmac_sha1;
else
- ses->ses_axf = &auth_hash_hmac_sha1;
+ axf = &auth_hash_hmac_sha1;
break;
case CRYPTO_RIPEMD160_HMAC:
- ses->ses_axf = &auth_hash_hmac_ripemd_160;
+ axf = &auth_hash_hmac_ripemd_160;
break;
case CRYPTO_SHA2_256_HMAC:
if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
- ses->ses_axf = &padlock_hmac_sha256;
+ axf = &padlock_hmac_sha256;
else
- ses->ses_axf = &auth_hash_hmac_sha2_256;
+ axf = &auth_hash_hmac_sha2_256;
break;
case CRYPTO_SHA2_384_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_384;
+ axf = &auth_hash_hmac_sha2_384;
break;
case CRYPTO_SHA2_512_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_512;
+ axf = &auth_hash_hmac_sha2_512;
+ break;
+ default:
+ axf = NULL;
break;
}
+ return (axf);
+}
+
+bool
+padlock_hash_check(const struct crypto_session_params *csp)
+{
+
+ return (padlock_hash_lookup(csp->csp_auth_alg) != NULL);
+}
+
+int
+padlock_hash_setup(struct padlock_session *ses,
+ const struct crypto_session_params *csp)
+{
+
+ ses->ses_axf = padlock_hash_lookup(csp->csp_auth_alg);
+ if (csp->csp_auth_mlen == 0)
+ ses->ses_mlen = ses->ses_axf->hashsize;
+ else
+ ses->ses_mlen = csp->csp_auth_mlen;
/* Allocate memory for HMAC inner and outer contexts. */
ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_PADLOCK,
@@ -362,26 +379,27 @@
return (ENOMEM);
/* Setup key if given. */
- if (macini->cri_key != NULL) {
- padlock_hash_key_setup(ses, macini->cri_key,
- macini->cri_klen);
+ if (csp->csp_auth_key != NULL) {
+ padlock_hash_key_setup(ses, csp->csp_auth_key,
+ csp->csp_auth_klen);
}
return (0);
}
int
-padlock_hash_process(struct padlock_session *ses, struct cryptodesc *maccrd,
- struct cryptop *crp)
+padlock_hash_process(struct padlock_session *ses, struct cryptop *crp,
+ const struct crypto_session_params *csp)
{
struct thread *td;
int error;
td = curthread;
fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR);
- if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
- padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+ if (crp->crp_auth_key != NULL)
+ padlock_hash_key_setup(ses, crp->crp_auth_key,
+ csp->csp_auth_klen);
- error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+ error = padlock_authcompute(ses, crp);
fpu_kern_leave(td, ses->ses_fpu_ctx);
return (error);
}
Index: sys/dev/cesa/cesa.h
===================================================================
--- sys/dev/cesa/cesa.h
+++ sys/dev/cesa/cesa.h
@@ -194,7 +194,6 @@
struct cesa_session {
uint32_t cs_config;
- unsigned int cs_klen;
unsigned int cs_ivlen;
unsigned int cs_hlen;
unsigned int cs_mblen;
@@ -208,8 +207,6 @@
struct cesa_sa_data *cr_csd;
bus_addr_t cr_csd_paddr;
struct cryptop *cr_crp;
- struct cryptodesc *cr_enc;
- struct cryptodesc *cr_mac;
struct cesa_session *cr_cs;
bus_dmamap_t cr_dmap;
int cr_dmap_loaded;
@@ -272,8 +269,6 @@
struct cesa_chain_info {
struct cesa_softc *cci_sc;
struct cesa_request *cci_cr;
- struct cryptodesc *cci_enc;
- struct cryptodesc *cci_mac;
uint32_t cci_config;
int cci_error;
};
Index: sys/dev/cesa/cesa.c
===================================================================
--- sys/dev/cesa/cesa.c
+++ sys/dev/cesa/cesa.c
@@ -69,6 +69,7 @@
#include <crypto/sha2/sha256.h>
#include <crypto/rijndael/rijndael.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform.h>
#include "cryptodev_if.h"
#include <arm/mv/mvreg.h>
@@ -80,7 +81,10 @@
static int cesa_attach_late(device_t);
static int cesa_detach(device_t);
static void cesa_intr(void *);
-static int cesa_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int cesa_probesession(device_t,
+ const struct crypto_session_params *);
+static int cesa_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int cesa_process(device_t, struct cryptop *, int);
static struct resource_spec cesa_res_spec[] = {
@@ -97,6 +101,7 @@
DEVMETHOD(device_detach, cesa_detach),
/* Crypto device methods */
+ DEVMETHOD(cryptodev_probesession, cesa_probesession),
DEVMETHOD(cryptodev_newsession, cesa_newsession),
DEVMETHOD(cryptodev_process, cesa_process),
@@ -417,78 +422,68 @@
return (0);
}
-static int
+static void
cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
{
- uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
- uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
- SHA1_CTX sha1ctx;
- SHA256_CTX sha256ctx;
- MD5_CTX md5ctx;
+ union authctx auth_ctx;
uint32_t *hout;
uint32_t *hin;
int i;
- memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
- memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
- for (i = 0; i < mklen; i++) {
- ipad[i] ^= mkey[i];
- opad[i] ^= mkey[i];
- }
-
hin = (uint32_t *)cs->cs_hiv_in;
hout = (uint32_t *)cs->cs_hiv_out;
switch (alg) {
case CRYPTO_MD5_HMAC:
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN);
- memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, opad, MD5_BLOCK_LEN);
- memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
+ hmac_init_ipad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+ memcpy(hin, auth_ctx.md5ctx.state,
+ sizeof(auth_ctx.md5ctx.state));
+ hmac_init_opad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+ memcpy(hout, auth_ctx.md5ctx.state,
+ sizeof(auth_ctx.md5ctx.state));
break;
case CRYPTO_SHA1_HMAC:
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN);
- memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN);
- memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
+ hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+ memcpy(hin, auth_ctx.sha1ctx.h.b32,
+ sizeof(auth_ctx.sha1ctx.h.b32));
+ hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+ memcpy(hout, auth_ctx.sha1ctx.h.b32,
+ sizeof(auth_ctx.sha1ctx.h.b32));
break;
case CRYPTO_SHA2_256_HMAC:
- SHA256_Init(&sha256ctx);
- SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN);
- memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state));
- SHA256_Init(&sha256ctx);
- SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN);
- memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state));
+ hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen,
+ &auth_ctx);
+ memcpy(hin, auth_ctx.sha256ctx.state,
+ sizeof(auth_ctx.sha256ctx.state));
+ hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen,
+ &auth_ctx);
+ memcpy(hout, auth_ctx.sha256ctx.state,
+ sizeof(auth_ctx.sha256ctx.state));
break;
default:
- return (EINVAL);
+ panic("shouldn't get here");
}
for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
hin[i] = htobe32(hin[i]);
hout[i] = htobe32(hout[i]);
}
-
- return (0);
}
static int
-cesa_prep_aes_key(struct cesa_session *cs)
+cesa_prep_aes_key(struct cesa_session *cs,
+ const struct crypto_session_params *csp)
{
uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
uint32_t *dkey;
int i;
- rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
+ rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8);
cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
dkey = (uint32_t *)cs->cs_aes_dkey;
- switch (cs->cs_klen) {
+ switch (csp->csp_cipher_klen) {
case 16:
cs->cs_config |= CESA_CSH_AES_KLEN_128;
for (i = 0; i < 4; i++)
@@ -515,22 +510,6 @@
return (0);
}
-static int
-cesa_is_hash(int alg)
-{
-
- switch (alg) {
- case CRYPTO_MD5:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- return (1);
- default:
- return (0);
- }
-}
-
static void
cesa_start_packet(struct cesa_packet *cp, unsigned int size)
{
@@ -584,6 +563,7 @@
unsigned int skip, len;
struct cesa_sa_desc *csd;
struct cesa_request *cr;
+ struct cryptop *crp;
struct cesa_softc *sc;
struct cesa_packet cp;
bus_dma_segment_t seg;
@@ -593,73 +573,107 @@
cci = arg;
sc = cci->cci_sc;
cr = cci->cci_cr;
+ crp = cr->cr_crp;
if (error) {
cci->cci_error = error;
return;
}
- elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
- eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
- mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
- mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
-
- if (elen && mlen &&
- ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
- (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
- (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
+ /*
+ * Only do a combined op if the AAD is adjacent to the payload
+ * and the AAD length is a multiple of the IV length. The
+ * checks against 'config' are to avoid recursing when the
+ * logic below invokes separate operations.
+ */
+ config = cci->cci_config;
+ if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC ||
+ (config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) &&
+ crp->crp_aad_length != 0 &&
+ (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) {
/*
* Data alignment in the request does not meet CESA requiremnts
* for combined encryption/decryption and hashing. We have to
* split the request to separate operations and process them
* one by one.
*/
- config = cci->cci_config;
if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_MAC;
- cci->cci_enc = NULL;
- cci->cci_mac = cr->cr_mac;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_ENC;
- cci->cci_enc = cr->cr_enc;
- cci->cci_mac = NULL;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
} else {
config &= ~CESA_CSHD_OP_MASK;
cci->cci_config = config | CESA_CSHD_ENC;
- cci->cci_enc = cr->cr_enc;
- cci->cci_mac = NULL;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
cci->cci_config = config | CESA_CSHD_MAC;
- cci->cci_enc = NULL;
- cci->cci_mac = cr->cr_mac;
- cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+ cesa_create_chain_cb(cci, segs, nseg, 0);
}
return;
}
+ mskip = mlen = eskip = elen = 0;
+
+ if (crp->crp_aad_length == 0) {
+ skip = crp->crp_payload_start;
+ len = crp->crp_payload_length;
+ switch (config & CESA_CSHD_OP_MASK) {
+ case CESA_CSHD_ENC:
+ eskip = skip;
+ elen = len;
+ break;
+ case CESA_CSHD_MAC:
+ mskip = skip;
+ mlen = len;
+ break;
+ default:
+ eskip = skip;
+ elen = len;
+ mskip = skip;
+ mlen = len;
+ break;
+ }
+ } else {
+ /*
+ * For an encryption-only separate request, only
+ * process the payload. For combined requests and
+ * hash-only requests, process the entire region.
+ */
+ switch (config & CESA_CSHD_OP_MASK) {
+ case CESA_CSHD_ENC:
+ skip = crp->crp_payload_start;
+ len = crp->crp_payload_length;
+ eskip = skip;
+ elen = len;
+ break;
+ case CESA_CSHD_MAC:
+ skip = crp->crp_aad_start;
+ len = crp->crp_aad_length + crp->crp_payload_length;
+ mskip = skip;
+ mlen = len;
+ break;
+ default:
+ skip = crp->crp_aad_start;
+ len = crp->crp_aad_length + crp->crp_payload_length;
+ mskip = skip;
+ mlen = len;
+ eskip = crp->crp_payload_start;
+ elen = crp->crp_payload_length;
+ break;
+ }
+ }
+
tmlen = mlen;
fragmented = 0;
mpsize = CESA_MAX_PACKET_SIZE;
mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
- if (elen && mlen) {
- skip = MIN(eskip, mskip);
- len = MAX(elen + eskip, mlen + mskip) - skip;
- } else if (elen) {
- skip = eskip;
- len = elen;
- } else {
- skip = mskip;
- len = mlen;
- }
-
/* Start first packet in chain */
cesa_start_packet(&cp, MIN(mpsize, len));
@@ -777,16 +791,9 @@
}
}
-static void
-cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
- bus_size_t size, int error)
-{
-
- cesa_create_chain_cb(arg, segs, nseg, error);
-}
-
static int
-cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
+cesa_create_chain(struct cesa_softc *sc,
+ const struct crypto_session_params *csp, struct cesa_request *cr)
{
struct cesa_chain_info cci;
struct cesa_tdma_desc *ctd;
@@ -797,17 +804,17 @@
CESA_LOCK_ASSERT(sc, sessions);
/* Create request metadata */
- if (cr->cr_enc) {
- if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
- (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+ if (csp->csp_cipher_klen != 0) {
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC &&
+ !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
- cr->cr_cs->cs_klen);
+ csp->csp_cipher_klen);
else
memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
- cr->cr_cs->cs_klen);
+ csp->csp_cipher_klen);
}
- if (cr->cr_mac) {
+ if (csp->csp_auth_klen != 0) {
memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
CESA_MAX_HASH_LEN);
memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
@@ -823,37 +830,30 @@
/* Prepare SA configuration */
config = cr->cr_cs->cs_config;
- if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+ if (csp->csp_cipher_alg != 0 &&
+ !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
config |= CESA_CSHD_DECRYPT;
- if (cr->cr_enc && !cr->cr_mac)
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
config |= CESA_CSHD_ENC;
- if (!cr->cr_enc && cr->cr_mac)
+ break;
+ case CSP_MODE_DIGEST:
config |= CESA_CSHD_MAC;
- if (cr->cr_enc && cr->cr_mac)
+ break;
+ case CSP_MODE_ETA:
config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
CESA_CSHD_ENC_AND_MAC;
+ break;
+ }
/* Create data packets */
cci.cci_sc = sc;
cci.cci_cr = cr;
- cci.cci_enc = cr->cr_enc;
- cci.cci_mac = cr->cr_mac;
cci.cci_config = config;
cci.cci_error = 0;
- if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
- error = bus_dmamap_load_uio(sc->sc_data_dtag,
- cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
- cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
- else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
- error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
- cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
- cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
- else
- error = bus_dmamap_load(sc->sc_data_dtag,
- cr->cr_dmap, cr->cr_crp->crp_buf,
- cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
- BUS_DMA_NOWAIT);
+ error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp,
+ cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT);
if (!error)
cr->cr_dmap_loaded = 1;
@@ -1385,18 +1385,6 @@
goto err8;
}
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- if (sc->sc_soc_id == MV_DEV_88F6828 ||
- sc->sc_soc_id == MV_DEV_88F6820 ||
- sc->sc_soc_id == MV_DEV_88F6810)
- crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-
return (0);
err8:
for (i = 0; i < CESA_REQUESTS; i++)
@@ -1487,6 +1475,7 @@
struct cesa_request *cr, *tmp;
struct cesa_softc *sc;
uint32_t ecr, icr;
+ uint8_t hash[HASH_MAX_LEN];
int blocked;
sc = arg;
@@ -1547,11 +1536,19 @@
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
cr->cr_crp->crp_etype = sc->sc_error;
- if (cr->cr_mac)
- crypto_copyback(cr->cr_crp->crp_flags,
- cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
- cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
-
+ if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) {
+ if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(cr->cr_crp,
+ cr->cr_crp->crp_digest_start,
+ cr->cr_cs->cs_hlen, hash);
+ if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash,
+ cr->cr_cs->cs_hlen) != 0)
+ cr->cr_crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(cr->cr_crp,
+ cr->cr_crp->crp_digest_start,
+ cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
+ }
crypto_done(cr->cr_crp);
cesa_free_request(sc, cr);
}
@@ -1571,43 +1568,99 @@
crypto_unblock(sc->sc_cid, blocked);
}
+static bool
+cesa_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_DES_CBC:
+ if (csp->csp_ivlen != DES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_3DES_CBC:
+ if (csp->csp_ivlen != DES3_BLOCK_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+
+ if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN)
+ return (false);
+
+ return (true);
+}
+
+static bool
+cesa_auth_supported(struct cesa_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA2_256_HMAC:
+ if (!(sc->sc_soc_id == MV_DEV_88F6828 ||
+ sc->sc_soc_id == MV_DEV_88F6820 ||
+ sc->sc_soc_id == MV_DEV_88F6810))
+ return (false);
+ /* FALLTHROUGH */
+ case CRYPTO_MD5:
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA1_HMAC:
+ break;
+ default:
+ return (false);
+ }
+
+ if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN)
+ return (false);
+
+ return (true);
+}
+
static int
-cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+cesa_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct cesa_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!cesa_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!cesa_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!cesa_auth_supported(sc, csp) ||
+ !cesa_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+cesa_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
struct cesa_session *cs;
struct cesa_softc *sc;
- struct cryptoini *enc;
- struct cryptoini *mac;
int error;
sc = device_get_softc(dev);
- enc = NULL;
- mac = NULL;
error = 0;
- /* Check and parse input */
- if (cesa_is_hash(cri->cri_alg))
- mac = cri;
- else
- enc = cri;
-
- cri = cri->cri_next;
-
- if (cri) {
- if (!enc && !cesa_is_hash(cri->cri_alg))
- enc = cri;
-
- if (!mac && cesa_is_hash(cri->cri_alg))
- mac = cri;
-
- if (cri->cri_next || !(enc && mac))
- return (EINVAL);
- }
-
- if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
- (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
- return (E2BIG);
-
/* Allocate session */
cs = crypto_get_driver_session(cses);
@@ -1616,106 +1669,89 @@
cs->cs_ivlen = 1;
cs->cs_mblen = 1;
- if (enc) {
- switch (enc->cri_alg) {
- case CRYPTO_AES_CBC:
- cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
- cs->cs_ivlen = AES_BLOCK_LEN;
- break;
- case CRYPTO_DES_CBC:
- cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
- cs->cs_ivlen = DES_BLOCK_LEN;
- break;
- case CRYPTO_3DES_CBC:
- cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
- CESA_CSHD_CBC;
- cs->cs_ivlen = DES3_BLOCK_LEN;
- break;
- default:
- error = EINVAL;
- break;
- }
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
+ cs->cs_ivlen = AES_BLOCK_LEN;
+ break;
+ case CRYPTO_DES_CBC:
+ cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
+ cs->cs_ivlen = DES_BLOCK_LEN;
+ break;
+ case CRYPTO_3DES_CBC:
+ cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
+ CESA_CSHD_CBC;
+ cs->cs_ivlen = DES3_BLOCK_LEN;
+ break;
}
- if (!error && mac) {
- switch (mac->cri_alg) {
- case CRYPTO_MD5:
- cs->cs_mblen = 1;
- cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_MD5;
- break;
- case CRYPTO_MD5_HMAC:
- cs->cs_mblen = MD5_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_MD5_HMAC;
- if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
- cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
- break;
- case CRYPTO_SHA1:
- cs->cs_mblen = 1;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA1;
- break;
- case CRYPTO_SHA1_HMAC:
- cs->cs_mblen = SHA1_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA1_HMAC;
- if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
- cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
- break;
- case CRYPTO_SHA2_256_HMAC:
- cs->cs_mblen = SHA2_256_BLOCK_LEN;
- cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN :
- mac->cri_mlen;
- cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
- break;
- default:
- error = EINVAL;
- break;
- }
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5:
+ cs->cs_mblen = 1;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_MD5;
+ break;
+ case CRYPTO_MD5_HMAC:
+ cs->cs_mblen = MD5_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_MD5_HMAC;
+ if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+ cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+ break;
+ case CRYPTO_SHA1:
+ cs->cs_mblen = 1;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA1;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ cs->cs_mblen = SHA1_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA1_HMAC;
+ if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+ cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+ break;
+ case CRYPTO_SHA2_256_HMAC:
+ cs->cs_mblen = SHA2_256_BLOCK_LEN;
+ cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN :
+ csp->csp_auth_mlen;
+ cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
+ break;
}
/* Save cipher key */
- if (!error && enc && enc->cri_key) {
- cs->cs_klen = enc->cri_klen / 8;
- memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
- if (enc->cri_alg == CRYPTO_AES_CBC)
- error = cesa_prep_aes_key(cs);
+ if (csp->csp_cipher_key != NULL) {
+ memcpy(cs->cs_key, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+ error = cesa_prep_aes_key(cs, csp);
}
/* Save digest key */
- if (!error && mac && mac->cri_key)
- error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
- mac->cri_klen / 8);
+ if (csp->csp_auth_key != NULL)
+ cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key,
+ csp->csp_auth_klen);
- if (error)
- return (error);
-
- return (0);
+ return (error);
}
static int
cesa_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct cesa_request *cr;
struct cesa_session *cs;
- struct cryptodesc *crd;
- struct cryptodesc *enc;
- struct cryptodesc *mac;
struct cesa_softc *sc;
int error;
sc = device_get_softc(dev);
- crd = crp->crp_desc;
- enc = NULL;
- mac = NULL;
error = 0;
cs = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
/* Check and parse input */
if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
@@ -1724,25 +1760,16 @@
return (0);
}
- if (cesa_is_hash(crd->crd_alg))
- mac = crd;
- else
- enc = crd;
-
- crd = crd->crd_next;
-
- if (crd) {
- if (!enc && !cesa_is_hash(crd->crd_alg))
- enc = crd;
-
- if (!mac && cesa_is_hash(crd->crd_alg))
- mac = crd;
-
- if (crd->crd_next || !(enc && mac)) {
- crp->crp_etype = EINVAL;
- crypto_done(crp);
- return (0);
- }
+ /*
+ * For requests with AAD, only requests where the AAD is
+ * immediately adjacent to the payload are supported.
+ */
+ if (crp->crp_aad_length != 0 &&
+ (crp->crp_aad_start + crp->crp_aad_length) !=
+ crp->crp_payload_start) {
+ crp->crp_etype = EINVAL;
+ crypto_done(crp);
+ return (0);
}
/*
@@ -1759,51 +1786,37 @@
/* Prepare request */
cr->cr_crp = crp;
- cr->cr_enc = enc;
- cr->cr_mac = mac;
cr->cr_cs = cs;
CESA_LOCK(sc, sessions);
cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
- if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(cr->cr_csd->csd_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ cr->cr_csd->csd_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(cr->cr_csd->csd_iv, crp->crp_iv, csp->csp_ivlen);
else
- arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
-
- if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
- } else if (enc) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ cr->cr_csd->csd_iv);
}
- if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
- cs->cs_klen = enc->crd_klen / 8;
- memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
- if (enc->crd_alg == CRYPTO_AES_CBC)
- error = cesa_prep_aes_key(cs);
- } else
- error = E2BIG;
+ if (crp->crp_cipher_key != NULL) {
+ memcpy(cs->cs_key, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
+ if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+ error = cesa_prep_aes_key(cs, csp);
}
- if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
- error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
- mac->crd_klen / 8);
- else
- error = E2BIG;
- }
+ if (!error && crp->crp_auth_key != NULL)
+ cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key,
+ csp->csp_auth_klen);
/* Convert request to chain of TDMA and SA descriptors */
if (!error)
- error = cesa_create_chain(sc, cr);
+ error = cesa_create_chain(sc, csp, cr);
cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
CESA_UNLOCK(sc, sessions);
Index: sys/dev/cxgbe/adapter.h
===================================================================
--- sys/dev/cxgbe/adapter.h
+++ sys/dev/cxgbe/adapter.h
@@ -1199,7 +1199,7 @@
void t4_aes_getdeckey(void *, const void *, unsigned int);
void t4_copy_partial_hash(int, union authctx *, void *);
void t4_init_gmac_hash(const char *, int, char *);
-void t4_init_hmac_digest(struct auth_hash *, u_int, char *, int, char *);
+void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *);
#ifdef DEV_NETMAP
/* t4_netmap.c */
Index: sys/dev/cxgbe/crypto/t4_crypto.c
===================================================================
--- sys/dev/cxgbe/crypto/t4_crypto.c
+++ sys/dev/cxgbe/crypto/t4_crypto.c
@@ -165,7 +165,7 @@
struct ccr_session {
bool active;
int pending;
- enum { HASH, HMAC, BLKCIPHER, AUTHENC, GCM, CCM } mode;
+ enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
union {
struct ccr_session_hmac hmac;
struct ccr_session_gmac gmac;
@@ -208,8 +208,8 @@
uint64_t stats_blkcipher_decrypt;
uint64_t stats_hash;
uint64_t stats_hmac;
- uint64_t stats_authenc_encrypt;
- uint64_t stats_authenc_decrypt;
+ uint64_t stats_eta_encrypt;
+ uint64_t stats_eta_decrypt;
uint64_t stats_gcm_encrypt;
uint64_t stats_gcm_decrypt;
uint64_t stats_ccm_encrypt;
@@ -230,9 +230,9 @@
* Non-hash-only requests require a PHYS_DSGL that describes the
* location to store the results of the encryption or decryption
* operation. This SGL uses a different format (PHYS_DSGL) and should
- * exclude the crd_skip bytes at the start of the data as well as
- * any AAD or IV. For authenticated encryption requests it should
- * cover include the destination of the hash or tag.
+ * exclude the skip bytes at the start of the data as well as any AAD
+ * or IV. For authenticated encryption requests it should include the
+ * destination of the hash or tag.
*
* The input payload may either be supplied inline as immediate data,
* or via a standard ULP_TX SGL. This SGL should include AAD,
@@ -251,12 +251,19 @@
int error;
sglist_reset(sg);
- if (crp->crp_flags & CRYPTO_F_IMBUF)
- error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
- else if (crp->crp_flags & CRYPTO_F_IOV)
- error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
- else
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ error = sglist_append_mbuf(sg, crp->crp_mbuf);
+ break;
+ case CRYPTO_BUF_UIO:
+ error = sglist_append_uio(sg, crp->crp_uio);
+ break;
+ case CRYPTO_BUF_CONTIG:
error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+ break;
+ default:
+ error = EINVAL;
+ }
return (error);
}
@@ -436,16 +443,13 @@
struct chcr_wr *crwr;
struct wrqe *wr;
struct auth_hash *axf;
- struct cryptodesc *crd;
char *dst;
u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len;
u_int hmac_ctrl, imm_len, iopad_size;
int error, sgl_nsegs, sgl_len, use_opad;
- crd = crp->crp_desc;
-
/* Reject requests with too large of an input buffer. */
- if (crd->crd_len > MAX_REQUEST_SIZE)
+ if (crp->crp_payload_length > MAX_REQUEST_SIZE)
return (EFBIG);
axf = s->hmac.auth_hash;
@@ -471,19 +475,19 @@
hash_size_in_response = axf->hashsize;
transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
- if (crd->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
imm_len = axf->blocksize;
sgl_nsegs = 0;
sgl_len = 0;
- } else if (ccr_use_imm_data(transhdr_len, crd->crd_len)) {
- imm_len = crd->crd_len;
+ } else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) {
+ imm_len = crp->crp_payload_length;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg;
@@ -512,8 +516,8 @@
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(0));
- crwr->sec_cpl.pldlen = htobe32(crd->crd_len == 0 ? axf->blocksize :
- crd->crd_len);
+ crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ?
+ axf->blocksize : crp->crp_payload_length);
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0));
@@ -527,7 +531,8 @@
V_SCMD_HMAC_CTRL(hmac_ctrl));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_LAST_FRAG(0) |
- V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1));
+ V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) |
+ V_SCMD_MAC_ONLY(1));
memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len);
@@ -540,14 +545,14 @@
V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1));
dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES;
- if (crd->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
dst[0] = 0x80;
if (s->mode == HMAC)
*(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) =
htobe64(axf->blocksize << 3);
} else if (imm_len != 0)
- crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, dst);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -561,15 +566,20 @@
ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
const struct cpl_fw6_pld *cpl, int error)
{
- struct cryptodesc *crd;
+ uint8_t hash[HASH_MAX_LEN];
- crd = crp->crp_desc;
- if (error == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- s->hmac.hash_len, (c_caddr_t)(cpl + 1));
- }
+ if (error)
+ return (error);
- return (error);
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
+ hash);
+ if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
+ (cpl + 1));
+ return (0);
}
static int
@@ -578,34 +588,31 @@
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
struct wrqe *wr;
- struct cryptodesc *crd;
char *dst;
u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
- u_int imm_len;
+ u_int imm_len, iv_len;
int dsgl_nsegs, dsgl_len;
int sgl_nsegs, sgl_len;
int error;
- crd = crp->crp_desc;
-
- if (s->blkcipher.key_len == 0 || crd->crd_len == 0)
+ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
return (EINVAL);
- if (crd->crd_alg == CRYPTO_AES_CBC &&
- (crd->crd_len % AES_BLOCK_LEN) != 0)
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+ (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
return (EINVAL);
/* Reject requests with too large of an input buffer. */
- if (crd->crd_len > MAX_REQUEST_SIZE)
+ if (crp->crp_payload_length > MAX_REQUEST_SIZE)
return (EFBIG);
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
sglist_reset(sc->sg_dsgl);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip,
- crd->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
@@ -617,23 +624,28 @@
kctx_len = roundup2(s->blkcipher.key_len, 16);
transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
- if (ccr_use_imm_data(transhdr_len, crd->crd_len +
- s->blkcipher.iv_len)) {
- imm_len = crd->crd_len;
+ /* For AES-XTS we send a 16-byte IV in the work request. */
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ iv_len = AES_BLOCK_LEN;
+ else
+ iv_len = s->blkcipher.iv_len;
+
+ if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) {
+ imm_len = crp->crp_payload_length;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crd->crd_skip, crd->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
sgl_nsegs = sc->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
}
- wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
+ wr_len = roundup2(transhdr_len, 16) + iv_len +
roundup2(imm_len, 16) + sgl_len;
if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG);
@@ -647,24 +659,20 @@
/*
* Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
+ * one if none is provided.
*/
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, s->blkcipher.iv_len, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, s->blkcipher.iv_len, 0);
+ crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+
+ /* Zero the remainder of the IV for AES-XTS. */
+ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
crp);
@@ -677,10 +685,10 @@
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
- crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len);
+ crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length);
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
- V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) |
+ V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) |
V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0));
@@ -693,7 +701,7 @@
V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
- V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+ V_SCMD_IV_SIZE(iv_len / 2) |
V_SCMD_NUM_IVS(0));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_IV_GEN_CTRL(0) |
@@ -701,24 +709,24 @@
V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
else
memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_ICM:
+ case SCMD_CIPH_MODE_AES_CTR:
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_XTS:
+ case SCMD_CIPH_MODE_AES_XTS:
key_half = s->blkcipher.key_len / 2;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
key_half);
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key + key_half,
s->blkcipher.enckey, key_half);
else
@@ -730,11 +738,11 @@
dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
- memcpy(dst, iv, s->blkcipher.iv_len);
- dst += s->blkcipher.iv_len;
+ memcpy(dst, iv, iv_len);
+ dst += iv_len;
if (imm_len != 0)
- crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
- crd->crd_len, dst);
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -775,8 +783,7 @@
}
static int
-ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
@@ -784,9 +791,9 @@
struct auth_hash *axf;
char *dst;
u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
- u_int hash_size_in_response, imm_len, iopad_size;
- u_int aad_start, aad_len, aad_stop;
- u_int auth_start, auth_stop, auth_insert;
+ u_int hash_size_in_response, imm_len, iopad_size, iv_len;
+ u_int aad_start, aad_stop;
+ u_int auth_insert;
u_int cipher_start, cipher_stop;
u_int hmac_ctrl, input_len;
int dsgl_nsegs, dsgl_len;
@@ -797,34 +804,24 @@
* If there is a need in the future, requests with an empty
* payload could be supported as HMAC-only requests.
*/
- if (s->blkcipher.key_len == 0 || crde->crd_len == 0)
+ if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
return (EINVAL);
- if (crde->crd_alg == CRYPTO_AES_CBC &&
- (crde->crd_len % AES_BLOCK_LEN) != 0)
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+ (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
return (EINVAL);
- /*
- * Compute the length of the AAD (data covered by the
- * authentication descriptor but not the encryption
- * descriptor). To simplify the logic, AAD is only permitted
- * before the cipher/plain text, not after. This is true of
- * all currently-generated requests.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
- return (EINVAL);
- if (crda->crd_skip < crde->crd_skip) {
- if (crda->crd_skip + crda->crd_len > crde->crd_skip)
- aad_len = (crde->crd_skip - crda->crd_skip);
- else
- aad_len = crda->crd_len;
- } else
- aad_len = 0;
- if (aad_len + s->blkcipher.iv_len > MAX_AAD_LEN)
+ /* For AES-XTS we send a 16-byte IV in the work request. */
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ iv_len = AES_BLOCK_LEN;
+ else
+ iv_len = s->blkcipher.iv_len;
+
+ if (crp->crp_aad_length + iv_len > MAX_AAD_LEN)
return (EINVAL);
axf = s->hmac.auth_hash;
hash_size_in_response = s->hmac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -839,26 +836,26 @@
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (s->blkcipher.iv_len + aad_len + crde->crd_len +
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (s->blkcipher.iv_len + aad_len + crde->crd_len >
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0,
- s->blkcipher.iv_len + aad_len);
+ iv_len + crp->crp_aad_length);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -888,7 +885,7 @@
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = aad_len + crde->crd_len;
+ input_len = crp->crp_aad_length + crp->crp_payload_length;
/*
* The firmware hangs if sent a request which is a
@@ -902,26 +899,27 @@
return (EFBIG);
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
- if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) {
+
+ if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len;
sgl_nsegs = 0;
sgl_len = 0;
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
- if (aad_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, aad_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -934,37 +932,25 @@
* Auth-data that overlaps with the cipher region is placed in
* the auth section.
*/
- if (aad_len != 0) {
- aad_start = s->blkcipher.iv_len + 1;
- aad_stop = aad_start + aad_len - 1;
+ if (crp->crp_aad_length != 0) {
+ aad_start = iv_len + 1;
+ aad_stop = aad_start + crp->crp_aad_length - 1;
} else {
aad_start = 0;
aad_stop = 0;
}
- cipher_start = s->blkcipher.iv_len + aad_len + 1;
+ cipher_start = iv_len + crp->crp_aad_length + 1;
if (op_type == CHCR_DECRYPT_OP)
cipher_stop = hash_size_in_response;
else
cipher_stop = 0;
- if (aad_len == crda->crd_len) {
- auth_start = 0;
- auth_stop = 0;
- } else {
- if (aad_len != 0)
- auth_start = cipher_start;
- else
- auth_start = s->blkcipher.iv_len + crda->crd_skip -
- crde->crd_skip + 1;
- auth_stop = (crde->crd_skip + crde->crd_len) -
- (crda->crd_skip + crda->crd_len) + cipher_stop;
- }
if (op_type == CHCR_DECRYPT_OP)
auth_insert = hash_size_in_response;
else
auth_insert = 0;
- wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
- roundup2(imm_len, 16) + sgl_len;
+ wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
+ sgl_len;
if (wr_len > SGE_MAX_WR_LEN)
return (EFBIG);
wr = alloc_wrqe(wr_len, sc->txq);
@@ -977,24 +963,20 @@
/*
* Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
+ * one if none is provided.
*/
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, s->blkcipher.iv_len, 0);
+ crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+ iv);
+
+ /* Zero the remainder of the IV for AES-XTS. */
+ memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len,
op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp);
@@ -1007,7 +989,7 @@
V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
- crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len);
+ crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
@@ -1016,8 +998,8 @@
V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
- V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
- V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
+ V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
+ V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
@@ -1030,7 +1012,7 @@
V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
V_SCMD_HMAC_CTRL(hmac_ctrl) |
- V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+ V_SCMD_IV_SIZE(iv_len / 2) |
V_SCMD_NUM_IVS(0));
crwr->sec_cpl.ivgen_hdrlen = htobe32(
V_SCMD_IV_GEN_CTRL(0) |
@@ -1038,24 +1020,24 @@
V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
- switch (crde->crd_alg) {
- case CRYPTO_AES_CBC:
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
else
memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_ICM:
+ case SCMD_CIPH_MODE_AES_CTR:
memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
s->blkcipher.key_len);
break;
- case CRYPTO_AES_XTS:
+ case SCMD_CIPH_MODE_AES_XTS:
key_half = s->blkcipher.key_len / 2;
memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
key_half);
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
memcpy(crwr->key_ctx.key + key_half,
s->blkcipher.enckey, key_half);
else
@@ -1070,20 +1052,20 @@
dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
- memcpy(dst, iv, s->blkcipher.iv_len);
- dst += s->blkcipher.iv_len;
+ memcpy(dst, iv, iv_len);
+ dst += iv_len;
if (imm_len != 0) {
- if (aad_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, aad_len, dst);
- dst += aad_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -1094,38 +1076,19 @@
}
static int
-ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s,
+ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s,
struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
{
- struct cryptodesc *crd;
/*
* The updated IV to permit chained requests is at
* cpl->data[2], but OCF doesn't permit chained requests.
- *
- * For a decryption request, the hardware may do a verification
- * of the HMAC which will fail if the existing HMAC isn't in the
- * buffer. If that happens, clear the error and copy the HMAC
- * from the CPL reply into the buffer.
- *
- * For encryption requests, crd should be the cipher request
- * which will have CRD_F_ENCRYPT set. For decryption
- * requests, crp_desc will be the HMAC request which should
- * not have this flag set.
*/
- crd = crp->crp_desc;
- if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) &&
- !(crd->crd_flags & CRD_F_ENCRYPT)) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- s->hmac.hash_len, (c_caddr_t)(cpl + 1));
- error = 0;
- }
return (error);
}
static int
-ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct chcr_wr *crwr;
@@ -1146,21 +1109,14 @@
* The crypto engine doesn't handle GCM requests with an empty
* payload, so handle those in software instead.
*/
- if (crde->crd_len == 0)
+ if (crp->crp_payload_length == 0)
return (EMSGSIZE);
- /*
- * AAD is only permitted before the cipher/plain text, not
- * after.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
- return (EMSGSIZE);
-
- if (crda->crd_len + AES_BLOCK_LEN > MAX_AAD_LEN)
+ if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN)
return (EMSGSIZE);
hash_size_in_response = s->gmac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -1186,6 +1142,12 @@
else
iv_len = s->blkcipher.iv_len;
+ /*
+ * GCM requests should always provide an explicit IV.
+ */
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+ return (EINVAL);
+
/*
* The output buffer consists of the cipher text followed by
* the tag when encrypting. For decryption it only contains
@@ -1196,25 +1158,26 @@
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (iv_len + crda->crd_len + crde->crd_len +
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (iv_len + crda->crd_len + crde->crd_len > MAX_REQUEST_SIZE)
+ if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
+ MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
- crda->crd_len);
+ crp->crp_aad_length);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1241,7 +1204,7 @@
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = crda->crd_len + crde->crd_len;
+ input_len = crp->crp_aad_length + crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
if (input_len > MAX_REQUEST_SIZE)
@@ -1253,19 +1216,19 @@
} else {
imm_len = 0;
sglist_reset(sc->sg_ulptx);
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, crda->crd_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1273,14 +1236,14 @@
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
}
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
aad_start = iv_len + 1;
- aad_stop = aad_start + crda->crd_len - 1;
+ aad_stop = aad_start + crp->crp_aad_length - 1;
} else {
aad_start = 0;
aad_stop = 0;
}
- cipher_start = iv_len + crda->crd_len + 1;
+ cipher_start = iv_len + crp->crp_aad_length + 1;
if (op_type == CHCR_DECRYPT_OP)
cipher_stop = hash_size_in_response;
else
@@ -1302,29 +1265,7 @@
crwr = wrtod(wr);
memset(crwr, 0, wr_len);
- /*
- * Read the existing IV from the request or generate a random
- * one if none is provided. Optionally copy the generated IV
- * into the output buffer if requested.
- *
- * If the input IV is 12 bytes, append an explicit 4-byte
- * counter of 1.
- */
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- arc4rand(iv, s->blkcipher.iv_len, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, s->blkcipher.iv_len, iv);
- }
+ memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
if (s->blkcipher.iv_len == 12)
*(uint32_t *)&iv[12] = htobe32(1);
@@ -1343,13 +1284,12 @@
/*
* NB: cipherstop is explicitly set to 0. On encrypt it
- * should normally be set to 0 anyway (as the encrypt crd ends
- * at the end of the input). However, for decrypt the cipher
- * ends before the tag in the AUTHENC case (and authstop is
- * set to stop before the tag), but for GCM the cipher still
- * runs to the end of the buffer. Not sure if this is
- * intentional or a firmware quirk, but it is required for
- * working tag validation with GCM decryption.
+ * should normally be set to 0 anyway. However, for decrypt
+ * the cipher ends before the tag in the ETA case (and
+ * authstop is set to stop before the tag), but for GCM the
+ * cipher still runs to the end of the buffer. Not sure if
+ * this is intentional or a firmware quirk, but it is required
+ * for working tag validation with GCM decryption.
*/
crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
@@ -1390,17 +1330,17 @@
memcpy(dst, iv, iv_len);
dst += iv_len;
if (imm_len != 0) {
- if (crda->crd_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, crda->crd_len, dst);
- dst += crda->crd_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else
ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
@@ -1429,8 +1369,7 @@
* performing the operation in software. Derived from swcr_authenc().
*/
static void
-ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp)
{
struct auth_hash *axf;
struct enc_xform *exf;
@@ -1478,30 +1417,19 @@
* This assumes a 12-byte IV from the crp. See longer comment
* above in ccr_gcm() for more details.
*/
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- arc4rand(iv, 12, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, 12);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, 12, iv);
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+ error = EINVAL;
+ goto out;
}
+ memcpy(iv, crp->crp_iv, 12);
*(uint32_t *)&iv[12] = htobe32(1);
axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */
- for (i = 0; i < crda->crd_len; i += sizeof(block)) {
- len = imin(crda->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+ len = imin(crp->crp_aad_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_aad_start + i, len, block);
bzero(block + len, sizeof(block) - len);
axf->Update(auth_ctx, block, sizeof(block));
}
@@ -1509,16 +1437,15 @@
exf->reinit(kschedule, iv);
/* Do encryption with MAC */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len);
- if (crde->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
exf->encrypt(kschedule, block);
axf->Update(auth_ctx, block, len);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ block);
} else {
axf->Update(auth_ctx, block, len);
}
@@ -1526,35 +1453,37 @@
/* Length block. */
bzero(block, sizeof(block));
- ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
- ((uint32_t *)block)[3] = htobe32(crde->crd_len * 8);
+ ((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
+ ((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8);
axf->Update(auth_ctx, block, sizeof(block));
/* Finalize MAC. */
axf->Final(digest, auth_ctx);
/* Inject or validate tag. */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest), digest);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+ digest);
error = 0;
} else {
char digest2[GMAC_DIGEST_LEN];
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest2), digest2);
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+ digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
error = 0;
/* Tag matches, decrypt data. */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ for (i = 0; i < crp->crp_payload_length;
+ i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i,
+ sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i,
+ len, block);
bzero(block + len, sizeof(block) - len);
exf->decrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i,
+ len, block);
}
} else
error = EBADMSG;
@@ -1571,8 +1500,8 @@
}
static void
-generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde,
- u_int hash_size_in_response, const char *iv, char *b0)
+generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response,
+ const char *iv, char *b0)
{
u_int i, payload_len;
@@ -1583,7 +1512,7 @@
b0[0] |= (((hash_size_in_response - 2) / 2) << 3);
/* Store the payload length as a big-endian value. */
- payload_len = crde->crd_len;
+ payload_len = crp->crp_payload_length;
for (i = 0; i < iv[0]; i++) {
b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len;
payload_len >>= 8;
@@ -1595,15 +1524,14 @@
* start of block 1. This only assumes a 16-bit AAD length
* since T6 doesn't support large AAD sizes.
*/
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
b0[0] |= (1 << 6);
- *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crda->crd_len);
+ *(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length);
}
}
static int
-ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
{
char iv[CHCR_MAX_CRYPTO_IV_LEN];
struct ulptx_idata *idata;
@@ -1625,14 +1553,7 @@
* The crypto engine doesn't handle CCM requests with an empty
* payload, so handle those in software instead.
*/
- if (crde->crd_len == 0)
- return (EMSGSIZE);
-
- /*
- * AAD is only permitted before the cipher/plain text, not
- * after.
- */
- if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+ if (crp->crp_payload_length == 0)
return (EMSGSIZE);
/*
@@ -1640,14 +1561,21 @@
* request.
*/
b0_len = CCM_B0_SIZE;
- if (crda->crd_len != 0)
+ if (crp->crp_aad_length != 0)
b0_len += CCM_AAD_FIELD_SIZE;
- aad_len = b0_len + crda->crd_len;
+ aad_len = b0_len + crp->crp_aad_length;
/*
- * Always assume a 12 byte input IV for now since that is what
- * OCF always generates. The full IV in the work request is
- * 16 bytes.
+ * CCM requests should always provide an explicit IV (really
+ * the nonce).
+ */
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+ return (EINVAL);
+
+ /*
+ * Always assume a 12 byte input nonce for now since that is
+ * what OCF always generates. The full IV in the work request
+ * is 16 bytes.
*/
iv_len = AES_BLOCK_LEN;
@@ -1655,7 +1583,7 @@
return (EMSGSIZE);
hash_size_in_response = s->ccm_mac.hash_len;
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
op_type = CHCR_ENCRYPT_OP;
else
op_type = CHCR_DECRYPT_OP;
@@ -1670,11 +1598,12 @@
* output buffer.
*/
if (op_type == CHCR_ENCRYPT_OP) {
- if (iv_len + aad_len + crde->crd_len + hash_size_in_response >
- MAX_REQUEST_SIZE)
+ if (iv_len + aad_len + crp->crp_payload_length +
+ hash_size_in_response > MAX_REQUEST_SIZE)
return (EFBIG);
} else {
- if (iv_len + aad_len + crde->crd_len > MAX_REQUEST_SIZE)
+ if (iv_len + aad_len + crp->crp_payload_length >
+ MAX_REQUEST_SIZE)
return (EFBIG);
}
sglist_reset(sc->sg_dsgl);
@@ -1682,13 +1611,13 @@
aad_len);
if (error)
return (error);
- error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
- crde->crd_len);
+ error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_ENCRYPT_OP) {
error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1715,7 +1644,7 @@
* inside of the AAD region, so a second copy is always
* required.
*/
- input_len = aad_len + crde->crd_len;
+ input_len = aad_len + crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
input_len += hash_size_in_response;
if (input_len > MAX_REQUEST_SIZE)
@@ -1729,19 +1658,19 @@
imm_len = b0_len;
sglist_reset(sc->sg_ulptx);
- if (crda->crd_len != 0) {
+ if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_skip, crda->crd_len);
+ crp->crp_aad_start, crp->crp_aad_length);
if (error)
return (error);
}
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crde->crd_skip, crde->crd_len);
+ crp->crp_payload_start, crp->crp_payload_length);
if (error)
return (error);
if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
- crda->crd_inject, hash_size_in_response);
+ crp->crp_digest_start, hash_size_in_response);
if (error)
return (error);
}
@@ -1774,27 +1703,12 @@
memset(crwr, 0, wr_len);
/*
- * Read the nonce from the request or generate a random one if
- * none is provided. Use the nonce to generate the full IV
- * with the counter set to 0.
+ * Read the nonce from the request. Use the nonce to generate
+ * the full IV with the counter set to 0.
*/
memset(iv, 0, iv_len);
iv[0] = (15 - AES_CCM_IV_LEN) - 1;
- if (op_type == CHCR_ENCRYPT_OP) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
- else
- arc4rand(iv + 1, AES_CCM_IV_LEN, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
- }
+ memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN);
ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
crp);
@@ -1851,20 +1765,20 @@
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len);
dst += iv_len;
- generate_ccm_b0(crda, crde, hash_size_in_response, iv, dst);
+ generate_ccm_b0(crp, hash_size_in_response, iv, dst);
if (sgl_nsegs == 0) {
dst += b0_len;
- if (crda->crd_len != 0) {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_skip, crda->crd_len, dst);
- dst += crda->crd_len;
+ if (crp->crp_aad_length != 0) {
+ crypto_copydata(crp, crp->crp_aad_start,
+ crp->crp_aad_length, dst);
+ dst += crp->crp_aad_length;
}
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
- crde->crd_len, dst);
- dst += crde->crd_len;
+ crypto_copydata(crp, crp->crp_payload_start,
+ crp->crp_payload_length, dst);
+ dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP)
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crda->crd_inject, hash_size_in_response, dst);
+ crypto_copydata(crp, crp->crp_digest_start,
+ hash_size_in_response, dst);
} else {
dst += CCM_B0_SIZE;
if (b0_len > CCM_B0_SIZE) {
@@ -1911,8 +1825,7 @@
* performing the operation in software. Derived from swcr_authenc().
*/
static void
-ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
- struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp)
{
struct auth_hash *axf;
struct enc_xform *exf;
@@ -1956,31 +1869,20 @@
if (error)
goto out;
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
- else
- arc4rand(iv, AES_CCM_IV_LEN, 0);
- if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv);
- } else {
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
- else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_inject, AES_CCM_IV_LEN, iv);
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+ error = EINVAL;
+ goto out;
}
+ memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN);
- auth_ctx->aes_cbc_mac_ctx.authDataLength = crda->crd_len;
- auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
+ auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
+ auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */
- for (i = 0; i < crda->crd_len; i += sizeof(block)) {
- len = imin(crda->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+ len = imin(crp->crp_aad_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_aad_start + i, len, block);
bzero(block + len, sizeof(block) - len);
axf->Update(auth_ctx, block, sizeof(block));
}
@@ -1988,16 +1890,15 @@
exf->reinit(kschedule, iv);
/* Do encryption/decryption with MAC */
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
- i, len, block);
+ for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i, sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len);
- if (crde->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
axf->Update(auth_ctx, block, len);
exf->encrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ block);
} else {
exf->decrypt(kschedule, block);
axf->Update(auth_ctx, block, len);
@@ -2008,28 +1909,30 @@
axf->Final(digest, auth_ctx);
/* Inject or validate tag. */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest), digest);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+ digest);
error = 0;
} else {
- char digest2[GMAC_DIGEST_LEN];
+ char digest2[AES_CBC_MAC_HASH_LEN];
- crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
- sizeof(digest2), digest2);
+ crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+ digest2);
if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
error = 0;
/* Tag matches, decrypt data. */
exf->reinit(kschedule, iv);
- for (i = 0; i < crde->crd_len; i += sizeof(block)) {
- len = imin(crde->crd_len - i, sizeof(block));
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ for (i = 0; i < crp->crp_payload_length;
+ i += sizeof(block)) {
+ len = imin(crp->crp_payload_length - i,
+ sizeof(block));
+ crypto_copydata(crp, crp->crp_payload_start + i,
+ len, block);
bzero(block + len, sizeof(block) - len);
exf->decrypt(kschedule, block);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crde->crd_skip + i, len, block);
+ crypto_copyback(crp, crp->crp_payload_start + i,
+ len, block);
}
} else
error = EBADMSG;
@@ -2096,11 +1999,11 @@
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD,
&sc->stats_blkcipher_decrypt, 0,
"Cipher decryption requests submitted");
- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD,
- &sc->stats_authenc_encrypt, 0,
+ SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD,
+ &sc->stats_eta_encrypt, 0,
"Combined AES+HMAC encryption requests submitted");
- SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD,
- &sc->stats_authenc_decrypt, 0,
+ SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD,
+ &sc->stats_eta_decrypt, 0,
"Combined AES+HMAC decryption requests submitted");
SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD,
&sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted");
@@ -2161,25 +2064,6 @@
sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
ccr_sysctls(sc);
- crypto_register(cid, CRYPTO_SHA1, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_224, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_256, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_384, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_512, 0, 0);
- crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_224_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_CBC, 0, 0);
- crypto_register(cid, CRYPTO_AES_ICM, 0, 0);
- crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
- crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
- crypto_register(cid, CRYPTO_AES_XTS, 0, 0);
- crypto_register(cid, CRYPTO_AES_CCM_16, 0, 0);
- crypto_register(cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
return (0);
}
@@ -2207,48 +2091,48 @@
}
static void
-ccr_init_hash_digest(struct ccr_session *s, int cri_alg)
+ccr_init_hash_digest(struct ccr_session *s)
{
union authctx auth_ctx;
struct auth_hash *axf;
axf = s->hmac.auth_hash;
axf->Init(&auth_ctx);
- t4_copy_partial_hash(cri_alg, &auth_ctx, s->hmac.pads);
+ t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads);
}
-static int
+static bool
ccr_aes_check_keylen(int alg, int klen)
{
- switch (klen) {
+ switch (klen * 8) {
case 128:
case 192:
if (alg == CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
case 256:
break;
case 512:
if (alg != CRYPTO_AES_XTS)
- return (EINVAL);
+ return (false);
break;
default:
- return (EINVAL);
+ return (false);
}
- return (0);
+ return (true);
}
static void
-ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
+ccr_aes_setkey(struct ccr_session *s, const void *key, int klen)
{
unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size;
unsigned int opad_present;
- if (alg == CRYPTO_AES_XTS)
- kbits = klen / 2;
+ if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+ kbits = (klen / 2) * 8;
else
- kbits = klen;
+ kbits = klen * 8;
switch (kbits) {
case 128:
ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
@@ -2263,18 +2147,18 @@
panic("should not get here");
}
- s->blkcipher.key_len = klen / 8;
+ s->blkcipher.key_len = klen;
memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
- switch (alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_XTS:
+ switch (s->blkcipher.cipher_mode) {
+ case SCMD_CIPH_MODE_AES_CBC:
+ case SCMD_CIPH_MODE_AES_XTS:
t4_aes_getdeckey(s->blkcipher.deckey, key, kbits);
break;
}
kctx_len = roundup2(s->blkcipher.key_len, 16);
switch (s->mode) {
- case AUTHENC:
+ case ETA:
mk_size = s->hmac.mk_size;
opad_present = 1;
iopad_size = roundup2(s->hmac.partial_digest_len, 16);
@@ -2309,171 +2193,220 @@
}
kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
- V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) |
+ V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode ==
+ SCMD_CIPH_MODE_AES_XTS) |
V_KEY_CONTEXT_OPAD_PRESENT(opad_present) |
V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) |
V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1));
}
+static bool
+ccr_auth_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA2_384:
+ case CRYPTO_SHA2_512:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+ccr_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_ICM:
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_XTS_IV_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (ccr_aes_check_keylen(csp->csp_cipher_alg,
+ csp->csp_cipher_klen));
+}
+
static int
-ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
-{
- struct ccr_softc *sc;
- struct ccr_session *s;
- struct auth_hash *auth_hash;
- struct cryptoini *c, *hash, *cipher;
- unsigned int auth_mode, cipher_mode, iv_len, mk_size;
- unsigned int partial_digest_len;
- int error;
- bool gcm_hash, hmac;
-
- if (cri == NULL)
+ccr_cipher_mode(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ return (SCMD_CIPH_MODE_AES_CBC);
+ case CRYPTO_AES_ICM:
+ return (SCMD_CIPH_MODE_AES_CTR);
+ case CRYPTO_AES_NIST_GCM_16:
+ return (SCMD_CIPH_MODE_AES_GCM);
+ case CRYPTO_AES_XTS:
+ return (SCMD_CIPH_MODE_AES_XTS);
+ case CRYPTO_AES_CCM_16:
+ return (SCMD_CIPH_MODE_AES_CCM);
+ default:
+ return (SCMD_CIPH_MODE_NOP);
+ }
+}
+
+static int
+ccr_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ unsigned int cipher_mode;
+
+ if (csp->csp_flags != 0)
return (EINVAL);
-
- gcm_hash = false;
- hmac = false;
- cipher = NULL;
- hash = NULL;
- auth_hash = NULL;
- auth_mode = SCMD_AUTH_MODE_NOP;
- cipher_mode = SCMD_CIPH_MODE_NOP;
- iv_len = 0;
- mk_size = 0;
- partial_digest_len = 0;
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_SHA1:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_512:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- case CRYPTO_AES_CCM_CBC_MAC:
- if (hash)
- return (EINVAL);
- hash = c;
- switch (c->cri_alg) {
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- auth_hash = &auth_hash_hmac_sha1;
- auth_mode = SCMD_AUTH_MODE_SHA1;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
- partial_digest_len = SHA1_HASH_LEN;
- break;
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_224_HMAC:
- auth_hash = &auth_hash_hmac_sha2_224;
- auth_mode = SCMD_AUTH_MODE_SHA224;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
- partial_digest_len = SHA2_256_HASH_LEN;
- break;
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_256_HMAC:
- auth_hash = &auth_hash_hmac_sha2_256;
- auth_mode = SCMD_AUTH_MODE_SHA256;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
- partial_digest_len = SHA2_256_HASH_LEN;
- break;
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_384_HMAC:
- auth_hash = &auth_hash_hmac_sha2_384;
- auth_mode = SCMD_AUTH_MODE_SHA512_384;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_SHA2_512:
- case CRYPTO_SHA2_512_HMAC:
- auth_hash = &auth_hash_hmac_sha2_512;
- auth_mode = SCMD_AUTH_MODE_SHA512_512;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
- partial_digest_len = SHA2_512_HASH_LEN;
- break;
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- gcm_hash = true;
- auth_mode = SCMD_AUTH_MODE_GHASH;
- mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
- break;
- case CRYPTO_AES_CCM_CBC_MAC:
- auth_mode = SCMD_AUTH_MODE_CBCMAC;
- break;
- }
- switch (c->cri_alg) {
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- hmac = true;
- break;
- }
- break;
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!ccr_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!ccr_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_XTS:
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (EINVAL);
+ if (csp->csp_auth_mlen < 0 ||
+ csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+ return (EINVAL);
+ break;
case CRYPTO_AES_CCM_16:
- if (cipher)
+ if (csp->csp_ivlen != AES_CCM_IV_LEN)
+ return (EINVAL);
+ if (csp->csp_auth_mlen < 0 ||
+ csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN)
return (EINVAL);
- cipher = c;
- switch (c->cri_alg) {
- case CRYPTO_AES_CBC:
- cipher_mode = SCMD_CIPH_MODE_AES_CBC;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_ICM:
- cipher_mode = SCMD_CIPH_MODE_AES_CTR;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_NIST_GCM_16:
- cipher_mode = SCMD_CIPH_MODE_AES_GCM;
- iv_len = AES_GCM_IV_LEN;
- break;
- case CRYPTO_AES_XTS:
- cipher_mode = SCMD_CIPH_MODE_AES_XTS;
- iv_len = AES_BLOCK_LEN;
- break;
- case CRYPTO_AES_CCM_16:
- cipher_mode = SCMD_CIPH_MODE_AES_CCM;
- iv_len = AES_CCM_IV_LEN;
- break;
- }
- if (c->cri_key != NULL) {
- error = ccr_aes_check_keylen(c->cri_alg,
- c->cri_klen);
- if (error)
- return (error);
- }
break;
default:
return (EINVAL);
}
- }
- if (gcm_hash != (cipher_mode == SCMD_CIPH_MODE_AES_GCM))
- return (EINVAL);
- if ((auth_mode == SCMD_AUTH_MODE_CBCMAC) !=
- (cipher_mode == SCMD_CIPH_MODE_AES_CCM))
- return (EINVAL);
- if (hash == NULL && cipher == NULL)
+ break;
+ case CSP_MODE_ETA:
+ if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
return (EINVAL);
- if (hash != NULL) {
- if (hmac || gcm_hash || auth_mode == SCMD_AUTH_MODE_CBCMAC) {
- if (hash->cri_key == NULL)
- return (EINVAL);
- } else {
- if (hash->cri_key != NULL)
- return (EINVAL);
- }
}
+ if (csp->csp_cipher_klen != 0) {
+ cipher_mode = ccr_cipher_mode(csp);
+ if (cipher_mode == SCMD_CIPH_MODE_NOP)
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+ccr_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct ccr_softc *sc;
+ struct ccr_session *s;
+ struct auth_hash *auth_hash;
+ unsigned int auth_mode, cipher_mode, mk_size;
+ unsigned int partial_digest_len;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA1_HMAC:
+ auth_hash = &auth_hash_hmac_sha1;
+ auth_mode = SCMD_AUTH_MODE_SHA1;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+ partial_digest_len = SHA1_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_224_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_224;
+ auth_mode = SCMD_AUTH_MODE_SHA224;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ partial_digest_len = SHA2_256_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA2_256_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_256;
+ auth_mode = SCMD_AUTH_MODE_SHA256;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+ partial_digest_len = SHA2_256_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_384:
+ case CRYPTO_SHA2_384_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_384;
+ auth_mode = SCMD_AUTH_MODE_SHA512_384;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ partial_digest_len = SHA2_512_HASH_LEN;
+ break;
+ case CRYPTO_SHA2_512:
+ case CRYPTO_SHA2_512_HMAC:
+ auth_hash = &auth_hash_hmac_sha2_512;
+ auth_mode = SCMD_AUTH_MODE_SHA512_512;
+ mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+ partial_digest_len = SHA2_512_HASH_LEN;
+ break;
+ default:
+ auth_hash = NULL;
+ auth_mode = SCMD_AUTH_MODE_NOP;
+ mk_size = 0;
+ partial_digest_len = 0;
+ break;
+ }
+
+ cipher_mode = ccr_cipher_mode(csp);
+
+#ifdef INVARIANTS
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+ cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+ cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid cipher algo");
+ break;
+ case CSP_MODE_DIGEST:
+ if (auth_mode == SCMD_AUTH_MODE_NOP)
+ panic("invalid auth algo");
+ break;
+ case CSP_MODE_AEAD:
+ if (cipher_mode != SCMD_CIPH_MODE_AES_GCM &&
+ cipher_mode != SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid aead cipher algo");
+ if (auth_mode != SCMD_AUTH_MODE_NOP)
+ panic("invalid aead auth aglo");
+ break;
+ case CSP_MODE_ETA:
+ if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+ cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+ cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ panic("invalid cipher algo");
+ if (auth_mode == SCMD_AUTH_MODE_NOP)
+ panic("invalid auth algo");
+ break;
+ default:
+ panic("invalid csp mode");
+ }
+#endif
+
sc = device_get_softc(dev);
/*
@@ -2493,54 +2426,61 @@
s = crypto_get_driver_session(cses);
- if (gcm_hash)
- s->mode = GCM;
- else if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
- s->mode = CCM;
- else if (hash != NULL && cipher != NULL)
- s->mode = AUTHENC;
- else if (hash != NULL) {
- if (hmac)
+ switch (csp->csp_mode) {
+ case CSP_MODE_AEAD:
+ if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+ s->mode = CCM;
+ else
+ s->mode = GCM;
+ break;
+ case CSP_MODE_ETA:
+ s->mode = ETA;
+ break;
+ case CSP_MODE_DIGEST:
+ if (csp->csp_auth_klen != 0)
s->mode = HMAC;
else
s->mode = HASH;
- } else {
- MPASS(cipher != NULL);
+ break;
+ case CSP_MODE_CIPHER:
s->mode = BLKCIPHER;
+ break;
}
- if (gcm_hash) {
- if (hash->cri_mlen == 0)
+
+ if (s->mode == GCM) {
+ if (csp->csp_auth_mlen == 0)
s->gmac.hash_len = AES_GMAC_HASH_LEN;
else
- s->gmac.hash_len = hash->cri_mlen;
- t4_init_gmac_hash(hash->cri_key, hash->cri_klen,
+ s->gmac.hash_len = csp->csp_auth_mlen;
+ t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen,
s->gmac.ghash_h);
- } else if (auth_mode == SCMD_AUTH_MODE_CBCMAC) {
- if (hash->cri_mlen == 0)
+ } else if (s->mode == CCM) {
+ if (csp->csp_auth_mlen == 0)
s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN;
else
- s->ccm_mac.hash_len = hash->cri_mlen;
- } else if (hash != NULL) {
+ s->ccm_mac.hash_len = csp->csp_auth_mlen;
+ } else if (auth_mode != SCMD_AUTH_MODE_NOP) {
s->hmac.auth_hash = auth_hash;
s->hmac.auth_mode = auth_mode;
s->hmac.mk_size = mk_size;
s->hmac.partial_digest_len = partial_digest_len;
- if (hash->cri_mlen == 0)
+ if (csp->csp_auth_mlen == 0)
s->hmac.hash_len = auth_hash->hashsize;
else
- s->hmac.hash_len = hash->cri_mlen;
- if (hmac)
+ s->hmac.hash_len = csp->csp_auth_mlen;
+ if (csp->csp_auth_key != NULL)
t4_init_hmac_digest(auth_hash, partial_digest_len,
- hash->cri_key, hash->cri_klen, s->hmac.pads);
+ csp->csp_auth_key, csp->csp_auth_klen,
+ s->hmac.pads);
else
- ccr_init_hash_digest(s, hash->cri_alg);
+ ccr_init_hash_digest(s);
}
- if (cipher != NULL) {
+ if (cipher_mode != SCMD_CIPH_MODE_NOP) {
s->blkcipher.cipher_mode = cipher_mode;
- s->blkcipher.iv_len = iv_len;
- if (cipher->cri_key != NULL)
- ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
- cipher->cri_klen);
+ s->blkcipher.iv_len = csp->csp_ivlen;
+ if (csp->csp_cipher_key != NULL)
+ ccr_aes_setkey(s, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
}
s->active = true;
@@ -2568,15 +2508,12 @@
static int
ccr_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct ccr_softc *sc;
struct ccr_session *s;
- struct cryptodesc *crd, *crda, *crde;
int error;
- if (crp == NULL)
- return (EINVAL);
-
- crd = crp->crp_desc;
+ csp = crypto_get_params(crp->crp_session);
s = crypto_get_driver_session(crp->crp_session);
sc = device_get_softc(dev);
@@ -2594,141 +2531,82 @@
sc->stats_hash++;
break;
case HMAC:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
+ if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash,
- s->hmac.partial_digest_len, crd->crd_key,
- crd->crd_klen, s->hmac.pads);
+ s->hmac.partial_digest_len, crp->crp_auth_key,
+ csp->csp_auth_klen, s->hmac.pads);
error = ccr_hash(sc, s, crp);
if (error == 0)
sc->stats_hmac++;
break;
case BLKCIPHER:
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crd->crd_alg,
- crd->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crd->crd_alg, crd->crd_key,
- crd->crd_klen);
- }
+ if (crp->crp_cipher_key != NULL)
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
error = ccr_blkcipher(sc, s, crp);
if (error == 0) {
- if (crd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_blkcipher_encrypt++;
else
sc->stats_blkcipher_decrypt++;
}
break;
- case AUTHENC:
- error = 0;
- switch (crd->crd_alg) {
- case CRYPTO_AES_CBC:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_XTS:
- /* Only encrypt-then-authenticate supported. */
- crde = crd;
- crda = crd->crd_next;
- if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
- error = EINVAL;
- break;
- }
- break;
- default:
- crda = crd;
- crde = crd->crd_next;
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- error = EINVAL;
- break;
- }
- break;
- }
- if (error)
- break;
- if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
+ case ETA:
+ if (crp->crp_auth_key != NULL)
t4_init_hmac_digest(s->hmac.auth_hash,
- s->hmac.partial_digest_len, crda->crd_key,
- crda->crd_klen, s->hmac.pads);
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
- }
- error = ccr_authenc(sc, s, crp, crda, crde);
+ s->hmac.partial_digest_len, crp->crp_auth_key,
+ csp->csp_auth_klen, s->hmac.pads);
+ if (crp->crp_cipher_key != NULL)
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
+ error = ccr_eta(sc, s, crp);
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
- sc->stats_authenc_encrypt++;
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ sc->stats_eta_encrypt++;
else
- sc->stats_authenc_decrypt++;
+ sc->stats_eta_decrypt++;
}
break;
case GCM:
- error = 0;
- if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
- crde = crd;
- crda = crd->crd_next;
- } else {
- crda = crd;
- crde = crd->crd_next;
+ if (crp->crp_cipher_key != NULL) {
+ t4_init_gmac_hash(crp->crp_cipher_key,
+ csp->csp_cipher_klen, s->gmac.ghash_h);
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
- if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
- t4_init_gmac_hash(crda->crd_key, crda->crd_klen,
- s->gmac.ghash_h);
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
- }
- if (crde->crd_len == 0) {
+ if (crp->crp_payload_length == 0) {
mtx_unlock(&sc->lock);
- ccr_gcm_soft(s, crp, crda, crde);
+ ccr_gcm_soft(s, crp);
return (0);
}
- error = ccr_gcm(sc, s, crp, crda, crde);
+ error = ccr_gcm(sc, s, crp);
if (error == EMSGSIZE) {
sc->stats_sw_fallback++;
mtx_unlock(&sc->lock);
- ccr_gcm_soft(s, crp, crda, crde);
+ ccr_gcm_soft(s, crp);
return (0);
}
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_gcm_encrypt++;
else
sc->stats_gcm_decrypt++;
}
break;
case CCM:
- error = 0;
- if (crd->crd_alg == CRYPTO_AES_CCM_16) {
- crde = crd;
- crda = crd->crd_next;
- } else {
- crda = crd;
- crde = crd->crd_next;
+ if (crp->crp_cipher_key != NULL) {
+ ccr_aes_setkey(s, crp->crp_cipher_key,
+ csp->csp_cipher_klen);
}
- if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
- error = ccr_aes_check_keylen(crde->crd_alg,
- crde->crd_klen);
- if (error)
- break;
- ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
- crde->crd_klen);
- }
- error = ccr_ccm(sc, s, crp, crda, crde);
+ error = ccr_ccm(sc, s, crp);
if (error == EMSGSIZE) {
sc->stats_sw_fallback++;
mtx_unlock(&sc->lock);
- ccr_ccm_soft(s, crp, crda, crde);
+ ccr_ccm_soft(s, crp);
return (0);
}
if (error == 0) {
- if (crde->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
sc->stats_ccm_encrypt++;
else
sc->stats_ccm_decrypt++;
@@ -2789,8 +2667,8 @@
case BLKCIPHER:
error = ccr_blkcipher_done(sc, s, crp, cpl, error);
break;
- case AUTHENC:
- error = ccr_authenc_done(sc, s, crp, cpl, error);
+ case ETA:
+ error = ccr_eta_done(sc, s, crp, cpl, error);
break;
case GCM:
error = ccr_gcm_done(sc, s, crp, cpl, error);
@@ -2835,6 +2713,7 @@
DEVMETHOD(device_attach, ccr_attach),
DEVMETHOD(device_detach, ccr_detach),
+ DEVMETHOD(cryptodev_probesession, ccr_probesession),
DEVMETHOD(cryptodev_newsession, ccr_newsession),
DEVMETHOD(cryptodev_freesession, ccr_freesession),
DEVMETHOD(cryptodev_process, ccr_process),
Index: sys/dev/cxgbe/crypto/t4_keyctx.c
===================================================================
--- sys/dev/cxgbe/crypto/t4_keyctx.c
+++ sys/dev/cxgbe/crypto/t4_keyctx.c
@@ -73,7 +73,7 @@
uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)];
int rounds;
- rounds = rijndaelKeySetupEnc(keysched, key, klen);
+ rounds = rijndaelKeySetupEnc(keysched, key, klen * 8);
rijndaelEncrypt(keysched, rounds, zeroes, ghash);
}
@@ -118,45 +118,19 @@
void
t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len,
- char *key, int klen, char *dst)
+ const char *key, int klen, char *dst)
{
union authctx auth_ctx;
- char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN];
- u_int i;
- /*
- * If the key is larger than the block size, use the digest of
- * the key as the key instead.
- */
- klen /= 8;
- if (klen > axf->blocksize) {
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, key, klen);
- axf->Final(ipad, &auth_ctx);
- klen = axf->hashsize;
- } else
- memcpy(ipad, key, klen);
-
- memset(ipad + klen, 0, axf->blocksize - klen);
- memcpy(opad, ipad, axf->blocksize);
-
- for (i = 0; i < axf->blocksize; i++) {
- ipad[i] ^= HMAC_IPAD_VAL;
- opad[i] ^= HMAC_OPAD_VAL;
- }
-
- /*
- * Hash the raw ipad and opad and store the partial results in
- * the key context.
- */
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, ipad, axf->blocksize);
+ hmac_init_ipad(axf, key, klen, &auth_ctx);
t4_copy_partial_hash(axf->type, &auth_ctx, dst);
dst += roundup2(partial_digest_len, 16);
- axf->Init(&auth_ctx);
- axf->Update(&auth_ctx, opad, axf->blocksize);
+
+ hmac_init_opad(axf, key, klen, &auth_ctx);
t4_copy_partial_hash(axf->type, &auth_ctx, dst);
+
+ explicit_bzero(&auth_ctx, sizeof(auth_ctx));
}
/*
Index: sys/dev/cxgbe/tom/t4_tls.c
===================================================================
--- sys/dev/cxgbe/tom/t4_tls.c
+++ sys/dev/cxgbe/tom/t4_tls.c
@@ -892,7 +892,7 @@
k_ctx->tx_key_info_size += GMAC_BLOCK_LEN;
memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
t4_init_gmac_hash(tls->params.cipher_key,
- tls->params.cipher_key_len * 8, hash);
+ tls->params.cipher_key_len, hash);
} else {
switch (tls->params.auth_algorithm) {
case CRYPTO_SHA1_HMAC:
@@ -920,7 +920,7 @@
k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2;
k_ctx->mac_secret_size = mac_key_size;
t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
- tls->params.auth_key_len * 8, hash);
+ tls->params.auth_key_len, hash);
}
k_ctx->frag_size = tls->params.max_frame_len;
Index: sys/dev/glxsb/glxsb.h
===================================================================
--- sys/dev/glxsb/glxsb.h
+++ sys/dev/glxsb/glxsb.h
@@ -37,8 +37,6 @@
struct glxsb_session {
uint32_t ses_key[4]; /* key */
- uint8_t ses_iv[SB_AES_BLOCK_SIZE]; /* initialization vector */
- int ses_klen; /* key len */
struct auth_hash *ses_axf;
uint8_t *ses_ictx;
uint8_t *ses_octx;
@@ -46,10 +44,10 @@
};
int glxsb_hash_setup(struct glxsb_session *ses,
- struct cryptoini *macini);
+ const struct crypto_session_params *csp);
int glxsb_hash_process(struct glxsb_session *ses,
- struct cryptodesc *maccrd, struct cryptop *crp);
+ const struct crypto_session_params *csp, struct cryptop *crp);
void glxsb_hash_free(struct glxsb_session *ses);
Index: sys/dev/glxsb/glxsb.c
===================================================================
--- sys/dev/glxsb/glxsb.c
+++ sys/dev/glxsb/glxsb.c
@@ -51,7 +51,6 @@
#include <dev/pci/pcireg.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
#include <opencrypto/xform.h>
#include "cryptodev_if.h"
@@ -172,8 +171,6 @@
struct glxsb_taskop {
struct glxsb_session *to_ses; /* crypto session */
struct cryptop *to_crp; /* cryptop to perfom */
- struct cryptodesc *to_enccrd; /* enccrd to perform */
- struct cryptodesc *to_maccrd; /* maccrd to perform */
};
struct glxsb_softc {
@@ -204,13 +201,16 @@
static void glxsb_rnd(void *);
static int glxsb_crypto_setup(struct glxsb_softc *);
-static int glxsb_crypto_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int glxsb_crypto_probesession(device_t,
+ const struct crypto_session_params *);
+static int glxsb_crypto_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static void glxsb_crypto_freesession(device_t, crypto_session_t);
static int glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t,
- uint32_t, void *, int, void *);
+ uint32_t, const void *, int, const void *);
-static int glxsb_crypto_encdec(struct cryptop *, struct cryptodesc *,
- struct glxsb_session *, struct glxsb_softc *);
+static int glxsb_crypto_encdec(struct cryptop *, struct glxsb_session *,
+ struct glxsb_softc *);
static void glxsb_crypto_task(void *, int);
static int glxsb_crypto_process(device_t, struct cryptop *, int);
@@ -222,6 +222,7 @@
DEVMETHOD(device_detach, glxsb_detach),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, glxsb_crypto_probesession),
DEVMETHOD(cryptodev_newsession, glxsb_crypto_newsession),
DEVMETHOD(cryptodev_freesession, glxsb_crypto_freesession),
DEVMETHOD(cryptodev_process, glxsb_crypto_process),
@@ -477,91 +478,67 @@
mtx_init(&sc->sc_task_mtx, "glxsb_crypto_mtx", NULL, MTX_DEF);
- if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0) != 0)
- goto crypto_fail;
- if (crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0) != 0)
- goto crypto_fail;
-
return (0);
-
-crypto_fail:
- device_printf(sc->sc_dev, "cannot register crypto\n");
- crypto_unregister_all(sc->sc_cid);
- mtx_destroy(&sc->sc_task_mtx);
- return (ENOMEM);
}
static int
-glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri)
+glxsb_crypto_probesession(device_t dev, const struct crypto_session_params *csp)
{
- struct glxsb_softc *sc = device_get_softc(dev);
- struct glxsb_session *ses;
- struct cryptoini *encini, *macini;
- int error;
- if (sc == NULL || cri == NULL)
+ if (csp->csp_flags != 0)
return (EINVAL);
- encini = macini = NULL;
- for (; cri != NULL; cri = cri->cri_next) {
- switch(cri->cri_alg) {
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- if (macini != NULL)
- return (EINVAL);
- macini = cri;
- break;
- case CRYPTO_AES_CBC:
- if (encini != NULL)
- return (EINVAL);
- encini = cri;
- break;
- default:
- return (EINVAL);
- }
- }
-
/*
* We only support HMAC algorithms to be able to work with
* ipsec(4), so if we are asked only for authentication without
- * encryption, don't pretend we can accellerate it.
+ * encryption, don't pretend we can accelerate it.
*/
- if (encini == NULL)
+ switch (csp->csp_mode) {
+ case CSP_MODE_ETA:
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_NULL_HMAC:
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_RIPEMD160_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ break;
+ default:
+ return (EINVAL);
+ }
+ /* FALLTHROUGH */
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ if (csp->csp_cipher_klen * 8 != 128)
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ default:
return (EINVAL);
-
- ses = crypto_get_driver_session(cses);
- if (encini->cri_alg == CRYPTO_AES_CBC) {
- if (encini->cri_klen != 128) {
- glxsb_crypto_freesession(sc->sc_dev, cses);
- return (EINVAL);
- }
- arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
- ses->ses_klen = encini->cri_klen;
-
- /* Copy the key (Geode LX wants the primary key only) */
- bcopy(encini->cri_key, ses->ses_key, sizeof(ses->ses_key));
}
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
- if (macini != NULL) {
- error = glxsb_hash_setup(ses, macini);
+static int
+glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
+ struct glxsb_softc *sc = device_get_softc(dev);
+ struct glxsb_session *ses;
+ int error;
+
+ ses = crypto_get_driver_session(cses);
+
+ /* Copy the key (Geode LX wants the primary key only) */
+ if (csp->csp_cipher_key != NULL)
+ bcopy(csp->csp_cipher_key, ses->ses_key, sizeof(ses->ses_key));
+
+ if (csp->csp_auth_alg != 0) {
+ error = glxsb_hash_setup(ses, csp);
if (error != 0) {
glxsb_crypto_freesession(sc->sc_dev, cses);
return (error);
@@ -574,19 +551,15 @@
static void
glxsb_crypto_freesession(device_t dev, crypto_session_t cses)
{
- struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
- if (sc == NULL)
- return;
-
ses = crypto_get_driver_session(cses);
glxsb_hash_free(ses);
}
static int
glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
- uint32_t pdst, void *key, int len, void *iv)
+ uint32_t pdst, const void *key, int len, const void *iv)
{
uint32_t status;
int i;
@@ -652,23 +625,24 @@
}
static int
-glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
- struct glxsb_session *ses, struct glxsb_softc *sc)
+glxsb_crypto_encdec(struct cryptop *crp, struct glxsb_session *ses,
+ struct glxsb_softc *sc)
{
char *op_src, *op_dst;
+ const void *key;
uint32_t op_psrc, op_pdst;
- uint8_t op_iv[SB_AES_BLOCK_SIZE], *piv;
+ uint8_t op_iv[SB_AES_BLOCK_SIZE];
int error;
int len, tlen, xlen;
int offset;
uint32_t control;
- if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0)
+ if ((crp->crp_payload_length % SB_AES_BLOCK_SIZE) != 0)
return (EINVAL);
/* How much of our buffer will we need to use? */
- xlen = crd->crd_len > GLXSB_MAX_AES_LEN ?
- GLXSB_MAX_AES_LEN : crd->crd_len;
+ xlen = crp->crp_payload_length > GLXSB_MAX_AES_LEN ?
+ GLXSB_MAX_AES_LEN : crp->crp_payload_length;
/*
* XXX Check if we can have input == output on Geode LX.
@@ -680,73 +654,57 @@
op_psrc = sc->sc_dma.dma_paddr;
op_pdst = sc->sc_dma.dma_paddr + xlen;
- if (crd->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
control = SB_CTL_ENC;
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
- else
- bcopy(ses->ses_iv, op_iv, sizeof(op_iv));
-
- if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, sizeof(op_iv), op_iv);
- }
- } else {
+ else
control = SB_CTL_DEC;
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
- else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, sizeof(op_iv), op_iv);
- }
- }
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(op_iv, sizeof(op_iv), 0);
+ crypto_copyback(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(op_iv, crp->crp_iv, sizeof(op_iv));
+ else
+ crypto_copydata(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+
offset = 0;
- tlen = crd->crd_len;
- piv = op_iv;
+ tlen = crp->crp_payload_length;
+
+ if (crp->crp_cipher_key != NULL)
+ key = crp->crp_cipher_key;
+ else
+ key = ses->ses_key;
/* Process the data in GLXSB_MAX_AES_LEN chunks */
while (tlen > 0) {
len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + offset, len, op_src);
+ crypto_copydata(crp, crp->crp_payload_start + offset, len,
+ op_src);
glxsb_dma_pre_op(sc, &sc->sc_dma);
- error = glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key,
- len, op_iv);
+ error = glxsb_aes(sc, control, op_psrc, op_pdst, key, len,
+ op_iv);
glxsb_dma_post_op(sc, &sc->sc_dma);
if (error != 0)
return (error);
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + offset, len, op_dst);
+ crypto_copyback(crp, crp->crp_payload_start + offset, len,
+ op_dst);
offset += len;
tlen -= len;
- if (tlen <= 0) { /* Ideally, just == 0 */
- /* Finished - put the IV in session IV */
- piv = ses->ses_iv;
- }
-
/*
- * Copy out last block for use as next iteration/session IV.
- *
- * piv is set to op_iv[] before the loop starts, but is
- * set to ses->ses_iv if we're going to exit the loop this
- * time.
+ * Copy out last block for use as next iteration IV.
*/
- if (crd->crd_flags & CRD_F_ENCRYPT)
- bcopy(op_dst + len - sizeof(op_iv), piv, sizeof(op_iv));
- else {
- /* Decryption, only need this if another iteration */
- if (tlen > 0) {
- bcopy(op_src + len - sizeof(op_iv), piv,
- sizeof(op_iv));
- }
- }
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ bcopy(op_dst + len - sizeof(op_iv), op_iv,
+ sizeof(op_iv));
+ else
+ bcopy(op_src + len - sizeof(op_iv), op_iv,
+ sizeof(op_iv));
} /* while */
/* All AES processing has now been done. */
@@ -759,30 +717,31 @@
glxsb_crypto_task(void *arg, int pending)
{
struct glxsb_softc *sc = arg;
+ const struct crypto_session_params *csp;
struct glxsb_session *ses;
struct cryptop *crp;
- struct cryptodesc *enccrd, *maccrd;
int error;
- maccrd = sc->sc_to.to_maccrd;
- enccrd = sc->sc_to.to_enccrd;
crp = sc->sc_to.to_crp;
ses = sc->sc_to.to_ses;
+ csp = crypto_get_params(crp->crp_session);
/* Perform data authentication if requested before encryption */
- if (maccrd != NULL && maccrd->crd_next == enccrd) {
- error = glxsb_hash_process(ses, maccrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = glxsb_hash_process(ses, csp, crp);
if (error != 0)
goto out;
}
- error = glxsb_crypto_encdec(crp, enccrd, ses, sc);
+ error = glxsb_crypto_encdec(crp, ses, sc);
if (error != 0)
goto out;
/* Perform data authentication if requested after encryption */
- if (maccrd != NULL && enccrd->crd_next == maccrd) {
- error = glxsb_hash_process(ses, maccrd, crp);
+ if (csp->csp_mode == CSP_MODE_ETA &&
+ CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = glxsb_hash_process(ses, csp, crp);
if (error != 0)
goto out;
}
@@ -801,52 +760,6 @@
{
struct glxsb_softc *sc = device_get_softc(dev);
struct glxsb_session *ses;
- struct cryptodesc *crd, *enccrd, *maccrd;
- int error = 0;
-
- enccrd = maccrd = NULL;
-
- /* Sanity check. */
- if (crp == NULL)
- return (EINVAL);
-
- if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
- error = EINVAL;
- goto fail;
- }
-
- for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
- switch (crd->crd_alg) {
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- if (maccrd != NULL) {
- error = EINVAL;
- goto fail;
- }
- maccrd = crd;
- break;
- case CRYPTO_AES_CBC:
- if (enccrd != NULL) {
- error = EINVAL;
- goto fail;
- }
- enccrd = crd;
- break;
- default:
- error = EINVAL;
- goto fail;
- }
- }
-
- if (enccrd == NULL || enccrd->crd_len % AES_BLOCK_LEN != 0) {
- error = EINVAL;
- goto fail;
- }
ses = crypto_get_driver_session(crp->crp_session);
@@ -857,17 +770,10 @@
}
sc->sc_task_count++;
- sc->sc_to.to_maccrd = maccrd;
- sc->sc_to.to_enccrd = enccrd;
sc->sc_to.to_crp = crp;
sc->sc_to.to_ses = ses;
mtx_unlock(&sc->sc_task_mtx);
taskqueue_enqueue(sc->sc_tq, &sc->sc_cryptotask);
return(0);
-
-fail:
- crp->crp_etype = error;
- crypto_done(crp);
- return (error);
}
Index: sys/dev/glxsb/glxsb_hash.c
===================================================================
--- sys/dev/glxsb/glxsb_hash.c
+++ sys/dev/glxsb/glxsb_hash.c
@@ -33,7 +33,6 @@
#include <sys/systm.h>
#include <sys/malloc.h>
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
#include <opencrypto/xform.h>
#include "glxsb.h"
@@ -51,92 +50,66 @@
MALLOC_DECLARE(M_GLXSB);
static void
-glxsb_hash_key_setup(struct glxsb_session *ses, caddr_t key, int klen)
+glxsb_hash_key_setup(struct glxsb_session *ses, const char *key, int klen)
{
struct auth_hash *axf;
- int i;
- klen /= 8;
axf = ses->ses_axf;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
-
- axf->Init(ses->ses_ictx);
- axf->Update(ses->ses_ictx, key, klen);
- axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- axf->Init(ses->ses_octx);
- axf->Update(ses->ses_octx, key, klen);
- axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ hmac_init_ipad(axf, key, klen, ses->ses_ictx);
+ hmac_init_opad(axf, key, klen, ses->ses_octx);
}
/*
* Compute keyed-hash authenticator.
*/
static int
-glxsb_authcompute(struct glxsb_session *ses, struct cryptodesc *crd,
- caddr_t buf, int flags)
+glxsb_authcompute(struct glxsb_session *ses, struct cryptop *crp)
{
- u_char hash[HASH_MAX_LEN];
+ u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN];
struct auth_hash *axf;
union authctx ctx;
int error;
axf = ses->ses_axf;
bcopy(ses->ses_ictx, &ctx, axf->ctxsize);
- error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+ error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
(int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
if (error != 0)
return (error);
+ error = crypto_apply(crp, crp->crp_payload_start,
+ crp->crp_payload_length,
+ (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+ if (error != 0)
+ return (error);
+
axf->Final(hash, &ctx);
bcopy(ses->ses_octx, &ctx, axf->ctxsize);
axf->Update(&ctx, hash, axf->hashsize);
axf->Final(hash, &ctx);
- /* Inject the authentication data */
- crypto_copyback(flags, buf, crd->crd_inject,
- ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+ /* Verify or inject the authentication data */
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash2);
+ if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0)
+ return (EBADMSG);
+ } else
+ crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen,
+ hash);
return (0);
}
int
-glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini)
+glxsb_hash_setup(struct glxsb_session *ses,
+ const struct crypto_session_params *csp)
{
- ses->ses_mlen = macini->cri_mlen;
-
- /* Find software structure which describes HMAC algorithm. */
- switch (macini->cri_alg) {
- case CRYPTO_NULL_HMAC:
- ses->ses_axf = &auth_hash_null;
- break;
- case CRYPTO_MD5_HMAC:
- ses->ses_axf = &auth_hash_hmac_md5;
- break;
- case CRYPTO_SHA1_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha1;
- break;
- case CRYPTO_RIPEMD160_HMAC:
- ses->ses_axf = &auth_hash_hmac_ripemd_160;
- break;
- case CRYPTO_SHA2_256_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_256;
- break;
- case CRYPTO_SHA2_384_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_384;
- break;
- case CRYPTO_SHA2_512_HMAC:
- ses->ses_axf = &auth_hash_hmac_sha2_512;
- break;
- }
+ ses->ses_axf = crypto_auth_hash(csp);
+ if (csp->csp_auth_mlen == 0)
+ ses->ses_mlen = ses->ses_axf->hashsize;
+ else
+ ses->ses_mlen = csp->csp_auth_mlen;
/* Allocate memory for HMAC inner and outer contexts. */
ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_GLXSB,
@@ -147,23 +120,24 @@
return (ENOMEM);
/* Setup key if given. */
- if (macini->cri_key != NULL) {
- glxsb_hash_key_setup(ses, macini->cri_key,
- macini->cri_klen);
+ if (csp->csp_auth_key != NULL) {
+ glxsb_hash_key_setup(ses, csp->csp_auth_key,
+ csp->csp_auth_klen);
}
return (0);
}
int
-glxsb_hash_process(struct glxsb_session *ses, struct cryptodesc *maccrd,
- struct cryptop *crp)
+glxsb_hash_process(struct glxsb_session *ses,
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
int error;
- if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
- glxsb_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+ if (crp->crp_auth_key != NULL)
+ glxsb_hash_key_setup(ses, crp->crp_auth_key,
+ csp->csp_auth_klen);
- error = glxsb_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+ error = glxsb_authcompute(ses, crp);
return (error);
}
Index: sys/dev/hifn/hifn7751.c
===================================================================
--- sys/dev/hifn/hifn7751.c
+++ sys/dev/hifn/hifn7751.c
@@ -61,6 +61,7 @@
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -71,6 +72,7 @@
#include <sys/rman.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -102,7 +104,9 @@
static int hifn_resume(device_t);
static int hifn_shutdown(device_t);
-static int hifn_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int hifn_probesession(device_t, const struct crypto_session_params *);
+static int hifn_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int hifn_process(device_t, struct cryptop *, int);
static device_method_t hifn_methods[] = {
@@ -115,6 +119,7 @@
DEVMETHOD(device_shutdown, hifn_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, hifn_probesession),
DEVMETHOD(cryptodev_newsession, hifn_newsession),
DEVMETHOD(cryptodev_process, hifn_process),
@@ -356,7 +361,7 @@
caddr_t kva;
int rseg, rid;
char rbase;
- u_int16_t ena, rev;
+ uint16_t rev;
sc->sc_dev = dev;
@@ -558,33 +563,22 @@
2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
printf("\n");
- sc->sc_cid = crypto_get_driverid(dev, sizeof(struct hifn_session),
- CRYPTOCAP_F_HARDWARE);
- if (sc->sc_cid < 0) {
- device_printf(dev, "could not get crypto driver id\n");
- goto fail_intr;
- }
-
WRITE_REG_0(sc, HIFN_0_PUCNFG,
READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
- ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
+ sc->sc_ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
- switch (ena) {
+ switch (sc->sc_ena) {
case HIFN_PUSTAT_ENA_2:
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
- if (sc->sc_flags & HIFN_HAS_AES)
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
- /*FALLTHROUGH*/
case HIFN_PUSTAT_ENA_1:
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
+ sc->sc_cid = crypto_get_driverid(dev,
+ sizeof(struct hifn_session), CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto fail_intr;
+ }
break;
}
-
+
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1547,6 +1541,7 @@
static u_int
hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
{
+ struct cryptop *crp;
u_int8_t *buf_pos;
hifn_base_command_t *base_cmd;
hifn_mac_command_t *mac_cmd;
@@ -1554,6 +1549,7 @@
int using_mac, using_crypt, len, ivlen;
u_int32_t dlen, slen;
+ crp = cmd->crp;
buf_pos = buf;
using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
@@ -1576,24 +1572,27 @@
if (using_mac) {
mac_cmd = (hifn_mac_command_t *)buf_pos;
- dlen = cmd->maccrd->crd_len;
+ dlen = crp->crp_aad_length + crp->crp_payload_length;
mac_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
mac_cmd->masks = htole16(cmd->mac_masks |
((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
- mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
+ if (crp->crp_aad_length != 0)
+ mac_cmd->header_skip = htole16(crp->crp_aad_start);
+ else
+ mac_cmd->header_skip = htole16(crp->crp_payload_start);
mac_cmd->reserved = 0;
buf_pos += sizeof(hifn_mac_command_t);
}
if (using_crypt) {
cry_cmd = (hifn_crypt_command_t *)buf_pos;
- dlen = cmd->enccrd->crd_len;
+ dlen = crp->crp_payload_length;
cry_cmd->source_count = htole16(dlen & 0xffff);
dlen >>= 16;
cry_cmd->masks = htole16(cmd->cry_masks |
((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
- cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
+ cry_cmd->header_skip = htole16(crp->crp_payload_length);
cry_cmd->reserved = 0;
buf_pos += sizeof(hifn_crypt_command_t);
}
@@ -1782,15 +1781,30 @@
return (idx);
}
+static bus_size_t
+hifn_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct hifn_operand *op = arg;
KASSERT(nsegs <= MAX_SCATTER,
("hifn_op_cb: too many DMA segments (%u > %u) "
"returned when mapping operand", nsegs, MAX_SCATTER));
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -1832,130 +1846,110 @@
return (ENOMEM);
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
- cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_srcmap1;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
- cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_srcmap1;
- }
- } else {
- err = EINVAL;
+ if (bus_dmamap_load_crp(sc->sc_dmat, cmd->src_map, crp, hifn_op_cb,
+ &cmd->src, BUS_DMA_NOWAIT)) {
+ hifnstats.hst_nomem_load++;
+ err = ENOMEM;
goto err_srcmap1;
}
+ cmd->src_mapsize = hifn_crp_length(crp);
if (hifn_dmamap_aligned(&cmd->src)) {
cmd->sloplen = cmd->src_mapsize & 3;
cmd->dst = cmd->src;
- } else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- err = EINVAL;
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *m0, *mlast;
+
+ KASSERT(cmd->dst_m == NULL,
+ ("hifn_crypto: dst_m initialized improperly"));
+ hifnstats.hst_unaligned++;
+
+ /*
+ * Source is not aligned on a longword boundary.
+ * Copy the data to insure alignment. If we fail
+ * to allocate mbufs or clusters while doing this
+ * we return ERESTART so the operation is requeued
+ * at the crypto later, but only if there are
+ * ops already posted to the hardware; otherwise we
+ * have no guarantee that we'll be re-entered.
+ */
+ totlen = cmd->src_mapsize;
+ if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ len = MHLEN;
+ MGETHDR(m0, M_NOWAIT, MT_DATA);
+ if (m0 && !m_dup_pkthdr(m0, crp->crp_mbuf, M_NOWAIT)) {
+ m_free(m0);
+ m0 = NULL;
+ }
+ } else {
+ len = MLEN;
+ MGET(m0, M_NOWAIT, MT_DATA);
+ }
+ if (m0 == NULL) {
+ hifnstats.hst_nomem_mbuf++;
+ err = sc->sc_cmdu ? ERESTART : ENOMEM;
goto err_srcmap;
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- int totlen, len;
- struct mbuf *m, *m0, *mlast;
-
- KASSERT(cmd->dst_m == cmd->src_m,
- ("hifn_crypto: dst_m initialized improperly"));
- hifnstats.hst_unaligned++;
- /*
- * Source is not aligned on a longword boundary.
- * Copy the data to insure alignment. If we fail
- * to allocate mbufs or clusters while doing this
- * we return ERESTART so the operation is requeued
- * at the crypto later, but only if there are
- * ops already posted to the hardware; otherwise we
- * have no guarantee that we'll be re-entered.
- */
- totlen = cmd->src_mapsize;
- if (cmd->src_m->m_flags & M_PKTHDR) {
- len = MHLEN;
- MGETHDR(m0, M_NOWAIT, MT_DATA);
- if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
- m_free(m0);
- m0 = NULL;
- }
- } else {
- len = MLEN;
- MGET(m0, M_NOWAIT, MT_DATA);
+ }
+ if (totlen >= MINCLSIZE) {
+ if (!(MCLGET(m0, M_NOWAIT))) {
+ hifnstats.hst_nomem_mcl++;
+ err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ m_freem(m0);
+ goto err_srcmap;
}
- if (m0 == NULL) {
+ len = MCLBYTES;
+ }
+ totlen -= len;
+ m0->m_pkthdr.len = m0->m_len = len;
+ mlast = m0;
+
+ while (totlen > 0) {
+ MGET(m, M_NOWAIT, MT_DATA);
+ if (m == NULL) {
hifnstats.hst_nomem_mbuf++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ m_freem(m0);
goto err_srcmap;
}
+ len = MLEN;
if (totlen >= MINCLSIZE) {
- if (!(MCLGET(m0, M_NOWAIT))) {
+ if (!(MCLGET(m, M_NOWAIT))) {
hifnstats.hst_nomem_mcl++;
err = sc->sc_cmdu ? ERESTART : ENOMEM;
+ mlast->m_next = m;
m_freem(m0);
goto err_srcmap;
}
len = MCLBYTES;
}
+
+ m->m_len = len;
+ m0->m_pkthdr.len += len;
totlen -= len;
- m0->m_pkthdr.len = m0->m_len = len;
- mlast = m0;
- while (totlen > 0) {
- MGET(m, M_NOWAIT, MT_DATA);
- if (m == NULL) {
- hifnstats.hst_nomem_mbuf++;
- err = sc->sc_cmdu ? ERESTART : ENOMEM;
- m_freem(m0);
- goto err_srcmap;
- }
- len = MLEN;
- if (totlen >= MINCLSIZE) {
- if (!(MCLGET(m, M_NOWAIT))) {
- hifnstats.hst_nomem_mcl++;
- err = sc->sc_cmdu ? ERESTART : ENOMEM;
- mlast->m_next = m;
- m_freem(m0);
- goto err_srcmap;
- }
- len = MCLBYTES;
- }
-
- m->m_len = len;
- m0->m_pkthdr.len += len;
- totlen -= len;
-
- mlast->m_next = m;
- mlast = m;
- }
- cmd->dst_m = m0;
+ mlast->m_next = m;
+ mlast = m;
}
- }
+ cmd->dst_m = m0;
- if (cmd->dst_map == NULL) {
- if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
+ if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+ &cmd->dst_map)) {
hifnstats.hst_nomem_map++;
err = ENOMEM;
goto err_srcmap;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
- cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_map++;
- err = ENOMEM;
- goto err_dstmap1;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
- cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
- hifnstats.hst_nomem_load++;
- err = ENOMEM;
- goto err_dstmap1;
- }
+
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dmat, cmd->dst_map, m0,
+ cmd->dst_segs, &cmd->dst_nsegs, 0)) {
+ hifnstats.hst_nomem_map++;
+ err = ENOMEM;
+ goto err_dstmap1;
}
+ cmd->dst_mapsize = m0->m_pkthdr.len;
+ } else {
+ err = EINVAL;
+ goto err_srcmap;
}
#ifdef HIFN_DEBUG
@@ -2111,8 +2105,8 @@
if (cmd->src_map != cmd->dst_map)
bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
err_srcmap:
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (cmd->src_m != cmd->dst_m)
+ if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ if (cmd->dst_m != NULL)
m_freem(cmd->dst_m);
}
bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
@@ -2307,67 +2301,121 @@
}
}
+static bool
+hifn_auth_supported(struct hifn_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (sc->sc_ena) {
+ case HIFN_PUSTAT_ENA_2:
+ case HIFN_PUSTAT_ENA_1:
+ break;
+ default:
+ return (false);
+ }
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+ break;
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ if (csp->csp_auth_klen > HIFN_MAC_KEY_LENGTH)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+
+ return (true);
+}
+
+static bool
+hifn_cipher_supported(struct hifn_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_cipher_klen == 0)
+ return (false);
+ if (csp->csp_ivlen > HIFN_MAX_IV_LENGTH)
+ return (false);
+ switch (sc->sc_ena) {
+ case HIFN_PUSTAT_ENA_2:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_3DES_CBC:
+ case CRYPTO_ARC4:
+ break;
+ case CRYPTO_AES_CBC:
+ if ((sc->sc_flags & HIFN_HAS_AES) == 0)
+ return (false);
+ switch (csp->csp_cipher_klen) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+ }
+ /*FALLTHROUGH*/
+ case HIFN_PUSTAT_ENA_1:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ return (true);
+ }
+ break;
+ }
+ return (false);
+}
+
+static int
+hifn_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct hifn_softc *sc;
+
+ sc = device_get_softc(dev);
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!hifn_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!hifn_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!hifn_auth_supported(sc, csp) ||
+ !hifn_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
+ * Allocate a new 'session'.
*/
static int
-hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+hifn_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
- struct hifn_softc *sc = device_get_softc(dev);
- struct cryptoini *c;
- int mac = 0, cry = 0;
struct hifn_session *ses;
- KASSERT(sc != NULL, ("hifn_newsession: null softc"));
- if (cri == NULL || sc == NULL)
- return (EINVAL);
-
ses = crypto_get_driver_session(cses);
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_SHA1:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- if (mac)
- return (EINVAL);
- mac = 1;
- ses->hs_mlen = c->cri_mlen;
- if (ses->hs_mlen == 0) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_MD5_HMAC:
- ses->hs_mlen = 16;
- break;
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- ses->hs_mlen = 20;
- break;
- }
- }
- break;
- case CRYPTO_DES_CBC:
- case CRYPTO_3DES_CBC:
- case CRYPTO_AES_CBC:
- /* XXX this may read fewer, does it matter? */
- read_random(ses->hs_iv,
- c->cri_alg == CRYPTO_AES_CBC ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- /*FALLTHROUGH*/
- case CRYPTO_ARC4:
- if (cry)
- return (EINVAL);
- cry = 1;
- break;
- default:
- return (EINVAL);
- }
+ if (csp->csp_auth_alg != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
+ else
+ ses->hs_mlen = csp->csp_auth_mlen;
}
- if (mac == 0 && cry == 0)
- return (EINVAL);
+
return (0);
}
@@ -2379,18 +2427,15 @@
static int
hifn_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct hifn_softc *sc = device_get_softc(dev);
struct hifn_command *cmd = NULL;
- int err, ivlen;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+ const void *mackey;
+ int err, ivlen, keylen;
struct hifn_session *ses;
- if (crp == NULL || crp->crp_callback == NULL) {
- hifnstats.hst_invalid++;
- return (EINVAL);
- }
-
ses = crypto_get_driver_session(crp->crp_session);
+
cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
if (cmd == NULL) {
hifnstats.hst_nomem++;
@@ -2398,80 +2443,26 @@
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- cmd->src_m = (struct mbuf *)crp->crp_buf;
- cmd->dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- cmd->src_io = (struct uio *)crp->crp_buf;
- cmd->dst_io = (struct uio *)crp->crp_buf;
- } else {
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous buffers! */
- }
+ csp = crypto_get_params(crp->crp_session);
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
+ /*
+ * The driver only supports ETA requests where there is no
+ * gap between the AAD and payload.
+ */
+ if (csp->csp_mode == CSP_MODE_ETA && crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
err = EINVAL;
goto errout;
}
- crd2 = crd1->crd_next;
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1 ||
- crd1->crd_alg == CRYPTO_MD5) {
- maccrd = crd1;
- enccrd = NULL;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4) {
- if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
- cmd->base_masks |= HIFN_BASE_CMD_DECODE;
- maccrd = NULL;
- enccrd = crd1;
- } else {
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_MD5 ||
- crd1->crd_alg == CRYPTO_SHA1) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC ||
- crd2->crd_alg == CRYPTO_AES_CBC ||
- crd2->crd_alg == CRYPTO_ARC4) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- cmd->base_masks = HIFN_BASE_CMD_DECODE;
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4 ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC ||
- crd2->crd_alg == CRYPTO_MD5 ||
- crd2->crd_alg == CRYPTO_SHA1) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- /*
- * We cannot order the 7751 as requested
- */
- err = EINVAL;
- goto errout;
- }
- }
-
- if (enccrd) {
- cmd->enccrd = enccrd;
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ case CSP_MODE_ETA:
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+ cmd->base_masks |= HIFN_BASE_CMD_DECODE;
cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
- switch (enccrd->crd_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_ARC4:
cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
break;
@@ -2494,36 +2485,24 @@
err = EINVAL;
goto errout;
}
- if (enccrd->crd_alg != CRYPTO_ARC4) {
- ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- else
- bcopy(ses->hs_iv, cmd->iv, ivlen);
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
- == 0) {
- crypto_copyback(crp->crp_flags,
- crp->crp_buf, enccrd->crd_inject,
- ivlen, cmd->iv);
- }
- } else {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- else {
- crypto_copydata(crp->crp_flags,
- crp->crp_buf, enccrd->crd_inject,
- ivlen, cmd->iv);
- }
- }
+ if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+ ivlen = csp->csp_ivlen;
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(cmd->iv, ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, ivlen,
+ cmd->iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(cmd->iv, crp->crp_iv, ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, ivlen,
+ cmd->iv);
}
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
- cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
- cmd->ck = enccrd->crd_key;
- cmd->cklen = enccrd->crd_klen >> 3;
+ if (crp->crp_cipher_key != NULL)
+ cmd->ck = crp->crp_cipher_key;
+ else
+ cmd->ck = csp->csp_cipher_key;
+ cmd->cklen = csp->csp_cipher_klen;
cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
/*
@@ -2546,13 +2525,15 @@
goto errout;
}
}
+ break;
}
- if (maccrd) {
- cmd->maccrd = maccrd;
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ case CSP_MODE_ETA:
cmd->base_masks |= HIFN_BASE_CMD_MAC;
- switch (maccrd->crd_alg) {
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5:
cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
@@ -2575,12 +2556,16 @@
break;
}
- if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
- maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+ if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC ||
+ csp->csp_auth_alg == CRYPTO_MD5_HMAC) {
cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
- bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
- bzero(cmd->mac + (maccrd->crd_klen >> 3),
- HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
+ if (crp->crp_auth_key != NULL)
+ mackey = crp->crp_auth_key;
+ else
+ mackey = csp->csp_auth_key;
+ keylen = csp->csp_auth_klen;
+ bcopy(mackey, cmd->mac, keylen);
+ bzero(cmd->mac + keylen, HIFN_MAC_KEY_LENGTH - keylen);
}
}
@@ -2655,9 +2640,8 @@
BUS_DMASYNC_POSTREAD);
}
- if (cmd->src_m != cmd->dst_m) {
- m_freem(cmd->src_m);
- crp->crp_buf = (caddr_t)cmd->dst_m;
+ if (cmd->dst_m != NULL) {
+ m_freem(cmd->dst_m);
}
/* non-shared buffers cannot be restarted */
@@ -2696,9 +2680,9 @@
{
struct hifn_dma *dma = sc->sc_dma;
struct cryptop *crp = cmd->crp;
- struct cryptodesc *crd;
+ uint8_t macbuf2[SHA1_HASH_LEN];
struct mbuf *m;
- int totlen, i, u, ivlen;
+ int totlen, i, u;
if (cmd->src_map == cmd->dst_map) {
bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
@@ -2710,9 +2694,8 @@
BUS_DMASYNC_POSTREAD);
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (cmd->src_m != cmd->dst_m) {
- crp->crp_buf = (caddr_t)cmd->dst_m;
+ if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ if (cmd->dst_m != NULL) {
totlen = cmd->src_mapsize;
for (m = cmd->dst_m; m != NULL; m = m->m_next) {
if (totlen < m->m_len) {
@@ -2721,15 +2704,15 @@
} else
totlen -= m->m_len;
}
- cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
- m_freem(cmd->src_m);
+ cmd->dst_m->m_pkthdr.len = crp->crp_mbuf->m_pkthdr.len;
+ m_freem(crp->crp_mbuf);
+ crp->crp_mbuf = cmd->dst_m;
}
}
if (cmd->sloplen != 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
- (caddr_t)&dma->slop[cmd->slopidx]);
+ crypto_copyback(crp, cmd->src_mapsize - cmd->sloplen,
+ cmd->sloplen, &dma->slop[cmd->slopidx]);
}
i = sc->sc_dstk; u = sc->sc_dstu;
@@ -2749,37 +2732,16 @@
hifnstats.hst_obytes += cmd->dst_mapsize;
- if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
- HIFN_BASE_CMD_CRYPT) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_DES_CBC &&
- crd->crd_alg != CRYPTO_3DES_CBC &&
- crd->crd_alg != CRYPTO_AES_CBC)
- continue;
- ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
- HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - ivlen, ivlen,
- cmd->session->hs_iv);
- break;
- }
- }
-
if (macbuf != NULL) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- int len;
-
- if (crd->crd_alg != CRYPTO_MD5 &&
- crd->crd_alg != CRYPTO_SHA1 &&
- crd->crd_alg != CRYPTO_MD5_HMAC &&
- crd->crd_alg != CRYPTO_SHA1_HMAC) {
- continue;
- }
- len = cmd->session->hs_mlen;
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, len, macbuf);
- break;
- }
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ cmd->session->hs_mlen, macbuf2);
+ if (timingsafe_bcmp(macbuf, macbuf2,
+ cmd->session->hs_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ cmd->session->hs_mlen, macbuf);
}
if (cmd->src_map != cmd->dst_map) {
Index: sys/dev/hifn/hifn7751var.h
===================================================================
--- sys/dev/hifn/hifn7751var.h
+++ sys/dev/hifn/hifn7751var.h
@@ -105,7 +105,6 @@
struct hifn_session {
- u_int8_t hs_iv[HIFN_MAX_IV_LENGTH];
int hs_mlen;
};
@@ -160,6 +159,7 @@
int sc_cmdk, sc_srck, sc_dstk, sc_resk;
int32_t sc_cid;
+ uint16_t sc_ena;
int sc_maxses;
int sc_ramsize;
int sc_flags;
@@ -257,10 +257,6 @@
*
*/
struct hifn_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -269,27 +265,24 @@
struct hifn_command {
struct hifn_session *session;
u_int16_t base_masks, cry_masks, mac_masks;
- u_int8_t iv[HIFN_MAX_IV_LENGTH], *ck, mac[HIFN_MAC_KEY_LENGTH];
+ u_int8_t iv[HIFN_MAX_IV_LENGTH], mac[HIFN_MAC_KEY_LENGTH];
+ const uint8_t *ck;
int cklen;
int sloplen, slopidx;
struct hifn_operand src;
struct hifn_operand dst;
+ struct mbuf *dst_m;
struct hifn_softc *softc;
struct cryptop *crp;
- struct cryptodesc *enccrd, *maccrd;
};
-#define src_m src.u.m
-#define src_io src.u.io
#define src_map src.map
#define src_mapsize src.mapsize
#define src_segs src.segs
#define src_nsegs src.nsegs
-#define dst_m dst.u.m
-#define dst_io dst.u.io
#define dst_map dst.map
#define dst_mapsize dst.mapsize
#define dst_segs dst.segs
Index: sys/dev/safe/safe.c
===================================================================
--- sys/dev/safe/safe.c
+++ sys/dev/safe/safe.c
@@ -47,6 +47,7 @@
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -56,10 +57,8 @@
#include <sys/bus.h>
#include <sys/rman.h>
-#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -88,7 +87,9 @@
static int safe_resume(device_t);
static int safe_shutdown(device_t);
-static int safe_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int safe_probesession(device_t, const struct crypto_session_params *);
+static int safe_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int safe_process(device_t, struct cryptop *, int);
static device_method_t safe_methods[] = {
@@ -101,6 +102,7 @@
DEVMETHOD(device_shutdown, safe_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, safe_probesession),
DEVMETHOD(cryptodev_newsession, safe_newsession),
DEVMETHOD(cryptodev_process, safe_process),
@@ -221,7 +223,7 @@
{
struct safe_softc *sc = device_get_softc(dev);
u_int32_t raddr;
- u_int32_t i, devinfo;
+ u_int32_t i;
int rid;
bzero(sc, sizeof (*sc));
@@ -374,12 +376,12 @@
device_printf(sc->sc_dev, "%s", safe_partname(sc));
- devinfo = READ_REG(sc, SAFE_DEVINFO);
- if (devinfo & SAFE_DEVINFO_RNG) {
+ sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO);
+ if (sc->sc_devinfo & SAFE_DEVINFO_RNG) {
sc->sc_flags |= SAFE_FLAGS_RNG;
printf(" rng");
}
- if (devinfo & SAFE_DEVINFO_PKEY) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) {
#if 0
printf(" key");
sc->sc_flags |= SAFE_FLAGS_KEY;
@@ -387,26 +389,18 @@
crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
#endif
}
- if (devinfo & SAFE_DEVINFO_DES) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_DES) {
printf(" des/3des");
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_AES) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_AES) {
printf(" aes");
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_MD5) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_MD5) {
printf(" md5");
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
}
- if (devinfo & SAFE_DEVINFO_SHA1) {
+ if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) {
printf(" sha1");
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
}
- printf(" null");
- crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
/* XXX other supported algorithms */
printf("\n");
@@ -629,11 +623,11 @@
#define N(a) (sizeof(a) / sizeof (a[0]))
static void
-safe_setup_enckey(struct safe_session *ses, caddr_t key)
+safe_setup_enckey(struct safe_session *ses, const void *key)
{
int i;
- bcopy(key, ses->ses_key, ses->ses_klen / 8);
+ bcopy(key, ses->ses_key, ses->ses_klen);
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_key); i++)
@@ -641,47 +635,30 @@
}
static void
-safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
+safe_setup_mackey(struct safe_session *ses, int algo, const uint8_t *key,
+ int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
int i;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
-
if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
- } else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_ipad_buffer,
- SHA1_BLOCK_LEN - klen);
- bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
- }
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+ explicit_bzero(&md5ctx, sizeof(md5ctx));
} else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_opad_buffer,
- SHA1_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+ bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+ hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
- }
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ explicit_bzero(&sha1ctx, sizeof(sha1ctx));
+ }
/* PE is little-endian, insure proper byte order */
for (i = 0; i < N(ses->ses_hminner); i++) {
@@ -691,90 +668,140 @@
}
#undef N
+static bool
+safe_auth_supported(struct safe_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_MD5) == 0)
+ return (false);
+ break;
+ case CRYPTO_SHA1_HMAC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+safe_cipher_supported(struct safe_softc *sc,
+ const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_DES) == 0)
+ return (false);
+ if (csp->csp_ivlen != 8)
+ return (false);
+ if (csp->csp_cipher_alg == CRYPTO_DES_CBC) {
+ if (csp->csp_cipher_klen != 8)
+ return (false);
+ } else {
+ if (csp->csp_cipher_klen != 24)
+ return (false);
+ }
+ break;
+ case CRYPTO_AES_CBC:
+ if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0)
+ return (false);
+ if (csp->csp_ivlen != 16)
+ return (false);
+ if (csp->csp_cipher_klen != 16 &&
+ csp->csp_cipher_klen != 24 &&
+ csp->csp_cipher_klen != 32)
+ return (false);
+ break;
+ }
+ return (true);
+}
+
+static int
+safe_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+ struct safe_softc *sc = device_get_softc(dev);
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!safe_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!safe_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!safe_auth_supported(sc, csp) ||
+ !safe_cipher_supported(sc, csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
+ * Allocate a new 'session'.
*/
static int
-safe_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+safe_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
- struct safe_softc *sc = device_get_softc(dev);
- struct cryptoini *c, *encini = NULL, *macini = NULL;
- struct safe_session *ses = NULL;
-
- if (cri == NULL || sc == NULL)
- return (EINVAL);
-
- for (c = cri; c != NULL; c = c->cri_next) {
- if (c->cri_alg == CRYPTO_MD5_HMAC ||
- c->cri_alg == CRYPTO_SHA1_HMAC ||
- c->cri_alg == CRYPTO_NULL_HMAC) {
- if (macini)
- return (EINVAL);
- macini = c;
- } else if (c->cri_alg == CRYPTO_DES_CBC ||
- c->cri_alg == CRYPTO_3DES_CBC ||
- c->cri_alg == CRYPTO_AES_CBC ||
- c->cri_alg == CRYPTO_NULL_CBC) {
- if (encini)
- return (EINVAL);
- encini = c;
- } else
- return (EINVAL);
- }
- if (encini == NULL && macini == NULL)
- return (EINVAL);
- if (encini) { /* validate key length */
- switch (encini->cri_alg) {
- case CRYPTO_DES_CBC:
- if (encini->cri_klen != 64)
- return (EINVAL);
- break;
- case CRYPTO_3DES_CBC:
- if (encini->cri_klen != 192)
- return (EINVAL);
- break;
- case CRYPTO_AES_CBC:
- if (encini->cri_klen != 128 &&
- encini->cri_klen != 192 &&
- encini->cri_klen != 256)
- return (EINVAL);
- break;
- }
- }
+ struct safe_session *ses;
ses = crypto_get_driver_session(cses);
- if (encini) {
- /* get an IV */
- /* XXX may read fewer than requested */
- read_random(ses->ses_iv, sizeof(ses->ses_iv));
-
- ses->ses_klen = encini->cri_klen;
- if (encini->cri_key != NULL)
- safe_setup_enckey(ses, encini->cri_key);
+ if (csp->csp_cipher_alg != 0) {
+ ses->ses_klen = csp->csp_cipher_klen;
+ if (csp->csp_cipher_key != NULL)
+ safe_setup_enckey(ses, csp->csp_cipher_key);
}
- if (macini) {
- ses->ses_mlen = macini->cri_mlen;
+ if (csp->csp_auth_alg != 0) {
+ ses->ses_mlen = csp->csp_auth_mlen;
if (ses->ses_mlen == 0) {
- if (macini->cri_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
- if (macini->cri_key != NULL) {
- safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
- macini->cri_klen / 8);
+ if (csp->csp_auth_key != NULL) {
+ safe_setup_mackey(ses, csp->csp_auth_alg,
+ csp->csp_auth_key, csp->csp_auth_klen);
}
}
return (0);
}
+static bus_size_t
+safe_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct safe_operand *op = arg;
@@ -782,7 +809,6 @@
(u_int) mapsize, nsegs, error));
if (error != 0)
return;
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -790,11 +816,10 @@
static int
safe_process(device_t dev, struct cryptop *crp, int hint)
{
- struct safe_softc *sc = device_get_softc(dev);
+ struct safe_softc *sc = device_get_softc(dev);
+ const struct crypto_session_params *csp;
int err = 0, i, nicealign, uniform;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
- int bypass, oplen, ivsize;
- caddr_t iv;
+ int bypass, oplen;
int16_t coffset;
struct safe_session *ses;
struct safe_ringentry *re;
@@ -802,11 +827,6 @@
struct safe_pdesc *pd;
u_int32_t cmd0, cmd1, staterec;
- if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
- safestats.st_invalid++;
- return (EINVAL);
- }
-
mtx_lock(&sc->sc_ringmtx);
if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
safestats.st_ringfull++;
@@ -823,104 +843,46 @@
re->re_crp = crp;
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- re->re_src_m = (struct mbuf *)crp->crp_buf;
- re->re_dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- re->re_src_io = (struct uio *)crp->crp_buf;
- re->re_dst_io = (struct uio *)crp->crp_buf;
- } else {
- safestats.st_badflags++;
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous blocks! */
- }
-
sa = &re->re_sa;
ses = crypto_get_driver_session(crp->crp_session);
-
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
- safestats.st_nodesc++;
- err = EINVAL;
- goto errout;
- }
- crd2 = crd1->crd_next;
+ csp = crypto_get_params(crp->crp_session);
cmd0 = SAFE_SA_CMD0_BASIC; /* basic group operation */
cmd1 = 0;
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_NULL_HMAC) {
- maccrd = crd1;
- enccrd = NULL;
- cmd0 |= SAFE_SA_CMD0_OP_HASH;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_NULL_CBC) {
- maccrd = NULL;
- enccrd = crd1;
- cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
- } else {
- safestats.st_badalg++;
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_NULL_HMAC) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC ||
- crd2->crd_alg == CRYPTO_AES_CBC ||
- crd2->crd_alg == CRYPTO_NULL_CBC) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_NULL_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC ||
- crd2->crd_alg == CRYPTO_NULL_HMAC) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- safestats.st_badalg++;
- err = EINVAL;
- goto errout;
- }
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ cmd0 |= SAFE_SA_CMD0_OP_HASH;
+ break;
+ case CSP_MODE_CIPHER:
+ cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
+ break;
+ case CSP_MODE_ETA:
cmd0 |= SAFE_SA_CMD0_OP_BOTH;
+ break;
}
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
- safe_setup_enckey(ses, enccrd->crd_key);
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_cipher_key != NULL)
+ safe_setup_enckey(ses, crp->crp_cipher_key);
- if (enccrd->crd_alg == CRYPTO_DES_CBC) {
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
cmd0 |= SAFE_SA_CMD0_DES;
cmd1 |= SAFE_SA_CMD1_CBC;
- ivsize = 2*sizeof(u_int32_t);
- } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
+ break;
+ case CRYPTO_3DES_CBC:
cmd0 |= SAFE_SA_CMD0_3DES;
cmd1 |= SAFE_SA_CMD1_CBC;
- ivsize = 2*sizeof(u_int32_t);
- } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
+ break;
+ case CRYPTO_AES_CBC:
cmd0 |= SAFE_SA_CMD0_AES;
cmd1 |= SAFE_SA_CMD1_CBC;
- if (ses->ses_klen == 128)
+ if (ses->ses_klen * 8 == 128)
cmd1 |= SAFE_SA_CMD1_AES128;
- else if (ses->ses_klen == 192)
+ else if (ses->ses_klen * 8 == 192)
cmd1 |= SAFE_SA_CMD1_AES192;
else
cmd1 |= SAFE_SA_CMD1_AES256;
- ivsize = 4*sizeof(u_int32_t);
- } else {
- cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
- ivsize = 0;
}
/*
@@ -932,32 +894,28 @@
* in the state record and set the hash/crypt offset to
* copy both the header+IV.
*/
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(re->re_sastate.sa_saved_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ re->re_sastate.sa_saved_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(re->re_sastate.sa_saved_iv, crp->crp_iv,
+ csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ re->re_sastate.sa_saved_iv);
+ cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
+
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
cmd0 |= SAFE_SA_CMD0_OUTBOUND;
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- iv = enccrd->crd_iv;
- else
- iv = (caddr_t) ses->ses_iv;
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivsize, iv);
- }
- bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
- cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
- re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
+ /*
+ * XXX: I suspect we don't need this since we
+ * don't save the returned IV.
+ */
+ cmd0 |= SAFE_SA_CMD0_SAVEIV;
} else {
cmd0 |= SAFE_SA_CMD0_INBOUND;
-
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
- bcopy(enccrd->crd_iv,
- re->re_sastate.sa_saved_iv, ivsize);
- } else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivsize,
- (caddr_t)re->re_sastate.sa_saved_iv);
- }
- cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
}
/*
* For basic encryption use the zero pad algorithm.
@@ -973,21 +931,23 @@
bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
}
- if (maccrd) {
- if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- safe_setup_mackey(ses, maccrd->crd_alg,
- maccrd->crd_key, maccrd->crd_klen / 8);
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_auth_key != NULL) {
+ safe_setup_mackey(ses, csp->csp_auth_alg,
+ crp->crp_auth_key, csp->csp_auth_klen);
}
- if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
cmd0 |= SAFE_SA_CMD0_MD5;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
- } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
+ break;
+ case CRYPTO_SHA1_HMAC:
cmd0 |= SAFE_SA_CMD0_SHA1;
cmd1 |= SAFE_SA_CMD1_HMAC; /* NB: enable HMAC */
- } else {
- cmd0 |= SAFE_SA_CMD0_HASH_NULL;
+ break;
}
+
/*
* Digest data is loaded from the SA and the hash
* result is saved to the state block where we
@@ -1003,38 +963,32 @@
re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
}
- if (enccrd && maccrd) {
+ if (csp->csp_mode == CSP_MODE_ETA) {
/*
- * The offset from hash data to the start of
- * crypt data is the difference in the skips.
+ * The driver only supports ETA requests where there
+ * is no gap between the AAD and payload.
*/
- bypass = maccrd->crd_skip;
- coffset = enccrd->crd_skip - maccrd->crd_skip;
- if (coffset < 0) {
- DPRINTF(("%s: hash does not precede crypt; "
- "mac skip %u enc skip %u\n",
- __func__, maccrd->crd_skip, enccrd->crd_skip));
- safestats.st_skipmismatch++;
- err = EINVAL;
- goto errout;
- }
- oplen = enccrd->crd_skip + enccrd->crd_len;
- if (maccrd->crd_skip + maccrd->crd_len != oplen) {
- DPRINTF(("%s: hash amount %u != crypt amount %u\n",
- __func__, maccrd->crd_skip + maccrd->crd_len,
- oplen));
+ if (crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
safestats.st_lenmismatch++;
err = EINVAL;
goto errout;
}
+ if (crp->crp_aad_length != 0)
+ bypass = crp->crp_aad_start;
+ else
+ bypass = crp->crp_payload_start;
+ coffset = crp->crp_aad_length;
+ oplen = crp->crp_payload_start + crp->crp_payload_length;
#ifdef SAFE_DEBUG
if (safe_debug) {
- printf("mac: skip %d, len %d, inject %d\n",
- maccrd->crd_skip, maccrd->crd_len,
- maccrd->crd_inject);
- printf("enc: skip %d, len %d, inject %d\n",
- enccrd->crd_skip, enccrd->crd_len,
- enccrd->crd_inject);
+ printf("AAD: skip %d, len %d, digest %d\n",
+ crp->crp_aad_start, crp->crp_aad_length,
+ crp->crp_digest_start);
+ printf("payload: skip %d, len %d, IV %d\n",
+ crp->crp_payload_start, crp->crp_payload_length,
+ crp->crp_iv_start);
printf("bypass %d coffset %d oplen %d\n",
bypass, coffset, oplen);
}
@@ -1070,13 +1024,8 @@
*/
cmd1 |= SAFE_SA_CMD1_MUTABLE;
} else {
- if (enccrd) {
- bypass = enccrd->crd_skip;
- oplen = bypass + enccrd->crd_len;
- } else {
- bypass = maccrd->crd_skip;
- oplen = bypass + maccrd->crd_len;
- }
+ bypass = crp->crp_payload_start;
+ oplen = bypass + crp->crp_payload_length;
coffset = 0;
}
/* XXX verify multiple of 4 when using s/g */
@@ -1092,27 +1041,15 @@
err = ENOMEM;
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map,
- re->re_src_m, safe_op_cb,
- &re->re_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- re->re_src_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map,
- re->re_src_io, safe_op_cb,
- &re->re_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- re->re_src_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
+ if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,
+ &re->re_src, BUS_DMA_NOWAIT) != 0) {
+ bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
+ re->re_src_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
}
+ re->re_src_mapsize = safe_crp_length(crp);
nicealign = safe_dmamap_aligned(&re->re_src);
uniform = safe_dmamap_uniform(&re->re_src);
@@ -1143,211 +1080,175 @@
re->re_desc.d_src = re->re_src_segs[0].ds_addr;
}
- if (enccrd == NULL && maccrd != NULL) {
+ if (csp->csp_mode == CSP_MODE_DIGEST) {
/*
* Hash op; no destination needed.
*/
} else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- if (!nicealign) {
- safestats.st_iovmisaligned++;
- err = EINVAL;
+ if (nicealign && uniform == 1) {
+ /*
+ * Source layout is suitable for direct
+ * sharing of the DMA map and segment list.
+ */
+ re->re_dst = re->re_src;
+ } else if (nicealign && uniform == 2) {
+ /*
+ * The source is properly aligned but requires a
+ * different particle list to handle DMA of the
+ * result. Create a new map and do the load to
+ * create the segment list. The particle
+ * descriptor setup code below will handle the
+ * rest.
+ */
+ if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT,
+ &re->re_dst_map)) {
+ safestats.st_nomap++;
+ err = ENOMEM;
goto errout;
}
- if (uniform != 1) {
- /*
- * Source is not suitable for direct use as
- * the destination. Create a new scatter/gather
- * list based on the destination requirements
- * and check if that's ok.
- */
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map)) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_uio(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_io,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- uniform = safe_dmamap_uniform(&re->re_dst);
- if (!uniform) {
- /*
- * There's no way to handle the DMA
- * requirements with this uio. We
- * could create a separate DMA area for
- * the result and then copy it back,
- * but for now we just bail and return
- * an error. Note that uio requests
- * > SAFE_MAX_DSIZE are handled because
- * the DMA map and segment list for the
- * destination wil result in a
- * destination particle list that does
- * the necessary scatter DMA.
- */
- safestats.st_iovnotuniform++;
- err = EINVAL;
- goto errout;
- }
- } else
- re->re_dst = re->re_src;
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (nicealign && uniform == 1) {
- /*
- * Source layout is suitable for direct
- * sharing of the DMA map and segment list.
- */
- re->re_dst = re->re_src;
- } else if (nicealign && uniform == 2) {
- /*
- * The source is properly aligned but requires a
- * different particle list to handle DMA of the
- * result. Create a new map and do the load to
- * create the segment list. The particle
- * descriptor setup code below will handle the
- * rest.
- */
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map)) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
+ if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,
+ crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=
+ 0) {
+ bus_dmamap_destroy(sc->sc_dstdmat,
+ re->re_dst_map);
+ re->re_dst_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
+ }
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *top, **mp;
+
+ /*
+ * DMA constraints require that we allocate a
+ * new mbuf chain for the destination. We
+ * allocate an entire new set of mbufs of
+ * optimal/required size and then tell the
+ * hardware to copy any bits that are not
+ * created as a byproduct of the operation.
+ */
+ if (!nicealign)
+ safestats.st_unaligned++;
+ if (!uniform)
+ safestats.st_notuniform++;
+ totlen = re->re_src_mapsize;
+ if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ len = MHLEN;
+ MGETHDR(m, M_NOWAIT, MT_DATA);
+ if (m && !m_dup_pkthdr(m, crp->crp_mbuf,
+ M_NOWAIT)) {
+ m_free(m);
+ m = NULL;
}
- if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_m,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
+ } else {
+ len = MLEN;
+ MGET(m, M_NOWAIT, MT_DATA);
+ }
+ if (m == NULL) {
+ safestats.st_nombuf++;
+ err = sc->sc_nqchip ? ERESTART : ENOMEM;
+ goto errout;
+ }
+ if (totlen >= MINCLSIZE) {
+ if (!(MCLGET(m, M_NOWAIT))) {
+ m_free(m);
+ safestats.st_nomcl++;
+ err = sc->sc_nqchip ?
+ ERESTART : ENOMEM;
goto errout;
}
- } else { /* !(aligned and/or uniform) */
- int totlen, len;
- struct mbuf *m, *top, **mp;
+ len = MCLBYTES;
+ }
+ m->m_len = len;
+ top = NULL;
+ mp = &top;
- /*
- * DMA constraints require that we allocate a
- * new mbuf chain for the destination. We
- * allocate an entire new set of mbufs of
- * optimal/required size and then tell the
- * hardware to copy any bits that are not
- * created as a byproduct of the operation.
- */
- if (!nicealign)
- safestats.st_unaligned++;
- if (!uniform)
- safestats.st_notuniform++;
- totlen = re->re_src_mapsize;
- if (re->re_src_m->m_flags & M_PKTHDR) {
- len = MHLEN;
- MGETHDR(m, M_NOWAIT, MT_DATA);
- if (m && !m_dup_pkthdr(m, re->re_src_m,
- M_NOWAIT)) {
- m_free(m);
- m = NULL;
- }
- } else {
- len = MLEN;
+ while (totlen > 0) {
+ if (top) {
MGET(m, M_NOWAIT, MT_DATA);
+ if (m == NULL) {
+ m_freem(top);
+ safestats.st_nombuf++;
+ err = sc->sc_nqchip ?
+ ERESTART : ENOMEM;
+ goto errout;
+ }
+ len = MLEN;
}
- if (m == NULL) {
- safestats.st_nombuf++;
- err = sc->sc_nqchip ? ERESTART : ENOMEM;
- goto errout;
- }
- if (totlen >= MINCLSIZE) {
+ if (top && totlen >= MINCLSIZE) {
if (!(MCLGET(m, M_NOWAIT))) {
- m_free(m);
+ *mp = m;
+ m_freem(top);
safestats.st_nomcl++;
err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
+ ERESTART : ENOMEM;
goto errout;
}
len = MCLBYTES;
}
- m->m_len = len;
- top = NULL;
- mp = &top;
-
- while (totlen > 0) {
- if (top) {
- MGET(m, M_NOWAIT, MT_DATA);
- if (m == NULL) {
- m_freem(top);
- safestats.st_nombuf++;
- err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
- goto errout;
- }
- len = MLEN;
- }
- if (top && totlen >= MINCLSIZE) {
- if (!(MCLGET(m, M_NOWAIT))) {
- *mp = m;
- m_freem(top);
- safestats.st_nomcl++;
- err = sc->sc_nqchip ?
- ERESTART : ENOMEM;
- goto errout;
- }
- len = MCLBYTES;
- }
- m->m_len = len = min(totlen, len);
- totlen -= len;
- *mp = m;
- mp = &m->m_next;
- }
- re->re_dst_m = top;
- if (bus_dmamap_create(sc->sc_dstdmat,
- BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
- safestats.st_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
- re->re_dst_map, re->re_dst_m,
- safe_op_cb, &re->re_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dstdmat,
- re->re_dst_map);
- re->re_dst_map = NULL;
- safestats.st_noload++;
- err = ENOMEM;
- goto errout;
- }
- if (re->re_src.mapsize > oplen) {
- /*
- * There's data following what the
- * hardware will copy for us. If this
- * isn't just the ICV (that's going to
- * be written on completion), copy it
- * to the new mbufs
- */
- if (!(maccrd &&
- (re->re_src.mapsize-oplen) == 12 &&
- maccrd->crd_inject == oplen))
- safe_mcopy(re->re_src_m,
- re->re_dst_m,
- oplen);
- else
- safestats.st_noicvcopy++;
- }
+ m->m_len = len = min(totlen, len);
+ totlen -= len;
+ *mp = m;
+ mp = &m->m_next;
+ }
+ re->re_dst_m = top;
+ if (bus_dmamap_create(sc->sc_dstdmat,
+ BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
+ safestats.st_nomap++;
+ err = ENOMEM;
+ goto errout;
+ }
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat,
+ re->re_dst_map, top, re->re_dst_segs,
+ &re->re_dst_nsegs, 0) != 0) {
+ bus_dmamap_destroy(sc->sc_dstdmat,
+ re->re_dst_map);
+ re->re_dst_map = NULL;
+ safestats.st_noload++;
+ err = ENOMEM;
+ goto errout;
+ }
+ re->re_dst_mapsize = re->re_src_mapsize;
+ if (re->re_src.mapsize > oplen) {
+ /*
+ * There's data following what the
+ * hardware will copy for us. If this
+ * isn't just the ICV (that's going to
+ * be written on completion), copy it
+ * to the new mbufs
+ */
+ if (!(csp->csp_mode == CSP_MODE_ETA &&
+ (re->re_src.mapsize-oplen) == ses->ses_mlen &&
+ crp->crp_digest_start == oplen))
+ safe_mcopy(crp->crp_mbuf, re->re_dst_m,
+ oplen);
+ else
+ safestats.st_noicvcopy++;
}
} else {
- safestats.st_badflags++;
- err = EINVAL;
- goto errout;
+ if (!nicealign) {
+ safestats.st_iovmisaligned++;
+ err = EINVAL;
+ goto errout;
+ } else {
+ /*
+ * There's no way to handle the DMA
+ * requirements with this uio. We
+ * could create a separate DMA area for
+ * the result and then copy it back,
+ * but for now we just bail and return
+ * an error. Note that uio requests
+ * > SAFE_MAX_DSIZE are handled because
+ * the DMA map and segment list for the
+ * destination wil result in a
+ * destination particle list that does
+ * the necessary scatter DMA.
+ */
+ safestats.st_iovnotuniform++;
+ err = EINVAL;
+ goto errout;
+ }
}
if (re->re_dst.nsegs > 1) {
@@ -1393,7 +1294,7 @@
* ready for processing.
*/
re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
- if (maccrd)
+ if (csp->csp_auth_alg != 0)
re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
re->re_desc.d_len = oplen
| SAFE_PE_LEN_READY
@@ -1412,7 +1313,7 @@
return (0);
errout:
- if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+ if (re->re_dst_m != NULL)
m_freem(re->re_dst_m);
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
@@ -1436,11 +1337,13 @@
static void
safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
{
+ const struct crypto_session_params *csp;
struct cryptop *crp = (struct cryptop *)re->re_crp;
struct safe_session *ses;
- struct cryptodesc *crd;
+ uint8_t hash[HASH_MAX_LEN];
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
safestats.st_opackets++;
safestats.st_obytes += re->re_dst.mapsize;
@@ -1454,6 +1357,9 @@
safestats.st_peoperr++;
crp->crp_etype = EIO; /* something more meaningful? */
}
+
+ /* XXX: Should crp_mbuf be updated to re->re_dst_m if it is non-NULL? */
+
if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
BUS_DMASYNC_POSTREAD);
@@ -1464,58 +1370,29 @@
bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
- /*
- * If result was written to a differet mbuf chain, swap
- * it in as the return value and reclaim the original.
- */
- if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
- m_freem(re->re_src_m);
- crp->crp_buf = (caddr_t)re->re_dst_m;
- }
-
- if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
- /* copy out IV for future use */
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- int ivsize;
-
- if (crd->crd_alg == CRYPTO_DES_CBC ||
- crd->crd_alg == CRYPTO_3DES_CBC) {
- ivsize = 2*sizeof(u_int32_t);
- } else if (crd->crd_alg == CRYPTO_AES_CBC) {
- ivsize = 4*sizeof(u_int32_t);
- } else
- continue;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - ivsize, ivsize,
- (caddr_t)ses->ses_iv);
- break;
- }
- }
-
if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
- /* copy out ICV result */
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
- crd->crd_alg == CRYPTO_SHA1_HMAC ||
- crd->crd_alg == CRYPTO_NULL_HMAC))
- continue;
- if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
- /*
- * SHA-1 ICV's are byte-swapped; fix 'em up
- * before copy them to their destination.
- */
- re->re_sastate.sa_saved_indigest[0] =
- bswap32(re->re_sastate.sa_saved_indigest[0]);
- re->re_sastate.sa_saved_indigest[1] =
- bswap32(re->re_sastate.sa_saved_indigest[1]);
- re->re_sastate.sa_saved_indigest[2] =
- bswap32(re->re_sastate.sa_saved_indigest[2]);
- }
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- crd->crd_inject, ses->ses_mlen,
- (caddr_t)re->re_sastate.sa_saved_indigest);
- break;
+ if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) {
+ /*
+ * SHA-1 ICV's are byte-swapped; fix 'em up
+ * before copying them to their destination.
+ */
+ re->re_sastate.sa_saved_indigest[0] =
+ bswap32(re->re_sastate.sa_saved_indigest[0]);
+ re->re_sastate.sa_saved_indigest[1] =
+ bswap32(re->re_sastate.sa_saved_indigest[1]);
+ re->re_sastate.sa_saved_indigest[2] =
+ bswap32(re->re_sastate.sa_saved_indigest[2]);
}
+
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ ses->ses_mlen, hash);
+ if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,
+ hash, ses->ses_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ ses->ses_mlen, re->re_sastate.sa_saved_indigest);
}
crypto_done(crp);
}
@@ -1921,7 +1798,7 @@
/*
* Free header MCR
*/
- if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+ if (re->re_dst_m != NULL)
m_freem(re->re_dst_m);
crp = (struct cryptop *)re->re_crp;
Index: sys/dev/safe/safevar.h
===================================================================
--- sys/dev/safe/safevar.h
+++ sys/dev/safe/safevar.h
@@ -75,10 +75,6 @@
* where each is mapped for DMA.
*/
struct safe_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -109,22 +105,18 @@
struct safe_operand re_src; /* source operand */
struct safe_operand re_dst; /* destination operand */
+ struct mbuf *re_dst_m;
int unused;
int re_flags;
-#define SAFE_QFLAGS_COPYOUTIV 0x1 /* copy back on completion */
#define SAFE_QFLAGS_COPYOUTICV 0x2 /* copy back on completion */
};
-#define re_src_m re_src.u.m
-#define re_src_io re_src.u.io
#define re_src_map re_src.map
#define re_src_nsegs re_src.nsegs
#define re_src_segs re_src.segs
#define re_src_mapsize re_src.mapsize
-#define re_dst_m re_dst.u.m
-#define re_dst_io re_dst.u.io
#define re_dst_map re_dst.map
#define re_dst_nsegs re_dst.nsegs
#define re_dst_segs re_dst.segs
@@ -138,7 +130,6 @@
u_int32_t ses_mlen; /* hmac length in bytes */
u_int32_t ses_hminner[5]; /* hmac inner state */
u_int32_t ses_hmouter[5]; /* hmac outer state */
- u_int32_t ses_iv[4]; /* DES/3DES/AES iv */
};
struct safe_softc {
@@ -157,6 +148,7 @@
int sc_suspended;
int sc_needwakeup; /* notify crypto layer */
int32_t sc_cid; /* crypto tag */
+ uint32_t sc_devinfo;
struct safe_dma_alloc sc_ringalloc; /* PE ring allocation state */
struct safe_ringentry *sc_ring; /* PE ring */
struct safe_ringentry *sc_ringtop; /* PE ring top */
Index: sys/dev/sec/sec.h
===================================================================
--- sys/dev/sec/sec.h
+++ sys/dev/sec/sec.h
@@ -98,6 +98,7 @@
uint8_t shd_iv[SEC_MAX_IV_LEN];
uint8_t shd_key[SEC_MAX_KEY_LEN];
uint8_t shd_mkey[SEC_MAX_KEY_LEN];
+ uint8_t shd_digest[HASH_MAX_LEN];
} __packed__;
#define shd_eu_sel0 shd_control.request.eu_sel0
@@ -144,21 +145,17 @@
};
struct sec_eu_methods {
- int (*sem_newsession)(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc,
- struct cryptoini *mac);
+ bool (*sem_newsession)(const struct crypto_session_params *csp);
int (*sem_make_desc)(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc,
- struct cryptop *crp, int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
};
struct sec_session {
struct sec_eu_methods *ss_eu;
uint8_t ss_key[SEC_MAX_KEY_LEN];
uint8_t ss_mkey[SEC_MAX_KEY_LEN];
- u_int ss_klen;
- u_int ss_mklen;
- u_int ss_ivlen;
+ int ss_mlen;
};
struct sec_desc_map_info {
@@ -319,11 +316,6 @@
(((sc)->sc_lt_free_cnt - (sc)->sc_lt_alloc_cnt - 1) \
& (SEC_LT_ENTRIES - 1))
-/* DMA Maping defines */
-#define SEC_MEMORY 0
-#define SEC_UIO 1
-#define SEC_MBUF 2
-
/* Size of SEC registers area */
#define SEC_IO_SIZE 0x10000
Index: sys/dev/sec/sec.c
===================================================================
--- sys/dev/sec/sec.c
+++ sys/dev/sec/sec.c
@@ -51,6 +51,7 @@
#include <machine/resource.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
#include "cryptodev_if.h"
#include <dev/ofw/ofw_bus_subr.h>
@@ -74,7 +75,7 @@
static int sec_alloc_dma_mem(struct sec_softc *sc,
struct sec_dma_mem *dma_mem, bus_size_t size);
static int sec_desc_map_dma(struct sec_softc *sc,
- struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
+ struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size,
struct sec_desc_map_info *sdmi);
static void sec_free_dma_mem(struct sec_dma_mem *dma_mem);
static void sec_enqueue(struct sec_softc *sc);
@@ -82,48 +83,43 @@
int channel);
static int sec_eu_channel(struct sec_softc *sc, int eu);
static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
- u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
+ u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize);
static int sec_make_pointer_direct(struct sec_softc *sc,
struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
+static int sec_probesession(device_t dev,
+ const struct crypto_session_params *csp);
static int sec_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri);
+ const struct crypto_session_params *csp);
static int sec_process(device_t dev, struct cryptop *crp, int hint);
-static int sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
- struct cryptoini **mac);
-static int sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
- struct cryptodesc **mac);
static int sec_build_common_ns_desc(struct sec_softc *sc,
- struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
- struct cryptodesc *enc, int buftype);
+ struct sec_desc *desc, const struct crypto_session_params *csp,
+ struct cryptop *crp);
static int sec_build_common_s_desc(struct sec_softc *sc,
- struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
- struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
+ struct sec_desc *desc, const struct crypto_session_params *csp,
+ struct cryptop *crp);
static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
/* AESU */
-static int sec_aesu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_aesu_newsession(const struct crypto_session_params *csp);
static int sec_aesu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
/* DEU */
-static int sec_deu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_deu_newsession(const struct crypto_session_params *csp);
static int sec_deu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
/* MDEU */
-static int sec_mdeu_can_handle(u_int alg);
-static int sec_mdeu_config(struct cryptodesc *crd,
+static bool sec_mdeu_can_handle(u_int alg);
+static int sec_mdeu_config(const struct crypto_session_params *csp,
u_int *eu, u_int *mode, u_int *hashlen);
-static int sec_mdeu_newsession(struct sec_softc *sc,
- struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool sec_mdeu_newsession(const struct crypto_session_params *csp);
static int sec_mdeu_make_desc(struct sec_softc *sc,
- struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
- int buftype);
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp);
static device_method_t sec_methods[] = {
/* Device interface */
@@ -136,6 +132,7 @@
DEVMETHOD(device_shutdown, sec_shutdown),
/* Crypto methods */
+ DEVMETHOD(cryptodev_probesession, sec_probesession),
DEVMETHOD(cryptodev_newsession, sec_newsession),
DEVMETHOD(cryptodev_process, sec_process),
@@ -362,24 +359,6 @@
if (error)
goto fail6;
- /* Register in OCF (AESU) */
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-
- /* Register in OCF (DEU) */
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-
- /* Register in OCF (MDEU) */
- crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
- if (sc->sc_version >= 3) {
- crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
- }
-
return (0);
fail6:
@@ -545,9 +524,12 @@
static void
sec_primary_intr(void *arg)
{
+ struct sec_session *ses;
struct sec_softc *sc = arg;
struct sec_desc *desc;
+ struct cryptop *crp;
uint64_t isr;
+ uint8_t hash[HASH_MAX_LEN];
int i, wakeup = 0;
SEC_LOCK(sc, controller);
@@ -595,7 +577,26 @@
SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
BUS_DMASYNC_PREWRITE);
- desc->sd_crp->crp_etype = desc->sd_error;
+ crp = desc->sd_crp;
+ crp->crp_etype = desc->sd_error;
+ if (crp->crp_etype == 0) {
+ ses = crypto_get_driver_session(crp->crp_session);
+ if (ses->ss_mlen != 0) {
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp,
+ crp->crp_digest_start,
+ ses->ss_mlen, hash);
+ if (timingsafe_bcmp(
+ desc->sd_desc->shd_digest,
+ hash, ses->ss_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp,
+ crp->crp_digest_start,
+ ses->ss_mlen,
+ desc->sd_desc->shd_digest);
+ }
+ }
crypto_done(desc->sd_crp);
SEC_DESC_FREE_POINTERS(desc);
@@ -786,14 +787,6 @@
sdmi->sdmi_lt_last = lt;
}
-static void
-sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
- bus_size_t size, int error)
-{
-
- sec_dma_map_desc_cb(arg, segs, nseg, error);
-}
-
static int
sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
bus_size_t size)
@@ -851,22 +844,22 @@
}
static int
-sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
- bus_size_t size, int type, struct sec_desc_map_info *sdmi)
+sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
+ struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi)
{
int error;
if (dma_mem->dma_vaddr != NULL)
return (EBUSY);
- switch (type) {
- case SEC_MEMORY:
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_CONTIG:
break;
- case SEC_UIO:
+ case CRYPTO_BUF_UIO:
size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
break;
- case SEC_MBUF:
- size = m_length((struct mbuf*)mem, NULL);
+ case CRYPTO_BUF_MBUF:
+ size = m_length(crp->crp_mbuf, NULL);
break;
default:
return (EINVAL);
@@ -899,20 +892,8 @@
return (error);
}
- switch (type) {
- case SEC_MEMORY:
- error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
- mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
- break;
- case SEC_UIO:
- error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
- mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
- break;
- case SEC_MBUF:
- error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
- mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
- break;
- }
+ error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp,
+ sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
if (error) {
device_printf(sc->sc_dev, "cannot get address of the DMA"
@@ -923,7 +904,7 @@
}
dma_mem->dma_is_map = 1;
- dma_mem->dma_vaddr = mem;
+ dma_mem->dma_vaddr = crp;
return (0);
}
@@ -1130,7 +1111,7 @@
static int
sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
- u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
+ u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize)
{
struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
struct sec_hw_desc_ptr *ptr;
@@ -1138,14 +1119,8 @@
SEC_LOCK_ASSERT(sc, descriptors);
- /* For flat memory map only requested region */
- if (dtype == SEC_MEMORY) {
- data = (uint8_t*)(data) + doffset;
- sdmi.sdmi_offset = 0;
- }
-
- error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
- dtype, &sdmi);
+ error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize,
+ &sdmi);
if (error)
return (error);
@@ -1162,115 +1137,116 @@
return (0);
}
-static int
-sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
- struct cryptoini **mac)
+static bool
+sec_cipher_supported(const struct crypto_session_params *csp)
{
- struct cryptoini *e, *m;
-
- e = cri;
- m = cri->cri_next;
-
- /* We can haldle only two operations */
- if (m && m->cri_next)
- return (EINVAL);
- if (sec_mdeu_can_handle(e->cri_alg)) {
- cri = m;
- m = e;
- e = cri;
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_CBC:
+ /* AESU */
+ if (csp->csp_ivlen != AES_BLOCK_LEN)
+ return (false);
+ break;
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ /* DEU */
+ if (csp->csp_ivlen != DES_BLOCK_LEN)
+ return (false);
+ break;
+ default:
+ return (false);
}
- if (m && !sec_mdeu_can_handle(m->cri_alg))
- return (EINVAL);
+ if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN)
+ return (false);
- *enc = e;
- *mac = m;
-
- return (0);
+ return (true);
}
-static int
-sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
- struct cryptodesc **mac)
+static bool
+sec_auth_supported(struct sec_softc *sc,
+ const struct crypto_session_params *csp)
{
- struct cryptodesc *e, *m, *t;
- e = crp->crp_desc;
- m = e->crd_next;
-
- /* We can haldle only two operations */
- if (m && m->crd_next)
- return (EINVAL);
-
- if (sec_mdeu_can_handle(e->crd_alg)) {
- t = m;
- m = e;
- e = t;
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ if (sc->sc_version < 3)
+ return (false);
+ /* FALLTHROUGH */
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ if (csp->csp_auth_klen > SEC_MAX_KEY_LEN)
+ return (false);
+ break;
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+ break;
+ default:
+ return (false);
}
-
- if (m && !sec_mdeu_can_handle(m->crd_alg))
- return (EINVAL);
-
- *enc = e;
- *mac = m;
-
- return (0);
+ return (true);
}
static int
-sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+sec_probesession(device_t dev, const struct crypto_session_params *csp)
{
struct sec_softc *sc = device_get_softc(dev);
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!sec_auth_supported(sc, csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+sec_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
+{
struct sec_eu_methods *eu = sec_eus;
- struct cryptoini *enc = NULL;
- struct cryptoini *mac = NULL;
struct sec_session *ses;
- int error = -1;
-
- error = sec_split_cri(cri, &enc, &mac);
- if (error)
- return (error);
-
- /* Check key lengths */
- if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
- return (E2BIG);
-
- if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
- return (E2BIG);
-
- /* Only SEC 3.0 supports digests larger than 256 bits */
- if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
- return (E2BIG);
ses = crypto_get_driver_session(cses);
/* Find EU for this session */
while (eu->sem_make_desc != NULL) {
- error = eu->sem_newsession(sc, ses, enc, mac);
- if (error >= 0)
+ if (eu->sem_newsession(csp))
break;
-
eu++;
}
-
- /* If not found, return EINVAL */
- if (error < 0)
- return (EINVAL);
+ KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session"));
/* Save cipher key */
- if (enc && enc->cri_key) {
- ses->ss_klen = enc->cri_klen / 8;
- memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
- }
+ if (csp->csp_cipher_key != NULL)
+ memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen);
/* Save digest key */
- if (mac && mac->cri_key) {
- ses->ss_mklen = mac->cri_klen / 8;
- memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
+ if (csp->csp_auth_key != NULL)
+ memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen);
+
+ if (csp->csp_auth_alg != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->ss_mlen = crypto_auth_hash(csp)->hashsize;
+ else
+ ses->ss_mlen = csp->csp_auth_mlen;
}
- ses->ss_eu = eu;
return (0);
}
@@ -1279,11 +1255,12 @@
{
struct sec_softc *sc = device_get_softc(dev);
struct sec_desc *desc = NULL;
- struct cryptodesc *mac, *enc;
+ const struct crypto_session_params *csp;
struct sec_session *ses;
- int buftype, error = 0;
+ int error = 0;
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
/* Check for input length */
if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
@@ -1292,13 +1269,6 @@
return (0);
}
- /* Get descriptors */
- if (sec_split_crp(crp, &enc, &mac)) {
- crp->crp_etype = EINVAL;
- crypto_done(crp);
- return (0);
- }
-
SEC_LOCK(sc, descriptors);
SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1315,56 +1285,29 @@
desc->sd_error = 0;
desc->sd_crp = crp;
- if (crp->crp_flags & CRYPTO_F_IOV)
- buftype = SEC_UIO;
- else if (crp->crp_flags & CRYPTO_F_IMBUF)
- buftype = SEC_MBUF;
- else
- buftype = SEC_MEMORY;
-
- if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
- ses->ss_ivlen);
- else
- arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
-
- if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, ses->ss_ivlen,
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(desc->sd_desc->shd_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
desc->sd_desc->shd_iv);
- } else if (enc) {
- if (enc->crd_flags & CRD_F_IV_EXPLICIT)
- memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
- ses->ss_ivlen);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(desc->sd_desc->shd_iv, crp->crp_iv,
+ csp->csp_ivlen);
else
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enc->crd_inject, ses->ss_ivlen,
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
desc->sd_desc->shd_iv);
}
- if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
- ses->ss_klen = enc->crd_klen / 8;
- memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
- } else
- error = E2BIG;
- }
+ if (crp->crp_cipher_key != NULL)
+ memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen);
- if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
- if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
- ses->ss_mklen = mac->crd_klen / 8;
- memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
- } else
- error = E2BIG;
- }
+ if (crp->crp_auth_key != NULL)
+ memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen);
- if (!error) {
- memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
- memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
+ memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen);
+ memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen);
- error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
- }
+ error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp);
if (error) {
SEC_DESC_FREE_POINTERS(desc);
@@ -1400,8 +1343,7 @@
static int
sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
- struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
- int buftype)
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
int error;
@@ -1417,25 +1359,25 @@
/* Pointer 1: IV IN */
error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+ offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
if (error)
return (error);
/* Pointer 2: Cipher Key */
error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+ offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
if (error)
return (error);
/* Pointer 3: Data IN */
- error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 4: Data OUT */
- error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
@@ -1452,20 +1394,13 @@
static int
sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
- struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
- struct cryptodesc *mac, int buftype)
+ const struct crypto_session_params *csp, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
u_int eu, mode, hashlen;
int error;
- if (mac->crd_len < enc->crd_len)
- return (EINVAL);
-
- if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
- return (EINVAL);
-
- error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
+ error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
if (error)
return (error);
@@ -1475,144 +1410,107 @@
/* Pointer 0: HMAC Key */
error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
+ offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen);
if (error)
return (error);
/* Pointer 1: HMAC-Only Data IN */
- error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
- mac->crd_len - enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start,
+ crp->crp_aad_length);
if (error)
return (error);
/* Pointer 2: Cipher Key */
error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+ offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
if (error)
return (error);
/* Pointer 3: IV IN */
error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
- offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+ offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
if (error)
return (error);
/* Pointer 4: Data IN */
- error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 5: Data OUT */
- error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
- enc->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
/* Pointer 6: HMAC OUT */
- error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
- hashlen, buftype);
+ error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr +
+ offsetof(struct sec_hw_desc, shd_digest), hashlen);
return (error);
}
/* AESU */
-static int
-sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_aesu_newsession(const struct crypto_session_params *csp)
{
- if (enc == NULL)
- return (-1);
-
- if (enc->cri_alg != CRYPTO_AES_CBC)
- return (-1);
-
- ses->ss_ivlen = AES_BLOCK_LEN;
-
- return (0);
+ return (csp->csp_cipher_alg == CRYPTO_AES_CBC);
}
static int
-sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_aesu_make_desc(struct sec_softc *sc,
+ const struct crypto_session_params *csp, struct sec_desc *desc,
+ struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
- struct cryptodesc *enc, *mac;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (!enc)
- return (EINVAL);
-
hd->shd_eu_sel0 = SEC_EU_AESU;
hd->shd_mode0 = SEC_AESU_MODE_CBC;
- if (enc->crd_alg != CRYPTO_AES_CBC)
- return (EINVAL);
-
- if (enc->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
hd->shd_mode0 |= SEC_AESU_MODE_ED;
hd->shd_dir = 0;
} else
hd->shd_dir = 1;
- if (mac)
- error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
- buftype);
+ if (csp->csp_mode == CSP_MODE_ETA)
+ error = sec_build_common_s_desc(sc, desc, csp, crp);
else
- error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
- buftype);
+ error = sec_build_common_ns_desc(sc, desc, csp, crp);
return (error);
}
/* DEU */
-static int
-sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_deu_newsession(const struct crypto_session_params *csp)
{
- if (enc == NULL)
- return (-1);
-
- switch (enc->cri_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_DES_CBC:
case CRYPTO_3DES_CBC:
- break;
+ return (true);
default:
- return (-1);
+ return (false);
}
-
- ses->ss_ivlen = DES_BLOCK_LEN;
-
- return (0);
}
static int
-sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_deu_make_desc(struct sec_softc *sc, const struct crypto_session_params *csp,
+ struct sec_desc *desc, struct cryptop *crp)
{
struct sec_hw_desc *hd = desc->sd_desc;
- struct cryptodesc *enc, *mac;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (!enc)
- return (EINVAL);
-
hd->shd_eu_sel0 = SEC_EU_DEU;
hd->shd_mode0 = SEC_DEU_MODE_CBC;
- switch (enc->crd_alg) {
+ switch (csp->csp_cipher_alg) {
case CRYPTO_3DES_CBC:
hd->shd_mode0 |= SEC_DEU_MODE_TS;
break;
@@ -1622,25 +1520,23 @@
return (EINVAL);
}
- if (enc->crd_flags & CRD_F_ENCRYPT) {
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
hd->shd_mode0 |= SEC_DEU_MODE_ED;
hd->shd_dir = 0;
} else
hd->shd_dir = 1;
- if (mac)
- error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
- buftype);
+ if (csp->csp_mode == CSP_MODE_ETA)
+ error = sec_build_common_s_desc(sc, desc, csp, crp);
else
- error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
- buftype);
+ error = sec_build_common_ns_desc(sc, desc, csp, crp);
return (error);
}
/* MDEU */
-static int
+static bool
sec_mdeu_can_handle(u_int alg)
{
switch (alg) {
@@ -1651,20 +1547,21 @@
case CRYPTO_SHA2_256_HMAC:
case CRYPTO_SHA2_384_HMAC:
case CRYPTO_SHA2_512_HMAC:
- return (1);
+ return (true);
default:
- return (0);
+ return (false);
}
}
static int
-sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
+sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode,
+ u_int *hashlen)
{
*mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
*eu = SEC_EU_NONE;
- switch (crd->crd_alg) {
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5_HMAC:
*mode |= SEC_MDEU_MODE_HMAC;
/* FALLTHROUGH */
@@ -1703,34 +1600,23 @@
return (0);
}
-static int
-sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
- struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_mdeu_newsession(const struct crypto_session_params *csp)
{
- if (mac && sec_mdeu_can_handle(mac->cri_alg))
- return (0);
-
- return (-1);
+ return (sec_mdeu_can_handle(csp->csp_auth_alg));
}
static int
-sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
- struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_mdeu_make_desc(struct sec_softc *sc,
+ const struct crypto_session_params *csp,
+ struct sec_desc *desc, struct cryptop *crp)
{
- struct cryptodesc *enc, *mac;
struct sec_hw_desc *hd = desc->sd_desc;
u_int eu, mode, hashlen;
int error;
- error = sec_split_crp(crp, &enc, &mac);
- if (error)
- return (error);
-
- if (enc)
- return (EINVAL);
-
- error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
+ error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
if (error)
return (error);
@@ -1754,7 +1640,7 @@
if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
error = sec_make_pointer_direct(sc, desc, 2,
desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
- shd_mkey), ses->ss_mklen);
+ shd_mkey), csp->csp_auth_klen);
else
error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
@@ -1762,8 +1648,8 @@
return (error);
/* Pointer 3: Input Data */
- error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
- mac->crd_len, buftype);
+ error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+ crp->crp_payload_length);
if (error)
return (error);
@@ -1773,8 +1659,8 @@
return (error);
/* Pointer 5: Hash out */
- error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
- mac->crd_inject, hashlen, buftype);
+ error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr +
+ offsetof(struct sec_hw_desc, shd_digest), hashlen);
if (error)
return (error);
Index: sys/dev/ubsec/ubsec.c
===================================================================
--- sys/dev/ubsec/ubsec.c
+++ sys/dev/ubsec/ubsec.c
@@ -61,6 +61,7 @@
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <sys/endian.h>
+#include <sys/uio.h>
#include <vm/vm.h>
#include <vm/pmap.h>
@@ -70,10 +71,8 @@
#include <sys/bus.h>
#include <sys/rman.h>
-#include <crypto/sha1.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
#include <sys/random.h>
#include <sys/kobj.h>
@@ -111,7 +110,9 @@
static int ubsec_resume(device_t);
static int ubsec_shutdown(device_t);
-static int ubsec_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int ubsec_probesession(device_t, const struct crypto_session_params *);
+static int ubsec_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static int ubsec_process(device_t, struct cryptop *, int);
static int ubsec_kprocess(device_t, struct cryptkop *, int);
@@ -125,6 +126,7 @@
DEVMETHOD(device_shutdown, ubsec_shutdown),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, ubsec_probesession),
DEVMETHOD(cryptodev_newsession, ubsec_newsession),
DEVMETHOD(cryptodev_process, ubsec_process),
DEVMETHOD(cryptodev_kprocess, ubsec_kprocess),
@@ -348,13 +350,6 @@
goto bad2;
}
- sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
- CRYPTOCAP_F_HARDWARE);
- if (sc->sc_cid < 0) {
- device_printf(dev, "could not get crypto driver id\n");
- goto bad3;
- }
-
/*
* Setup DMA descriptor area.
*/
@@ -370,7 +365,7 @@
NULL, NULL, /* lockfunc, lockarg */
&sc->sc_dmat)) {
device_printf(dev, "cannot allocate DMA tag\n");
- goto bad4;
+ goto bad3;
}
SIMPLEQ_INIT(&sc->sc_freequeue);
dmap = sc->sc_dmaa;
@@ -404,11 +399,6 @@
device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-
/*
* Reset Broadcom chip
*/
@@ -424,6 +414,13 @@
*/
ubsec_init_board(sc);
+ sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
+ CRYPTOCAP_F_HARDWARE);
+ if (sc->sc_cid < 0) {
+ device_printf(dev, "could not get crypto driver id\n");
+ goto bad4;
+ }
+
#ifndef UBSEC_NO_RNG
if (sc->sc_flags & UBS_FLAGS_RNG) {
sc->sc_statmask |= BS_STAT_MCR2_DONE;
@@ -477,7 +474,15 @@
}
return (0);
bad4:
- crypto_unregister_all(sc->sc_cid);
+ while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
+ struct ubsec_q *q;
+
+ q = SIMPLEQ_FIRST(&sc->sc_freequeue);
+ SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
+ ubsec_dma_free(sc, &q->q_dma->d_alloc);
+ free(q, M_DEVBUF);
+ }
+ bus_dma_tag_destroy(sc->sc_dmat);
bad3:
bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bad2:
@@ -498,13 +503,14 @@
/* XXX wait/abort active ops */
+ crypto_unregister_all(sc->sc_cid);
+
/* disable interrupts */
WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
callout_stop(&sc->sc_rngto);
-
- crypto_unregister_all(sc->sc_cid);
+ bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
#ifdef UBSEC_RNDTEST
if (sc->sc_rndtest)
@@ -531,7 +537,6 @@
mtx_destroy(&sc->sc_mcr2lock);
bus_generic_detach(dev);
- bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
bus_dma_tag_destroy(sc->sc_dmat);
@@ -826,7 +831,7 @@
}
static void
-ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
+ubsec_setup_enckey(struct ubsec_session *ses, int algo, const void *key)
{
/* Go ahead and compute key in ubsec's byte order */
@@ -846,112 +851,134 @@
}
static void
-ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
+ubsec_setup_mackey(struct ubsec_session *ses, int algo, const char *key,
+ int klen)
{
MD5_CTX md5ctx;
SHA1_CTX sha1ctx;
- int i;
-
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_IPAD_VAL;
if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
- } else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_ipad_buffer,
- SHA1_BLOCK_LEN - klen);
- bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
- }
- for (i = 0; i < klen; i++)
- key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- if (algo == CRYPTO_MD5_HMAC) {
- MD5Init(&md5ctx);
- MD5Update(&md5ctx, key, klen);
- MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
+ hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+ explicit_bzero(&md5ctx, sizeof(md5ctx));
} else {
- SHA1Init(&sha1ctx);
- SHA1Update(&sha1ctx, key, klen);
- SHA1Update(&sha1ctx, hmac_opad_buffer,
- SHA1_BLOCK_LEN - klen);
+ hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+ bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+ hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+
+ explicit_bzero(&sha1ctx, sizeof(sha1ctx));
+ }
+}
+
+static bool
+ubsec_auth_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ return (true);
+ default:
+ return (false);
+ }
+}
+
+static bool
+ubsec_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ return (csp->csp_ivlen == 8);
+ default:
+ return (false);
+ }
+}
+
+static int
+ubsec_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!ubsec_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!ubsec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!ubsec_auth_supported(csp) ||
+ !ubsec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
}
- for (i = 0; i < klen; i++)
- key[i] ^= HMAC_OPAD_VAL;
+ return (CRYPTODEV_PROBE_HARDWARE);
}
/*
- * Allocate a new 'session' and return an encoded session id. 'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
+ * Allocate a new 'session'.
*/
static int
-ubsec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ubsec_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
- struct ubsec_softc *sc = device_get_softc(dev);
- struct cryptoini *c, *encini = NULL, *macini = NULL;
- struct ubsec_session *ses = NULL;
-
- if (cri == NULL || sc == NULL)
- return (EINVAL);
-
- for (c = cri; c != NULL; c = c->cri_next) {
- if (c->cri_alg == CRYPTO_MD5_HMAC ||
- c->cri_alg == CRYPTO_SHA1_HMAC) {
- if (macini)
- return (EINVAL);
- macini = c;
- } else if (c->cri_alg == CRYPTO_DES_CBC ||
- c->cri_alg == CRYPTO_3DES_CBC) {
- if (encini)
- return (EINVAL);
- encini = c;
- } else
- return (EINVAL);
- }
- if (encini == NULL && macini == NULL)
- return (EINVAL);
+ struct ubsec_session *ses;
ses = crypto_get_driver_session(cses);
- if (encini) {
- /* get an IV, network byte order */
- /* XXX may read fewer than requested */
- read_random(ses->ses_iv, sizeof(ses->ses_iv));
+ if (csp->csp_cipher_alg != 0 && csp->csp_cipher_key != NULL)
+ ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+ csp->csp_cipher_key);
- if (encini->cri_key != NULL) {
- ubsec_setup_enckey(ses, encini->cri_alg,
- encini->cri_key);
- }
- }
-
- if (macini) {
- ses->ses_mlen = macini->cri_mlen;
+ if (csp->csp_auth_alg != 0) {
+ ses->ses_mlen = csp->csp_auth_mlen;
if (ses->ses_mlen == 0) {
- if (macini->cri_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ses->ses_mlen = MD5_HASH_LEN;
else
ses->ses_mlen = SHA1_HASH_LEN;
}
- if (macini->cri_key != NULL) {
- ubsec_setup_mackey(ses, macini->cri_alg,
- macini->cri_key, macini->cri_klen / 8);
+ if (csp->csp_auth_key != NULL) {
+ ubsec_setup_mackey(ses, csp->csp_auth_alg,
+ csp->csp_auth_key, csp->csp_auth_klen);
}
}
return (0);
}
+static bus_size_t
+ubsec_crp_length(struct cryptop *crp)
+{
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (crp->crp_mbuf->m_pkthdr.len);
+ case CRYPTO_BUF_UIO:
+ return (crp->crp_uio->uio_resid);
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_ilen);
+ default:
+ panic("bad crp buffer type");
+ }
+}
+
static void
-ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
{
struct ubsec_operand *op = arg;
@@ -959,12 +986,11 @@
("Too many DMA segments returned when mapping operand"));
#ifdef UBSEC_DEBUG
if (ubsec_debug)
- printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
- (u_int) mapsize, nsegs, error);
+ printf("ubsec_op_cb: nsegs %d error %d\n",
+ nsegs, error);
#endif
if (error != 0)
return;
- op->mapsize = mapsize;
op->nsegs = nsegs;
bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
}
@@ -972,22 +998,17 @@
static int
ubsec_process(device_t dev, struct cryptop *crp, int hint)
{
+ const struct crypto_session_params *csp;
struct ubsec_softc *sc = device_get_softc(dev);
struct ubsec_q *q = NULL;
int err = 0, i, j, nicealign;
- struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
- int encoffset = 0, macoffset = 0, cpskip, cpoffset;
+ int cpskip, cpoffset;
int sskip, dskip, stheend, dtheend;
int16_t coffset;
struct ubsec_session *ses;
struct ubsec_pktctx ctx;
struct ubsec_dma *dmap = NULL;
- if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
- ubsecstats.hst_invalid++;
- return (EINVAL);
- }
-
mtx_lock(&sc->sc_freeqlock);
if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
ubsecstats.hst_queuefull++;
@@ -1006,103 +1027,34 @@
q->q_dma = dmap;
ses = crypto_get_driver_session(crp->crp_session);
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- q->q_src_m = (struct mbuf *)crp->crp_buf;
- q->q_dst_m = (struct mbuf *)crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- q->q_src_io = (struct uio *)crp->crp_buf;
- q->q_dst_io = (struct uio *)crp->crp_buf;
- } else {
- ubsecstats.hst_badflags++;
- err = EINVAL;
- goto errout; /* XXX we don't handle contiguous blocks! */
- }
-
bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
dmap->d_dma->d_mcr.mcr_flags = 0;
q->q_crp = crp;
- crd1 = crp->crp_desc;
- if (crd1 == NULL) {
- ubsecstats.hst_nodesc++;
- err = EINVAL;
- goto errout;
- }
- crd2 = crd1->crd_next;
+ csp = crypto_get_params(crp->crp_session);
- if (crd2 == NULL) {
- if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC) {
- maccrd = crd1;
- enccrd = NULL;
- } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC) {
- maccrd = NULL;
- enccrd = crd1;
- } else {
- ubsecstats.hst_badalg++;
- err = EINVAL;
- goto errout;
- }
- } else {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC) &&
- ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
- maccrd = crd1;
- enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
- (crd1->crd_flags & CRD_F_ENCRYPT)) {
- enccrd = crd1;
- maccrd = crd2;
- } else {
- /*
- * We cannot order the ubsec as requested
- */
- ubsecstats.hst_badalg++;
- err = EINVAL;
- goto errout;
- }
- }
-
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- ubsec_setup_enckey(ses, enccrd->crd_alg,
- enccrd->crd_key);
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_cipher_key != NULL) {
+ ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+ crp->crp_cipher_key);
}
- encoffset = enccrd->crd_skip;
ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
- q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(ctx.pc_iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start,
+ csp->csp_ivlen, ctx.pc_iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(ctx.pc_iv, crp->crp_iv, csp->csp_ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ ctx.pc_iv);
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
- else {
- ctx.pc_iv[0] = ses->ses_iv[0];
- ctx.pc_iv[1] = ses->ses_iv[1];
- }
-
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
- }
- } else {
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
-
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
- else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
- }
}
ctx.pc_deskey[0] = ses->ses_deskey[0];
@@ -1115,15 +1067,13 @@
SWAP32(ctx.pc_iv[1]);
}
- if (maccrd) {
- if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- ubsec_setup_mackey(ses, maccrd->crd_alg,
- maccrd->crd_key, maccrd->crd_klen / 8);
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_auth_key != NULL) {
+ ubsec_setup_mackey(ses, csp->csp_auth_alg,
+ crp->crp_auth_key, csp->csp_auth_klen);
}
- macoffset = maccrd->crd_skip;
-
- if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
+ if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
else
ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
@@ -1137,35 +1087,37 @@
}
}
- if (enccrd && maccrd) {
+ if (csp->csp_mode == CSP_MODE_ETA) {
/*
- * ubsec cannot handle packets where the end of encryption
- * and authentication are not the same, or where the
- * encrypted part begins before the authenticated part.
+ * ubsec only supports ETA requests where there is no
+ * gap between the AAD and payload.
*/
- if ((encoffset + enccrd->crd_len) !=
- (macoffset + maccrd->crd_len)) {
+ if (crp->crp_aad_length != 0 &&
+ crp->crp_aad_start + crp->crp_aad_length !=
+ crp->crp_payload_start) {
ubsecstats.hst_lenmismatch++;
err = EINVAL;
goto errout;
}
- if (enccrd->crd_skip < maccrd->crd_skip) {
- ubsecstats.hst_skipmismatch++;
- err = EINVAL;
- goto errout;
+
+ if (crp->crp_aad_length != 0) {
+ sskip = crp->crp_aad_start;
+ } else {
+ sskip = crp->crp_payload_start;
}
- sskip = maccrd->crd_skip;
- cpskip = dskip = enccrd->crd_skip;
- stheend = maccrd->crd_len;
- dtheend = enccrd->crd_len;
- coffset = enccrd->crd_skip - maccrd->crd_skip;
+ cpskip = dskip = crp->crp_payload_start;
+ stheend = crp->crp_aad_length + crp->crp_payload_length;
+ dtheend = crp->crp_payload_length;
+ coffset = crp->crp_aad_length;
cpoffset = cpskip + dtheend;
#ifdef UBSEC_DEBUG
if (ubsec_debug) {
- printf("mac: skip %d, len %d, inject %d\n",
- maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
- printf("enc: skip %d, len %d, inject %d\n",
- enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
+ printf("AAD: start %d, len %d, digest %d\n",
+ crp->crp_aad_start, crp->crp_aad_length,
+ crp->crp_digest_start);
+ printf("payload: start %d, len %d, IV %d\n",
+ crp->crp_payload_start, crp->crp_payload_length,
+ crp->crp_iv_start);
printf("src: skip %d, len %d\n", sskip, stheend);
printf("dst: skip %d, len %d\n", dskip, dtheend);
printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
@@ -1173,8 +1125,8 @@
}
#endif
} else {
- cpskip = dskip = sskip = macoffset + encoffset;
- dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
+ cpskip = dskip = sskip = crp->crp_payload_start;
+ dtheend = stheend = crp->crp_payload_length;
cpoffset = cpskip + dtheend;
coffset = 0;
}
@@ -1185,25 +1137,15 @@
err = ENOMEM;
goto errout;
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
- q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- q->q_src_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
- q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- q->q_src_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
+ if (bus_dmamap_load_crp(sc->sc_dmat, q->q_src_map, crp, ubsec_op_cb,
+ &q->q_src, BUS_DMA_NOWAIT) != 0) {
+ bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
+ q->q_src_map = NULL;
+ ubsecstats.hst_noload++;
+ err = ENOMEM;
+ goto errout;
}
+ q->q_src_mapsize = ubsec_crp_length(crp);
nicealign = ubsec_dmamap_aligned(&q->q_src);
dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
@@ -1257,7 +1199,7 @@
j++;
}
- if (enccrd == NULL && maccrd != NULL) {
+ if (csp->csp_mode == CSP_MODE_DIGEST) {
dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
@@ -1270,104 +1212,79 @@
dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
#endif
} else {
- if (crp->crp_flags & CRYPTO_F_IOV) {
- if (!nicealign) {
- ubsecstats.hst_iovmisaligned++;
- err = EINVAL;
- goto errout;
- }
- if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
- &q->q_dst_map)) {
- ubsecstats.hst_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
- q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
- q->q_dst_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
- } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
- if (nicealign) {
- q->q_dst = q->q_src;
+ if (nicealign) {
+ q->q_dst = q->q_src;
+ } else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+ int totlen, len;
+ struct mbuf *m, *top, **mp;
+
+ ubsecstats.hst_unaligned++;
+ totlen = q->q_src_mapsize;
+ if (totlen >= MINCLSIZE) {
+ m = m_getcl(M_NOWAIT, MT_DATA,
+ crp->crp_mbuf->m_flags & M_PKTHDR);
+ len = MCLBYTES;
+ } else if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+ m = m_gethdr(M_NOWAIT, MT_DATA);
+ len = MHLEN;
} else {
- int totlen, len;
- struct mbuf *m, *top, **mp;
+ m = m_get(M_NOWAIT, MT_DATA);
+ len = MLEN;
+ }
+ if (m && crp->crp_mbuf->m_flags & M_PKTHDR &&
+ !m_dup_pkthdr(m, crp->crp_mbuf, M_NOWAIT)) {
+ m_free(m);
+ m = NULL;
+ }
+ if (m == NULL) {
+ ubsecstats.hst_nombuf++;
+ err = sc->sc_nqueue ? ERESTART : ENOMEM;
+ goto errout;
+ }
+ m->m_len = len = min(totlen, len);
+ totlen -= len;
+ top = m;
+ mp = &top;
- ubsecstats.hst_unaligned++;
- totlen = q->q_src_mapsize;
+ while (totlen > 0) {
if (totlen >= MINCLSIZE) {
- m = m_getcl(M_NOWAIT, MT_DATA,
- q->q_src_m->m_flags & M_PKTHDR);
+ m = m_getcl(M_NOWAIT, MT_DATA, 0);
len = MCLBYTES;
- } else if (q->q_src_m->m_flags & M_PKTHDR) {
- m = m_gethdr(M_NOWAIT, MT_DATA);
- len = MHLEN;
} else {
m = m_get(M_NOWAIT, MT_DATA);
len = MLEN;
}
- if (m && q->q_src_m->m_flags & M_PKTHDR &&
- !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
- m_free(m);
- m = NULL;
- }
if (m == NULL) {
+ m_freem(top);
ubsecstats.hst_nombuf++;
err = sc->sc_nqueue ? ERESTART : ENOMEM;
goto errout;
}
m->m_len = len = min(totlen, len);
totlen -= len;
- top = m;
- mp = &top;
-
- while (totlen > 0) {
- if (totlen >= MINCLSIZE) {
- m = m_getcl(M_NOWAIT,
- MT_DATA, 0);
- len = MCLBYTES;
- } else {
- m = m_get(M_NOWAIT, MT_DATA);
- len = MLEN;
- }
- if (m == NULL) {
- m_freem(top);
- ubsecstats.hst_nombuf++;
- err = sc->sc_nqueue ? ERESTART : ENOMEM;
- goto errout;
- }
- m->m_len = len = min(totlen, len);
- totlen -= len;
- *mp = m;
- mp = &m->m_next;
- }
- q->q_dst_m = top;
- ubsec_mcopy(q->q_src_m, q->q_dst_m,
- cpskip, cpoffset);
- if (bus_dmamap_create(sc->sc_dmat,
- BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
- ubsecstats.hst_nomap++;
- err = ENOMEM;
- goto errout;
- }
- if (bus_dmamap_load_mbuf(sc->sc_dmat,
- q->q_dst_map, q->q_dst_m,
- ubsec_op_cb, &q->q_dst,
- BUS_DMA_NOWAIT) != 0) {
- bus_dmamap_destroy(sc->sc_dmat,
- q->q_dst_map);
- q->q_dst_map = NULL;
- ubsecstats.hst_noload++;
- err = ENOMEM;
- goto errout;
- }
+ *mp = m;
+ mp = &m->m_next;
}
+ q->q_dst_m = top;
+ ubsec_mcopy(crp->crp_mbuf, q->q_dst_m, cpskip, cpoffset);
+ if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+ &q->q_dst_map) != 0) {
+ ubsecstats.hst_nomap++;
+ err = ENOMEM;
+ goto errout;
+ }
+ if (bus_dmamap_load_mbuf_sg(sc->sc_dmat,
+ q->q_dst_map, q->q_dst_m, q->q_dst_segs,
+ &q->q_dst_nsegs, 0) != 0) {
+ bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
+ q->q_dst_map = NULL;
+ ubsecstats.hst_noload++;
+ err = ENOMEM;
+ goto errout;
+ }
+ q->q_dst_mapsize = q->q_src_mapsize;
} else {
- ubsecstats.hst_badflags++;
+ ubsecstats.hst_iovmisaligned++;
err = EINVAL;
goto errout;
}
@@ -1414,7 +1331,7 @@
pb->pb_len = htole32(packl);
if ((i + 1) == q->q_dst_nsegs) {
- if (maccrd)
+ if (csp->csp_auth_alg != 0)
pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
offsetof(struct ubsec_dmachunk, d_macbuf[0]));
else
@@ -1465,7 +1382,7 @@
errout:
if (q != NULL) {
- if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+ if (q->q_dst_m != NULL)
m_freem(q->q_dst_m);
if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
@@ -1495,12 +1412,14 @@
static void
ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
{
+ const struct crypto_session_params *csp;
struct cryptop *crp = (struct cryptop *)q->q_crp;
struct ubsec_session *ses;
- struct cryptodesc *crd;
struct ubsec_dma *dmap = q->q_dma;
+ char hash[SHA1_HASH_LEN];
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
ubsecstats.hst_opackets++;
ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
@@ -1517,31 +1436,21 @@
bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
- if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
- m_freem(q->q_src_m);
- crp->crp_buf = (caddr_t)q->q_dst_m;
+ if (q->q_dst_m != NULL) {
+ m_freem(crp->crp_mbuf);
+ crp->crp_mbuf = q->q_dst_m;
}
- /* copy out IV for future use */
- if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_DES_CBC &&
- crd->crd_alg != CRYPTO_3DES_CBC)
- continue;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- crd->crd_skip + crd->crd_len - 8, 8,
- (caddr_t)ses->ses_iv);
- break;
- }
- }
-
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- if (crd->crd_alg != CRYPTO_MD5_HMAC &&
- crd->crd_alg != CRYPTO_SHA1_HMAC)
- continue;
- crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
- ses->ses_mlen, (caddr_t)dmap->d_dma->d_macbuf);
- break;
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ ses->ses_mlen, hash);
+ if (timingsafe_bcmp(dmap->d_dma->d_macbuf, hash,
+ ses->ses_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ ses->ses_mlen, dmap->d_dma->d_macbuf);
}
mtx_lock(&sc->sc_freeqlock);
SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
@@ -1942,7 +1851,7 @@
if(q->q_stacked_mcr[i]) {
q2 = q->q_stacked_mcr[i];
- if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
+ if (q2->q_dst_m != NULL)
m_freem(q2->q_dst_m);
crp = (struct cryptop *)q2->q_crp;
@@ -1959,7 +1868,7 @@
/*
* Free header MCR
*/
- if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+ if (q->q_dst_m != NULL)
m_freem(q->q_dst_m);
crp = (struct cryptop *)q->q_crp;
Index: sys/dev/ubsec/ubsecvar.h
===================================================================
--- sys/dev/ubsec/ubsecvar.h
+++ sys/dev/ubsec/ubsecvar.h
@@ -134,10 +134,6 @@
#define UBS_FLAGS_RNG 0x10 /* hardware rng */
struct ubsec_operand {
- union {
- struct mbuf *m;
- struct uio *io;
- } u;
bus_dmamap_t map;
bus_size_t mapsize;
int nsegs;
@@ -153,19 +149,16 @@
struct ubsec_operand q_src;
struct ubsec_operand q_dst;
+ struct mbuf *q_dst_m;
int q_flags;
};
-#define q_src_m q_src.u.m
-#define q_src_io q_src.u.io
#define q_src_map q_src.map
#define q_src_nsegs q_src.nsegs
#define q_src_segs q_src.segs
#define q_src_mapsize q_src.mapsize
-#define q_dst_m q_dst.u.m
-#define q_dst_io q_dst.u.io
#define q_dst_map q_dst.map
#define q_dst_nsegs q_dst.nsegs
#define q_dst_segs q_dst.segs
@@ -215,7 +208,6 @@
u_int32_t ses_mlen; /* hmac length */
u_int32_t ses_hminner[5]; /* hmac inner state */
u_int32_t ses_hmouter[5]; /* hmac outer state */
- u_int32_t ses_iv[2]; /* [3]DES iv */
};
#endif /* _KERNEL */
Index: sys/geom/eli/g_eli.h
===================================================================
--- sys/geom/eli/g_eli.h
+++ sys/geom/eli/g_eli.h
@@ -163,6 +163,7 @@
struct g_eli_worker {
struct g_eli_softc *w_softc;
struct proc *w_proc;
+ void *w_first_key;
u_int w_number;
crypto_session_t w_sid;
boolean_t w_active;
@@ -573,6 +574,25 @@
}
}
+static __inline u_int
+g_eli_ivlen(u_int algo)
+{
+
+ switch (algo) {
+ case CRYPTO_AES_XTS:
+ return (AES_XTS_IV_LEN);
+ case CRYPTO_AES_CBC:
+ return (AES_BLOCK_LEN);
+ case CRYPTO_BLF_CBC:
+ return (BLOWFISH_BLOCK_LEN);
+ case CRYPTO_CAMELLIA_CBC:
+ return (CAMELLIA_BLOCK_LEN);
+ case CRYPTO_3DES_CBC:
+ return (DES3_BLOCK_LEN);
+ }
+ return (0);
+}
+
static __inline u_int
g_eli_hashlen(u_int algo)
{
Index: sys/geom/eli/g_eli.c
===================================================================
--- sys/geom/eli/g_eli.c
+++ sys/geom/eli/g_eli.c
@@ -487,41 +487,44 @@
g_eli_newsession(struct g_eli_worker *wr)
{
struct g_eli_softc *sc;
- struct cryptoini crie, cria;
+ struct crypto_session_params csp;
int error;
+ void *key;
sc = wr->w_softc;
- bzero(&crie, sizeof(crie));
- crie.cri_alg = sc->sc_ealgo;
- crie.cri_klen = sc->sc_ekeylen;
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_cipher_alg = sc->sc_ealgo;
+ csp.csp_ivlen = g_eli_ivlen(sc->sc_ealgo);
+ csp.csp_cipher_klen = sc->sc_ekeylen / 8;
if (sc->sc_ealgo == CRYPTO_AES_XTS)
- crie.cri_klen <<= 1;
+ csp.csp_cipher_klen <<= 1;
if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
- crie.cri_key = g_eli_key_hold(sc, 0,
+ key = g_eli_key_hold(sc, 0,
LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize);
+ csp.csp_cipher_key = key;
} else {
- crie.cri_key = sc->sc_ekey;
+ key = NULL;
+ csp.csp_cipher_key = sc->sc_ekey;
}
if (sc->sc_flags & G_ELI_FLAG_AUTH) {
- bzero(&cria, sizeof(cria));
- cria.cri_alg = sc->sc_aalgo;
- cria.cri_klen = sc->sc_akeylen;
- cria.cri_key = sc->sc_akey;
- crie.cri_next = &cria;
+ csp.csp_mode = CSP_MODE_ETA;
+ csp.csp_auth_alg = sc->sc_aalgo;
+ csp.csp_auth_klen = G_ELI_AUTH_SECKEYLEN;
}
switch (sc->sc_crypto) {
case G_ELI_CRYPTO_SW:
- error = crypto_newsession(&wr->w_sid, &crie,
+ error = crypto_newsession(&wr->w_sid, &csp,
CRYPTOCAP_F_SOFTWARE);
break;
case G_ELI_CRYPTO_HW:
- error = crypto_newsession(&wr->w_sid, &crie,
+ error = crypto_newsession(&wr->w_sid, &csp,
CRYPTOCAP_F_HARDWARE);
break;
case G_ELI_CRYPTO_UNKNOWN:
- error = crypto_newsession(&wr->w_sid, &crie,
+ error = crypto_newsession(&wr->w_sid, &csp,
CRYPTOCAP_F_HARDWARE);
if (error == 0) {
mtx_lock(&sc->sc_queue_mtx);
@@ -529,7 +532,7 @@
sc->sc_crypto = G_ELI_CRYPTO_HW;
mtx_unlock(&sc->sc_queue_mtx);
} else {
- error = crypto_newsession(&wr->w_sid, &crie,
+ error = crypto_newsession(&wr->w_sid, &csp,
CRYPTOCAP_F_SOFTWARE);
mtx_lock(&sc->sc_queue_mtx);
if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
@@ -541,8 +544,12 @@
panic("%s: invalid condition", __func__);
}
- if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0)
- g_eli_key_drop(sc, crie.cri_key);
+ if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
+ if (error)
+ g_eli_key_drop(sc, key);
+ else
+ wr->w_first_key = key;
+ }
return (error);
}
@@ -550,8 +557,14 @@
static void
g_eli_freesession(struct g_eli_worker *wr)
{
+ struct g_eli_softc *sc;
crypto_freesession(wr->w_sid);
+ if (wr->w_first_key != NULL) {
+ sc = wr->w_softc;
+ g_eli_key_drop(sc, wr->w_first_key);
+ wr->w_first_key = NULL;
+ }
}
static void
Index: sys/geom/eli/g_eli_crypto.c
===================================================================
--- sys/geom/eli/g_eli_crypto.c
+++ sys/geom/eli/g_eli_crypto.c
@@ -61,50 +61,40 @@
g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize,
const u_char *key, size_t keysize)
{
- struct cryptoini cri;
+ struct crypto_session_params csp;
struct cryptop *crp;
- struct cryptodesc *crd;
crypto_session_t sid;
- u_char *p;
int error;
KASSERT(algo != CRYPTO_AES_XTS,
("%s: CRYPTO_AES_XTS unexpected here", __func__));
- bzero(&cri, sizeof(cri));
- cri.cri_alg = algo;
- cri.cri_key = __DECONST(void *, key);
- cri.cri_klen = keysize;
- error = crypto_newsession(&sid, &cri, CRYPTOCAP_F_SOFTWARE);
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_cipher_alg = algo;
+ csp.csp_ivlen = g_eli_ivlen(algo);
+ csp.csp_cipher_key = key;
+ csp.csp_cipher_klen = keysize / 8;
+ error = crypto_newsession(&sid, &csp, CRYPTOCAP_F_SOFTWARE);
if (error != 0)
return (error);
- p = malloc(sizeof(*crp) + sizeof(*crd), M_ELI, M_NOWAIT | M_ZERO);
- if (p == NULL) {
+ crp = crypto_getreq(sid, M_NOWAIT);
+ if (crp == NULL) {
crypto_freesession(sid);
return (ENOMEM);
}
- crp = (struct cryptop *)p; p += sizeof(*crp);
- crd = (struct cryptodesc *)p; p += sizeof(*crd);
- crd->crd_skip = 0;
- crd->crd_len = datasize;
- crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- if (enc)
- crd->crd_flags |= CRD_F_ENCRYPT;
- crd->crd_alg = algo;
- crd->crd_key = __DECONST(void *, key);
- crd->crd_klen = keysize;
- bzero(crd->crd_iv, sizeof(crd->crd_iv));
- crd->crd_next = NULL;
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = datasize;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
+ crp->crp_op = enc ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+ memset(crp->crp_iv, 0, sizeof(crp->crp_iv));
- crp->crp_session = sid;
- crp->crp_ilen = datasize;
- crp->crp_olen = datasize;
crp->crp_opaque = NULL;
crp->crp_callback = g_eli_crypto_done;
+ crp->crp_buf_type = CRYPTO_BUF_CONTIG;
+ crp->crp_ilen = datasize;
crp->crp_buf = (void *)data;
- crp->crp_flags = CRYPTO_F_CBIFSYNC;
- crp->crp_desc = crd;
error = crypto_dispatch(crp);
if (error == 0) {
@@ -113,7 +103,7 @@
error = crp->crp_etype;
}
- free(crp, M_ELI);
+ crypto_freereq(crp);
crypto_freesession(sid);
return (error);
}
Index: sys/geom/eli/g_eli_integrity.c
===================================================================
--- sys/geom/eli/g_eli_integrity.c
+++ sys/geom/eli/g_eli_integrity.c
@@ -141,93 +141,30 @@
bp = (struct bio *)crp->crp_opaque;
bp->bio_inbed++;
if (crp->crp_etype == 0) {
- bp->bio_completed += crp->crp_olen;
+ bp->bio_completed += crp->crp_ilen;
G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%jd completed=%jd).",
- bp->bio_inbed, bp->bio_children, (intmax_t)crp->crp_olen, (intmax_t)bp->bio_completed);
+ bp->bio_inbed, bp->bio_children, (intmax_t)crp->crp_ilen, (intmax_t)bp->bio_completed);
} else {
- G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
- bp->bio_inbed, bp->bio_children, crp->crp_etype);
+ if (crp->crp_etype == EBADMSG)
+ G_ELI_DEBUG(0,
+ "Crypto READ request failed to authenticate (%d/%d),",
+ bp->bio_inbed, bp->bio_children);
+ else
+ G_ELI_DEBUG(1,
+ "Crypto READ request failed (%d/%d) error=%d.",
+ bp->bio_inbed, bp->bio_children, crp->crp_etype);
if (bp->bio_error == 0)
bp->bio_error = crp->crp_etype;
}
sc = bp->bio_to->geom->softc;
- g_eli_key_drop(sc, crp->crp_desc->crd_next->crd_key);
+ if (crp->crp_cipher_key != NULL)
+ g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+ crypto_freereq(crp);
/*
* Do we have all sectors already?
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
- if (bp->bio_error == 0) {
- u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
- u_char *srcdata, *dstdata, *auth;
- off_t coroff, corsize;
-
- /*
- * Verify data integrity based on calculated and read HMACs.
- */
- /* Sectorsize of decrypted provider eg. 4096. */
- decr_secsize = bp->bio_to->sectorsize;
- /* The real sectorsize of encrypted provider, eg. 512. */
- encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
- /* Number of data bytes in one encrypted sector, eg. 480. */
- data_secsize = sc->sc_data_per_sector;
- /* Number of sectors from decrypted provider, eg. 2. */
- nsec = bp->bio_length / decr_secsize;
- /* Number of sectors from encrypted provider, eg. 18. */
- nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
- /* Last sector number in every big sector, eg. 9. */
- lsec = sc->sc_bytes_per_sector / encr_secsize;
-
- srcdata = bp->bio_driver2;
- dstdata = bp->bio_data;
- auth = srcdata + encr_secsize * nsec;
- coroff = -1;
- corsize = 0;
-
- for (i = 1; i <= nsec; i++) {
- data_secsize = sc->sc_data_per_sector;
- if ((i % lsec) == 0)
- data_secsize = decr_secsize % data_secsize;
- if (bcmp(srcdata, auth, sc->sc_alen) != 0) {
- /*
- * Curruption detected, remember the offset if
- * this is the first corrupted sector and
- * increase size.
- */
- if (bp->bio_error == 0)
- bp->bio_error = -1;
- if (coroff == -1) {
- coroff = bp->bio_offset +
- (dstdata - (u_char *)bp->bio_data);
- }
- corsize += data_secsize;
- } else {
- /*
- * No curruption, good.
- * Report previous corruption if there was one.
- */
- if (coroff != -1) {
- G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
- "bytes of data at offset %jd.",
- sc->sc_name, (intmax_t)corsize,
- (intmax_t)coroff);
- coroff = -1;
- corsize = 0;
- }
- bcopy(srcdata + sc->sc_alen, dstdata,
- data_secsize);
- }
- srcdata += encr_secsize;
- dstdata += data_secsize;
- auth += sc->sc_alen;
- }
- /* Report previous corruption if there was one. */
- if (coroff != -1) {
- G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
- "bytes of data at offset %jd.",
- sc->sc_name, (intmax_t)corsize, (intmax_t)coroff);
- }
- }
free(bp->bio_driver2, M_ELI);
bp->bio_driver2 = NULL;
if (bp->bio_error != 0) {
@@ -277,7 +214,9 @@
bp->bio_error = crp->crp_etype;
}
sc = bp->bio_to->geom->softc;
- g_eli_key_drop(sc, crp->crp_desc->crd_key);
+ if (crp->crp_cipher_key != NULL)
+ g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+ crypto_freereq(crp);
/*
* All sectors are already encrypted?
*/
@@ -361,9 +300,6 @@
cbp->bio_length = cp->provider->sectorsize * nsec;
size = cbp->bio_length;
- size += sc->sc_alen * nsec;
- size += sizeof(struct cryptop) * nsec;
- size += sizeof(struct cryptodesc) * nsec * 2;
size += G_ELI_AUTH_SECKEYLEN * nsec;
cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK);
@@ -408,10 +344,9 @@
{
struct g_eli_softc *sc;
struct cryptop *crp;
- struct cryptodesc *crde, *crda;
u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
off_t dstoff;
- u_char *p, *data, *auth, *authkey, *plaindata;
+ u_char *p, *data, *authkey, *plaindata;
int error;
G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -433,19 +368,14 @@
/* Destination offset, used for IV generation. */
dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
- auth = NULL; /* Silence compiler warning. */
plaindata = bp->bio_data;
if (bp->bio_cmd == BIO_READ) {
data = bp->bio_driver2;
- auth = data + encr_secsize * nsec;
- p = auth + sc->sc_alen * nsec;
+ p = data + encr_secsize * nsec;
} else {
size_t size;
size = encr_secsize * nsec;
- size += sizeof(*crp) * nsec;
- size += sizeof(*crde) * nsec;
- size += sizeof(*crda) * nsec;
size += G_ELI_AUTH_SECKEYLEN * nsec;
size += sizeof(uintptr_t); /* Space for alignment. */
data = malloc(size, M_ELI, M_WAITOK);
@@ -460,9 +390,7 @@
#endif
for (i = 1; i <= nsec; i++, dstoff += encr_secsize) {
- crp = (struct cryptop *)p; p += sizeof(*crp);
- crde = (struct cryptodesc *)p; p += sizeof(*crde);
- crda = (struct cryptodesc *)p; p += sizeof(*crda);
+ crp = crypto_getreq(wr->w_sid, M_WAITOK);
authkey = (u_char *)p; p += G_ELI_AUTH_SECKEYLEN;
data_secsize = sc->sc_data_per_sector;
@@ -477,21 +405,14 @@
encr_secsize - sc->sc_alen - data_secsize);
}
- if (bp->bio_cmd == BIO_READ) {
- /* Remember read HMAC. */
- bcopy(data, auth, sc->sc_alen);
- auth += sc->sc_alen;
- /* TODO: bzero(9) can be commented out later. */
- bzero(data, sc->sc_alen);
- } else {
+ if (bp->bio_cmd == BIO_WRITE) {
bcopy(plaindata, data + sc->sc_alen, data_secsize);
plaindata += data_secsize;
}
- crp->crp_session = wr->w_sid;
crp->crp_ilen = sc->sc_alen + data_secsize;
- crp->crp_olen = data_secsize;
crp->crp_opaque = (void *)bp;
+ crp->crp_buf_type = CRYPTO_BUF_CONTIG;
crp->crp_buf = (void *)data;
data += encr_secsize;
crp->crp_flags = CRYPTO_F_CBIFSYNC;
@@ -499,41 +420,28 @@
crp->crp_flags |= CRYPTO_F_BATCH;
if (bp->bio_cmd == BIO_WRITE) {
crp->crp_callback = g_eli_auth_write_done;
- crp->crp_desc = crde;
- crde->crd_next = crda;
- crda->crd_next = NULL;
+ crp->crp_op = CRYPTO_OP_ENCRYPT |
+ CRYPTO_OP_COMPUTE_DIGEST;
} else {
crp->crp_callback = g_eli_auth_read_done;
- crp->crp_desc = crda;
- crda->crd_next = crde;
- crde->crd_next = NULL;
+ crp->crp_op = CRYPTO_OP_DECRYPT |
+ CRYPTO_OP_VERIFY_DIGEST;
}
- crde->crd_skip = sc->sc_alen;
- crde->crd_len = data_secsize;
- crde->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0)
- crde->crd_flags |= CRD_F_KEY_EXPLICIT;
- if (bp->bio_cmd == BIO_WRITE)
- crde->crd_flags |= CRD_F_ENCRYPT;
- crde->crd_alg = sc->sc_ealgo;
- crde->crd_key = g_eli_key_hold(sc, dstoff, encr_secsize);
- crde->crd_klen = sc->sc_ekeylen;
- if (sc->sc_ealgo == CRYPTO_AES_XTS)
- crde->crd_klen <<= 1;
- g_eli_crypto_ivgen(sc, dstoff, crde->crd_iv,
- sizeof(crde->crd_iv));
+ crp->crp_digest_start = 0;
+ crp->crp_payload_start = sc->sc_alen;
+ crp->crp_payload_length = data_secsize;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+ if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) {
+ crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+ encr_secsize);
+ }
+ g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+ sizeof(crp->crp_iv));
- crda->crd_skip = sc->sc_alen;
- crda->crd_len = data_secsize;
- crda->crd_inject = 0;
- crda->crd_flags = CRD_F_KEY_EXPLICIT;
- crda->crd_alg = sc->sc_aalgo;
g_eli_auth_keygen(sc, dstoff, authkey);
- crda->crd_key = authkey;
- crda->crd_klen = G_ELI_AUTH_SECKEYLEN * 8;
+ crp->crp_auth_key = authkey;
- crp->crp_etype = 0;
error = crypto_dispatch(crp);
KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
error));
Index: sys/geom/eli/g_eli_privacy.c
===================================================================
--- sys/geom/eli/g_eli_privacy.c
+++ sys/geom/eli/g_eli_privacy.c
@@ -82,7 +82,7 @@
if (crp->crp_etype == 0) {
G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
bp->bio_inbed, bp->bio_children);
- bp->bio_completed += crp->crp_olen;
+ bp->bio_completed += crp->crp_ilen;
} else {
G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
bp->bio_inbed, bp->bio_children, crp->crp_etype);
@@ -90,8 +90,9 @@
bp->bio_error = crp->crp_etype;
}
sc = bp->bio_to->geom->softc;
- if (sc != NULL)
- g_eli_key_drop(sc, crp->crp_desc->crd_key);
+ if (sc != NULL && crp->crp_cipher_key != NULL)
+ g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+ crypto_freereq(crp);
/*
* Do we have all sectors already?
*/
@@ -143,7 +144,9 @@
}
gp = bp->bio_to->geom;
sc = gp->softc;
- g_eli_key_drop(sc, crp->crp_desc->crd_key);
+ if (crp->crp_cipher_key != NULL)
+ g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+ crypto_freereq(crp);
/*
* All sectors are already encrypted?
*/
@@ -233,11 +236,9 @@
{
struct g_eli_softc *sc;
struct cryptop *crp;
- struct cryptodesc *crd;
u_int i, nsec, secsize;
off_t dstoff;
- size_t size;
- u_char *p, *data;
+ u_char *data;
int error;
G_ELI_LOGREQ(3, bp, "%s", __func__);
@@ -247,71 +248,49 @@
secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
nsec = bp->bio_length / secsize;
- /*
- * Calculate how much memory do we need.
- * We need separate crypto operation for every single sector.
- * It is much faster to calculate total amount of needed memory here and
- * do the allocation once instead of allocating memory in pieces (many,
- * many pieces).
- */
- size = sizeof(*crp) * nsec;
- size += sizeof(*crd) * nsec;
+ bp->bio_inbed = 0;
+ bp->bio_children = nsec;
+
/*
* If we write the data we cannot destroy current bio_data content,
* so we need to allocate more memory for encrypted data.
*/
- if (bp->bio_cmd == BIO_WRITE)
- size += bp->bio_length;
- p = malloc(size, M_ELI, M_WAITOK);
-
- bp->bio_inbed = 0;
- bp->bio_children = nsec;
- bp->bio_driver2 = p;
-
- if (bp->bio_cmd == BIO_READ)
- data = bp->bio_data;
- else {
- data = p;
- p += bp->bio_length;
+ if (bp->bio_cmd == BIO_WRITE) {
+ data = malloc(bp->bio_length, M_ELI, M_WAITOK);
+ bp->bio_driver2 = data;
bcopy(bp->bio_data, data, bp->bio_length);
- }
+ } else
+ data = bp->bio_data;
for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
- crp = (struct cryptop *)p; p += sizeof(*crp);
- crd = (struct cryptodesc *)p; p += sizeof(*crd);
+ crp = crypto_getreq(wr->w_sid, M_WAITOK);
- crp->crp_session = wr->w_sid;
crp->crp_ilen = secsize;
- crp->crp_olen = secsize;
crp->crp_opaque = (void *)bp;
+ crp->crp_buf_type = CRYPTO_BUF_CONTIG;
crp->crp_buf = (void *)data;
data += secsize;
- if (bp->bio_cmd == BIO_WRITE)
+ if (bp->bio_cmd == BIO_WRITE) {
+ crp->crp_op = CRYPTO_OP_ENCRYPT;
crp->crp_callback = g_eli_crypto_write_done;
- else /* if (bp->bio_cmd == BIO_READ) */
+ } else /* if (bp->bio_cmd == BIO_READ) */ {
+ crp->crp_op = CRYPTO_OP_DECRYPT;
crp->crp_callback = g_eli_crypto_read_done;
+ }
crp->crp_flags = CRYPTO_F_CBIFSYNC;
if (g_eli_batch)
crp->crp_flags |= CRYPTO_F_BATCH;
- crp->crp_desc = crd;
- crd->crd_skip = 0;
- crd->crd_len = secsize;
- crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0)
- crd->crd_flags |= CRD_F_KEY_EXPLICIT;
- if (bp->bio_cmd == BIO_WRITE)
- crd->crd_flags |= CRD_F_ENCRYPT;
- crd->crd_alg = sc->sc_ealgo;
- crd->crd_key = g_eli_key_hold(sc, dstoff, secsize);
- crd->crd_klen = sc->sc_ekeylen;
- if (sc->sc_ealgo == CRYPTO_AES_XTS)
- crd->crd_klen <<= 1;
- g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv,
- sizeof(crd->crd_iv));
- crd->crd_next = NULL;
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = secsize;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+ if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) {
+ crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+ secsize);
+ }
+ g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+ sizeof(crp->crp_iv));
- crp->crp_etype = 0;
error = crypto_dispatch(crp);
KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
error));
Index: sys/kern/subr_bus_dma.c
===================================================================
--- sys/kern/subr_bus_dma.c
+++ sys/kern/subr_bus_dma.c
@@ -54,6 +54,8 @@
#include <cam/cam.h>
#include <cam/cam_ccb.h>
+#include <opencrypto/cryptodev.h>
+
#include <machine/bus.h>
/*
@@ -635,3 +637,52 @@
return (0);
}
+
+int
+bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map, struct cryptop *crp,
+ bus_dmamap_callback_t *callback, void *callback_arg, int flags)
+{
+ bus_dma_segment_t *segs;
+ int error;
+ int nsegs;
+
+ flags |= BUS_DMA_NOWAIT;
+ nsegs = -1;
+ error = 0;
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_CONTIG:
+ error = _bus_dmamap_load_buffer(dmat, map, crp->crp_buf,
+ crp->crp_ilen, kernel_pmap, flags, NULL, &nsegs);
+ break;
+ case CRYPTO_BUF_MBUF:
+ error = _bus_dmamap_load_mbuf_sg(dmat, map, crp->crp_mbuf,
+ NULL, &nsegs, flags);
+ break;
+ case CRYPTO_BUF_UIO:
+ error = _bus_dmamap_load_uio(dmat, map, crp->crp_uio, &nsegs,
+ flags);
+ break;
+ }
+ nsegs++;
+
+ CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
+ __func__, dmat, flags, error, nsegs);
+
+ if (error == EINPROGRESS)
+ return (error);
+
+ segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
+ if (error)
+ (*callback)(callback_arg, segs, 0, error);
+ else
+ (*callback)(callback_arg, segs, nsegs, 0);
+
+ /*
+ * Return ENOMEM to the caller so that it can pass it up the stack.
+ * This error only happens when NOWAIT is set, so deferral is disabled.
+ */
+ if (error == ENOMEM)
+ return (error);
+
+ return (0);
+}
Index: sys/kern/uipc_ktls.c
===================================================================
--- sys/kern/uipc_ktls.c
+++ sys/kern/uipc_ktls.c
@@ -437,9 +437,12 @@
*/
switch (en->auth_algorithm) {
case 0:
+#ifdef COMPAT_FREEBSD12
+ /* XXX: Really 13.0-current COMPAT. */
case CRYPTO_AES_128_NIST_GMAC:
case CRYPTO_AES_192_NIST_GMAC:
case CRYPTO_AES_256_NIST_GMAC:
+#endif
break;
default:
return (EINVAL);
Index: sys/kgssapi/krb5/kcrypto_aes.c
===================================================================
--- sys/kgssapi/krb5/kcrypto_aes.c
+++ sys/kgssapi/krb5/kcrypto_aes.c
@@ -77,7 +77,7 @@
{
void *kp = ks->ks_key;
struct aes_state *as = ks->ks_priv;
- struct cryptoini cri;
+ struct crypto_session_params csp;
if (kp != in)
bcopy(in, kp, ks->ks_class->ec_keylen);
@@ -90,22 +90,22 @@
/*
* We only want the first 96 bits of the HMAC.
*/
- bzero(&cri, sizeof(cri));
- cri.cri_alg = CRYPTO_SHA1_HMAC;
- cri.cri_klen = ks->ks_class->ec_keybits;
- cri.cri_mlen = 12;
- cri.cri_key = ks->ks_key;
- cri.cri_next = NULL;
- crypto_newsession(&as->as_session_sha1, &cri,
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_DIGEST;
+ csp.csp_auth_alg = CRYPTO_SHA1_HMAC;
+ csp.csp_auth_klen = ks->ks_class->ec_keybits / 8;
+ csp.csp_auth_mlen = 12;
+ csp.csp_auth_key = ks->ks_key;
+ crypto_newsession(&as->as_session_sha1, &csp,
CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
- bzero(&cri, sizeof(cri));
- cri.cri_alg = CRYPTO_AES_CBC;
- cri.cri_klen = ks->ks_class->ec_keybits;
- cri.cri_mlen = 0;
- cri.cri_key = ks->ks_key;
- cri.cri_next = NULL;
- crypto_newsession(&as->as_session_aes, &cri,
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_cipher_alg = CRYPTO_AES_CBC;
+ csp.csp_cipher_klen = ks->ks_class->ec_keybits / 8;
+ csp.csp_cipher_key = ks->ks_key;
+ csp.csp_ivlen = 16;
+ crypto_newsession(&as->as_session_aes, &csp,
CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
}
@@ -138,31 +138,27 @@
static void
aes_encrypt_1(const struct krb5_key_state *ks, int buftype, void *buf,
- size_t skip, size_t len, void *ivec, int encdec)
+ size_t skip, size_t len, void *ivec, bool encrypt)
{
struct aes_state *as = ks->ks_priv;
struct cryptop *crp;
- struct cryptodesc *crd;
int error;
- crp = crypto_getreq(1);
- crd = crp->crp_desc;
+ crp = crypto_getreq(as->as_session_aes, M_WAITOK);
- crd->crd_skip = skip;
- crd->crd_len = len;
- crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = len;
+ crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
if (ivec) {
- bcopy(ivec, crd->crd_iv, 16);
+ memcpy(crp->crp_iv, ivec, 16);
} else {
- bzero(crd->crd_iv, 16);
+ memset(crp->crp_iv, 0, 16);
}
- crd->crd_next = NULL;
- crd->crd_alg = CRYPTO_AES_CBC;
- crp->crp_session = as->as_session_aes;
- crp->crp_flags = buftype | CRYPTO_F_CBIFSYNC;
+ crp->crp_buf_type = buftype;
crp->crp_buf = buf;
- crp->crp_opaque = (void *) as;
+ crp->crp_opaque = as;
crp->crp_callback = aes_crypto_cb;
error = crypto_dispatch(crp);
@@ -204,16 +200,16 @@
/*
* Note: caller will ensure len >= blocklen.
*/
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
- CRD_F_ENCRYPT);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+ true);
} else if (plen == 0) {
/*
* This is equivalent to CBC mode followed by swapping
* the last two blocks. We assume that neither of the
* last two blocks cross iov boundaries.
*/
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
- CRD_F_ENCRYPT);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+ true);
off = skip + len - 2 * blocklen;
m_copydata(inout, off, 2 * blocklen, (void*) &last2);
m_copyback(inout, off, blocklen, last2.cn);
@@ -227,8 +223,8 @@
* the encrypted versions of the last two blocks, we
* reshuffle to create the final result.
*/
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
- ivec, CRD_F_ENCRYPT);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len - plen,
+ ivec, true);
/*
* Copy out the last two blocks, pad the last block
@@ -241,8 +237,8 @@
m_copydata(inout, off, blocklen + plen, (void*) &last2);
for (i = plen; i < blocklen; i++)
last2.cn[i] = 0;
- aes_encrypt_1(ks, 0, last2.cn, 0, blocklen, last2.cn_1,
- CRD_F_ENCRYPT);
+ aes_encrypt_1(ks, CRYPTO_BUF_CONTIG, last2.cn, 0, blocklen,
+ last2.cn_1, true);
m_copyback(inout, off, blocklen, last2.cn);
m_copyback(inout, off + blocklen, plen, last2.cn_1);
}
@@ -274,7 +270,8 @@
/*
* Note: caller will ensure len >= blocklen.
*/
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+ false);
} else if (plen == 0) {
/*
* This is equivalent to CBC mode followed by swapping
@@ -284,7 +281,8 @@
m_copydata(inout, off, 2 * blocklen, (void*) &last2);
m_copyback(inout, off, blocklen, last2.cn);
m_copyback(inout, off + blocklen, blocklen, last2.cn_1);
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+ false);
} else {
/*
* This is the difficult case. We first decrypt the
@@ -298,8 +296,8 @@
* decrypted with the rest in CBC mode.
*/
off = skip + len - plen - blocklen;
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, off, blocklen,
- NULL, 0);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, off, blocklen,
+ NULL, false);
m_copydata(inout, off, blocklen + plen, (void*) &last2);
for (i = 0; i < plen; i++) {
@@ -309,8 +307,8 @@
}
m_copyback(inout, off, blocklen + plen, (void*) &last2);
- aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
- ivec, 0);
+ aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len - plen,
+ ivec, false);
}
}
@@ -321,26 +319,17 @@
{
struct aes_state *as = ks->ks_priv;
struct cryptop *crp;
- struct cryptodesc *crd;
int error;
- crp = crypto_getreq(1);
- crd = crp->crp_desc;
+ crp = crypto_getreq(as->as_session_sha1, M_WAITOK);
- crd->crd_skip = skip;
- crd->crd_len = inlen;
- crd->crd_inject = skip + inlen;
- crd->crd_flags = 0;
- crd->crd_next = NULL;
- crd->crd_alg = CRYPTO_SHA1_HMAC;
-
- crp->crp_session = as->as_session_sha1;
- crp->crp_ilen = inlen;
- crp->crp_olen = 12;
- crp->crp_etype = 0;
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
- crp->crp_buf = (void *) inout;
- crp->crp_opaque = (void *) as;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = inlen;
+ crp->crp_digest_start = skip + inlen;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
+ crp->crp_mbuf = inout;
+ crp->crp_opaque = as;
crp->crp_callback = aes_crypto_cb;
error = crypto_dispatch(crp);
Index: sys/kgssapi/krb5/kcrypto_des.c
===================================================================
--- sys/kgssapi/krb5/kcrypto_des.c
+++ sys/kgssapi/krb5/kcrypto_des.c
@@ -78,25 +78,24 @@
static void
des1_set_key(struct krb5_key_state *ks, const void *in)
{
+ struct crypto_session_params csp;
void *kp = ks->ks_key;
struct des1_state *ds = ks->ks_priv;
- struct cryptoini cri[1];
+
+ if (ds->ds_session)
+ crypto_freesession(ds->ds_session);
if (kp != in)
bcopy(in, kp, ks->ks_class->ec_keylen);
- if (ds->ds_session)
- crypto_freesession(ds->ds_session);
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_ivlen = 8;
+ csp.csp_cipher_alg = CRYPTO_DES_CBC;
+ csp.csp_cipher_klen = 8;
+ csp.csp_cipher_key = ks->ks_key;
- bzero(cri, sizeof(cri));
-
- cri[0].cri_alg = CRYPTO_DES_CBC;
- cri[0].cri_klen = 64;
- cri[0].cri_mlen = 0;
- cri[0].cri_key = ks->ks_key;
- cri[0].cri_next = NULL;
-
- crypto_newsession(&ds->ds_session, cri,
+ crypto_newsession(&ds->ds_session, &csp,
CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
}
@@ -163,32 +162,27 @@
}
static void
-des1_encrypt_1(const struct krb5_key_state *ks, int buftype, void *buf,
- size_t skip, size_t len, void *ivec, int encdec)
+des1_encrypt_1(const struct krb5_key_state *ks, int buf_type, void *buf,
+ size_t skip, size_t len, void *ivec, bool encrypt)
{
struct des1_state *ds = ks->ks_priv;
struct cryptop *crp;
- struct cryptodesc *crd;
int error;
- crp = crypto_getreq(1);
- crd = crp->crp_desc;
+ crp = crypto_getreq(ds->ds_session, M_WAITOK);
- crd->crd_skip = skip;
- crd->crd_len = len;
- crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = len;
+ crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
if (ivec) {
- bcopy(ivec, crd->crd_iv, 8);
+ memcpy(crp->crp_iv, ivec, 8);
} else {
- bzero(crd->crd_iv, 8);
+ memset(crp->crp_iv, 0, 8);
}
- crd->crd_next = NULL;
- crd->crd_alg = CRYPTO_DES_CBC;
-
- crp->crp_session = ds->ds_session;
- crp->crp_flags = buftype | CRYPTO_F_CBIFSYNC;
+ crp->crp_buf_type = buf_type;
crp->crp_buf = buf;
- crp->crp_opaque = (void *) ds;
+ crp->crp_opaque = ds;
crp->crp_callback = des1_crypto_cb;
error = crypto_dispatch(crp);
@@ -208,8 +202,7 @@
size_t skip, size_t len, void *ivec, size_t ivlen)
{
- des1_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
- CRD_F_ENCRYPT);
+ des1_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec, true);
}
static void
@@ -217,7 +210,7 @@
size_t skip, size_t len, void *ivec, size_t ivlen)
{
- des1_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+ des1_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec, false);
}
static int
@@ -244,7 +237,7 @@
m_apply(inout, skip, inlen, MD5Update_int, &md5);
MD5Final(hash, &md5);
- des1_encrypt_1(ks, 0, hash, 0, 16, NULL, CRD_F_ENCRYPT);
+ des1_encrypt_1(ks, CRYPTO_BUF_CONTIG, hash, 0, 16, NULL, true);
m_copyback(inout, skip + inlen, outlen, hash + 8);
}
Index: sys/kgssapi/krb5/kcrypto_des3.c
===================================================================
--- sys/kgssapi/krb5/kcrypto_des3.c
+++ sys/kgssapi/krb5/kcrypto_des3.c
@@ -48,7 +48,8 @@
struct des3_state {
struct mtx ds_lock;
- crypto_session_t ds_session;
+ crypto_session_t ds_cipher_session;
+ crypto_session_t ds_hmac_session;
};
static void
@@ -69,8 +70,10 @@
{
struct des3_state *ds = ks->ks_priv;
- if (ds->ds_session)
- crypto_freesession(ds->ds_session);
+ if (ds->ds_cipher_session) {
+ crypto_freesession(ds->ds_cipher_session);
+ crypto_freesession(ds->ds_hmac_session);
+ }
mtx_destroy(&ds->ds_lock);
free(ks->ks_priv, M_GSSAPI);
}
@@ -78,31 +81,35 @@
static void
des3_set_key(struct krb5_key_state *ks, const void *in)
{
+ struct crypto_session_params csp;
void *kp = ks->ks_key;
struct des3_state *ds = ks->ks_priv;
- struct cryptoini cri[2];
+
+ if (ds->ds_cipher_session) {
+ crypto_freesession(ds->ds_cipher_session);
+ crypto_freesession(ds->ds_hmac_session);
+ }
if (kp != in)
bcopy(in, kp, ks->ks_class->ec_keylen);
- if (ds->ds_session)
- crypto_freesession(ds->ds_session);
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_DIGEST;
+ csp.csp_auth_alg = CRYPTO_SHA1_HMAC;
+ csp.csp_auth_klen = 24;
+ csp.csp_auth_key = ks->ks_key;
- bzero(cri, sizeof(cri));
+ crypto_newsession(&ds->ds_hmac_session, &csp,
+ CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
- cri[0].cri_alg = CRYPTO_SHA1_HMAC;
- cri[0].cri_klen = 192;
- cri[0].cri_mlen = 0;
- cri[0].cri_key = ks->ks_key;
- cri[0].cri_next = &cri[1];
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_CIPHER;
+ csp.csp_cipher_alg = CRYPTO_3DES_CBC;
+ csp.csp_cipher_klen = 24;
+ csp.csp_cipher_key = ks->ks_key;
+ csp.csp_ivlen = 8;
- cri[1].cri_alg = CRYPTO_3DES_CBC;
- cri[1].cri_klen = 192;
- cri[1].cri_mlen = 0;
- cri[1].cri_key = ks->ks_key;
- cri[1].cri_next = NULL;
-
- crypto_newsession(&ds->ds_session, cri,
+ crypto_newsession(&ds->ds_cipher_session, &csp,
CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
}
@@ -158,7 +165,7 @@
int error;
struct des3_state *ds = (struct des3_state *) crp->crp_opaque;
- if (crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC)
+ if (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)
return (0);
error = crp->crp_etype;
@@ -174,36 +181,31 @@
static void
des3_encrypt_1(const struct krb5_key_state *ks, struct mbuf *inout,
- size_t skip, size_t len, void *ivec, int encdec)
+ size_t skip, size_t len, void *ivec, bool encrypt)
{
struct des3_state *ds = ks->ks_priv;
struct cryptop *crp;
- struct cryptodesc *crd;
int error;
- crp = crypto_getreq(1);
- crd = crp->crp_desc;
+ crp = crypto_getreq(ds->ds_cipher_session, M_WAITOK);
- crd->crd_skip = skip;
- crd->crd_len = len;
- crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = len;
+ crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
if (ivec) {
- bcopy(ivec, crd->crd_iv, 8);
+ memcpy(crp->crp_iv, ivec, 8);
} else {
- bzero(crd->crd_iv, 8);
+ memset(crp->crp_iv, 0, 8);
}
- crd->crd_next = NULL;
- crd->crd_alg = CRYPTO_3DES_CBC;
-
- crp->crp_session = ds->ds_session;
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
- crp->crp_buf = (void *) inout;
- crp->crp_opaque = (void *) ds;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
+ crp->crp_mbuf = inout;
+ crp->crp_opaque = ds;
crp->crp_callback = des3_crypto_cb;
error = crypto_dispatch(crp);
- if ((crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC) == 0) {
+ if ((crypto_ses2caps(ds->ds_cipher_session) & CRYPTOCAP_F_SYNC) == 0) {
mtx_lock(&ds->ds_lock);
if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
error = msleep(crp, &ds->ds_lock, 0, "gssdes3", 0);
@@ -218,7 +220,7 @@
size_t skip, size_t len, void *ivec, size_t ivlen)
{
- des3_encrypt_1(ks, inout, skip, len, ivec, CRD_F_ENCRYPT);
+ des3_encrypt_1(ks, inout, skip, len, ivec, true);
}
static void
@@ -226,7 +228,7 @@
size_t skip, size_t len, void *ivec, size_t ivlen)
{
- des3_encrypt_1(ks, inout, skip, len, ivec, 0);
+ des3_encrypt_1(ks, inout, skip, len, ivec, false);
}
static void
@@ -235,31 +237,23 @@
{
struct des3_state *ds = ks->ks_priv;
struct cryptop *crp;
- struct cryptodesc *crd;
int error;
- crp = crypto_getreq(1);
- crd = crp->crp_desc;
+ crp = crypto_getreq(ds->ds_hmac_session, M_WAITOK);
- crd->crd_skip = skip;
- crd->crd_len = inlen;
- crd->crd_inject = skip + inlen;
- crd->crd_flags = 0;
- crd->crd_next = NULL;
- crd->crd_alg = CRYPTO_SHA1_HMAC;
-
- crp->crp_session = ds->ds_session;
- crp->crp_ilen = inlen;
- crp->crp_olen = 20;
- crp->crp_etype = 0;
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
- crp->crp_buf = (void *) inout;
- crp->crp_opaque = (void *) ds;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = inlen;
+ crp->crp_digest_start = skip + inlen;
+ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
+ crp->crp_mbuf = inout;
+ crp->crp_opaque = ds;
crp->crp_callback = des3_crypto_cb;
error = crypto_dispatch(crp);
- if ((crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC) == 0) {
+ if ((crypto_ses2caps(ds->ds_hmac_session) & CRYPTOCAP_F_SYNC) == 0) {
mtx_lock(&ds->ds_lock);
if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
error = msleep(crp, &ds->ds_lock, 0, "gssdes3", 0);
Index: sys/mips/cavium/cryptocteon/cavium_crypto.c
===================================================================
--- sys/mips/cavium/cryptocteon/cavium_crypto.c
+++ sys/mips/cavium/cryptocteon/cavium_crypto.c
@@ -328,7 +328,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
uint64_t *data;
int data_i, data_l;
@@ -339,8 +339,8 @@
(crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -387,7 +387,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
uint64_t *data;
int data_i, data_l;
@@ -398,8 +398,8 @@
(crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -447,7 +447,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
uint64_t *data, *pdata;
int data_i, data_l;
@@ -458,8 +458,8 @@
(crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -516,7 +516,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
uint64_t *data, *pdata;
int data_i, data_l;
@@ -527,8 +527,8 @@
(crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -587,7 +587,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
uint64_t *data;
@@ -600,8 +600,8 @@
(auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -667,13 +667,9 @@
CVMX_MT_HSH_STARTMD5(tmp1);
/* save the HMAC */
- IOV_INIT(iov, data, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data, data_i, data_l);
- icv_off -= 8;
- }
+ data = (uint64_t *)icv;
CVMX_MF_HSH_IV(*data, 0);
- IOV_CONSUME(iov, data, data_i, data_l);
+ data++;
CVMX_MF_HSH_IV(tmp1, 1);
*(uint32_t *)data = (uint32_t) (tmp1 >> 32);
@@ -689,7 +685,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
uint64_t *data;
@@ -702,8 +698,8 @@
(auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -772,13 +768,9 @@
CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
/* save the HMAC */
- IOV_INIT(iov, data, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data, data_i, data_l);
- icv_off -= 8;
- }
+ data = (uint64_t *)icv;
CVMX_MF_HSH_IV(*data, 0);
- IOV_CONSUME(iov, data, data_i, data_l);
+ data++;
CVMX_MF_HSH_IV(tmp1, 1);
*(uint32_t *)data = (uint32_t) (tmp1 >> 32);
@@ -794,7 +786,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -815,8 +807,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -920,16 +912,12 @@
CVMX_MT_HSH_STARTMD5(tmp1);
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -942,7 +930,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -963,8 +951,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1068,16 +1056,12 @@
CVMX_MT_HSH_STARTMD5(tmp1);
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1093,7 +1077,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1114,8 +1098,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1222,16 +1206,12 @@
CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1244,7 +1224,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1265,8 +1245,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1372,16 +1352,12 @@
CVMX_MT_HSH_DATZ(6);
CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1397,7 +1373,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1419,8 +1395,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1552,16 +1528,12 @@
CVMX_MT_HSH_STARTMD5(tmp1);
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1574,7 +1546,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1596,8 +1568,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1725,16 +1697,12 @@
CVMX_MT_HSH_STARTMD5(tmp1);
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1750,7 +1718,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1772,8 +1740,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -1924,16 +1892,12 @@
#endif
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
@@ -1946,7 +1910,7 @@
struct iovec *iov, size_t iovcnt, size_t iovlen,
int auth_off, int auth_len,
int crypt_off, int crypt_len,
- int icv_off, uint8_t *ivp)
+ uint8_t *icv, uint8_t *ivp)
{
int next = 0;
union {
@@ -1968,8 +1932,8 @@
(auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
- "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ "icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
return -EINVAL;
}
@@ -2119,16 +2083,12 @@
#endif
/* save the HMAC */
- IOV_INIT(iov, data32, data_i, data_l);
- while (icv_off > 0) {
- IOV_CONSUME(iov, data32, data_i, data_l);
- icv_off -= 4;
- }
+ data32 = (uint32_t *)icv;
CVMX_MF_HSH_IV(tmp1, 0);
*data32 = (uint32_t) (tmp1 >> 32);
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
*data32 = (uint32_t) tmp1;
- IOV_CONSUME(iov, data32, data_i, data_l);
+ data32++;
CVMX_MF_HSH_IV(tmp1, 1);
*data32 = (uint32_t) (tmp1 >> 32);
Index: sys/mips/cavium/cryptocteon/cryptocteon.c
===================================================================
--- sys/mips/cavium/cryptocteon/cryptocteon.c
+++ sys/mips/cavium/cryptocteon/cryptocteon.c
@@ -59,7 +59,10 @@
static int cryptocteon_attach(device_t);
static int cryptocteon_process(device_t, struct cryptop *, int);
-static int cryptocteon_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int cryptocteon_probesession(device_t,
+ const struct crypto_session_params *);
+static int cryptocteon_newsession(device_t, crypto_session_t,
+ const struct crypto_session_params *);
static void
cryptocteon_identify(driver_t *drv, device_t parent)
@@ -89,168 +92,187 @@
return (ENXIO);
}
- crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
- crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-
return (0);
}
-/*
- * Generate a new octo session. We artifically limit it to a single
- * hash/cipher or hash-cipher combo just to make it easier, most callers
- * do not expect more than this anyway.
- */
+static bool
+cryptocteon_auth_supported(const struct crypto_session_params *csp)
+{
+ u_int hash_len;
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ hash_len = MD5_HASH_LEN;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ hash_len = SHA1_HASH_LEN;
+ break;
+ default:
+ return (false);
+ }
+
+ if (csp->csp_auth_klen > hash_len)
+ return (false);
+ return (true);
+}
+
+static bool
+cryptocteon_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ if (csp->csp_ivlen != 8)
+ return (false);
+ if (csp->csp_cipher_klen != 8 &&
+ csp->csp_cipher_klen != 24)
+ return (false);
+ break;
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != 16)
+ return (false);
+ if (csp->csp_cipher_klen != 16 &&
+ csp->csp_cipher_klen != 24 &&
+ csp->csp_cipher_klen != 32)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+
+ return (true);
+}
+
+static int
+cryptocteon_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!cryptocteon_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!cryptocteon_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!cryptocteon_auth_supported(csp) ||
+ !cryptocteon_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
+
+static void
+cryptocteon_calc_hash(const struct crypto_session_params *csp, const char *key,
+ struct octo_sess *ocd)
+{
+ char hash_key[SHA1_HASH_LEN];
+
+ memset(hash_key, 0, sizeof(hash_key));
+ memcpy(hash_key, key, csp->csp_auth_klen);
+ octo_calc_hash(csp->csp_auth_alg == CRYPTO_SHA1_HMAC, hash_key,
+ ocd->octo_hminner, ocd->octo_hmouter);
+}
+
+/* Generate a new octo session. */
static int
cryptocteon_newsession(device_t dev, crypto_session_t cses,
- struct cryptoini *cri)
+ const struct crypto_session_params *csp)
{
- struct cryptoini *c, *encini = NULL, *macini = NULL;
struct cryptocteon_softc *sc;
struct octo_sess *ocd;
- int i;
sc = device_get_softc(dev);
- if (cri == NULL || sc == NULL)
- return (EINVAL);
-
- /*
- * To keep it simple, we only handle hash, cipher or hash/cipher in a
- * session, you cannot currently do multiple ciphers/hashes in one
- * session even though it would be possibel to code this driver to
- * handle it.
- */
- for (i = 0, c = cri; c && i < 2; i++) {
- if (c->cri_alg == CRYPTO_MD5_HMAC ||
- c->cri_alg == CRYPTO_SHA1_HMAC ||
- c->cri_alg == CRYPTO_NULL_HMAC) {
- if (macini) {
- break;
- }
- macini = c;
- }
- if (c->cri_alg == CRYPTO_DES_CBC ||
- c->cri_alg == CRYPTO_3DES_CBC ||
- c->cri_alg == CRYPTO_AES_CBC ||
- c->cri_alg == CRYPTO_NULL_CBC) {
- if (encini) {
- break;
- }
- encini = c;
- }
- c = c->cri_next;
- }
- if (!macini && !encini) {
- dprintf("%s,%d - EINVAL bad cipher/hash or combination\n",
- __FILE__, __LINE__);
- return EINVAL;
- }
- if (c) {
- dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n",
- __FILE__, __LINE__);
- return EINVAL;
- }
-
- /*
- * So we have something we can do, lets setup the session
- */
ocd = crypto_get_driver_session(cses);
- if (encini && encini->cri_key) {
- ocd->octo_encklen = (encini->cri_klen + 7) / 8;
- memcpy(ocd->octo_enckey, encini->cri_key, ocd->octo_encklen);
- }
-
- if (macini && macini->cri_key) {
- ocd->octo_macklen = (macini->cri_klen + 7) / 8;
- memcpy(ocd->octo_mackey, macini->cri_key, ocd->octo_macklen);
- }
+ ocd->octo_encklen = csp->csp_cipher_klen;
+ if (csp->csp_cipher_key != NULL)
+ memcpy(ocd->octo_enckey, csp->csp_cipher_key,
+ ocd->octo_encklen);
- ocd->octo_mlen = 0;
- if (encini && encini->cri_mlen)
- ocd->octo_mlen = encini->cri_mlen;
- else if (macini && macini->cri_mlen)
- ocd->octo_mlen = macini->cri_mlen;
- else
- ocd->octo_mlen = 12;
+ if (csp->csp_auth_key != NULL)
+ cryptocteon_calc_hash(csp, csp->csp_auth_key, ocd);
- /*
- * point c at the enc if it exists, otherwise the mac
- */
- c = encini ? encini : macini;
+ ocd->octo_mlen = csp->csp_auth_mlen;
+ if (csp->csp_auth_mlen == 0) {
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ ocd->octo_mlen = MD5_HASH_LEN;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ ocd->octo_mlen = SHA1_HASH_LEN;
+ break;
+ }
+ }
- switch (c->cri_alg) {
- case CRYPTO_DES_CBC:
- case CRYPTO_3DES_CBC:
- ocd->octo_ivsize = 8;
- switch (macini ? macini->cri_alg : -1) {
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5_HMAC:
- ocd->octo_encrypt = octo_des_cbc_md5_encrypt;
- ocd->octo_decrypt = octo_des_cbc_md5_decrypt;
- octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
+ ocd->octo_encrypt = octo_null_md5_encrypt;
+ ocd->octo_decrypt = octo_null_md5_encrypt;
break;
case CRYPTO_SHA1_HMAC:
- ocd->octo_encrypt = octo_des_cbc_sha1_encrypt;
- ocd->octo_decrypt = octo_des_cbc_sha1_encrypt;
- octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
+ ocd->octo_encrypt = octo_null_sha1_encrypt;
+ ocd->octo_decrypt = octo_null_sha1_encrypt;
break;
- case -1:
+ }
+ break;
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
ocd->octo_encrypt = octo_des_cbc_encrypt;
ocd->octo_decrypt = octo_des_cbc_decrypt;
break;
- default:
- dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
- return EINVAL;
- }
- break;
- case CRYPTO_AES_CBC:
- ocd->octo_ivsize = 16;
- switch (macini ? macini->cri_alg : -1) {
- case CRYPTO_MD5_HMAC:
- ocd->octo_encrypt = octo_aes_cbc_md5_encrypt;
- ocd->octo_decrypt = octo_aes_cbc_md5_decrypt;
- octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
- break;
- case CRYPTO_SHA1_HMAC:
- ocd->octo_encrypt = octo_aes_cbc_sha1_encrypt;
- ocd->octo_decrypt = octo_aes_cbc_sha1_decrypt;
- octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
- break;
- case -1:
+ case CRYPTO_AES_CBC:
ocd->octo_encrypt = octo_aes_cbc_encrypt;
ocd->octo_decrypt = octo_aes_cbc_decrypt;
break;
- default:
- dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
- return EINVAL;
}
break;
- case CRYPTO_MD5_HMAC:
- ocd->octo_encrypt = octo_null_md5_encrypt;
- ocd->octo_decrypt = octo_null_md5_encrypt;
- octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
- break;
- case CRYPTO_SHA1_HMAC:
- ocd->octo_encrypt = octo_null_sha1_encrypt;
- ocd->octo_decrypt = octo_null_sha1_encrypt;
- octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
- ocd->octo_hmouter);
+ case CSP_MODE_ETA:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ ocd->octo_encrypt = octo_des_cbc_md5_encrypt;
+ ocd->octo_decrypt = octo_des_cbc_md5_decrypt;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ ocd->octo_encrypt = octo_des_cbc_sha1_encrypt;
+ ocd->octo_decrypt = octo_des_cbc_sha1_encrypt;
+ break;
+ }
+ break;
+ case CRYPTO_AES_CBC:
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ ocd->octo_encrypt = octo_aes_cbc_md5_encrypt;
+ ocd->octo_decrypt = octo_aes_cbc_md5_decrypt;
+ break;
+ case CRYPTO_SHA1_HMAC:
+ ocd->octo_encrypt = octo_aes_cbc_sha1_encrypt;
+ ocd->octo_decrypt = octo_aes_cbc_sha1_decrypt;
+ break;
+ }
+ break;
+ }
break;
- default:
- dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
- return EINVAL;
}
- ocd->octo_encalg = encini ? encini->cri_alg : -1;
- ocd->octo_macalg = macini ? macini->cri_alg : -1;
+ KASSERT(ocd->octo_encrypt != NULL && ocd->octo_decrypt != NULL,
+ ("%s: missing function pointers", __func__));
return (0);
}
@@ -261,106 +283,107 @@
static int
cryptocteon_process(device_t dev, struct cryptop *crp, int hint)
{
- struct cryptodesc *crd;
+ const struct crypto_session_params *csp;
struct octo_sess *od;
size_t iovcnt, iovlen;
struct mbuf *m = NULL;
struct uio *uiop = NULL;
- struct cryptodesc *enccrd = NULL, *maccrd = NULL;
unsigned char *ivp = NULL;
- unsigned char iv_data[HASH_MAX_LEN];
- int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0;
+ unsigned char iv_data[16];
+ unsigned char icv[SHA1_HASH_LEN], icv2[SHA1_HASH_LEN];
+ int auth_off, auth_len, crypt_off, crypt_len;
struct cryptocteon_softc *sc;
sc = device_get_softc(dev);
- if (sc == NULL || crp == NULL)
- return EINVAL;
-
crp->crp_etype = 0;
- if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
- dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__);
- crp->crp_etype = EINVAL;
- goto done;
- }
-
od = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
+
+ /*
+ * The crypto routines assume that the regions to auth and
+ * cipher are exactly 8 byte multiples and aligned on 8
+ * byte logical boundaries within the iovecs.
+ */
+ if (crp->crp_aad_length % 8 != 0 || crp->crp_payload_length % 8 != 0) {
+ crp->crp_etype = EFBIG;
+ goto done;
+ }
+
+ /*
+ * As currently written, the crypto routines assume the AAD and
+ * payload are adjacent.
+ */
+ if (crp->crp_aad_length != 0 && crp->crp_payload_start !=
+ crp->crp_aad_start + crp->crp_aad_length) {
+ crp->crp_etype = EFBIG;
+ goto done;
+ }
+
+ crypt_off = crp->crp_payload_start;
+ crypt_len = crp->crp_payload_length;
+ if (crp->crp_aad_length != 0) {
+ auth_off = crp->crp_aad_start;
+ auth_len = crp->crp_aad_length + crp->crp_payload_length;
+ } else {
+ auth_off = crypt_off;
+ auth_len = crypt_len;
+ }
/*
* do some error checking outside of the loop for m and IOV processing
* this leaves us with valid m or uiop pointers for later
*/
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ {
unsigned frags;
- m = (struct mbuf *) crp->crp_buf;
+ m = crp->crp_mbuf;
for (frags = 0; m != NULL; frags++)
m = m->m_next;
if (frags >= UIO_MAXIOV) {
printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags);
+ crp->crp_etype = EFBIG;
goto done;
}
- m = (struct mbuf *) crp->crp_buf;
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
- uiop = (struct uio *) crp->crp_buf;
+ m = crp->crp_mbuf;
+ break;
+ }
+ case CRYPTO_BUF_UIO:
+ uiop = crp->crp_uio;
if (uiop->uio_iovcnt > UIO_MAXIOV) {
printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__,
uiop->uio_iovcnt);
+ crp->crp_etype = EFBIG;
goto done;
}
+ break;
}
- /* point our enccrd and maccrd appropriately */
- crd = crp->crp_desc;
- if (crd->crd_alg == od->octo_encalg)
- enccrd = crd;
- if (crd->crd_alg == od->octo_macalg)
- maccrd = crd;
- crd = crd->crd_next;
- if (crd) {
- if (crd->crd_alg == od->octo_encalg)
- enccrd = crd;
- if (crd->crd_alg == od->octo_macalg)
- maccrd = crd;
- crd = crd->crd_next;
- }
- if (crd) {
- crp->crp_etype = EINVAL;
- dprintf("%s,%d: ENOENT - descriptors do not match session\n",
- __FILE__, __LINE__);
- goto done;
- }
-
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
- ivp = enccrd->crd_iv;
- } else {
+ if (csp->csp_cipher_alg != 0) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv_data, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ iv_data);
+ ivp = iv_data;
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ ivp = crp->crp_iv;
+ else {
+ crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+ iv_data);
ivp = iv_data;
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp);
- }
-
- if (maccrd) {
- auth_off = maccrd->crd_skip;
- auth_len = maccrd->crd_len;
- icv_off = maccrd->crd_inject;
}
-
- crypt_off = enccrd->crd_skip;
- crypt_len = enccrd->crd_len;
- } else { /* if (maccrd) */
- auth_off = maccrd->crd_skip;
- auth_len = maccrd->crd_len;
- icv_off = maccrd->crd_inject;
}
/*
* setup the I/O vector to cover the buffer
*/
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
iovcnt = 0;
iovlen = 0;
@@ -371,7 +394,8 @@
m = m->m_next;
iovlen += od->octo_iov[iovcnt++].iov_len;
}
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ break;
+ case CRYPTO_BUF_UIO:
iovlen = 0;
for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) {
od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base;
@@ -379,44 +403,44 @@
iovlen += od->octo_iov[iovcnt].iov_len;
}
- } else {
+ break;
+ case CRYPTO_BUF_CONTIG:
iovlen = crp->crp_ilen;
od->octo_iov[0].iov_base = crp->crp_buf;
od->octo_iov[0].iov_len = crp->crp_ilen;
iovcnt = 1;
+ break;
+ default:
+ panic("can't happen");
}
/*
* setup a new explicit key
*/
- if (enccrd) {
- if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- od->octo_encklen = (enccrd->crd_klen + 7) / 8;
- memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen);
- }
- }
- if (maccrd) {
- if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
- od->octo_macklen = (maccrd->crd_klen + 7) / 8;
- memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen);
- od->octo_mackey_set = 0;
- }
- if (!od->octo_mackey_set) {
- octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1,
- maccrd->crd_key, od->octo_hminner, od->octo_hmouter);
- od->octo_mackey_set = 1;
- }
- }
+ if (crp->crp_cipher_key != NULL)
+ memcpy(od->octo_enckey, crp->crp_cipher_key, od->octo_encklen);
+ if (crp->crp_auth_key != NULL)
+ cryptocteon_calc_hash(csp, crp->crp_auth_key, od);
- if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT))
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
(*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
else
(*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen,
- auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+ auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
+ if (csp->csp_auth_alg != 0) {
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start,
+ od->octo_mlen, icv2);
+ if (timingsafe_bcmp(icv, icv2, od->octo_mlen) != 0)
+ crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(crp, crp->crp_digest_start,
+ od->octo_mlen, icv);
+ }
done:
crypto_done(crp);
return (0);
@@ -429,6 +453,7 @@
DEVMETHOD(device_attach, cryptocteon_attach),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, cryptocteon_probesession),
DEVMETHOD(cryptodev_newsession, cryptocteon_newsession),
DEVMETHOD(cryptodev_process, cryptocteon_process),
Index: sys/mips/cavium/cryptocteon/cryptocteonvar.h
===================================================================
--- sys/mips/cavium/cryptocteon/cryptocteonvar.h
+++ sys/mips/cavium/cryptocteon/cryptocteonvar.h
@@ -34,23 +34,15 @@
struct octo_sess;
-typedef int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
-typedef int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
+typedef int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, uint8_t *icv, uint8_t *ivp);
+typedef int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, uint8_t *icv, uint8_t *ivp);
struct octo_sess {
- int octo_encalg;
#define MAX_CIPHER_KEYLEN 64
char octo_enckey[MAX_CIPHER_KEYLEN];
int octo_encklen;
- int octo_macalg;
- #define MAX_HASH_KEYLEN 64
- char octo_mackey[MAX_HASH_KEYLEN];
- int octo_macklen;
- int octo_mackey_set;
-
int octo_mlen;
- int octo_ivsize;
octo_encrypt_t *octo_encrypt;
octo_decrypt_t *octo_decrypt;
Index: sys/mips/nlm/dev/sec/nlmrsa.c
===================================================================
--- sys/mips/nlm/dev/sec/nlmrsa.c
+++ sys/mips/nlm/dev/sec/nlmrsa.c
@@ -76,7 +76,6 @@
#endif
static int xlp_rsa_init(struct xlp_rsa_softc *sc, int node);
-static int xlp_rsa_newsession(device_t , crypto_session_t, struct cryptoini *);
static int xlp_rsa_kprocess(device_t , struct cryptkop *, int);
static int xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits);
static void xlp_free_cmd_params(struct xlp_rsa_command *cmd);
@@ -98,7 +97,6 @@
DEVMETHOD(bus_driver_added, bus_generic_driver_added),
/* crypto device methods */
- DEVMETHOD(cryptodev_newsession, xlp_rsa_newsession),
DEVMETHOD(cryptodev_kprocess, xlp_rsa_kprocess),
DEVMETHOD_END
@@ -313,20 +311,6 @@
return (0);
}
-/*
- * Allocate a new 'session' (unused).
- */
-static int
-xlp_rsa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
-{
- struct xlp_rsa_softc *sc = device_get_softc(dev);
-
- if (cri == NULL || sc == NULL)
- return (EINVAL);
-
- return (0);
-}
-
/*
* XXX freesession should run a zero'd mac/encrypt key into context ram.
* XXX to blow away any keys already stored there.
Index: sys/mips/nlm/dev/sec/nlmsec.c
===================================================================
--- sys/mips/nlm/dev/sec/nlmsec.c
+++ sys/mips/nlm/dev/sec/nlmsec.c
@@ -52,6 +52,7 @@
#include <dev/pci/pcivar.h>
#include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
#include "cryptodev_if.h"
@@ -71,13 +72,14 @@
unsigned int creditleft;
-void xlp_sec_print_data(struct cryptop *crp);
-
static int xlp_sec_init(struct xlp_sec_softc *sc);
-static int xlp_sec_newsession(device_t , crypto_session_t, struct cryptoini *);
+static int xlp_sec_probesession(device_t,
+ const struct crypto_session_params *);
+static int xlp_sec_newsession(device_t , crypto_session_t,
+ const struct crypto_session_params *);
static int xlp_sec_process(device_t , struct cryptop *, int);
-static int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
- struct cryptodesc *enccrd);
+static void xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
+ const struct crypto_session_params *);
static int xlp_get_nsegs(struct cryptop *, unsigned int *);
static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
static void xlp_free_cmd_params(struct xlp_sec_command *);
@@ -97,6 +99,7 @@
DEVMETHOD(bus_driver_added, bus_generic_driver_added),
/* crypto device methods */
+ DEVMETHOD(cryptodev_probesession, xlp_sec_probesession),
DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
DEVMETHOD(cryptodev_process, xlp_sec_process),
@@ -197,46 +200,6 @@
return;
}
-void
-xlp_sec_print_data(struct cryptop *crp)
-{
- int i, key_len;
- struct cryptodesc *crp_desc;
-
- printf("session = %p, crp_ilen = %d, crp_olen=%d \n", crp->crp_session,
- crp->crp_ilen, crp->crp_olen);
-
- printf("crp_flags = 0x%x\n", crp->crp_flags);
-
- printf("crp buf:\n");
- for (i = 0; i < crp->crp_ilen; i++) {
- printf("%c ", crp->crp_buf[i]);
- if (i % 10 == 0)
- printf("\n");
- }
-
- printf("\n");
- printf("****************** desc ****************\n");
- crp_desc = crp->crp_desc;
- printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
- crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
- crp_desc->crd_alg);
-
- key_len = crp_desc->crd_klen / 8;
- printf("key(%d) :\n", key_len);
- for (i = 0; i < key_len; i++)
- printf("%d", crp_desc->crd_key[i]);
- printf("\n");
-
- printf(" IV : \n");
- for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
- printf("%d", crp_desc->crd_iv[i]);
- printf("\n");
-
- printf("crd_next=%p\n", crp_desc->crd_next);
- return;
-}
-
void
print_cmd(struct xlp_sec_command *cmd)
{
@@ -289,8 +252,7 @@
{
struct xlp_sec_command *cmd = NULL;
struct xlp_sec_softc *sc = NULL;
- struct cryptodesc *crd = NULL;
- unsigned int ivlen = 0;
+ uint8_t hash[HASH_MAX_LEN];
KASSERT(code == FMN_SWCODE_CRYPTO,
("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
@@ -310,23 +272,6 @@
(unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
(int)CRYPTO_ERROR(msg->msg[1])));
- crd = cmd->enccrd;
- /* Copy the last 8 or 16 bytes to the session iv, so that in few
- * cases this will be used as IV for the next request
- */
- if (crd != NULL) {
- if ((crd->crd_alg == CRYPTO_DES_CBC ||
- crd->crd_alg == CRYPTO_3DES_CBC ||
- crd->crd_alg == CRYPTO_AES_CBC) &&
- (crd->crd_flags & CRD_F_ENCRYPT)) {
- ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
- XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
- crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
- crd->crd_skip + crd->crd_len - ivlen, ivlen,
- cmd->ses->ses_iv);
- }
- }
-
/* If there are not enough credits to send, then send request
* will fail with ERESTART and the driver will be blocked until it is
* unblocked here after knowing that there are sufficient credits to
@@ -339,10 +284,16 @@
sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
}
}
- if(cmd->maccrd) {
- crypto_copyback(cmd->crp->crp_flags,
- cmd->crp->crp_buf, cmd->maccrd->crd_inject,
- cmd->hash_dst_len, cmd->hashdest);
+ if (cmd->hash_dst_len != 0) {
+ if (cmd->crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(cmd->crp, cmd->crp->crp_digest_start,
+ cmd->hash_dst_len, hash);
+ if (timingsafe_bcmp(cmd->hashdest, hash,
+ cmd->hash_dst_len) != 0)
+ cmd->crp->crp_etype = EBADMSG;
+ } else
+ crypto_copyback(cmd->crp, cmd->crp->crp_digest_start,
+ cmd->hash_dst_len, cmd->hashdest);
}
/* This indicates completion of the crypto operation */
@@ -392,29 +343,6 @@
" id\n");
goto error_exit;
}
- if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
- printf("register failed for CRYPTO_DES_CBC\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
- printf("register failed for CRYPTO_3DES_CBC\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
- printf("register failed for CRYPTO_AES_CBC\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
- printf("register failed for CRYPTO_ARC4\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
- printf("register failed for CRYPTO_MD5\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
- printf("register failed for CRYPTO_SHA1\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
- printf("register failed for CRYPTO_MD5_HMAC\n");
-
- if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
- printf("register failed for CRYPTO_SHA1_HMAC\n");
base = nlm_get_sec_pcibase(node);
qstart = nlm_qidstart(base);
@@ -443,65 +371,88 @@
return (0);
}
+static bool
+xlp_sec_auth_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5:
+ case CRYPTO_SHA1:
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ break;
+ default:
+ return (false);
+ }
+ return (true);
+}
+
+static bool
+xlp_sec_cipher_supported(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ case CRYPTO_3DES_CBC:
+ if (csp->csp_ivlen != XLP_SEC_DES_IV_LENGTH)
+ return (false);
+ break;
+ case CRYPTO_AES_CBC:
+ if (csp->csp_ivlen != XLP_SEC_AES_IV_LENGTH)
+ return (false);
+ break;
+ case CRYPTO_ARC4:
+ if (csp->csp_ivlen != XLP_SEC_ARC4_IV_LENGTH)
+ return (false);
+ break;
+ default:
+ return (false);
+ }
+
+ return (true);
+}
+
+static int
+xlp_sec_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_DIGEST:
+ if (!xlp_sec_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!xlp_sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_ETA:
+ if (!xlp_sec_auth_supported(csp) ||
+ !xlp_sec_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+ return (CRYPTODEV_PROBE_HARDWARE);
+}
+
static int
-xlp_sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+xlp_sec_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
- struct cryptoini *c;
- struct xlp_sec_softc *sc = device_get_softc(dev);
- int mac = 0, cry = 0;
struct xlp_sec_session *ses;
- struct xlp_sec_command *cmd = NULL;
-
- if (cri == NULL || sc == NULL)
- return (EINVAL);
ses = crypto_get_driver_session(cses);
- cmd = &ses->cmd;
- for (c = cri; c != NULL; c = c->cri_next) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_SHA1:
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- if (mac)
- return (EINVAL);
- mac = 1;
- ses->hs_mlen = c->cri_mlen;
- if (ses->hs_mlen == 0) {
- switch (c->cri_alg) {
- case CRYPTO_MD5:
- case CRYPTO_MD5_HMAC:
- ses->hs_mlen = 16;
- break;
- case CRYPTO_SHA1:
- case CRYPTO_SHA1_HMAC:
- ses->hs_mlen = 20;
- break;
- }
- }
- break;
- case CRYPTO_DES_CBC:
- case CRYPTO_3DES_CBC:
- case CRYPTO_AES_CBC:
- /* XXX this may read fewer, does it matter? */
- read_random(ses->ses_iv, c->cri_alg ==
- CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
- XLP_SEC_DES_IV_LENGTH);
- /* FALLTHROUGH */
- case CRYPTO_ARC4:
- if (cry)
- return (EINVAL);
- cry = 1;
- break;
- default:
- return (EINVAL);
- }
+ if (csp->csp_auth_alg != 0) {
+ if (csp->csp_auth_mlen == 0)
+ ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
+ else
+ ses->hs_mlen = csp->csp_auth_mlen;
}
- if (mac == 0 && cry == 0)
- return (EINVAL);
- cmd->hash_dst_len = ses->hs_mlen;
return (0);
}
@@ -510,54 +461,42 @@
* ram. to blow away any keys already stored there.
*/
-static int
+static void
xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
- struct cryptodesc *enccrd)
+ const struct crypto_session_params *csp)
{
- unsigned int ivlen = 0;
struct cryptop *crp = NULL;
crp = cmd->crp;
- if (enccrd->crd_alg != CRYPTO_ARC4) {
- ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
- XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
- if (enccrd->crd_flags & CRD_F_ENCRYPT) {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- } else {
- bcopy(cmd->ses->ses_iv, cmd->iv, ivlen);
- }
- if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
- crypto_copyback(crp->crp_flags,
- crp->crp_buf, enccrd->crd_inject,
- ivlen, cmd->iv);
- }
- } else {
- if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
- bcopy(enccrd->crd_iv, cmd->iv, ivlen);
- } else {
- crypto_copydata(crp->crp_flags, crp->crp_buf,
- enccrd->crd_inject, ivlen, cmd->iv);
- }
- }
+ if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(cmd->iv, csp->csp_ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+ cmd->iv);
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ memcpy(cmd->iv, crp->crp_iv, csp->csp_ivlen);
}
- return (0);
}
static int
xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
{
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ {
struct mbuf *m = NULL;
- m = (struct mbuf *)crp->crp_buf;
+ m = crp->crp_mbuf;
while (m != NULL) {
*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
m = m->m_next;
}
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ break;
+ }
+ case CRYPTO_BUF_UIO:
+ {
struct uio *uio = NULL;
struct iovec *iov = NULL;
int iol = 0;
@@ -570,8 +509,13 @@
iol--;
iov++;
}
- } else {
+ break;
+ }
+ case CRYPTO_BUF_CONTIG:
*nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
+ break;
+ default:
+ return (EINVAL);
}
return (0);
}
@@ -638,20 +582,24 @@
xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
{
struct xlp_sec_softc *sc = device_get_softc(dev);
+ const struct crypto_session_params *csp;
struct xlp_sec_command *cmd = NULL;
int err = -1, ret = 0;
- struct cryptodesc *crd1, *crd2;
struct xlp_sec_session *ses;
unsigned int nsegs = 0;
- if (crp == NULL || crp->crp_callback == NULL) {
- return (EINVAL);
- }
- if (sc == NULL) {
- err = EINVAL;
- goto errout;
- }
ses = crypto_get_driver_session(crp->crp_session);
+ csp = crypto_get_params(crp->crp_session);
+
+ /*
+ * This device only support AAD requests where the AAD is
+ * adjacent to the payload.
+ */
+ if (crp->crp_aad_length != 0 && crp->crp_payload_start !=
+ crp->crp_aad_start + crp->crp_aad_length) {
+ err = EFBIG;
+ goto errout;
+ }
if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
M_NOWAIT | M_ZERO)) == NULL) {
@@ -663,18 +611,12 @@
cmd->ses = ses;
cmd->hash_dst_len = ses->hs_mlen;
- if ((crd1 = crp->crp_desc) == NULL) {
- err = EINVAL;
- goto errout;
- }
- crd2 = crd1->crd_next;
-
if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
err = EINVAL;
goto errout;
}
- if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
- ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
+
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
/* Since IV is given as separate segment to avoid copy */
nsegs += 1;
}
@@ -683,98 +625,70 @@
if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
goto errout;
- if ((crd1 != NULL) && (crd2 == NULL)) {
- if (crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4) {
- cmd->enccrd = crd1;
- cmd->maccrd = NULL;
- if ((ret = nlm_get_cipher_param(cmd)) != 0) {
- err = EINVAL;
- goto errout;
- }
- if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
- cmd->cipheroff = cmd->ivlen;
- else
- cmd->cipheroff = cmd->enccrd->crd_skip;
- cmd->cipherlen = cmd->enccrd->crd_len;
- if (crd1->crd_flags & CRD_F_IV_PRESENT)
- cmd->ivoff = 0;
- else
- cmd->ivoff = cmd->enccrd->crd_inject;
- if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
- goto errout;
- if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
- goto errout;
- } else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1 ||
- crd1->crd_alg == CRYPTO_MD5) {
- cmd->enccrd = NULL;
- cmd->maccrd = crd1;
- if ((ret = nlm_get_digest_param(cmd)) != 0) {
- err = EINVAL;
- goto errout;
- }
- cmd->hashoff = cmd->maccrd->crd_skip;
- cmd->hashlen = cmd->maccrd->crd_len;
- cmd->hmacpad = 0;
- cmd->hashsrc = 0;
- if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
- goto errout;
- } else {
+ switch (csp->csp_mode) {
+ case CSP_MODE_CIPHER:
+ if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
err = EINVAL;
goto errout;
}
- } else if( (crd1 != NULL) && (crd2 != NULL) ) {
- if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
- crd1->crd_alg == CRYPTO_SHA1_HMAC ||
- crd1->crd_alg == CRYPTO_MD5 ||
- crd1->crd_alg == CRYPTO_SHA1) &&
- (crd2->crd_alg == CRYPTO_DES_CBC ||
- crd2->crd_alg == CRYPTO_3DES_CBC ||
- crd2->crd_alg == CRYPTO_AES_CBC ||
- crd2->crd_alg == CRYPTO_ARC4)) {
- cmd->maccrd = crd1;
- cmd->enccrd = crd2;
- } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
- crd1->crd_alg == CRYPTO_ARC4 ||
- crd1->crd_alg == CRYPTO_3DES_CBC ||
- crd1->crd_alg == CRYPTO_AES_CBC) &&
- (crd2->crd_alg == CRYPTO_MD5_HMAC ||
- crd2->crd_alg == CRYPTO_SHA1_HMAC ||
- crd2->crd_alg == CRYPTO_MD5 ||
- crd2->crd_alg == CRYPTO_SHA1)) {
- cmd->enccrd = crd1;
- cmd->maccrd = crd2;
- } else {
+ cmd->cipheroff = crp->crp_payload_start;
+ cmd->cipherlen = crp->crp_payload_length;
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+ cmd->cipheroff += cmd->ivlen;
+ cmd->ivoff = 0;
+ } else
+ cmd->ivoff = crp->crp_iv_start;
+ xlp_copyiv(sc, cmd, csp);
+ if ((err = nlm_crypto_do_cipher(sc, cmd, csp)) != 0)
+ goto errout;
+ break;
+ case CSP_MODE_DIGEST:
+ if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
err = EINVAL;
goto errout;
}
- if ((ret = nlm_get_cipher_param(cmd)) != 0) {
+ cmd->hashoff = crp->crp_payload_start;
+ cmd->hashlen = crp->crp_payload_length;
+ cmd->hmacpad = 0;
+ cmd->hashsrc = 0;
+ if ((err = nlm_crypto_do_digest(sc, cmd, csp)) != 0)
+ goto errout;
+ break;
+ case CSP_MODE_ETA:
+ if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
err = EINVAL;
goto errout;
}
- if ((ret = nlm_get_digest_param(cmd)) != 0) {
+ if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
err = EINVAL;
goto errout;
}
- cmd->ivoff = cmd->enccrd->crd_inject;
- cmd->hashoff = cmd->maccrd->crd_skip;
- cmd->hashlen = cmd->maccrd->crd_len;
+ if (crp->crp_aad_length != 0) {
+ cmd->hashoff = crp->crp_aad_start;
+ cmd->hashlen = crp->crp_aad_length +
+ crp->crp_payload_length;
+ } else {
+ cmd->hashoff = crp->crp_payload_start;
+ cmd->hashlen = crp->crp_payload_length;
+ }
cmd->hmacpad = 0;
- if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
cmd->hashsrc = 1;
else
cmd->hashsrc = 0;
- cmd->cipheroff = cmd->enccrd->crd_skip;
- cmd->cipherlen = cmd->enccrd->crd_len;
- if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
+ cmd->cipheroff = crp->crp_payload_start;
+ cmd->cipherlen = crp->crp_payload_length;
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+ cmd->hashoff += cmd->ivlen;
+ cmd->cipheroff += cmd->ivlen;
+ cmd->ivoff = 0;
+ } else
+ cmd->ivoff = crp->crp_iv_start;
+ xlp_copyiv(sc, cmd, csp);
+ if ((err = nlm_crypto_do_cipher_digest(sc, cmd, csp)) != 0)
goto errout;
- if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
- goto errout;
- } else {
+ break;
+ default:
err = EINVAL;
goto errout;
}
Index: sys/mips/nlm/dev/sec/nlmseclib.h
===================================================================
--- sys/mips/nlm/dev/sec/nlmseclib.h
+++ sys/mips/nlm/dev/sec/nlmseclib.h
@@ -91,7 +91,6 @@
struct xlp_sec_command {
struct cryptop *crp;
- struct cryptodesc *enccrd, *maccrd;
struct xlp_sec_session *ses;
struct nlm_crypto_pkt_ctrl *ctrlp;
struct nlm_crypto_pkt_param *paramp;
@@ -116,8 +115,6 @@
struct xlp_sec_session {
int hs_mlen;
- uint8_t ses_iv[EALG_MAX_BLOCK_LEN];
- struct xlp_sec_command cmd;
};
/*
@@ -135,17 +132,22 @@
#ifdef NLM_SEC_DEBUG
void print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m);
-void xlp_sec_print_data(struct cryptop *crp);
void print_cmd(struct xlp_sec_command *cmd);
#endif
-int nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd);
+int nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
int nlm_crypto_do_cipher(struct xlp_sec_softc *sc,
- struct xlp_sec_command *cmd);
+ struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
int nlm_crypto_do_digest(struct xlp_sec_softc *sc,
- struct xlp_sec_command *cmd);
+ struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
int nlm_crypto_do_cipher_digest(struct xlp_sec_softc *sc,
- struct xlp_sec_command *cmd);
-int nlm_get_digest_param(struct xlp_sec_command *cmd);
-int nlm_get_cipher_param(struct xlp_sec_command *cmd);
+ struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
+int nlm_get_digest_param(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
+int nlm_get_cipher_param(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp);
#endif /* _NLMSECLIB_H_ */
Index: sys/mips/nlm/dev/sec/nlmseclib.c
===================================================================
--- sys/mips/nlm/dev/sec/nlmseclib.c
+++ sys/mips/nlm/dev/sec/nlmseclib.c
@@ -92,18 +92,17 @@
}
int
-nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd)
+nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp)
{
unsigned int srcseg = 0, dstseg = 0;
- struct cryptodesc *cipdesc = NULL;
struct cryptop *crp = NULL;
crp = cmd->crp;
- cipdesc = cmd->enccrd;
- if (cipdesc != NULL) {
+ if (csp->csp_mode != CSP_MODE_DIGEST) {
/* IV is given as ONE segment to avoid copy */
- if (cipdesc->crd_flags & CRD_F_IV_EXPLICIT) {
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
cmd->iv, cmd->ivlen);
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg,
@@ -111,32 +110,37 @@
}
}
- if (crp->crp_flags & CRYPTO_F_IMBUF) {
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ {
struct mbuf *m = NULL;
- m = (struct mbuf *)crp->crp_buf;
+ m = crp->crp_mbuf;
while (m != NULL) {
srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
mtod(m,caddr_t), m->m_len);
- if (cipdesc != NULL) {
+ if (csp->csp_mode != CSP_MODE_DIGEST) {
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
dstseg, mtod(m,caddr_t), m->m_len);
}
m = m->m_next;
}
- } else if (crp->crp_flags & CRYPTO_F_IOV) {
+ break;
+ }
+ case CRYPTO_BUF_UIO:
+ {
struct uio *uio = NULL;
struct iovec *iov = NULL;
int iol = 0;
- uio = (struct uio *)crp->crp_buf;
- iov = (struct iovec *)uio->uio_iov;
+ uio = crp->crp_uio;
+ iov = uio->uio_iov;
iol = uio->uio_iovcnt;
while (iol > 0) {
srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
(caddr_t)iov->iov_base, iov->iov_len);
- if (cipdesc != NULL) {
+ if (csp->csp_mode != CSP_MODE_DIGEST) {
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
dstseg, (caddr_t)iov->iov_base,
iov->iov_len);
@@ -144,67 +148,75 @@
iov++;
iol--;
}
- } else {
+ }
+ case CRYPTO_BUF_CONTIG:
srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
((caddr_t)crp->crp_buf), crp->crp_ilen);
- if (cipdesc != NULL) {
+ if (csp->csp_mode != CSP_MODE_DIGEST) {
dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg,
((caddr_t)crp->crp_buf), crp->crp_ilen);
}
+ break;
}
return (0);
}
int
-nlm_crypto_do_cipher(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd)
+nlm_crypto_do_cipher(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp)
{
- struct cryptodesc *cipdesc = NULL;
- unsigned char *cipkey = NULL;
+ const unsigned char *cipkey = NULL;
int ret = 0;
- cipdesc = cmd->enccrd;
- cipkey = (unsigned char *)cipdesc->crd_key;
+ if (cmd->crp->crp_cipher_key != NULL)
+ cipkey = cmd->crp->crp_cipher_key;
+ else
+ cipkey = csp->csp_cipher_key;
if (cmd->cipheralg == NLM_CIPHER_3DES) {
- if (!(cipdesc->crd_flags & CRD_F_ENCRYPT)) {
- uint64_t *k, *tkey;
- k = (uint64_t *)cipdesc->crd_key;
+ if (!CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op)) {
+ const uint64_t *k;
+ uint64_t *tkey;
+ k = (const uint64_t *)cipkey;
tkey = (uint64_t *)cmd->des3key;
tkey[2] = k[0];
tkey[1] = k[1];
tkey[0] = k[2];
- cipkey = (unsigned char *)tkey;
+ cipkey = (const unsigned char *)tkey;
}
}
nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, 0, NLM_HASH_BYPASS, 0,
cmd->cipheralg, cmd->ciphermode, cipkey,
- (cipdesc->crd_klen >> 3), NULL, 0);
+ csp->csp_cipher_klen, NULL, 0);
nlm_crypto_fill_cipher_pkt_param(cmd->ctrlp, cmd->paramp,
- (cipdesc->crd_flags & CRD_F_ENCRYPT) ? 1 : 0, cmd->ivoff,
+ CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op) ? 1 : 0, cmd->ivoff,
cmd->ivlen, cmd->cipheroff, cmd->cipherlen);
- nlm_crypto_form_srcdst_segs(cmd);
+ nlm_crypto_form_srcdst_segs(cmd, csp);
ret = nlm_crypto_complete_sec_request(sc, cmd);
return (ret);
}
int
-nlm_crypto_do_digest(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd)
+nlm_crypto_do_digest(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp)
{
- struct cryptodesc *digdesc = NULL;
+ const char *key;
int ret=0;
- digdesc = cmd->maccrd;
-
- nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, (digdesc->crd_klen) ? 1 : 0,
+ if (cmd->crp->crp_auth_key != NULL)
+ key = cmd->crp->crp_auth_key;
+ else
+ key = csp->csp_auth_key;
+ nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, csp->csp_auth_klen ? 1 : 0,
cmd->hashalg, cmd->hashmode, NLM_CIPHER_BYPASS, 0,
- NULL, 0, digdesc->crd_key, digdesc->crd_klen >> 3);
+ NULL, 0, key, csp->csp_auth_klen);
nlm_crypto_fill_auth_pkt_param(cmd->ctrlp, cmd->paramp,
cmd->hashoff, cmd->hashlen, cmd->hmacpad,
(unsigned char *)cmd->hashdest);
- nlm_crypto_form_srcdst_segs(cmd);
+ nlm_crypto_form_srcdst_segs(cmd, csp);
ret = nlm_crypto_complete_sec_request(sc, cmd);
@@ -213,48 +225,54 @@
int
nlm_crypto_do_cipher_digest(struct xlp_sec_softc *sc,
- struct xlp_sec_command *cmd)
+ struct xlp_sec_command *cmd, const struct crypto_session_params *csp)
{
- struct cryptodesc *cipdesc=NULL, *digdesc=NULL;
- unsigned char *cipkey = NULL;
+ const unsigned char *cipkey = NULL;
+ const char *authkey;
int ret=0;
- cipdesc = cmd->enccrd;
- digdesc = cmd->maccrd;
-
- cipkey = (unsigned char *)cipdesc->crd_key;
+ if (cmd->crp->crp_cipher_key != NULL)
+ cipkey = cmd->crp->crp_cipher_key;
+ else
+ cipkey = csp->csp_cipher_key;
+ if (cmd->crp->crp_auth_key != NULL)
+ authkey = cmd->crp->crp_auth_key;
+ else
+ authkey = csp->csp_auth_key;
if (cmd->cipheralg == NLM_CIPHER_3DES) {
- if (!(cipdesc->crd_flags & CRD_F_ENCRYPT)) {
- uint64_t *k, *tkey;
- k = (uint64_t *)cipdesc->crd_key;
+ if (!CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op)) {
+ const uint64_t *k;
+ uint64_t *tkey;
+ k = (const uint64_t *)cipkey;
tkey = (uint64_t *)cmd->des3key;
tkey[2] = k[0];
tkey[1] = k[1];
tkey[0] = k[2];
- cipkey = (unsigned char *)tkey;
+ cipkey = (const unsigned char *)tkey;
}
}
- nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, (digdesc->crd_klen) ? 1 : 0,
+ nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, csp->csp_auth_klen ? 1 : 0,
cmd->hashalg, cmd->hashmode, cmd->cipheralg, cmd->ciphermode,
- cipkey, (cipdesc->crd_klen >> 3),
- digdesc->crd_key, (digdesc->crd_klen >> 3));
+ cipkey, csp->csp_cipher_klen,
+ authkey, csp->csp_auth_klen);
nlm_crypto_fill_cipher_auth_pkt_param(cmd->ctrlp, cmd->paramp,
- (cipdesc->crd_flags & CRD_F_ENCRYPT) ? 1 : 0, cmd->hashsrc,
+ CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op) ? 1 : 0, cmd->hashsrc,
cmd->ivoff, cmd->ivlen, cmd->hashoff, cmd->hashlen,
cmd->hmacpad, cmd->cipheroff, cmd->cipherlen,
(unsigned char *)cmd->hashdest);
- nlm_crypto_form_srcdst_segs(cmd);
+ nlm_crypto_form_srcdst_segs(cmd, csp);
ret = nlm_crypto_complete_sec_request(sc, cmd);
return (ret);
}
int
-nlm_get_digest_param(struct xlp_sec_command *cmd)
+nlm_get_digest_param(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp)
{
- switch(cmd->maccrd->crd_alg) {
+ switch(csp->csp_auth_alg) {
case CRYPTO_MD5:
cmd->hashalg = NLM_HASH_MD5;
cmd->hashmode = NLM_HASH_MODE_SHA1;
@@ -278,9 +296,10 @@
return (0);
}
int
-nlm_get_cipher_param(struct xlp_sec_command *cmd)
+nlm_get_cipher_param(struct xlp_sec_command *cmd,
+ const struct crypto_session_params *csp)
{
- switch(cmd->enccrd->crd_alg) {
+ switch(csp->csp_cipher_alg) {
case CRYPTO_DES_CBC:
cmd->cipheralg = NLM_CIPHER_DES;
cmd->ciphermode = NLM_CIPHER_MODE_CBC;
Index: sys/mips/nlm/hal/nlmsaelib.h
===================================================================
--- sys/mips/nlm/hal/nlmsaelib.h
+++ sys/mips/nlm/hal/nlmsaelib.h
@@ -462,8 +462,8 @@
nlm_crypto_fill_pkt_ctrl(struct nlm_crypto_pkt_ctrl *ctrl, unsigned int hmac,
enum nlm_hash_algo hashalg, enum nlm_hash_mode hashmode,
enum nlm_cipher_algo cipheralg, enum nlm_cipher_mode ciphermode,
- unsigned char *cipherkey, unsigned int cipherkeylen,
- unsigned char *hashkey, unsigned int hashkeylen)
+ const unsigned char *cipherkey, unsigned int cipherkeylen,
+ const unsigned char *hashkey, unsigned int hashkeylen)
{
unsigned int taglen = 0, hklen = 0;
Index: sys/netipsec/xform.h
===================================================================
--- sys/netipsec/xform.h
+++ sys/netipsec/xform.h
@@ -107,10 +107,11 @@
void xform_detach(void *);
int xform_init(struct secasvar *, u_short);
-struct cryptoini;
+struct crypto_session_params;
/* XF_AH */
int xform_ah_authsize(const struct auth_hash *);
-extern int ah_init0(struct secasvar *, struct xformsw *, struct cryptoini *);
+int ah_init0(struct secasvar *, struct xformsw *,
+ struct crypto_session_params *);
extern int ah_zeroize(struct secasvar *sav);
extern size_t ah_hdrsiz(struct secasvar *);
Index: sys/netipsec/xform_ah.c
===================================================================
--- sys/netipsec/xform_ah.c
+++ sys/netipsec/xform_ah.c
@@ -128,9 +128,7 @@
alen = esph->hashsize / 2; /* RFC4868 2.3 */
break;
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
+ case CRYPTO_AES_NIST_GMAC:
alen = esph->hashsize;
break;
@@ -174,7 +172,8 @@
* NB: public for use by esp_init.
*/
int
-ah_init0(struct secasvar *sav, struct xformsw *xsp, struct cryptoini *cria)
+ah_init0(struct secasvar *sav, struct xformsw *xsp,
+ struct crypto_session_params *csp)
{
const struct auth_hash *thash;
int keylen;
@@ -235,11 +234,10 @@
sav->tdb_authalgxform = thash;
/* Initialize crypto session. */
- bzero(cria, sizeof (*cria));
- cria->cri_alg = sav->tdb_authalgxform->type;
- cria->cri_klen = _KEYBITS(sav->key_auth);
- cria->cri_key = sav->key_auth->key_data;
- cria->cri_mlen = AUTHSIZE(sav);
+ csp->csp_auth_alg = sav->tdb_authalgxform->type;
+ csp->csp_auth_klen = _KEYBITS(sav->key_auth) / 8;
+ csp->csp_auth_key = sav->key_auth->key_data;
+ csp->csp_auth_mlen = AUTHSIZE(sav);
return 0;
}
@@ -250,12 +248,14 @@
static int
ah_init(struct secasvar *sav, struct xformsw *xsp)
{
- struct cryptoini cria;
+ struct crypto_session_params csp;
int error;
- error = ah_init0(sav, xsp, &cria);
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_DIGEST;
+ error = ah_init0(sav, xsp, &csp);
return error ? error :
- crypto_newsession(&sav->tdb_cryptoid, &cria, V_crypto_support);
+ crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
}
/*
@@ -560,7 +560,6 @@
{
IPSEC_DEBUG_DECLARE(char buf[128]);
const struct auth_hash *ahx;
- struct cryptodesc *crda;
struct cryptop *crp;
struct xform_data *xd;
struct newah *ah;
@@ -628,7 +627,7 @@
AHSTAT_ADD(ahs_ibytes, m->m_pkthdr.len - skip - hl);
/* Get crypto descriptors. */
- crp = crypto_getreq(1);
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptor\n",
__func__));
@@ -637,17 +636,9 @@
goto bad;
}
- crda = crp->crp_desc;
- IPSEC_ASSERT(crda != NULL, ("null crypto descriptor"));
-
- crda->crd_skip = 0;
- crda->crd_len = m->m_pkthdr.len;
- crda->crd_inject = skip + rplen;
-
- /* Authentication operation. */
- crda->crd_alg = ahx->type;
- crda->crd_klen = _KEYBITS(sav->key_auth);
- crda->crd_key = sav->key_auth->key_data;
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = m->m_pkthdr.len;
+ crp->crp_digest_start = skip + rplen;
/* Allocate IPsec-specific opaque crypto info. */
xd = malloc(sizeof(*xd) + skip + rplen + authsize, M_XDATA,
@@ -686,13 +677,14 @@
/* Crypto operation descriptor. */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
if (V_async_crypto)
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = ah_input_cb;
- crp->crp_session = cryptoid;
- crp->crp_opaque = (caddr_t) xd;
+ crp->crp_opaque = xd;
/* These are passed as-is to the callback. */
xd->sav = sav;
@@ -725,8 +717,8 @@
int authsize, rplen, ahsize, error, skip, protoff;
uint8_t nxt;
- m = (struct mbuf *) crp->crp_buf;
- xd = (struct xform_data *) crp->crp_opaque;
+ m = crp->crp_mbuf;
+ xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
sav = xd->sav;
skip = xd->skip;
@@ -866,7 +858,6 @@
{
IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
const struct auth_hash *ahx;
- struct cryptodesc *crda;
struct xform_data *xd;
struct mbuf *mi;
struct cryptop *crp;
@@ -988,7 +979,7 @@
SECASVAR_UNLOCK(sav);
/* Get crypto descriptors. */
- crp = crypto_getreq(1);
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptors\n",
__func__));
@@ -997,15 +988,9 @@
goto bad;
}
- crda = crp->crp_desc;
- crda->crd_skip = 0;
- crda->crd_inject = skip + rplen;
- crda->crd_len = m->m_pkthdr.len;
-
- /* Authentication operation. */
- crda->crd_alg = ahx->type;
- crda->crd_key = sav->key_auth->key_data;
- crda->crd_klen = _KEYBITS(sav->key_auth);
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = m->m_pkthdr.len;
+ crp->crp_digest_start = skip + rplen;
/* Allocate IPsec-specific opaque crypto info. */
xd = malloc(sizeof(struct xform_data) + skip, M_XDATA,
@@ -1069,13 +1054,14 @@
/* Crypto operation descriptor. */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
if (V_async_crypto)
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = ah_output_cb;
- crp->crp_session = cryptoid;
- crp->crp_opaque = (caddr_t) xd;
+ crp->crp_opaque = xd;
/* These are passed as-is to the callback. */
xd->sp = sp;
Index: sys/netipsec/xform_esp.c
===================================================================
--- sys/netipsec/xform_esp.c
+++ sys/netipsec/xform_esp.c
@@ -137,7 +137,7 @@
esp_init(struct secasvar *sav, struct xformsw *xsp)
{
const struct enc_xform *txform;
- struct cryptoini cria, crie;
+ struct crypto_session_params csp;
int keylen;
int error;
@@ -193,11 +193,13 @@
else
sav->ivlen = txform->ivsize;
+ memset(&csp, 0, sizeof(csp));
+
/*
* Setup AH-related state.
*/
if (sav->alg_auth != 0) {
- error = ah_init0(sav, xsp, &cria);
+ error = ah_init0(sav, xsp, &csp);
if (error)
return error;
}
@@ -231,35 +233,20 @@
keylen, txform->name));
return EINVAL;
}
- bzero(&cria, sizeof(cria));
- cria.cri_alg = sav->tdb_authalgxform->type;
- cria.cri_key = sav->key_enc->key_data;
- cria.cri_klen = _KEYBITS(sav->key_enc) - SAV_ISGCM(sav) * 32;
- }
+ csp.csp_mode = CSP_MODE_AEAD;
+ } else if (sav->alg_auth != 0)
+ csp.csp_mode = CSP_MODE_ETA;
+ else
+ csp.csp_mode = CSP_MODE_CIPHER;
/* Initialize crypto session. */
- bzero(&crie, sizeof(crie));
- crie.cri_alg = sav->tdb_encalgxform->type;
- crie.cri_key = sav->key_enc->key_data;
- crie.cri_klen = _KEYBITS(sav->key_enc) - SAV_ISCTRORGCM(sav) * 32;
+ csp.csp_cipher_alg = sav->tdb_encalgxform->type;
+ csp.csp_cipher_key = sav->key_enc->key_data;
+ csp.csp_cipher_klen = _KEYBITS(sav->key_enc) / 8 -
+ SAV_ISCTRORGCM(sav) * 4;
+ csp.csp_ivlen = txform->ivsize;
- if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
- /* init both auth & enc */
- crie.cri_next = &cria;
- error = crypto_newsession(&sav->tdb_cryptoid,
- &crie, V_crypto_support);
- } else if (sav->tdb_encalgxform) {
- error = crypto_newsession(&sav->tdb_cryptoid,
- &crie, V_crypto_support);
- } else if (sav->tdb_authalgxform) {
- error = crypto_newsession(&sav->tdb_cryptoid,
- &cria, V_crypto_support);
- } else {
- /* XXX cannot happen? */
- DPRINTF(("%s: no encoding OR authentication xform!\n",
- __func__));
- error = EINVAL;
- }
+ error = crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
return error;
}
@@ -289,7 +276,6 @@
const struct auth_hash *esph;
const struct enc_xform *espx;
struct xform_data *xd;
- struct cryptodesc *crde;
struct cryptop *crp;
struct newesp *esp;
uint8_t *ivp;
@@ -369,7 +355,7 @@
ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
/* Get crypto descriptors */
- crp = crypto_getreq(esph && espx ? 2 : 1);
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
DPRINTF(("%s: failed to acquire crypto descriptors\n",
__func__));
@@ -379,7 +365,7 @@
}
/* Get IPsec-specific opaque pointer */
- xd = malloc(sizeof(*xd) + alen, M_XDATA, M_NOWAIT | M_ZERO);
+ xd = malloc(sizeof(*xd), M_XDATA, M_NOWAIT | M_ZERO);
if (xd == NULL) {
DPRINTF(("%s: failed to allocate xform_data\n", __func__));
ESPSTAT_INC(esps_crypto);
@@ -389,39 +375,24 @@
}
if (esph != NULL) {
- struct cryptodesc *crda = crp->crp_desc;
-
- IPSEC_ASSERT(crda != NULL, ("null ah crypto descriptor"));
-
- /* Authentication descriptor */
- crda->crd_skip = skip;
+ crp->crp_op = CRYPTO_OP_VERIFY_DIGEST;
+ crp->crp_aad_start = skip;
if (SAV_ISGCM(sav))
- crda->crd_len = 8; /* RFC4106 5, SPI + SN */
+ crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
else
- crda->crd_len = m->m_pkthdr.len - (skip + alen);
- crda->crd_inject = m->m_pkthdr.len - alen;
-
- crda->crd_alg = esph->type;
-
- /* Copy the authenticator */
- m_copydata(m, m->m_pkthdr.len - alen, alen,
- (caddr_t) (xd + 1));
-
- /* Chain authentication request */
- crde = crda->crd_next;
- } else {
- crde = crp->crp_desc;
+ crp->crp_aad_length = hlen;
+ crp->crp_digest_start = m->m_pkthdr.len - alen;
}
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
if (V_async_crypto)
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = esp_input_cb;
- crp->crp_session = cryptoid;
- crp->crp_opaque = (caddr_t) xd;
+ crp->crp_opaque = xd;
/* These are passed as-is to the callback */
xd->sav = sav;
@@ -431,13 +402,12 @@
xd->vnet = curvnet;
/* Decryption descriptor */
- IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
- crde->crd_skip = skip + hlen;
- crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
- crde->crd_inject = skip + hlen - sav->ivlen;
+ crp->crp_op |= CRYPTO_OP_DECRYPT;
+ crp->crp_payload_start = skip + hlen;
+ crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen);
if (SAV_ISCTRORGCM(sav)) {
- ivp = &crde->crd_iv[0];
+ ivp = &crp->crp_iv[0];
/* GCM IV Format: RFC4106 4 */
/* CTR IV Format: RFC3686 4 */
@@ -452,10 +422,9 @@
}
m_copydata(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
- crde->crd_flags |= CRD_F_IV_EXPLICIT;
- }
-
- crde->crd_alg = espx->type;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+ } else if (sav->ivlen != 0)
+ crp->crp_iv_start = skip + hlen - sav->ivlen;
return (crypto_dispatch(crp));
bad:
@@ -471,22 +440,17 @@
esp_input_cb(struct cryptop *crp)
{
IPSEC_DEBUG_DECLARE(char buf[128]);
- u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
+ uint8_t lastthree[3];
const struct auth_hash *esph;
struct mbuf *m;
- struct cryptodesc *crd;
struct xform_data *xd;
struct secasvar *sav;
struct secasindex *saidx;
- caddr_t ptr;
crypto_session_t cryptoid;
int hlen, skip, protoff, error, alen;
- crd = crp->crp_desc;
- IPSEC_ASSERT(crd != NULL, ("null crypto descriptor!"));
-
- m = (struct mbuf *) crp->crp_buf;
- xd = (struct xform_data *) crp->crp_opaque;
+ m = crp->crp_mbuf;
+ xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
sav = xd->sav;
skip = xd->skip;
@@ -505,10 +469,15 @@
CURVNET_RESTORE();
return (crypto_dispatch(crp));
}
- ESPSTAT_INC(esps_noxform);
- DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
- error = crp->crp_etype;
- goto bad;
+
+ /* EBADMSG indicates authentication failure. */
+ if (!(crp->crp_etype == EBADMSG && esph != NULL)) {
+ ESPSTAT_INC(esps_noxform);
+ DPRINTF(("%s: crypto error %d\n", __func__,
+ crp->crp_etype));
+ error = crp->crp_etype;
+ goto bad;
+ }
}
/* Shouldn't happen... */
@@ -524,12 +493,7 @@
if (esph != NULL) {
alen = xform_ah_authsize(esph);
AHSTAT_INC(ahs_hist[sav->alg_auth]);
- /* Copy the authenticator from the packet */
- m_copydata(m, m->m_pkthdr.len - alen, alen, aalg);
- ptr = (caddr_t) (xd + 1);
-
- /* Verify authenticator */
- if (timingsafe_bcmp(ptr, aalg, alen) != 0) {
+ if (crp->crp_etype == EBADMSG) {
DPRINTF(("%s: authentication hash mismatch for "
"packet in SA %s/%08lx\n", __func__,
ipsec_address(&saidx->dst, buf, sizeof(buf)),
@@ -666,7 +630,6 @@
u_int idx, int skip, int protoff)
{
IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
- struct cryptodesc *crde = NULL, *crda = NULL;
struct cryptop *crp;
const struct auth_hash *esph;
const struct enc_xform *espx;
@@ -825,10 +788,10 @@
prot = IPPROTO_ESP;
m_copyback(m, protoff, sizeof(u_int8_t), (u_char *) &prot);
- /* Get crypto descriptors. */
- crp = crypto_getreq(esph != NULL ? 2 : 1);
+ /* Get crypto descriptor. */
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
- DPRINTF(("%s: failed to acquire crypto descriptors\n",
+ DPRINTF(("%s: failed to acquire crypto descriptor\n",
__func__));
ESPSTAT_INC(esps_crypto);
error = ENOBUFS;
@@ -845,19 +808,14 @@
goto bad;
}
- crde = crp->crp_desc;
- crda = crde->crd_next;
-
/* Encryption descriptor. */
- crde->crd_skip = skip + hlen;
- crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
- crde->crd_flags = CRD_F_ENCRYPT;
- crde->crd_inject = skip + hlen - sav->ivlen;
+ crp->crp_payload_start = skip + hlen;
+ crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen);
+ crp->crp_op = CRYPTO_OP_ENCRYPT;
/* Encryption operation. */
- crde->crd_alg = espx->type;
if (SAV_ISCTRORGCM(sav)) {
- ivp = &crde->crd_iv[0];
+ ivp = &crp->crp_iv[0];
/* GCM IV Format: RFC4106 4 */
/* CTR IV Format: RFC3686 4 */
@@ -873,7 +831,10 @@
}
m_copyback(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
- crde->crd_flags |= CRD_F_IV_EXPLICIT|CRD_F_IV_PRESENT;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+ } else if (sav->ivlen != 0) {
+ crp->crp_iv_start = skip + hlen - sav->ivlen;
+ crp->crp_flags |= CRYPTO_F_IV_GENERATE;
}
/* Callback parameters */
@@ -885,23 +846,23 @@
/* Crypto operation descriptor. */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+ crp->crp_flags |= CRYPTO_F_CBIFSYNC;
if (V_async_crypto)
crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = esp_output_cb;
- crp->crp_opaque = (caddr_t) xd;
- crp->crp_session = cryptoid;
+ crp->crp_opaque = xd;
if (esph) {
/* Authentication descriptor. */
- crda->crd_alg = esph->type;
- crda->crd_skip = skip;
+ crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_aad_start = skip;
if (SAV_ISGCM(sav))
- crda->crd_len = 8; /* RFC4106 5, SPI + SN */
+ crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
else
- crda->crd_len = m->m_pkthdr.len - (skip + alen);
- crda->crd_inject = m->m_pkthdr.len - alen;
+ crp->crp_aad_length = hlen;
+ crp->crp_digest_start = m->m_pkthdr.len - alen;
}
return crypto_dispatch(crp);
Index: sys/netipsec/xform_ipcomp.c
===================================================================
--- sys/netipsec/xform_ipcomp.c
+++ sys/netipsec/xform_ipcomp.c
@@ -156,7 +156,7 @@
ipcomp_init(struct secasvar *sav, struct xformsw *xsp)
{
const struct comp_algo *tcomp;
- struct cryptoini cric;
+ struct crypto_session_params csp;
/* NB: algorithm really comes in alg_enc and not alg_comp! */
tcomp = comp_algorithm_lookup(sav->alg_enc);
@@ -170,10 +170,11 @@
sav->tdb_compalgxform = tcomp;
/* Initialize crypto session */
- bzero(&cric, sizeof (cric));
- cric.cri_alg = sav->tdb_compalgxform->type;
+ memset(&csp, 0, sizeof(csp));
+ csp.csp_mode = CSP_MODE_COMPRESS;
+ csp.csp_cipher_alg = sav->tdb_compalgxform->type;
- return crypto_newsession(&sav->tdb_cryptoid, &cric, V_crypto_support);
+ return crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
}
/*
@@ -195,9 +196,9 @@
ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
{
struct xform_data *xd;
- struct cryptodesc *crdc;
struct cryptop *crp;
struct ipcomp *ipcomp;
+ crypto_session_t cryptoid;
caddr_t addr;
int error, hlen = IPCOMP_HLENGTH;
@@ -222,8 +223,12 @@
goto bad;
}
+ SECASVAR_LOCK(sav);
+ cryptoid = sav->tdb_cryptoid;
+ SECASVAR_UNLOCK(sav);
+
/* Get crypto descriptors */
- crp = crypto_getreq(1);
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
DPRINTF(("%s: no crypto descriptors\n", __func__));
IPCOMPSTAT_INC(ipcomps_crypto);
@@ -237,28 +242,26 @@
crypto_freereq(crp);
goto bad;
}
- crdc = crp->crp_desc;
-
- crdc->crd_skip = skip + hlen;
- crdc->crd_len = m->m_pkthdr.len - (skip + hlen);
- crdc->crd_inject = skip;
/* Decompression operation */
- crdc->crd_alg = sav->tdb_compalgxform->type;
-
+ crp->crp_op = CRYPTO_OP_DECOMPRESS;
+ crp->crp_payload_start = skip + hlen;
+ crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen);
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = ipcomp_input_cb;
- crp->crp_opaque = (caddr_t) xd;
+ crp->crp_opaque = xd;
/* These are passed as-is to the callback */
xd->sav = sav;
xd->protoff = protoff;
xd->skip = skip;
xd->vnet = curvnet;
+ xd->cryptoid = cryptoid;
SECASVAR_LOCK(sav);
crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
@@ -288,8 +291,8 @@
int skip, protoff;
uint8_t nproto;
- m = (struct mbuf *) crp->crp_buf;
- xd = (struct xform_data *) crp->crp_opaque;
+ m = crp->crp_mbuf;
+ xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
sav = xd->sav;
skip = xd->skip;
@@ -396,9 +399,9 @@
{
IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
const struct comp_algo *ipcompx;
- struct cryptodesc *crdc;
struct cryptop *crp;
struct xform_data *xd;
+ crypto_session_t cryptoid;
int error, ralen, maxpacketsize;
IPSEC_ASSERT(sav != NULL, ("null SA"));
@@ -466,25 +469,23 @@
}
/* Ok now, we can pass to the crypto processing. */
+ SECASVAR_LOCK(sav);
+ cryptoid = sav->tdb_cryptoid;
+ SECASVAR_UNLOCK(sav);
/* Get crypto descriptors */
- crp = crypto_getreq(1);
+ crp = crypto_getreq(cryptoid, M_NOWAIT);
if (crp == NULL) {
IPCOMPSTAT_INC(ipcomps_crypto);
DPRINTF(("%s: failed to acquire crypto descriptor\n",__func__));
error = ENOBUFS;
goto bad;
}
- crdc = crp->crp_desc;
/* Compression descriptor */
- crdc->crd_skip = skip;
- crdc->crd_len = ralen;
- crdc->crd_flags = CRD_F_COMP;
- crdc->crd_inject = skip;
-
- /* Compression operation */
- crdc->crd_alg = ipcompx->type;
+ crp->crp_op = CRYPTO_OP_COMPRESS;
+ crp->crp_payload_start = skip;
+ crp->crp_payload_length = ralen;
/* IPsec-specific opaque crypto info */
xd = malloc(sizeof(struct xform_data), M_XDATA, M_NOWAIT | M_ZERO);
@@ -502,17 +503,15 @@
xd->skip = skip;
xd->protoff = protoff;
xd->vnet = curvnet;
+ xd->cryptoid = cryptoid;
/* Crypto operation descriptor */
crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
- crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
- crp->crp_buf = (caddr_t) m;
+ crp->crp_flags = CRYPTO_F_CBIFSYNC;
+ crp->crp_mbuf = m;
+ crp->crp_buf_type = CRYPTO_BUF_MBUF;
crp->crp_callback = ipcomp_output_cb;
- crp->crp_opaque = (caddr_t) xd;
-
- SECASVAR_LOCK(sav);
- crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
- SECASVAR_UNLOCK(sav);
+ crp->crp_opaque = xd;
return crypto_dispatch(crp);
bad:
@@ -538,8 +537,8 @@
u_int idx;
int error, skip, protoff;
- m = (struct mbuf *) crp->crp_buf;
- xd = (struct xform_data *) crp->crp_opaque;
+ m = crp->crp_mbuf;
+ xd = crp->crp_opaque;
CURVNET_SET(xd->vnet);
idx = xd->idx;
sp = xd->sp;
@@ -572,7 +571,7 @@
}
IPCOMPSTAT_INC(ipcomps_hist[sav->alg_comp]);
- if (crp->crp_ilen - skip > crp->crp_olen) {
+ if (crp->crp_payload_length > crp->crp_olen) {
struct mbuf *mo;
struct ipcomp *ipcomp;
int roff;
@@ -639,8 +638,8 @@
} else {
/* Compression was useless, we have lost time. */
IPCOMPSTAT_INC(ipcomps_uncompr);
- DPRINTF(("%s: compressions was useless %d - %d <= %d\n",
- __func__, crp->crp_ilen, skip, crp->crp_olen));
+ DPRINTF(("%s: compressions was useless %d <= %d\n",
+ __func__, crp->crp_payload_length, crp->crp_olen));
/* XXX remember state to not compress the next couple
* of packets, RFC 3173, 2.2. Non-Expansion Policy */
}
Index: sys/opencrypto/criov.c
===================================================================
--- sys/opencrypto/criov.c
+++ sys/opencrypto/criov.c
@@ -157,41 +157,62 @@
}
void
-crypto_copyback(int flags, caddr_t buf, int off, int size, c_caddr_t in)
+crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
{
- if ((flags & CRYPTO_F_IMBUF) != 0)
- m_copyback((struct mbuf *)buf, off, size, in);
- else if ((flags & CRYPTO_F_IOV) != 0)
- cuio_copyback((struct uio *)buf, off, size, in);
- else
- bcopy(in, buf + off, size);
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ m_copyback(crp->crp_mbuf, off, size, src);
+ break;
+ case CRYPTO_BUF_UIO:
+ cuio_copyback(crp->crp_uio, off, size, src);
+ break;
+ case CRYPTO_BUF_CONTIG:
+ bcopy(src, crp->crp_buf + off, size);
+ break;
+ default:
+ panic("invalid crp buf type %d", crp->crp_buf_type);
+ }
}
void
-crypto_copydata(int flags, caddr_t buf, int off, int size, caddr_t out)
+crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
{
- if ((flags & CRYPTO_F_IMBUF) != 0)
- m_copydata((struct mbuf *)buf, off, size, out);
- else if ((flags & CRYPTO_F_IOV) != 0)
- cuio_copydata((struct uio *)buf, off, size, out);
- else
- bcopy(buf + off, out, size);
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ m_copydata(crp->crp_mbuf, off, size, dst);
+ break;
+ case CRYPTO_BUF_UIO:
+ cuio_copydata(crp->crp_uio, off, size, dst);
+ break;
+ case CRYPTO_BUF_CONTIG:
+ bcopy(crp->crp_buf + off, dst, size);
+ break;
+ default:
+ panic("invalid crp buf type %d", crp->crp_buf_type);
+ }
}
int
-crypto_apply(int flags, caddr_t buf, int off, int len,
+crypto_apply(struct cryptop *crp, int off, int len,
int (*f)(void *, void *, u_int), void *arg)
{
int error;
- if ((flags & CRYPTO_F_IMBUF) != 0)
- error = m_apply((struct mbuf *)buf, off, len, f, arg);
- else if ((flags & CRYPTO_F_IOV) != 0)
- error = cuio_apply((struct uio *)buf, off, len, f, arg);
- else
- error = (*f)(arg, buf + off, len);
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ error = m_apply(crp->crp_mbuf, off, len, f, arg);
+ break;
+ case CRYPTO_BUF_UIO:
+ error = cuio_apply(crp->crp_uio, off, len, f, arg);
+ break;
+ case CRYPTO_BUF_CONTIG:
+ error = (*f)(arg, crp->crp_buf + off, len);
+ break;
+ default:
+ panic("invalid crp buf type %d", crp->crp_buf_type);
+ }
return (error);
}
@@ -279,17 +300,17 @@
}
void *
-crypto_contiguous_subsegment(int crp_flags, void *crpbuf,
- size_t skip, size_t len)
+crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
{
- if ((crp_flags & CRYPTO_F_IMBUF) != 0)
- return (m_contiguous_subsegment(crpbuf, skip, len));
- else if ((crp_flags & CRYPTO_F_IOV) != 0)
- return (cuio_contiguous_segment(crpbuf, skip, len));
- else {
- MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) !=
- (CRYPTO_F_IMBUF | CRYPTO_F_IOV));
- return ((char*)crpbuf + skip);
+
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ return (m_contiguous_subsegment(crp->crp_mbuf, skip, len));
+ case CRYPTO_BUF_UIO:
+ return (cuio_contiguous_segment(crp->crp_uio, skip, len));
+ case CRYPTO_BUF_CONTIG:
+ return (crp->crp_buf + skip);
+ default:
+ panic("invalid crp buf type %d", crp->crp_buf_type);
}
}
-
Index: sys/opencrypto/crypto.c
===================================================================
--- sys/opencrypto/crypto.c
+++ sys/opencrypto/crypto.c
@@ -56,6 +56,7 @@
#define CRYPTO_TIMING /* enable timing support */
+#include "opt_compat.h"
#include "opt_ddb.h"
#include <sys/param.h>
@@ -69,6 +70,7 @@
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/proc.h>
+#include <sys/refcount.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
@@ -79,7 +81,8 @@
#include <vm/uma.h>
#include <crypto/intake.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/xform.h> /* XXX for M_XDATA */
+#include <opencrypto/xform_auth.h>
+#include <opencrypto/xform_enc.h>
#include <sys/kobj.h>
#include <sys/bus.h>
@@ -89,19 +92,12 @@
#include <machine/pcb.h>
#endif
-struct crypto_session {
- device_t parent;
- void *softc;
- uint32_t hid;
- uint32_t capabilities;
-};
-
SDT_PROVIDER_DEFINE(opencrypto);
/*
* Crypto drivers register themselves by allocating a slot in the
* crypto_drivers table with crypto_get_driverid() and then registering
- * each algorithm they support with crypto_register() and crypto_kregister().
+ * each asym algorithm they support with crypto_kregister().
*/
static struct mtx crypto_drivers_mtx; /* lock on driver table */
#define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx)
@@ -117,15 +113,10 @@
* Not tagged fields are read-only.
*/
struct cryptocap {
- device_t cc_dev; /* (d) device/driver */
+ device_t cc_dev;
+ uint32_t cc_hid;
u_int32_t cc_sessions; /* (d) # of sessions */
u_int32_t cc_koperations; /* (d) # os asym operations */
- /*
- * Largest possible operator length (in bits) for each type of
- * encryption algorithm. XXX not used
- */
- u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
- u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1];
u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1];
int cc_flags; /* (d) flags */
@@ -133,9 +124,17 @@
int cc_qblocked; /* (q) symmetric q blocked */
int cc_kqblocked; /* (q) asymmetric q blocked */
size_t cc_session_size;
+ volatile int cc_refs;
+};
+
+static struct cryptocap **crypto_drivers = NULL;
+static int crypto_drivers_size = 0;
+
+struct crypto_session {
+ struct cryptocap *cap;
+ void *softc;
+ struct crypto_session_params csp;
};
-static struct cryptocap *crypto_drivers = NULL;
-static int crypto_drivers_num = 0;
/*
* There are two queues for crypto requests; one for symmetric (e.g.
@@ -151,6 +150,9 @@
#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
+static SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
+ "In-kernel cryptography");
+
/*
* Taskqueue used to dispatch the crypto requests
* that have the CRYPTO_F_ASYNC flag
@@ -187,22 +189,37 @@
(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
static int crypto_workers_num = 0;
+SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
+ &crypto_workers_num, 0,
+ "Number of crypto workers used to dispatch crypto jobs");
+#ifdef COMPAT_FREEBSD12
SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
&crypto_workers_num, 0,
"Number of crypto workers used to dispatch crypto jobs");
+#endif
static uma_zone_t cryptop_zone;
-static uma_zone_t cryptodesc_zone;
static uma_zone_t cryptoses_zone;
-int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */
+int crypto_userasymcrypto = 1;
+SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW,
+ &crypto_userasymcrypto, 0,
+ "Enable user-mode access to asymmetric crypto support");
+#ifdef COMPAT_FREEBSD12
SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
&crypto_userasymcrypto, 0,
"Enable/disable user-mode access to asymmetric crypto support");
-int crypto_devallowsoft = 0; /* only use hardware crypto */
+#endif
+
+int crypto_devallowsoft = 0;
+SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW,
+ &crypto_devallowsoft, 0,
+ "Enable use of software crypto by /dev/crypto");
+#ifdef COMPAT_FREEBSD12
SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
&crypto_devallowsoft, 0,
"Enable/disable use of software crypto by /dev/crypto");
+#endif
MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
@@ -211,13 +228,12 @@
static void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
static void crypto_destroy(void);
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
-static int crypto_kinvoke(struct cryptkop *krp, int flags);
-static void crypto_remove(struct cryptocap *cap);
+static int crypto_kinvoke(struct cryptkop *krp);
static void crypto_task_invoke(void *ctx, int pending);
static void crypto_batch_enqueue(struct cryptop *crp);
static struct cryptostats cryptostats;
-SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
+SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats,
cryptostats, "Crypto system statistics");
#ifdef CRYPTO_TIMING
@@ -257,6 +273,29 @@
return (keybuf);
}
+static struct cryptocap *
+cap_ref(struct cryptocap *cap)
+{
+
+ refcount_acquire(&cap->cc_refs);
+ return (cap);
+}
+
+static void
+cap_rele(struct cryptocap *cap)
+{
+
+ if (refcount_release(&cap->cc_refs) == 0)
+ return;
+
+ KASSERT(cap->cc_sessions == 0,
+ ("freeing crypto driver with active sessions"));
+ KASSERT(cap->cc_koperations == 0,
+ ("freeing crypto driver with active key operations"));
+
+ free(cap, M_CRYPTO_DATA);
+}
+
static int
crypto_init(void)
{
@@ -273,22 +312,18 @@
cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
0, 0, 0, 0,
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
- cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
- 0, 0, 0, 0,
- UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
cryptoses_zone = uma_zcreate("crypto_session",
sizeof(struct crypto_session), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
- if (cryptodesc_zone == NULL || cryptop_zone == NULL ||
- cryptoses_zone == NULL) {
+ if (cryptop_zone == NULL || cryptoses_zone == NULL) {
printf("crypto_init: cannot setup crypto zones\n");
error = ENOMEM;
goto bad;
}
- crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
- crypto_drivers = malloc(crypto_drivers_num *
+ crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
+ crypto_drivers = malloc(crypto_drivers_size *
sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
if (crypto_drivers == NULL) {
printf("crypto_init: cannot setup crypto drivers\n");
@@ -379,10 +414,57 @@
}
}
+static void
+hmac_init_pad(struct auth_hash *axf, const char *key, int klen, void *auth_ctx,
+ uint8_t padval)
+{
+ uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
+ u_int i;
+
+ KASSERT(axf->blocksize <= sizeof(hmac_key),
+ ("Invalid HMAC block size %d", axf->blocksize));
+
+ /*
+ * If the key is larger than the block size, use the digest of
+ * the key as the key instead.
+ */
+ memset(hmac_key, 0, sizeof(hmac_key));
+ if (klen > axf->blocksize) {
+ axf->Init(auth_ctx);
+ axf->Update(auth_ctx, key, klen);
+ axf->Final(hmac_key, auth_ctx);
+ klen = axf->hashsize;
+ } else
+ memcpy(hmac_key, key, klen);
+
+ for (i = 0; i < axf->blocksize; i++)
+ hmac_key[i] ^= padval;
+
+ axf->Init(auth_ctx);
+ axf->Update(auth_ctx, hmac_key, axf->blocksize);
+}
+
+void
+hmac_init_ipad(struct auth_hash *axf, const char *key, int klen,
+ void *auth_ctx)
+{
+
+ hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
+}
+
+void
+hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
+ void *auth_ctx)
+{
+
+ hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
+}
+
static void
crypto_destroy(void)
{
struct crypto_ret_worker *ret_worker;
+ int i;
/*
* Terminate any crypto threads.
@@ -400,13 +482,14 @@
/*
* Reclaim dynamically allocated resources.
*/
- if (crypto_drivers != NULL)
- free(crypto_drivers, M_CRYPTO_DATA);
+ for (i = 0; i < crypto_drivers_size; i++) {
+ if (crypto_drivers[i] != NULL)
+ cap_rele(crypto_drivers[i]);
+ }
+ free(crypto_drivers, M_CRYPTO_DATA);
if (cryptoses_zone != NULL)
uma_zdestroy(cryptoses_zone);
- if (cryptodesc_zone != NULL)
- uma_zdestroy(cryptodesc_zone);
if (cryptop_zone != NULL)
uma_zdestroy(cryptop_zone);
mtx_destroy(&crypto_q_mtx);
@@ -421,13 +504,13 @@
uint32_t
crypto_ses2hid(crypto_session_t crypto_session)
{
- return (crypto_session->hid);
+ return (crypto_session->cap->cc_hid);
}
uint32_t
crypto_ses2caps(crypto_session_t crypto_session)
{
- return (crypto_session->capabilities);
+ return (crypto_session->cap->cc_flags & 0xff000000);
}
void *
@@ -436,86 +519,411 @@
return (crypto_session->softc);
}
+const struct crypto_session_params *
+crypto_get_params(crypto_session_t crypto_session)
+{
+ return (&crypto_session->csp);
+}
+
+struct auth_hash *
+crypto_auth_hash(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ return (&auth_hash_hmac_md5);
+ case CRYPTO_SHA1_HMAC:
+ return (&auth_hash_hmac_sha1);
+ case CRYPTO_SHA2_224_HMAC:
+ return (&auth_hash_hmac_sha2_224);
+ case CRYPTO_SHA2_256_HMAC:
+ return (&auth_hash_hmac_sha2_256);
+ case CRYPTO_SHA2_384_HMAC:
+ return (&auth_hash_hmac_sha2_384);
+ case CRYPTO_SHA2_512_HMAC:
+ return (&auth_hash_hmac_sha2_512);
+ case CRYPTO_NULL_HMAC:
+ return (&auth_hash_null);
+ case CRYPTO_RIPEMD160_HMAC:
+ return (&auth_hash_hmac_ripemd_160);
+ case CRYPTO_MD5_KPDK:
+ return (&auth_hash_key_md5);
+ case CRYPTO_SHA1_KPDK:
+ return (&auth_hash_key_sha1);
+#ifdef notyet
+ case CRYPTO_MD5:
+ return (&auth_hash_md5);
+#endif
+ case CRYPTO_SHA1:
+ return (&auth_hash_sha1);
+ case CRYPTO_SHA2_224:
+ return (&auth_hash_sha2_224);
+ case CRYPTO_SHA2_256:
+ return (&auth_hash_sha2_256);
+ case CRYPTO_SHA2_384:
+ return (&auth_hash_sha2_384);
+ case CRYPTO_SHA2_512:
+ return (&auth_hash_sha2_512);
+ case CRYPTO_AES_NIST_GMAC:
+ switch (csp->csp_auth_klen) {
+ case 128 / 8:
+ return (&auth_hash_nist_gmac_aes_128);
+ case 192 / 8:
+ return (&auth_hash_nist_gmac_aes_192);
+ case 256 / 8:
+ return (&auth_hash_nist_gmac_aes_256);
+ default:
+ return (NULL);
+ }
+ case CRYPTO_BLAKE2B:
+ return (&auth_hash_blake2b);
+ case CRYPTO_BLAKE2S:
+ return (&auth_hash_blake2s);
+ case CRYPTO_POLY1305:
+ return (&auth_hash_poly1305);
+ case CRYPTO_AES_CCM_CBC_MAC:
+ switch (csp->csp_auth_klen) {
+ case 128 / 8:
+ return (&auth_hash_ccm_cbc_mac_128);
+ case 192 / 8:
+ return (&auth_hash_ccm_cbc_mac_192);
+ case 256 / 8:
+ return (&auth_hash_ccm_cbc_mac_256);
+ default:
+ return (NULL);
+ }
+ default:
+ return (NULL);
+ }
+}
+
+struct enc_xform *
+crypto_cipher(const struct crypto_session_params *csp)
+{
+
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DES_CBC:
+ return (&enc_xform_des);
+ case CRYPTO_3DES_CBC:
+ return (&enc_xform_3des);
+ case CRYPTO_BLF_CBC:
+ return (&enc_xform_blf);
+ case CRYPTO_CAST_CBC:
+ return (&enc_xform_cast5);
+ case CRYPTO_SKIPJACK_CBC:
+ return (&enc_xform_skipjack);
+ case CRYPTO_RIJNDAEL128_CBC:
+ return (&enc_xform_rijndael128);
+ case CRYPTO_AES_XTS:
+ return (&enc_xform_aes_xts);
+ case CRYPTO_AES_ICM:
+ return (&enc_xform_aes_icm);
+ case CRYPTO_AES_NIST_GCM_16:
+ return (&enc_xform_aes_nist_gcm);
+ case CRYPTO_CAMELLIA_CBC:
+ return (&enc_xform_camellia);
+ case CRYPTO_NULL_CBC:
+ return (&enc_xform_null);
+ case CRYPTO_CHACHA20:
+ return (&enc_xform_chacha20);
+ case CRYPTO_AES_CCM_16:
+ return (&enc_xform_ccm);
+ default:
+ return (NULL);
+ }
+}
+
static struct cryptocap *
crypto_checkdriver(u_int32_t hid)
{
- if (crypto_drivers == NULL)
- return NULL;
- return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
-}
-/*
- * Compare a driver's list of supported algorithms against another
- * list; return non-zero if all algorithms are supported.
- */
-static int
-driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
-{
- const struct cryptoini *cr;
-
- /* See if all the algorithms are supported. */
- for (cr = cri; cr; cr = cr->cri_next)
- if (cap->cc_alg[cr->cri_alg] == 0)
- return 0;
- return 1;
+ return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
}
/*
* Select a driver for a new session that supports the specified
* algorithms and, optionally, is constrained according to the flags.
- * The algorithm we use here is pretty stupid; just use the
- * first driver that supports all the algorithms we need. If there
- * are multiple drivers we choose the driver with the fewest active
- * sessions. We prefer hardware-backed drivers to software ones.
- *
- * XXX We need more smarts here (in real life too, but that's
- * XXX another story altogether).
*/
static struct cryptocap *
-crypto_select_driver(const struct cryptoini *cri, int flags)
+crypto_select_driver(const struct crypto_session_params *csp, int flags)
{
struct cryptocap *cap, *best;
- int match, hid;
+ int best_match, error, hid;
CRYPTO_DRIVER_ASSERT();
- /*
- * Look first for hardware crypto devices if permitted.
- */
- if (flags & CRYPTOCAP_F_HARDWARE)
- match = CRYPTOCAP_F_HARDWARE;
- else
- match = CRYPTOCAP_F_SOFTWARE;
best = NULL;
-again:
- for (hid = 0; hid < crypto_drivers_num; hid++) {
- cap = &crypto_drivers[hid];
+ for (hid = 0; hid < crypto_drivers_size; hid++) {
/*
- * If it's not initialized, is in the process of
- * going away, or is not appropriate (hardware
- * or software based on match), then skip.
+ * If there is no driver for this slot, or the driver
+ * is not appropriate (hardware or software based on
+ * match), then skip.
*/
- if (cap->cc_dev == NULL ||
- (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
- (cap->cc_flags & match) == 0)
+ cap = crypto_drivers[hid];
+ if (cap == NULL ||
+ (cap->cc_flags & flags) == 0)
continue;
- /* verify all the algorithms are supported. */
- if (driver_suitable(cap, cri)) {
- if (best == NULL ||
- cap->cc_sessions < best->cc_sessions)
- best = cap;
+ error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
+ if (error >= 0)
+ continue;
+
+ /*
+ * Use the driver with the highest probe value.
+ * Hardware drivers use a higher probe value than
+ * software. In case of a tie, prefer the driver with
+ * the fewest active sessions.
+ */
+ if (best == NULL || error > best_match ||
+ (error == best_match &&
+ cap->cc_sessions < best->cc_sessions)) {
+ best = cap;
+ best_match = error;
}
}
- if (best == NULL && match == CRYPTOCAP_F_HARDWARE &&
- (flags & CRYPTOCAP_F_SOFTWARE)) {
- /* sort of an Algol 68-style for loop */
- match = CRYPTOCAP_F_SOFTWARE;
- goto again;
- }
return best;
}
+static bool
+alg_is_compression(int alg)
+{
+
+ if (alg == CRYPTO_DEFLATE_COMP)
+ return (true);
+ return (false);
+}
+
+static bool
+alg_is_cipher(int alg)
+{
+
+ if (alg >= CRYPTO_DES_CBC && alg <= CRYPTO_SKIPJACK_CBC)
+ return (true);
+ if (alg >= CRYPTO_AES_CBC && alg <= CRYPTO_ARC4)
+ return (true);
+ if (alg == CRYPTO_NULL_CBC)
+ return (true);
+ if (alg >= CRYPTO_CAMELLIA_CBC && alg <= CRYPTO_AES_ICM)
+ return (true);
+ if (alg == CRYPTO_CHACHA20)
+ return (true);
+ return (false);
+}
+
+static bool
+alg_is_digest(int alg)
+{
+
+ if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK)
+ return (true);
+ if (alg >= CRYPTO_MD5 && alg <= CRYPTO_SHA1)
+ return (true);
+ if (alg == CRYPTO_NULL_HMAC)
+ return (true);
+ if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+ return (true);
+ if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+ return (true);
+ if (alg == CRYPTO_AES_NIST_GMAC)
+ return (true);
+ if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S)
+ return (true);
+ if (alg >= CRYPTO_SHA2_224_HMAC && alg <= CRYPTO_POLY1305)
+ return (true);
+ if (alg == CRYPTO_AES_CCM_CBC_MAC)
+ return (true);
+ return (false);
+}
+
+static bool
+alg_is_keyed_digest(int alg)
+{
+
+ if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK)
+ return (true);
+ if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+ return (true);
+ if (alg == CRYPTO_AES_NIST_GMAC)
+ return (true);
+ if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S)
+ return (true);
+ if (alg == CRYPTO_SHA2_224_HMAC)
+ return (true);
+ if (alg == CRYPTO_POLY1305)
+ return (true);
+ if (alg == CRYPTO_AES_CCM_CBC_MAC)
+ return (true);
+ return (false);
+}
+
+static bool
+alg_is_aead(int alg)
+{
+
+ if (alg == CRYPTO_AES_NIST_GCM_16)
+ return (true);
+ if (alg == CRYPTO_AES_CCM_16)
+ return (true);
+ return (false);
+}
+
+/* Various sanity checks on crypto session parameters. */
+static bool
+check_csp(const struct crypto_session_params *csp)
+{
+ struct auth_hash *axf;
+
+ /* Mode-independent checks. */
+ if (csp->csp_flags != 0)
+ return (false);
+ if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
+ csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
+ return (false);
+ if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
+ return (false);
+ if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
+ return (false);
+
+ switch (csp->csp_mode) {
+ case CSP_MODE_COMPRESS:
+ if (!alg_is_compression(csp->csp_cipher_alg))
+ return (false);
+ if (csp->csp_flags != 0)
+ return (false);
+ if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
+ csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
+ csp->csp_auth_mlen != 0)
+ return (false);
+ break;
+ case CSP_MODE_CIPHER:
+ if (!alg_is_cipher(csp->csp_cipher_alg))
+ return (false);
+ if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
+ if (csp->csp_cipher_klen == 0)
+ return (false);
+ if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+ if (csp->csp_ivlen == 0)
+ return (false);
+ }
+ }
+ if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+ return (false);
+ if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
+ csp->csp_auth_mlen != 0)
+ return (false);
+ break;
+ case CSP_MODE_DIGEST:
+ if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
+ return (false);
+
+ /* IV is optional for digests (e.g. GMAC). */
+ if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+ return (false);
+ if (!alg_is_digest(csp->csp_auth_alg))
+ return (false);
+
+ /* Key is optional for BLAKE2 digests. */
+ if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
+ csp->csp_auth_alg == CRYPTO_BLAKE2S)
+ ;
+ else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
+ if (csp->csp_auth_klen == 0)
+ return (false);
+ } else {
+ if (csp->csp_auth_klen != 0)
+ return (false);
+ }
+ if (csp->csp_auth_mlen != 0) {
+ axf = crypto_auth_hash(csp);
+ if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
+ return (false);
+ }
+ break;
+ case CSP_MODE_AEAD:
+ if (!alg_is_aead(csp->csp_cipher_alg))
+ return (false);
+ if (csp->csp_cipher_klen == 0)
+ return (false);
+ if (csp->csp_ivlen == 0 ||
+ csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+ return (false);
+ if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
+ return (false);
+
+ /*
+ * XXX: Would be nice to have a better way to get this
+ * value.
+ */
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ if (csp->csp_auth_mlen > 16)
+ return (false);
+ break;
+ }
+ break;
+ case CSP_MODE_ETA:
+ if (!alg_is_cipher(csp->csp_cipher_alg))
+ return (false);
+ if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
+ if (csp->csp_cipher_klen == 0)
+ return (false);
+ if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+ if (csp->csp_ivlen == 0)
+ return (false);
+ }
+ }
+ if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+ return (false);
+ if (!alg_is_digest(csp->csp_auth_alg))
+ return (false);
+
+ /* Key is optional for BLAKE2 digests. */
+ if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
+ csp->csp_auth_alg == CRYPTO_BLAKE2S)
+ ;
+ else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
+ if (csp->csp_auth_klen == 0)
+ return (false);
+ } else {
+ if (csp->csp_auth_klen != 0)
+ return (false);
+ }
+ if (csp->csp_auth_mlen != 0) {
+ axf = crypto_auth_hash(csp);
+ if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
+ return (false);
+ }
+ break;
+ default:
+ return (false);
+ }
+
+ return (true);
+}
+
+/*
+ * Delete a session after it has been detached from its driver.
+ */
+static void
+crypto_deletesession(crypto_session_t cses)
+{
+ struct cryptocap *cap;
+
+ cap = cses->cap;
+
+ explicit_bzero(cses->softc, cap->cc_session_size);
+ free(cses->softc, M_CRYPTO_DATA);
+ uma_zfree(cryptoses_zone, cses);
+
+ CRYPTO_DRIVER_LOCK();
+ cap->cc_sessions--;
+ if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
+ wakeup(cap);
+ CRYPTO_DRIVER_UNLOCK();
+ cap_rele(cap);
+}
+
/*
* Create a new session. The crid argument specifies a crypto
* driver to use or constraints on a driver to select (hardware
@@ -523,18 +931,17 @@
* must be capable of the requested crypto algorithms.
*/
int
-crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid)
+crypto_newsession(crypto_session_t *cses,
+ const struct crypto_session_params *csp, int crid)
{
crypto_session_t res;
- void *softc_mem;
struct cryptocap *cap;
- u_int32_t hid;
- size_t softc_size;
int err;
-restart:
+ if (!check_csp(csp))
+ return (EINVAL);
+
res = NULL;
- softc_mem = NULL;
CRYPTO_DRIVER_LOCK();
if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
@@ -542,76 +949,39 @@
* Use specified driver; verify it is capable.
*/
cap = crypto_checkdriver(crid);
- if (cap != NULL && !driver_suitable(cap, cri))
+ if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
cap = NULL;
} else {
/*
* No requested driver; select based on crid flags.
*/
- cap = crypto_select_driver(cri, crid);
- /*
- * if NULL then can't do everything in one session.
- * XXX Fix this. We need to inject a "virtual" session
- * XXX layer right about here.
- */
+ cap = crypto_select_driver(csp, crid);
}
if (cap == NULL) {
+ CRYPTO_DRIVER_UNLOCK();
CRYPTDEB("no driver");
- err = EOPNOTSUPP;
- goto out;
+ return (EOPNOTSUPP);
}
+ cap_ref(cap);
cap->cc_sessions++;
- softc_size = cap->cc_session_size;
- hid = cap - crypto_drivers;
- cap = NULL;
CRYPTO_DRIVER_UNLOCK();
- softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO);
res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
- res->softc = softc_mem;
-
- CRYPTO_DRIVER_LOCK();
- cap = crypto_checkdriver(hid);
- if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) {
- cap->cc_sessions--;
- crypto_remove(cap);
- cap = NULL;
- }
- if (cap == NULL) {
- free(softc_mem, M_CRYPTO_DATA);
- uma_zfree(cryptoses_zone, res);
- CRYPTO_DRIVER_UNLOCK();
- goto restart;
- }
+ res->cap = cap;
+ res->softc = malloc(cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK |
+ M_ZERO);
+ res->csp = *csp;
/* Call the driver initialization routine. */
- err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri);
+ err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
if (err != 0) {
CRYPTDEB("dev newsession failed: %d", err);
- goto out;
+ crypto_deletesession(res);
+ return (err);
}
- res->capabilities = cap->cc_flags & 0xff000000;
- res->hid = hid;
*cses = res;
-
-out:
- CRYPTO_DRIVER_UNLOCK();
- if (err != 0) {
- free(softc_mem, M_CRYPTO_DATA);
- if (res != NULL)
- uma_zfree(cryptoses_zone, res);
- }
- return err;
-}
-
-static void
-crypto_remove(struct cryptocap *cap)
-{
-
- mtx_assert(&crypto_drivers_mtx, MA_OWNED);
- if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
- bzero(cap, sizeof(*cap));
+ return (0);
}
/*
@@ -622,101 +992,81 @@
crypto_freesession(crypto_session_t cses)
{
struct cryptocap *cap;
- void *ses;
- size_t ses_size;
- u_int32_t hid;
if (cses == NULL)
return;
- CRYPTO_DRIVER_LOCK();
-
- hid = crypto_ses2hid(cses);
- KASSERT(hid < crypto_drivers_num,
- ("bogus crypto_session %p hid %u", cses, hid));
- cap = &crypto_drivers[hid];
-
- ses = cses->softc;
- ses_size = cap->cc_session_size;
-
- if (cap->cc_sessions)
- cap->cc_sessions--;
+ cap = cses->cap;
/* Call the driver cleanup routine, if available. */
CRYPTODEV_FREESESSION(cap->cc_dev, cses);
- explicit_bzero(ses, ses_size);
- free(ses, M_CRYPTO_DATA);
- uma_zfree(cryptoses_zone, cses);
-
- if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
- crypto_remove(cap);
-
- CRYPTO_DRIVER_UNLOCK();
+ crypto_deletesession(cses);
}
/*
- * Return an unused driver id. Used by drivers prior to registering
- * support for the algorithms they handle.
+ * Return a new driver id. Registers a driver with the system so that
+ * it can be probed by subsequent sessions.
*/
int32_t
crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
{
- struct cryptocap *newdrv;
+ struct cryptocap *cap, **newdrv;
int i;
if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
- printf("%s: no flags specified when registering driver\n",
- device_get_nameunit(dev));
+ device_printf(dev,
+ "no flags specified when registering driver\n");
return -1;
}
+ cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
+ cap->cc_dev = dev;
+ cap->cc_session_size = sessionsize;
+ cap->cc_flags = flags;
+ refcount_init(&cap->cc_refs, 1);
+
CRYPTO_DRIVER_LOCK();
-
- for (i = 0; i < crypto_drivers_num; i++) {
- if (crypto_drivers[i].cc_dev == NULL &&
- (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
- break;
+ for (;;) {
+ for (i = 0; i < crypto_drivers_size; i++) {
+ if (crypto_drivers[i] == NULL)
+ break;
}
- }
- /* Out of entries, allocate some more. */
- if (i == crypto_drivers_num) {
- /* Be careful about wrap-around. */
- if (2 * crypto_drivers_num <= crypto_drivers_num) {
+ if (i < crypto_drivers_size)
+ break;
+
+ /* Out of entries, allocate some more. */
+
+ if (2 * crypto_drivers_size <= crypto_drivers_size) {
CRYPTO_DRIVER_UNLOCK();
printf("crypto: driver count wraparound!\n");
- return -1;
+ cap_rele(cap);
+ return (-1);
}
+ CRYPTO_DRIVER_UNLOCK();
- newdrv = malloc(2 * crypto_drivers_num *
- sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
- if (newdrv == NULL) {
- CRYPTO_DRIVER_UNLOCK();
- printf("crypto: no space to expand driver table!\n");
- return -1;
- }
+ newdrv = malloc(2 * crypto_drivers_size *
+ sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
- bcopy(crypto_drivers, newdrv,
- crypto_drivers_num * sizeof(struct cryptocap));
+ CRYPTO_DRIVER_LOCK();
+ memcpy(newdrv, crypto_drivers,
+ crypto_drivers_size * sizeof(*crypto_drivers));
- crypto_drivers_num *= 2;
+ crypto_drivers_size *= 2;
free(crypto_drivers, M_CRYPTO_DATA);
crypto_drivers = newdrv;
}
- /* NB: state is zero'd on free */
- crypto_drivers[i].cc_sessions = 1; /* Mark */
- crypto_drivers[i].cc_dev = dev;
- crypto_drivers[i].cc_flags = flags;
- crypto_drivers[i].cc_session_size = sessionsize;
+ cap->cc_hid = i;
+ crypto_drivers[i] = cap;
+ CRYPTO_DRIVER_UNLOCK();
+
if (bootverbose)
printf("crypto: assign %s driver id %u, flags 0x%x\n",
device_get_nameunit(dev), i, flags);
- CRYPTO_DRIVER_UNLOCK();
-
return i;
}
@@ -729,20 +1079,22 @@
int
crypto_find_driver(const char *match)
{
+ struct cryptocap *cap;
int i, len = strlen(match);
CRYPTO_DRIVER_LOCK();
- for (i = 0; i < crypto_drivers_num; i++) {
- device_t dev = crypto_drivers[i].cc_dev;
- if (dev == NULL ||
- (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
+ for (i = 0; i < crypto_drivers_size; i++) {
+ if (crypto_drivers[i] == NULL)
continue;
- if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
- strncmp(match, device_get_name(dev), len) == 0)
- break;
+ cap = crypto_drivers[i];
+ if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
+ strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
+ CRYPTO_DRIVER_UNLOCK();
+ return (i);
+ }
}
CRYPTO_DRIVER_UNLOCK();
- return i < crypto_drivers_num ? i : -1;
+ return (-1);
}
/*
@@ -752,8 +1104,16 @@
device_t
crypto_find_device_byhid(int hid)
{
- struct cryptocap *cap = crypto_checkdriver(hid);
- return cap != NULL ? cap->cc_dev : NULL;
+ struct cryptocap *cap;
+ device_t dev;
+
+ dev = NULL;
+ CRYPTO_DRIVER_LOCK();
+ cap = crypto_checkdriver(hid);
+ if (cap != NULL)
+ dev = cap->cc_dev;
+ CRYPTO_DRIVER_UNLOCK();
+ return (dev);
}
/*
@@ -762,8 +1122,16 @@
int
crypto_getcaps(int hid)
{
- struct cryptocap *cap = crypto_checkdriver(hid);
- return cap != NULL ? cap->cc_flags : 0;
+ struct cryptocap *cap;
+ int flags;
+
+ flags = 0;
+ CRYPTO_DRIVER_LOCK();
+ cap = crypto_checkdriver(hid);
+ if (cap != NULL)
+ flags = cap->cc_flags;
+ CRYPTO_DRIVER_UNLOCK();
+ return (flags);
}
/*
@@ -802,103 +1170,6 @@
return err;
}
-/*
- * Register support for a non-key-related algorithm. This routine
- * is called once for each such algorithm supported by a driver.
- */
-int
-crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
- u_int32_t flags)
-{
- struct cryptocap *cap;
- int err;
-
- CRYPTO_DRIVER_LOCK();
-
- cap = crypto_checkdriver(driverid);
- /* NB: algorithms are in the range [1..max] */
- if (cap != NULL &&
- (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
- /*
- * XXX Do some performance testing to determine placing.
- * XXX We probably need an auxiliary data structure that
- * XXX describes relative performances.
- */
-
- cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
- cap->cc_max_op_len[alg] = maxoplen;
- if (bootverbose)
- printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
- , device_get_nameunit(cap->cc_dev)
- , alg
- , flags
- , maxoplen
- );
- cap->cc_sessions = 0; /* Unmark */
- err = 0;
- } else
- err = EINVAL;
-
- CRYPTO_DRIVER_UNLOCK();
- return err;
-}
-
-static void
-driver_finis(struct cryptocap *cap)
-{
- u_int32_t ses, kops;
-
- CRYPTO_DRIVER_ASSERT();
-
- ses = cap->cc_sessions;
- kops = cap->cc_koperations;
- bzero(cap, sizeof(*cap));
- if (ses != 0 || kops != 0) {
- /*
- * If there are pending sessions,
- * just mark as invalid.
- */
- cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
- cap->cc_sessions = ses;
- cap->cc_koperations = kops;
- }
-}
-
-/*
- * Unregister a crypto driver. If there are pending sessions using it,
- * leave enough information around so that subsequent calls using those
- * sessions will correctly detect the driver has been unregistered and
- * reroute requests.
- */
-int
-crypto_unregister(u_int32_t driverid, int alg)
-{
- struct cryptocap *cap;
- int i, err;
-
- CRYPTO_DRIVER_LOCK();
- cap = crypto_checkdriver(driverid);
- if (cap != NULL &&
- (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
- cap->cc_alg[alg] != 0) {
- cap->cc_alg[alg] = 0;
- cap->cc_max_op_len[alg] = 0;
-
- /* Was this the last algorithm ? */
- for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
- if (cap->cc_alg[i] != 0)
- break;
-
- if (i == CRYPTO_ALGORITHM_MAX + 1)
- driver_finis(cap);
- err = 0;
- } else
- err = EINVAL;
- CRYPTO_DRIVER_UNLOCK();
-
- return err;
-}
-
/*
* Unregister all algorithms associated with a crypto driver.
* If there are pending sessions using it, leave enough information
@@ -910,18 +1181,27 @@
crypto_unregister_all(u_int32_t driverid)
{
struct cryptocap *cap;
- int err;
CRYPTO_DRIVER_LOCK();
cap = crypto_checkdriver(driverid);
- if (cap != NULL) {
- driver_finis(cap);
- err = 0;
- } else
- err = EINVAL;
+ if (cap == NULL) {
+ CRYPTO_DRIVER_UNLOCK();
+ return (EINVAL);
+ }
+
+ cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
+ crypto_drivers[driverid] = NULL;
+
+ /*
+ * XXX: This doesn't do anything to kick sessions that
+ * have no pending operations.
+ */
+ while (cap->cc_sessions != 0 || cap->cc_koperations != 0)
+ mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
CRYPTO_DRIVER_UNLOCK();
+ cap_rele(cap);
- return err;
+ return (0);
}
/*
@@ -951,6 +1231,125 @@
return err;
}
+#ifdef INVARIANTS
+/* Various sanity checks on crypto requests. */
+static void
+crp_sanity(struct cryptop *crp)
+{
+ struct crypto_session_params *csp;
+
+ KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
+ KASSERT(crp->crp_ilen >= 0, ("incoming crp with -ve input length"));
+ KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
+ KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
+ ("incoming crp already done"));
+
+ csp = &crp->crp_session->csp;
+ switch (csp->csp_mode) {
+ case CSP_MODE_COMPRESS:
+ KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
+ crp->crp_op == CRYPTO_OP_DECOMPRESS,
+ ("invalid compression op %x", crp->crp_op));
+ break;
+ case CSP_MODE_CIPHER:
+ KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
+ crp->crp_op == CRYPTO_OP_DECRYPT,
+ ("invalid cipher op %x", crp->crp_op));
+ break;
+ case CSP_MODE_DIGEST:
+ KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
+ crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
+ ("invalid digest op %x", crp->crp_op));
+ break;
+ case CSP_MODE_AEAD:
+ KASSERT(crp->crp_op ==
+ (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
+ crp->crp_op ==
+ (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
+ ("invalid AEAD op %x", crp->crp_op));
+ if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
+ KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
+ ("GCM without a separate IV"));
+ if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
+ KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
+ ("CCM without a separate IV"));
+ break;
+ case CSP_MODE_ETA:
+ KASSERT(crp->crp_op ==
+ (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
+ crp->crp_op ==
+ (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
+ ("invalid ETA op %x", crp->crp_op));
+ break;
+ }
+ KASSERT((crp->crp_flags & CRYPTO_F_IV_GENERATE) == 0 ||
+ crp->crp_op == CRYPTO_OP_ENCRYPT ||
+ crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST),
+ ("IV_GENERATE set for non-encryption operation %x", crp->crp_op));
+ KASSERT((crp->crp_flags &
+ (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) !=
+ (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE),
+ ("crp with both IV_SEPARATE and IV_GENERATE set"));
+ KASSERT(crp->crp_buf_type >= CRYPTO_BUF_CONTIG &&
+ crp->crp_buf_type <= CRYPTO_BUF_MBUF,
+ ("invalid crp buffer type %d", crp->crp_buf_type));
+ if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
+ KASSERT(crp->crp_aad_start == 0 ||
+ crp->crp_aad_start < crp->crp_ilen,
+ ("invalid AAD start"));
+ KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0,
+ ("AAD with zero length and non-zero start"));
+ KASSERT(crp->crp_aad_length == 0 ||
+ crp->crp_aad_start + crp->crp_aad_length <= crp->crp_ilen,
+ ("AAD outside input length"));
+ } else {
+ KASSERT(crp->crp_aad_start == 0 && crp->crp_aad_length == 0,
+ ("AAD region in request not supporting AAD"));
+ }
+ if (csp->csp_ivlen == 0) {
+ KASSERT((crp->crp_flags &
+ (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) == 0,
+ ("IV_GENERATE or IV_SEPARATE set when IV isn't used"));
+ KASSERT(crp->crp_iv_start == 0,
+ ("crp_iv_start set when IV isn't used"));
+ } else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+ KASSERT(crp->crp_iv_start == 0,
+ ("IV_SEPARATE used with non-zero IV start"));
+ } else {
+ KASSERT(crp->crp_iv_start < crp->crp_ilen,
+ ("invalid IV start"));
+ KASSERT(crp->crp_iv_start + csp->csp_ivlen <= crp->crp_ilen,
+ ("IV outside input length"));
+ }
+ KASSERT(crp->crp_payload_start == 0 ||
+ crp->crp_payload_start < crp->crp_ilen,
+ ("invalid payload start"));
+ KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
+ crp->crp_ilen, ("payload outside input length"));
+ if (csp->csp_mode == CSP_MODE_DIGEST ||
+ csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
+ KASSERT(crp->crp_digest_start == 0 ||
+ crp->crp_digest_start < crp->crp_ilen,
+ ("invalid digest start"));
+ /* XXX: For the mlen == 0 case this check isn't perfect. */
+ KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <=
+ crp->crp_ilen,
+ ("digest outside input length"));
+ } else {
+ KASSERT(crp->crp_digest_start == 0,
+ ("non-zero digest start for request without a digest"));
+ }
+ if (csp->csp_cipher_klen != 0)
+ KASSERT(csp->csp_cipher_key != NULL ||
+ crp->crp_cipher_key != NULL,
+ ("cipher request without a key"));
+ if (csp->csp_auth_klen != 0)
+ KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
+ ("auth request without a key"));
+ KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
+}
+#endif
+
/*
* Add a crypto request to a queue, to be processed by the kernel thread.
*/
@@ -958,9 +1357,14 @@
crypto_dispatch(struct cryptop *crp)
{
struct cryptocap *cap;
- u_int32_t hid;
int result;
+#ifdef INVARIANTS
+ crp_sanity(crp);
+#endif
+
+ /* TODO: Handle CRYPTO_F_IV_GENERATE so drivers don't have to. */
+
cryptostats.cs_ops++;
#ifdef CRYPTO_TIMING
@@ -987,16 +1391,12 @@
}
if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
- hid = crypto_ses2hid(crp->crp_session);
-
/*
* Caller marked the request to be processed
* immediately; dispatch it directly to the
* driver unless the driver is currently blocked.
*/
- cap = crypto_checkdriver(hid);
- /* Driver cannot disappeared when there is an active session. */
- KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
+ cap = crp->crp_session->cap;
if (!cap->cc_qblocked) {
result = crypto_invoke(cap, crp, 0);
if (result != ERESTART)
@@ -1033,7 +1433,8 @@
cryptostats.cs_kops++;
- error = crypto_kinvoke(krp, krp->krp_crid);
+ krp->krp_cap = NULL;
+ error = crypto_kinvoke(krp);
if (error == ERESTART) {
CRYPTO_Q_LOCK();
TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
@@ -1081,15 +1482,14 @@
match = CRYPTOCAP_F_SOFTWARE;
best = NULL;
again:
- for (hid = 0; hid < crypto_drivers_num; hid++) {
- cap = &crypto_drivers[hid];
+ for (hid = 0; hid < crypto_drivers_size; hid++) {
/*
- * If it's not initialized, is in the process of
- * going away, or is not appropriate (hardware
- * or software based on match), then skip.
+ * If there is no driver for this slot, or the driver
+ * is not appropriate (hardware or software based on
+ * match), then skip.
*/
+ cap = crypto_drivers[hid];
if (cap->cc_dev == NULL ||
- (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
(cap->cc_flags & match) == 0)
continue;
@@ -1111,30 +1511,33 @@
}
/*
- * Dispatch an asymmetric crypto request.
+ * Choose a driver for an asymmetric crypto request.
*/
-static int
-crypto_kinvoke(struct cryptkop *krp, int crid)
+static struct cryptocap *
+crypto_lookup_kdriver(struct cryptkop *krp)
{
- struct cryptocap *cap = NULL;
- int error;
+ struct cryptocap *cap;
+ uint32_t crid;
- KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
- KASSERT(krp->krp_callback != NULL,
- ("%s: krp->crp_callback == NULL", __func__));
+ /* If this request is requeued, it might already have a driver. */
+ cap = krp->krp_cap;
+ if (cap != NULL)
+ return (cap);
- CRYPTO_DRIVER_LOCK();
+ /* Use krp_crid to choose a driver. */
+ crid = krp->krp_crid;
if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
cap = crypto_checkdriver(crid);
if (cap != NULL) {
/*
- * Driver present, it must support the necessary
- * algorithm and, if s/w drivers are excluded,
- * it must be registered as hardware-backed.
+ * Driver present, it must support the
+ * necessary algorithm and, if s/w drivers are
+ * excluded, it must be registered as
+ * hardware-backed.
*/
if (!kdriver_suitable(cap, krp) ||
(!crypto_devallowsoft &&
- (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
+ (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
cap = NULL;
}
} else {
@@ -1145,32 +1548,61 @@
crid &= ~CRYPTOCAP_F_SOFTWARE;
cap = crypto_select_kdriver(krp, crid);
}
- if (cap != NULL && !cap->cc_kqblocked) {
- krp->krp_hid = cap - crypto_drivers;
- cap->cc_koperations++;
- CRYPTO_DRIVER_UNLOCK();
- error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
- CRYPTO_DRIVER_LOCK();
- if (error == ERESTART) {
- cap->cc_koperations--;
- CRYPTO_DRIVER_UNLOCK();
- return (error);
- }
- } else {
- /*
- * NB: cap is !NULL if device is blocked; in
- * that case return ERESTART so the operation
- * is resubmitted if possible.
- */
- error = (cap == NULL) ? ENODEV : ERESTART;
+
+ if (cap != NULL) {
+ krp->krp_cap = cap_ref(cap);
+ krp->krp_hid = cap->cc_hid;
}
- CRYPTO_DRIVER_UNLOCK();
+ return (cap);
+}
- if (error) {
- krp->krp_status = error;
+/*
+ * Dispatch an asymmetric crypto request.
+ */
+static int
+crypto_kinvoke(struct cryptkop *krp)
+{
+ struct cryptocap *cap = NULL;
+ int error;
+
+ KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
+ KASSERT(krp->krp_callback != NULL,
+ ("%s: krp->crp_callback == NULL", __func__));
+
+ CRYPTO_DRIVER_LOCK();
+ cap = crypto_lookup_kdriver(krp);
+ if (cap == NULL) {
+ CRYPTO_DRIVER_UNLOCK();
+ krp->krp_status = ENODEV;
crypto_kdone(krp);
+ return (0);
}
- return 0;
+
+ /*
+ * If the device is blocked, return ERESTART to requeue it.
+ */
+ if (cap->cc_kqblocked) {
+ /*
+ * XXX: Previously this set krp_status to ERESTART and
+ * invoked crypto_kdone but the caller would still
+ * requeue it.
+ */
+ CRYPTO_DRIVER_UNLOCK();
+ return (ERESTART);
+ }
+
+ cap->cc_koperations++;
+ CRYPTO_DRIVER_UNLOCK();
+ error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
+ if (error == ERESTART) {
+ CRYPTO_DRIVER_LOCK();
+ cap->cc_koperations--;
+ CRYPTO_DRIVER_UNLOCK();
+ return (error);
+ }
+
+ KASSERT(error == 0, ("error %d returned from crypto_kprocess", error));
+ return (0);
}
#ifdef CRYPTO_TIMING
@@ -1204,13 +1636,10 @@
{
struct cryptocap *cap;
struct cryptop *crp;
- int hid, result;
+ int result;
crp = (struct cryptop *)ctx;
-
- hid = crypto_ses2hid(crp->crp_session);
- cap = crypto_checkdriver(hid);
-
+ cap = crp->crp_session->cap;
result = crypto_invoke(cap, crp, 0);
if (result == ERESTART)
crypto_batch_enqueue(crp);
@@ -1226,14 +1655,15 @@
KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
KASSERT(crp->crp_callback != NULL,
("%s: crp->crp_callback == NULL", __func__));
- KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
+ KASSERT(crp->crp_session != NULL,
+ ("%s: crp->crp_session == NULL", __func__));
#ifdef CRYPTO_TIMING
if (crypto_timing)
crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
#endif
if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
- struct cryptodesc *crd;
+ struct crypto_session_params csp;
crypto_session_t nses;
/*
@@ -1242,14 +1672,32 @@
*
* XXX: What if there are more already queued requests for this
* session?
+ *
+ * XXX: Real solution is to make sessions refcounted
+ * and force callers to hold a reference when
+ * assigning to crp_session. Could maybe change
+ * crypto_getreq to accept a session pointer to make
+ * that work. Alternatively, we could abandon the
+ * notion of rewriting crp_session in requests forcing
+ * the caller to deal with allocating a new session.
+ * Perhaps provide a method to allow a crp's session to
+ * be swapped that callers could use.
*/
+ csp = crp->crp_session->csp;
crypto_freesession(crp->crp_session);
- for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
- crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
-
- /* XXX propagate flags from initial session? */
- if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI),
+ /*
+ * XXX: Key pointers may no longer be valid. If we
+ * really want to support this we need to define the
+ * KPI such that 'csp' is required to be valid for the
+ * duration of a session by the caller perhaps.
+ *
+ * XXX: If the keys have been changed this will reuse
+ * the old keys. This probably suggests making
+ * rekeying more explicit and updating the key
+ * pointers in 'csp' when the keys change.
+ */
+ if (crypto_newsession(&nses, &csp,
CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
crp->crp_session = nses;
@@ -1264,13 +1712,9 @@
}
}
-/*
- * Release a set of crypto descriptors.
- */
void
crypto_freereq(struct cryptop *crp)
{
- struct cryptodesc *crd;
if (crp == NULL)
return;
@@ -1300,36 +1744,18 @@
}
#endif
- while ((crd = crp->crp_desc) != NULL) {
- crp->crp_desc = crd->crd_next;
- uma_zfree(cryptodesc_zone, crd);
- }
uma_zfree(cryptop_zone, crp);
}
-/*
- * Acquire a set of crypto descriptors.
- */
struct cryptop *
-crypto_getreq(int num)
+crypto_getreq(crypto_session_t cses, int how)
{
- struct cryptodesc *crd;
struct cryptop *crp;
- crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
- if (crp != NULL) {
- while (num--) {
- crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
- if (crd == NULL) {
- crypto_freereq(crp);
- return NULL;
- }
-
- crd->crd_next = crp->crp_desc;
- crp->crp_desc = crd;
- }
- }
- return crp;
+ MPASS(how == M_WAITOK || how == M_NOWAIT);
+ crp = uma_zalloc(cryptop_zone, how | M_ZERO);
+ crp->crp_session = cses;
+ return (crp);
}
/*
@@ -1432,15 +1858,14 @@
if (krp->krp_status != 0)
cryptostats.cs_kerrs++;
CRYPTO_DRIVER_LOCK();
- /* XXX: What if driver is loaded in the meantime? */
- if (krp->krp_hid < crypto_drivers_num) {
- cap = &crypto_drivers[krp->krp_hid];
- KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
- cap->cc_koperations--;
- if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
- crypto_remove(cap);
- }
+ cap = krp->krp_cap;
+ KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
+ cap->cc_koperations--;
+ if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
+ wakeup(cap);
CRYPTO_DRIVER_UNLOCK();
+ krp->krp_cap = NULL;
+ cap_rele(cap);
ret_worker = CRYPTO_RETW(0);
@@ -1457,11 +1882,12 @@
int hid, kalg, feat = 0;
CRYPTO_DRIVER_LOCK();
- for (hid = 0; hid < crypto_drivers_num; hid++) {
- const struct cryptocap *cap = &crypto_drivers[hid];
+ for (hid = 0; hid < crypto_drivers_size; hid++) {
+ const struct cryptocap *cap = crypto_drivers[hid];
- if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
- !crypto_devallowsoft) {
+ if (cap == NULL ||
+ ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
+ !crypto_devallowsoft)) {
continue;
}
for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
@@ -1500,7 +1926,6 @@
struct cryptop *crp, *submit;
struct cryptkop *krp;
struct cryptocap *cap;
- u_int32_t hid;
int result, hint;
#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
@@ -1517,15 +1942,14 @@
submit = NULL;
hint = 0;
TAILQ_FOREACH(crp, &crp_q, crp_next) {
- hid = crypto_ses2hid(crp->crp_session);
- cap = crypto_checkdriver(hid);
+ cap = crp->crp_session->cap;
/*
* Driver cannot disappeared when there is an active
* session.
*/
KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
__func__, __LINE__));
- if (cap == NULL || cap->cc_dev == NULL) {
+ if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
/* Op needs to be migrated, process it. */
if (submit == NULL)
submit = crp;
@@ -1541,7 +1965,7 @@
* better to just use a per-driver
* queue instead.
*/
- if (crypto_ses2hid(submit->crp_session) == hid)
+ if (submit->crp_session->cap == cap)
hint = CRYPTO_HINT_MORE;
break;
} else {
@@ -1554,11 +1978,12 @@
}
if (submit != NULL) {
TAILQ_REMOVE(&crp_q, submit, crp_next);
- hid = crypto_ses2hid(submit->crp_session);
- cap = crypto_checkdriver(hid);
+ cap = submit->crp_session->cap;
KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
__func__, __LINE__));
+ CRYPTO_Q_UNLOCK();
result = crypto_invoke(cap, submit, hint);
+ CRYPTO_Q_LOCK();
if (result == ERESTART) {
/*
* The driver ran out of resources, mark the
@@ -1569,8 +1994,7 @@
* at the front. This should be ok; putting
* it at the end does not work.
*/
- /* XXX validate sid again? */
- crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1;
+ cap->cc_qblocked = 1;
TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
cryptostats.cs_blocks++;
}
@@ -1578,19 +2002,15 @@
/* As above, but for key ops */
TAILQ_FOREACH(krp, &crp_kq, krp_next) {
- cap = crypto_checkdriver(krp->krp_hid);
- if (cap == NULL || cap->cc_dev == NULL) {
+ cap = krp->krp_cap;
+ if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
/*
- * Operation needs to be migrated, invalidate
- * the assigned device so it will reselect a
- * new one below. Propagate the original
- * crid selection flags if supplied.
+ * Operation needs to be migrated,
+ * clear krp_cap so a new driver is
+ * selected.
*/
- krp->krp_hid = krp->krp_crid &
- (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
- if (krp->krp_hid == 0)
- krp->krp_hid =
- CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
+ krp->krp_cap = NULL;
+ cap_rele(cap);
break;
}
if (!cap->cc_kqblocked)
@@ -1598,7 +2018,9 @@
}
if (krp != NULL) {
TAILQ_REMOVE(&crp_kq, krp, krp_next);
- result = crypto_kinvoke(krp, krp->krp_hid);
+ CRYPTO_Q_UNLOCK();
+ result = crypto_kinvoke(krp);
+ CRYPTO_Q_LOCK();
if (result == ERESTART) {
/*
* The driver ran out of resources, mark the
@@ -1609,8 +2031,7 @@
* at the front. This should be ok; putting
* it at the end does not work.
*/
- /* XXX validate sid again? */
- crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
+ krp->krp_cap->cc_kqblocked = 1;
TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
cryptostats.cs_kblocks++;
}
@@ -1731,9 +2152,9 @@
, "QB"
, "KB"
);
- for (hid = 0; hid < crypto_drivers_num; hid++) {
- const struct cryptocap *cap = &crypto_drivers[hid];
- if (cap->cc_dev == NULL)
+ for (hid = 0; hid < crypto_drivers_size; hid++) {
+ const struct cryptocap *cap = crypto_drivers[hid];
+ if (cap == NULL)
continue;
db_printf("%-12s %4u %4u %08x %2u %2u\n"
, device_get_nameunit(cap->cc_dev)
@@ -1756,15 +2177,15 @@
db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
"HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
- "Desc", "Callback");
+ "Device", "Callback");
TAILQ_FOREACH(crp, &crp_q, crp_next) {
db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
- , (int) crypto_ses2hid(crp->crp_session)
+ , crp->crp_session->cap->cc_hid
, (int) crypto_ses2caps(crp->crp_session)
, crp->crp_ilen, crp->crp_olen
, crp->crp_etype
, crp->crp_flags
- , crp->crp_desc
+ , device_get_nameunit(crp->crp_session->cap->cc_dev)
, crp->crp_callback
);
}
@@ -1775,7 +2196,7 @@
TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
db_printf("%8td %4u %4u %04x %8p\n"
, CRYPTO_RETW_ID(ret_worker)
- , (int) crypto_ses2hid(crp->crp_session)
+ , crp->crp_session->cap->cc_hid
, crp->crp_etype
, crp->crp_flags
, crp->crp_callback
Index: sys/opencrypto/cryptodev.h
===================================================================
--- sys/opencrypto/cryptodev.h
+++ sys/opencrypto/cryptodev.h
@@ -71,7 +71,6 @@
/* Some initial values */
#define CRYPTO_DRIVERS_INITIAL 4
-#define CRYPTO_SW_SESSIONS 32
/* Hash values */
#define NULL_HASH_LEN 16
@@ -189,11 +188,13 @@
#define CRYPTO_CAMELLIA_CBC 21
#define CRYPTO_AES_XTS 22
#define CRYPTO_AES_ICM 23 /* commonly known as CTR mode */
-#define CRYPTO_AES_NIST_GMAC 24 /* cipher side */
+#define CRYPTO_AES_NIST_GMAC 24 /* GMAC only */
#define CRYPTO_AES_NIST_GCM_16 25 /* 16 byte ICV */
+#ifdef _KERNEL
#define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */
#define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */
#define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */
+#endif
#define CRYPTO_BLAKE2B 29 /* Blake2b hash */
#define CRYPTO_BLAKE2S 30 /* Blake2s hash */
#define CRYPTO_CHACHA20 31 /* Chacha20 stream cipher */
@@ -378,6 +379,13 @@
#ifdef _KERNEL
+/*
+ * Return values for cryptodev_probesession methods.
+ */
+#define CRYPTODEV_PROBE_HARDWARE (-100)
+#define CRYPTODEV_PROBE_ACCEL_SOFTWARE (-200)
+#define CRYPTODEV_PROBE_SOFTWARE (-500)
+
#if 0
#define CRYPTDEB(s, ...) do { \
printf("%s:%d: " s "\n", __FILE__, __LINE__, ## __VA_ARGS__); \
@@ -386,40 +394,29 @@
#define CRYPTDEB(...) do { } while (0)
#endif
-/* Standard initialization structure beginning */
-struct cryptoini {
- int cri_alg; /* Algorithm to use */
- int cri_klen; /* Key length, in bits */
- int cri_mlen; /* Number of bytes we want from the
- entire hash. 0 means all. */
- caddr_t cri_key; /* key to use */
- u_int8_t cri_iv[EALG_MAX_BLOCK_LEN]; /* IV to use */
- struct cryptoini *cri_next;
-};
+struct crypto_session_params {
+ int csp_mode; /* Type of operations to perform. */
-/* Describe boundaries of a single crypto operation */
-struct cryptodesc {
- int crd_skip; /* How many bytes to ignore from start */
- int crd_len; /* How many bytes to process */
- int crd_inject; /* Where to inject results, if applicable */
- int crd_flags;
+#define CSP_MODE_NONE 0
+#define CSP_MODE_COMPRESS 1 /* Compression/decompression. */
+#define CSP_MODE_CIPHER 2 /* Encrypt/decrypt. */
+#define CSP_MODE_DIGEST 3 /* Compute/verify digest. */
+#define CSP_MODE_AEAD 4 /* Combined auth/encryption. */
+#define CSP_MODE_ETA 5 /* IPsec style encrypt-then-auth */
-#define CRD_F_ENCRYPT 0x01 /* Set when doing encryption */
-#define CRD_F_IV_PRESENT 0x02 /* When encrypting, IV is already in
- place, so don't copy. */
-#define CRD_F_IV_EXPLICIT 0x04 /* IV explicitly provided */
-#define CRD_F_DSA_SHA_NEEDED 0x08 /* Compute SHA-1 of buffer for DSA */
-#define CRD_F_COMP 0x0f /* Set when doing compression */
-#define CRD_F_KEY_EXPLICIT 0x10 /* Key explicitly provided */
+ int csp_flags;
- struct cryptoini CRD_INI; /* Initialization/context data */
-#define crd_esn CRD_INI.cri_esn
-#define crd_iv CRD_INI.cri_iv
-#define crd_key CRD_INI.cri_key
-#define crd_alg CRD_INI.cri_alg
-#define crd_klen CRD_INI.cri_klen
+ int csp_ivlen; /* IV length in bytes. */
- struct cryptodesc *crd_next;
+ int csp_cipher_alg;
+ int csp_cipher_klen; /* Key length in bytes. */
+ const void *csp_cipher_key;
+
+ int csp_auth_alg;
+ int csp_auth_klen; /* Key length in bytes. */
+ const void *csp_auth_key;
+ int csp_auth_mlen; /* Number of digest bytes to use.
+ 0 means all. */
};
/* Structure describing complete operation */
@@ -444,8 +441,6 @@
*/
int crp_flags;
-#define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains */
-#define CRYPTO_F_IOV 0x0002 /* Input/output are uio */
#define CRYPTO_F_BATCH 0x0008 /* Batch op if possible */
#define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */
#define CRYPTO_F_DONE 0x0020 /* Operation completed */
@@ -458,14 +453,35 @@
* order there are submitted. Applied only
* if CRYPTO_F_ASYNC flags is set
*/
+#define CRYPTO_F_IV_SEPARATE 0x0200 /* Use crp_iv[] as IV. */
+#define CRYPTO_F_IV_GENERATE 0x0400 /* Generate a random IV and store. */
+
+ int crp_op;
union {
caddr_t crp_buf; /* Data to be processed */
struct mbuf *crp_mbuf;
struct uio *crp_uio;
};
- void * crp_opaque; /* Opaque pointer, passed along */
- struct cryptodesc *crp_desc; /* Linked list of processing descriptors */
+ int crp_buf_type; /* Which union member describes data. */
+
+ int crp_aad_start; /* Location of AAD. */
+ int crp_aad_length; /* 0 => no AAD. */
+ int crp_iv_start; /* Location of IV. IV length is from
+ * the session.
+ */
+ int crp_payload_start; /* Location of ciphertext. */
+ int crp_payload_length;
+ int crp_digest_start; /* Location of MAC/tag. Length is
+ * from the session.
+ */
+
+ uint8_t crp_iv[EALG_MAX_BLOCK_LEN]; /* IV if IV_SEPARATE. */
+
+ const void *crp_cipher_key; /* New cipher key if non-NULL. */
+ const void *crp_auth_key; /* New auth key if non-NULL. */
+
+ void *crp_opaque; /* Opaque pointer, passed along */
int (*crp_callback)(struct cryptop *); /* Callback function */
@@ -485,11 +501,18 @@
(crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER)
#define CRYPTO_BUF_CONTIG 0x0
-#define CRYPTO_BUF_IOV 0x1
+#define CRYPTO_BUF_UIO 0x1
#define CRYPTO_BUF_MBUF 0x2
-#define CRYPTO_OP_DECRYPT 0x0
-#define CRYPTO_OP_ENCRYPT 0x1
+/* Flags in crp_op. */
+#define CRYPTO_OP_DECRYPT 0x0
+#define CRYPTO_OP_ENCRYPT 0x1
+#define CRYPTO_OP_IS_ENCRYPT(op) ((op) & CRYPTO_OP_ENCRYPT)
+#define CRYPTO_OP_COMPUTE_DIGEST 0x0
+#define CRYPTO_OP_VERIFY_DIGEST 0x2
+#define CRYPTO_OP_DECOMPRESS CRYPTO_OP_DECRYPT
+#define CRYPTO_OP_COMPRESS CRYPTO_OP_ENCRYPT
+#define CRYPTO_OP_IS_COMPRESS(op) ((op) & CRYPTO_OP_COMPRESS)
/*
* Hints passed to process methods.
@@ -504,18 +527,24 @@
u_short krp_iparams; /* # of input parameters */
u_short krp_oparams; /* # of output parameters */
u_int krp_crid; /* desired device, etc. */
- u_int32_t krp_hid;
+ uint32_t krp_hid; /* device used */
struct crparam krp_param[CRK_MAXPARAM]; /* kvm */
- int (*krp_callback)(struct cryptkop *);
+ void (*krp_callback)(struct cryptkop *);
+ struct cryptocap *krp_cap;
};
uint32_t crypto_ses2hid(crypto_session_t crypto_session);
uint32_t crypto_ses2caps(crypto_session_t crypto_session);
void *crypto_get_driver_session(crypto_session_t crypto_session);
+const struct crypto_session_params *crypto_get_params(
+ crypto_session_t crypto_session);
+struct auth_hash *crypto_auth_hash(const struct crypto_session_params *csp);
+struct enc_xform *crypto_cipher(const struct crypto_session_params *csp);
MALLOC_DECLARE(M_CRYPTO_DATA);
-extern int crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int hard);
+extern int crypto_newsession(crypto_session_t *cses,
+ const struct crypto_session_params *params, int hard);
extern void crypto_freesession(crypto_session_t cses);
#define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE
#define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE
@@ -525,10 +554,7 @@
extern int crypto_find_driver(const char *);
extern device_t crypto_find_device_byhid(int hid);
extern int crypto_getcaps(int hid);
-extern int crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
- u_int32_t flags);
extern int crypto_kregister(u_int32_t, int, u_int32_t);
-extern int crypto_unregister(u_int32_t driverid, int alg);
extern int crypto_unregister_all(u_int32_t driverid);
extern int crypto_dispatch(struct cryptop *crp);
extern int crypto_kdispatch(struct cryptkop *);
@@ -540,17 +566,30 @@
extern int crypto_getfeat(int *);
extern void crypto_freereq(struct cryptop *crp);
-extern struct cryptop *crypto_getreq(int num);
+extern struct cryptop *crypto_getreq(crypto_session_t cses, int how);
extern int crypto_usercrypto; /* userland may do crypto requests */
extern int crypto_userasymcrypto; /* userland may do asym crypto reqs */
extern int crypto_devallowsoft; /* only use hardware crypto */
+/* Helper routines for drivers to initialize auth contexts for HMAC. */
+struct auth_hash;
+
+void hmac_init_ipad(struct auth_hash *axf, const char *key, int klen,
+ void *auth_ctx);
+void hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
+ void *auth_ctx);
+
/*
* Crypto-related utility routines used mainly by drivers.
*
* XXX these don't really belong here; but for now they're
* kept apart from the rest of the system.
+ *
+ * Similar to m_copyback/data, *_copyback copy data from the 'src'
+ * buffer into the crypto request's data buffer while *_copydata copy
+ * data from the crypto request's data buffer into the the 'dst'
+ * buffer.
*/
struct uio;
extern void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp);
@@ -564,14 +603,13 @@
extern int crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr,
int *cnt, int *allocated);
-extern void crypto_copyback(int flags, caddr_t buf, int off, int size,
- c_caddr_t in);
-extern void crypto_copydata(int flags, caddr_t buf, int off, int size,
- caddr_t out);
-extern int crypto_apply(int flags, caddr_t buf, int off, int len,
+void crypto_copyback(struct cryptop *crp, int off, int size,
+ const void *src);
+void crypto_copydata(struct cryptop *crp, int off, int size, void *dst);
+int crypto_apply(struct cryptop *crp, int off, int len,
int (*f)(void *, void *, u_int), void *arg);
-
-extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t);
+void *crypto_contiguous_subsegment(struct cryptop *crp, size_t skip,
+ size_t len);
#endif /* _KERNEL */
#endif /* _CRYPTO_CRYPTO_H_ */
Index: sys/opencrypto/cryptodev.c
===================================================================
--- sys/opencrypto/cryptodev.c
+++ sys/opencrypto/cryptodev.c
@@ -51,7 +51,6 @@
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/errno.h>
-#include <sys/uio.h>
#include <sys/random.h>
#include <sys/conf.h>
#include <sys/kernel.h>
@@ -270,23 +269,19 @@
u_int32_t ses;
struct mtx lock; /* for op submission */
- u_int32_t cipher;
struct enc_xform *txform;
- u_int32_t mac;
- struct auth_hash *thash;
+ int hashsize;
+ int ivsize;
+ int mode;
- caddr_t key;
- int keylen;
-
- caddr_t mackey;
- int mackeylen;
+ void *key;
+ void *mackey;
};
struct cryptop_data {
struct csession *cse;
- struct iovec iovec[1];
- struct uio uio;
+ char *buf;
bool done;
};
@@ -326,9 +321,9 @@
static struct csession *csefind(struct fcrypt *, u_int);
static bool csedelete(struct fcrypt *, u_int);
-static struct csession *csecreate(struct fcrypt *, crypto_session_t, caddr_t,
- u_int64_t, caddr_t, u_int64_t, u_int32_t, u_int32_t, struct enc_xform *,
- struct auth_hash *);
+static struct csession *csecreate(struct fcrypt *, crypto_session_t,
+ struct crypto_session_params *, struct enc_xform *, void *,
+ struct auth_hash *, void *);
static void csefree(struct csession *);
static int cryptodev_op(struct csession *, struct crypt_op *,
@@ -375,7 +370,7 @@
struct thread *td)
{
#define SES2(p) ((struct session2_op *)p)
- struct cryptoini cria, crie;
+ struct crypto_session_params csp;
struct fcrypt *fcr = fp->f_data;
struct csession *cse;
struct session_op *sop;
@@ -383,6 +378,8 @@
struct crypt_aead *caead;
struct enc_xform *txform = NULL;
struct auth_hash *thash = NULL;
+ void *key = NULL;
+ void *mackey = NULL;
struct crypt_kop *kop;
crypto_session_t cses;
u_int32_t ses;
@@ -487,18 +484,37 @@
case CRYPTO_RIPEMD160_HMAC:
thash = &auth_hash_hmac_ripemd_160;
break;
+#ifdef COMPAT_FREEBSD12
case CRYPTO_AES_128_NIST_GMAC:
- thash = &auth_hash_nist_gmac_aes_128;
- break;
case CRYPTO_AES_192_NIST_GMAC:
- thash = &auth_hash_nist_gmac_aes_192;
- break;
case CRYPTO_AES_256_NIST_GMAC:
- thash = &auth_hash_nist_gmac_aes_256;
+ /* Should always be paired with GCM. */
+ if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
+ CRYPTDEB("GMAC without GCM");
+ return (EINVAL);
+ }
+ break;
+#endif
+ case CRYPTO_AES_NIST_GMAC:
+ switch (sop->mackeylen * 8) {
+ case 128:
+ thash = &auth_hash_nist_gmac_aes_128;
+ break;
+ case 192:
+ thash = &auth_hash_nist_gmac_aes_192;
+ break;
+ case 256:
+ thash = &auth_hash_nist_gmac_aes_256;
+ break;
+ default:
+ CRYPTDEB("invalid GMAC key length");
+ SDT_PROBE1(opencrypto, dev, ioctl, error,
+ __LINE__);
+ return (EINVAL);
+ }
break;
-
case CRYPTO_AES_CCM_CBC_MAC:
- switch (sop->keylen) {
+ switch (sop->mackeylen) {
case 16:
thash = &auth_hash_ccm_cbc_mac_128;
break;
@@ -554,12 +570,52 @@
return (EINVAL);
}
- bzero(&crie, sizeof(crie));
- bzero(&cria, sizeof(cria));
+ if (txform == NULL && thash == NULL)
+ return (EINVAL);
+
+ memset(&csp, 0, sizeof(csp));
+
+ if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
+ switch (sop->mac) {
+#ifdef COMPAT_FREEBSD12
+ case CRYPTO_AES_128_NIST_GMAC:
+ case CRYPTO_AES_192_NIST_GMAC:
+ case CRYPTO_AES_256_NIST_GMAC:
+ if (sop->keylen != sop->mackeylen)
+ return (EINVAL);
+ break;
+#endif
+ case 0:
+ break;
+ default:
+ return (EINVAL);
+ }
+ csp.csp_mode = CSP_MODE_AEAD;
+ } else if (sop->cipher == CRYPTO_AES_CCM_16) {
+ switch (sop->mac) {
+#ifdef COMPAT_FREEBSD12
+ case CRYPTO_AES_CCM_CBC_MAC:
+ if (sop->keylen != sop->mackeylen)
+ return (EINVAL);
+ thash = NULL;
+ break;
+#endif
+ case 0:
+ break;
+ default:
+ return (EINVAL);
+ }
+ csp.csp_mode = CSP_MODE_AEAD;
+ } else if (txform && thash)
+ csp.csp_mode = CSP_MODE_ETA;
+ else if (txform)
+ csp.csp_mode = CSP_MODE_CIPHER;
+ else
+ csp.csp_mode = CSP_MODE_DIGEST;
if (txform) {
- crie.cri_alg = txform->type;
- crie.cri_klen = sop->keylen * 8;
+ csp.csp_cipher_alg = txform->type;
+ csp.csp_cipher_klen = sop->keylen;
if (sop->keylen > txform->maxkey ||
sop->keylen < txform->minkey) {
CRYPTDEB("invalid cipher parameters");
@@ -569,22 +625,21 @@
goto bail;
}
- crie.cri_key = malloc(crie.cri_klen / 8,
- M_XDATA, M_WAITOK);
- if ((error = copyin(sop->key, crie.cri_key,
- crie.cri_klen / 8))) {
+ key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
+ error = copyin(sop->key, key, csp.csp_cipher_klen);
+ if (error) {
CRYPTDEB("invalid key");
SDT_PROBE1(opencrypto, dev, ioctl, error,
__LINE__);
goto bail;
}
- if (thash)
- crie.cri_next = &cria;
+ csp.csp_cipher_key = key;
+ csp.csp_ivlen = txform->ivsize;
}
if (thash) {
- cria.cri_alg = thash->type;
- cria.cri_klen = sop->mackeylen * 8;
+ csp.csp_auth_alg = thash->type;
+ csp.csp_auth_klen = sop->mackeylen;
if (sop->mackeylen > thash->keysize ||
sop->mackeylen < 0) {
CRYPTDEB("invalid mac key length");
@@ -594,17 +649,24 @@
goto bail;
}
- if (cria.cri_klen) {
- cria.cri_key = malloc(cria.cri_klen / 8,
- M_XDATA, M_WAITOK);
- if ((error = copyin(sop->mackey, cria.cri_key,
- cria.cri_klen / 8))) {
+ if (csp.csp_auth_klen) {
+ mackey = malloc(csp.csp_auth_klen, M_XDATA,
+ M_WAITOK);
+ error = copyin(sop->mackey, mackey,
+ csp.csp_auth_klen);
+ if (error) {
CRYPTDEB("invalid mac key");
SDT_PROBE1(opencrypto, dev, ioctl,
error, __LINE__);
goto bail;
}
+ csp.csp_auth_key = mackey;
}
+
+ if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
+ csp.csp_ivlen = AES_GCM_IV_LEN;
+ if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
+ csp.csp_ivlen = AES_CCM_IV_LEN;
}
/* NB: CIOCGSESSION2 has the crid */
@@ -623,16 +685,14 @@
}
} else
crid = CRYPTOCAP_F_HARDWARE;
- error = crypto_newsession(&cses, (txform ? &crie : &cria), crid);
+ error = crypto_newsession(&cses, &csp, crid);
if (error) {
CRYPTDEB("crypto_newsession");
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
- cse = csecreate(fcr, cses, crie.cri_key, crie.cri_klen,
- cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform,
- thash);
+ cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey);
if (cse == NULL) {
crypto_freesession(cses);
@@ -652,10 +712,8 @@
}
bail:
if (error) {
- if (crie.cri_key)
- free(crie.cri_key, M_XDATA);
- if (cria.cri_key)
- free(cria.cri_key, M_XDATA);
+ free(key, M_XDATA);
+ free(mackey, M_XDATA);
}
#ifdef COMPAT_FREEBSD32
else {
@@ -773,20 +831,11 @@
cod_alloc(struct csession *cse, size_t len, struct thread *td)
{
struct cryptop_data *cod;
- struct uio *uio;
cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
cod->cse = cse;
- uio = &cod->uio;
- uio->uio_iov = cod->iovec;
- uio->uio_iovcnt = 1;
- uio->uio_resid = len;
- uio->uio_segflg = UIO_SYSSPACE;
- uio->uio_rw = UIO_WRITE;
- uio->uio_td = td;
- uio->uio_iov[0].iov_len = len;
- uio->uio_iov[0].iov_base = malloc(len, M_XDATA, M_WAITOK);
+ cod->buf = malloc(len, M_XDATA, M_WAITOK);
return (cod);
}
@@ -794,7 +843,7 @@
cod_free(struct cryptop_data *cod)
{
- free(cod->uio.uio_iov[0].iov_base, M_XDATA);
+ free(cod->buf, M_XDATA);
free(cod, M_XDATA);
}
@@ -803,8 +852,10 @@
{
static struct timeval arc4warn, blfwarn, castwarn, deswarn, md5warn;
static struct timeval skipwarn, tdeswarn;
+ const struct crypto_session_params *csp;
- switch (cse->cipher) {
+ csp = crypto_get_params(cse->cses);
+ switch (csp->csp_cipher_alg) {
case CRYPTO_DES_CBC:
if (ratecheck(&deswarn, &warninterval))
gone_in(13, "DES cipher via /dev/crypto");
@@ -831,7 +882,7 @@
break;
}
- switch (cse->mac) {
+ switch (csp->csp_auth_alg) {
case CRYPTO_MD5_HMAC:
if (ratecheck(&md5warn, &warninterval))
gone_in(13, "MD5-HMAC authenticator via /dev/crypto");
@@ -848,7 +899,6 @@
{
struct cryptop_data *cod = NULL;
struct cryptop *crp = NULL;
- struct cryptodesc *crde = NULL, *crda = NULL;
int error;
if (cop->len > 256*1024-4) {
@@ -863,106 +913,135 @@
}
}
- if (cse->thash)
- cod = cod_alloc(cse, cop->len + cse->thash->hashsize, td);
- else
- cod = cod_alloc(cse, cop->len, td);
-
- crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL));
- if (crp == NULL) {
+ if (cop->mac && cse->hashsize == 0) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- error = ENOMEM;
+ error = EINVAL;
goto bail;
}
- if (cse->thash && cse->txform) {
- if (cop->flags & COP_F_CIPHER_FIRST) {
- crde = crp->crp_desc;
- crda = crde->crd_next;
- } else {
- crda = crp->crp_desc;
- crde = crda->crd_next;
+ /*
+ * The COP_F_CIPHER_FIRST flag predates explicit session
+ * modes, but the only way it was used was for EtA so allow it
+ * as long as it is consistent with EtA.
+ */
+ if (cop->flags & COP_F_CIPHER_FIRST) {
+ if (cop->op != COP_ENCRYPT) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ return (EINVAL);
}
- } else if (cse->thash) {
- crda = crp->crp_desc;
- } else if (cse->txform) {
- crde = crp->crp_desc;
- } else {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- error = EINVAL;
- goto bail;
}
- if ((error = copyin(cop->src, cod->uio.uio_iov[0].iov_base,
- cop->len))) {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- goto bail;
- }
+ cod = cod_alloc(cse, cop->len + cse->hashsize, td);
- if (crda) {
- crda->crd_skip = 0;
- crda->crd_len = cop->len;
- crda->crd_inject = cop->len;
+ crp = crypto_getreq(cse->cses, M_WAITOK);
- crda->crd_alg = cse->mac;
- crda->crd_key = cse->mackey;
- crda->crd_klen = cse->mackeylen * 8;
+ error = copyin(cop->src, cod->buf, cop->len);
+ if (error) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ goto bail;
}
+ crp->crp_payload_start = 0;
+ crp->crp_payload_length = cop->len;
+ if (cse->hashsize)
+ crp->crp_digest_start = cop->len;
- if (crde) {
- if (cop->op == COP_ENCRYPT)
- crde->crd_flags |= CRD_F_ENCRYPT;
- else
- crde->crd_flags &= ~CRD_F_ENCRYPT;
- crde->crd_len = cop->len;
- crde->crd_inject = 0;
-
- crde->crd_alg = cse->cipher;
- crde->crd_key = cse->key;
- crde->crd_klen = cse->keylen * 8;
+ switch (cse->mode) {
+ case CSP_MODE_COMPRESS:
+ switch (cop->op) {
+ case COP_ENCRYPT:
+ crp->crp_op = CRYPTO_OP_COMPRESS;
+ break;
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_DECOMPRESS;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ case CSP_MODE_CIPHER:
+ switch (cop->op) {
+ case COP_ENCRYPT:
+ crp->crp_op = CRYPTO_OP_ENCRYPT;
+ break;
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_DECRYPT;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ case CSP_MODE_DIGEST:
+ switch (cop->op) {
+ case 0:
+ case COP_ENCRYPT:
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ case CSP_MODE_ETA:
+ switch (cop->op) {
+ case COP_ENCRYPT:
+ crp->crp_op = CRYPTO_OP_ENCRYPT |
+ CRYPTO_OP_COMPUTE_DIGEST;
+ break;
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_DECRYPT |
+ CRYPTO_OP_VERIFY_DIGEST;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
}
- crp->crp_ilen = cop->len;
- crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
- | (cop->flags & COP_F_BATCH);
- crp->crp_uio = &cod->uio;
+ crp->crp_ilen = cop->len + cse->hashsize;
+ crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
+ crp->crp_buf = cod->buf;
+ crp->crp_buf_type = CRYPTO_BUF_CONTIG;
crp->crp_callback = cryptodev_cb;
- crp->crp_session = cse->cses;
crp->crp_opaque = cod;
if (cop->iv) {
- if (crde == NULL) {
+ if (cse->ivsize == 0) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
error = EINVAL;
goto bail;
}
- if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- error = EINVAL;
- goto bail;
- }
- if ((error = copyin(cop->iv, crde->crd_iv,
- cse->txform->ivsize))) {
+ error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
- crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- crde->crd_skip = 0;
- } else if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
- crde->crd_skip = 0;
- } else if (crde) {
- crde->crd_flags |= CRD_F_IV_PRESENT;
- crde->crd_skip = cse->txform->ivsize;
- crde->crd_len -= cse->txform->ivsize;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+ } else if (cse->ivsize != 0) {
+ crp->crp_iv_start = 0;
+ crp->crp_payload_start += cse->ivsize;
+ crp->crp_payload_length -= cse->ivsize;
}
- if (cop->mac && crda == NULL) {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- error = EINVAL;
- goto bail;
+ if (cop->mac != NULL) {
+ error = copyin(cop->mac, cod->buf + cop->len, cse->hashsize);
+ if (error) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ goto bail;
+ }
}
cryptodev_warn(cse);
-
again:
/*
* Let the dispatch run unlocked, then, interlock against the
@@ -995,18 +1074,20 @@
goto bail;
}
- if (cop->dst &&
- (error = copyout(cod->uio.uio_iov[0].iov_base, cop->dst,
- cop->len))) {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- goto bail;
+ if (cop->dst != NULL) {
+ error = copyout(cod->buf, cop->dst, cop->len);
+ if (error) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ goto bail;
+ }
}
- if (cop->mac &&
- (error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base + cop->len,
- cop->mac, cse->thash->hashsize))) {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- goto bail;
+ if (cop->mac != NULL) {
+ error = copyout(cod->buf + cop->len, cop->mac, cse->hashsize);
+ if (error) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ goto bail;
+ }
}
bail:
@@ -1027,7 +1108,6 @@
{
struct cryptop_data *cod = NULL;
struct cryptop *crp = NULL;
- struct cryptodesc *crde = NULL, *crda = NULL;
int error;
if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
@@ -1035,99 +1115,122 @@
return (E2BIG);
}
- if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL ||
+ if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
(caead->len % cse->txform->blocksize) != 0) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
return (EINVAL);
}
- cod = cod_alloc(cse, caead->aadlen + caead->len + cse->thash->hashsize,
- td);
-
- crp = crypto_getreq(2);
- if (crp == NULL) {
- error = ENOMEM;
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- goto bail;
- }
-
+ /*
+ * The COP_F_CIPHER_FIRST flag predates explicit session
+ * modes, but the only way it was used was for EtA so allow it
+ * as long as it is consistent with EtA.
+ */
if (caead->flags & COP_F_CIPHER_FIRST) {
- crde = crp->crp_desc;
- crda = crde->crd_next;
- } else {
- crda = crp->crp_desc;
- crde = crda->crd_next;
+ if (caead->op != COP_ENCRYPT) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ return (EINVAL);
+ }
}
- if ((error = copyin(caead->aad, cod->uio.uio_iov[0].iov_base,
- caead->aadlen))) {
+ cod = cod_alloc(cse, caead->aadlen + caead->len + cse->hashsize, td);
+
+ crp = crypto_getreq(cse->cses, M_WAITOK);
+
+ error = copyin(caead->aad, cod->buf, caead->aadlen);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
+ crp->crp_aad_start = 0;
+ crp->crp_aad_length = caead->aadlen;
- if ((error = copyin(caead->src, (char *)cod->uio.uio_iov[0].iov_base +
- caead->aadlen, caead->len))) {
+ error = copyin(caead->src, cod->buf + caead->aadlen, caead->len);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
+ crp->crp_payload_start = caead->aadlen;
+ crp->crp_payload_length = caead->len;
+ crp->crp_digest_start = caead->aadlen + caead->len;
- /*
- * For GCM/CCM, crd_len covers only the AAD. For other ciphers
- * chained with an HMAC, crd_len covers both the AAD and the
- * cipher text.
- */
- crda->crd_skip = 0;
- if (cse->cipher == CRYPTO_AES_NIST_GCM_16 ||
- cse->cipher == CRYPTO_AES_CCM_16)
- crda->crd_len = caead->aadlen;
- else
- crda->crd_len = caead->aadlen + caead->len;
- crda->crd_inject = caead->aadlen + caead->len;
-
- crda->crd_alg = cse->mac;
- crda->crd_key = cse->mackey;
- crda->crd_klen = cse->mackeylen * 8;
-
- if (caead->op == COP_ENCRYPT)
- crde->crd_flags |= CRD_F_ENCRYPT;
- else
- crde->crd_flags &= ~CRD_F_ENCRYPT;
- crde->crd_skip = caead->aadlen;
- crde->crd_len = caead->len;
- crde->crd_inject = caead->aadlen;
-
- crde->crd_alg = cse->cipher;
- crde->crd_key = cse->key;
- crde->crd_klen = cse->keylen * 8;
+ switch (cse->mode) {
+ case CSP_MODE_AEAD:
+ switch (caead->op) {
+ case COP_ENCRYPT:
+ crp->crp_op = CRYPTO_OP_ENCRYPT |
+ CRYPTO_OP_COMPUTE_DIGEST;
+ break;
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_DECRYPT |
+ CRYPTO_OP_VERIFY_DIGEST;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ case CSP_MODE_ETA:
+ switch (caead->op) {
+ case COP_ENCRYPT:
+ crp->crp_op = CRYPTO_OP_ENCRYPT |
+ CRYPTO_OP_COMPUTE_DIGEST;
+ break;
+ case COP_DECRYPT:
+ crp->crp_op = CRYPTO_OP_DECRYPT |
+ CRYPTO_OP_VERIFY_DIGEST;
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
+ break;
+ default:
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ error = EINVAL;
+ goto bail;
+ }
- crp->crp_ilen = caead->aadlen + caead->len;
- crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
- | (caead->flags & COP_F_BATCH);
- crp->crp_uio = &cod->uio;
+ crp->crp_ilen = caead->aadlen + caead->len + cse->hashsize;
+ crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
+ crp->crp_buf = cod->buf;
+ crp->crp_buf_type = CRYPTO_BUF_CONTIG;
crp->crp_callback = cryptodev_cb;
- crp->crp_session = cse->cses;
crp->crp_opaque = cod;
if (caead->iv) {
- if (caead->ivlen > sizeof(crde->crd_iv)) {
+ /*
+ * Permit a 16-byte IV for AES-XTS, but only use the
+ * first 8 bytes as a block number.
+ */
+ if (cse->mode == CSP_MODE_ETA &&
+ caead->ivlen == AES_BLOCK_LEN &&
+ cse->ivsize == AES_XTS_IV_LEN)
+ caead->ivlen = AES_XTS_IV_LEN;
+
+ if (caead->ivlen != cse->ivsize) {
error = EINVAL;
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
- if ((error = copyin(caead->iv, crde->crd_iv, caead->ivlen))) {
+ error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
- crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+ crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
} else {
- crde->crd_flags |= CRD_F_IV_PRESENT;
- crde->crd_skip += cse->txform->ivsize;
- crde->crd_len -= cse->txform->ivsize;
+ crp->crp_iv_start = crp->crp_payload_start;
+ crp->crp_payload_start += cse->ivsize;
+ crp->crp_payload_length -= cse->ivsize;
}
- if ((error = copyin(caead->tag, (caddr_t)cod->uio.uio_iov[0].iov_base +
- caead->len + caead->aadlen, cse->thash->hashsize))) {
+ error = copyin(caead->tag, cod->buf + caead->len + caead->aadlen,
+ cse->hashsize);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
@@ -1164,15 +1267,18 @@
goto bail;
}
- if (caead->dst && (error = copyout(
- (caddr_t)cod->uio.uio_iov[0].iov_base + caead->aadlen, caead->dst,
- caead->len))) {
- SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
- goto bail;
+ if (caead->dst != NULL) {
+ error = copyout(cod->buf + caead->aadlen, caead->dst,
+ caead->len);
+ if (error) {
+ SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+ goto bail;
+ }
}
- if ((error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base +
- caead->aadlen + caead->len, caead->tag, cse->thash->hashsize))) {
+ error = copyout(cod->buf + caead->aadlen + caead->len, caead->tag,
+ cse->hashsize);
+ if (error) {
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
goto bail;
}
@@ -1202,13 +1308,11 @@
return (0);
}
-static int
-cryptodevkey_cb(void *op)
+static void
+cryptodevkey_cb(struct cryptkop *krp)
{
- struct cryptkop *krp = (struct cryptkop *) op;
wakeup_one(krp);
- return (0);
}
static int
@@ -1267,7 +1371,7 @@
krp->krp_oparams = kop->crk_oparams;
krp->krp_crid = kop->crk_crid;
krp->krp_status = 0;
- krp->krp_callback = (int (*) (struct cryptkop *)) cryptodevkey_cb;
+ krp->krp_callback = cryptodevkey_cb;
for (i = 0; i < CRK_MAXPARAM; i++) {
if (kop->crk_param[i].crp_nbits > 65536) {
@@ -1303,7 +1407,7 @@
goto fail;
}
- kop->crk_crid = krp->krp_crid; /* device that did the work */
+ kop->crk_crid = krp->krp_hid; /* device that did the work */
if (krp->krp_status != 0) {
error = krp->krp_status;
SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
@@ -1429,9 +1533,9 @@
}
struct csession *
-csecreate(struct fcrypt *fcr, crypto_session_t cses, caddr_t key, u_int64_t keylen,
- caddr_t mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac,
- struct enc_xform *txform, struct auth_hash *thash)
+csecreate(struct fcrypt *fcr, crypto_session_t cses,
+ struct crypto_session_params *csp, struct enc_xform *txform,
+ void *key, struct auth_hash *thash, void *mackey)
{
struct csession *cse;
@@ -1441,14 +1545,17 @@
mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
refcount_init(&cse->refs, 1);
cse->key = key;
- cse->keylen = keylen/8;
cse->mackey = mackey;
- cse->mackeylen = mackeylen/8;
+ cse->mode = csp->csp_mode;
cse->cses = cses;
- cse->cipher = cipher;
- cse->mac = mac;
cse->txform = txform;
- cse->thash = thash;
+ if (thash != NULL)
+ cse->hashsize = thash->hashsize;
+ else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
+ cse->hashsize = AES_GMAC_HASH_LEN;
+ else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
+ cse->hashsize = AES_CBC_MAC_HASH_LEN;
+ cse->ivsize = csp->csp_ivlen;
mtx_lock(&fcr->lock);
TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
cse->ses = fcr->sesn++;
Index: sys/opencrypto/cryptodev_if.m
===================================================================
--- sys/opencrypto/cryptodev_if.m
+++ sys/opencrypto/cryptodev_if.m
@@ -40,32 +40,138 @@
};
/**
- * Crypto driver method to initialize a new session object with the given
- * initialization parameters (cryptoini). The driver's session memory object
- * is already allocated and zeroed, like driver softcs. It is accessed with
+ * @brief Probe to see if a crypto driver supports a session.
+ *
+ * The crypto framework invokes this method on each crypto driver when
+ * creating a session for symmetric crypto operations to determine if
+ * the driver supports the algorithms and mode requested by the
+ * session.
+ *
+ * If the driver does not support a session with the requested
+ * parameters, this function should fail with an error.
+ *
+ * If the driver does support a session with the requested parameters,
+ * this function should return a negative value indicating the
+ * priority of this driver. These negative values should be derived
+ * from one of the CRYPTODEV_PROBE_* constants in
+ * <opencrypto/cryptodev.h>.
+ *
+ * This function's return value is similar to that used by
+ * DEVICE_PROBE(9). However, a return value of zero is not supported
+ * and should not be used.
+ *
+ * @param dev the crypto driver device
+ * @param csp crypto session parameters
+ *
+ * @retval negative if the driver supports this session - the
+ * least negative value is used to select the
+ * driver for the session
+ * @retval EINVAL if the driver does not support the session
+ * @retval positive if some other error occurs
+ */
+METHOD int probesession {
+ device_t dev;
+ const struct crypto_session_params *csp;
+};
+
+/**
+ * @brief Initialize a new crypto session object
+ *
+ * Invoked by the crypto framework to initialize driver-specific data
+ * for a crypto session. The framework allocates and zeroes the
+ * driver's per-session memory object prior to invoking this method.
+ * The driver is able to access it's per-session memory object via
* crypto_get_driver_session().
+ *
+ * @param dev the crypto driver device
+ * @param crypto_session session being initialized
+ * @param csp crypto session parameters
+ *
+ * @retval 0 success
+ * @retval non-zero if some kind of error occurred
*/
METHOD int newsession {
device_t dev;
crypto_session_t crypto_session;
- struct cryptoini *cri;
+ const struct crypto_session_params *csp;
};
/**
- * Optional crypto driver method to release any additional allocations. OCF
- * owns session memory itself; it is zeroed before release.
+ * @brief Destroy a crypto session object
+ *
+ * The crypto framework invokes this method when tearing down a crypto
+ * session. After this callback returns, the frame will explicitly
+ * zero and free the drvier's per-session memory object. If the
+ * driver requires additional actions to destroy a session, it should
+ * perform those in this method. If the driver does not require
+ * additional actions it does not need to provide an implementation of
+ * this method.
+ *
+ * @param dev the crypto driver device
+ * @param crypto_session session being destroyed
*/
METHOD void freesession {
device_t dev;
crypto_session_t crypto_session;
} DEFAULT null_freesession;
+/**
+ * @brief Perform a symmetric crypto operation
+ *
+ * The crypto framework invokes this method for each symmetric crypto
+ * operation performed on a session. A reference to the containing
+ * session is stored as a member of 'struct cryptop'. This routine
+ * should not block, but queue the operation if necessary.
+ *
+ * This method may return ERESTART to indicate that any internal
+ * queues are full so the operation should be queued in the crypto
+ * framework and retried in the future.
+ *
+ * To report errors with a crypto operation, 'crp_etype' should be set
+ * and the operation completed by calling 'crypto_done'. This method
+ * should then return zero.
+ *
+ * @param dev the crypto driver device
+ * @param op crypto operation to perform
+ * @param flags set to CRYPTO_HINT_MORE if additional symmetric
+ * crypto operations are queued for this driver;
+ * otherwise set to zero.
+ *
+ * @retval 0 success
+ * @retval ERESTART internal queue is full
+ */
METHOD int process {
device_t dev;
struct cryptop *op;
int flags;
};
+/**
+ * @brief Perform an asymmetric crypto operation
+ *
+ * The crypto framework invokes this method for each asymmetric crypto
+ * operation. Each asymmetric crypto operation should be
+ * self-contained and is not assicated with any persistent session.
+ * This routine should not block, but queue the operation if
+ * necessary.
+ *
+ * This method may return ERESTART to indicate that any internal
+ * queues are full so the operation should be queued in the crypto
+ * framework and retried in the future.
+ *
+ * To report errors with a crypto operation, 'krp_status' should be set
+ * and the operation completed by calling 'crypto_kdone'. This method
+ * should then return zero.
+ *
+ * @param dev the crypto driver device
+ * @param op crypto operation to perform
+ * @param flags set to CRYPTO_HINT_MORE if additional asymmetric
+ * crypto operations are queued for this driver;
+ * otherwise set to zero.
+ *
+ * @retval 0 success
+ * @retval ERESTART internal queue is full
+ */
METHOD int kprocess {
device_t dev;
struct cryptkop *op;
Index: sys/opencrypto/cryptosoft.h
===================================================================
--- sys/opencrypto/cryptosoft.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/* $FreeBSD$ */
-/* $OpenBSD: cryptosoft.h,v 1.10 2002/04/22 23:10:09 deraadt Exp $ */
-
-/*-
- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
- *
- * This code was written by Angelos D. Keromytis in Athens, Greece, in
- * February 2000. Network Security Technologies Inc. (NSTI) kindly
- * supported the development of this code.
- *
- * Copyright (c) 2000 Angelos D. Keromytis
- *
- * Permission to use, copy, and modify this software with or without fee
- * is hereby granted, provided that this entire notice is included in
- * all source code copies of any software which is or includes a copy or
- * modification of this software.
- *
- * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
- * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
- * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
- * PURPOSE.
- */
-
-#ifndef _CRYPTO_CRYPTOSOFT_H_
-#define _CRYPTO_CRYPTOSOFT_H_
-
-/* Software session entry */
-struct swcr_data {
- int sw_alg; /* Algorithm */
- union {
- struct {
- u_int8_t *SW_ictx;
- u_int8_t *SW_octx;
- u_int16_t SW_klen;
- u_int16_t SW_mlen;
- struct auth_hash *SW_axf;
- } SWCR_AUTH;
- struct {
- u_int8_t *SW_kschedule;
- struct enc_xform *SW_exf;
- } SWCR_ENC;
- struct {
- u_int32_t SW_size;
- struct comp_algo *SW_cxf;
- } SWCR_COMP;
- } SWCR_UN;
-
-#define sw_ictx SWCR_UN.SWCR_AUTH.SW_ictx
-#define sw_octx SWCR_UN.SWCR_AUTH.SW_octx
-#define sw_klen SWCR_UN.SWCR_AUTH.SW_klen
-#define sw_mlen SWCR_UN.SWCR_AUTH.SW_mlen
-#define sw_axf SWCR_UN.SWCR_AUTH.SW_axf
-#define sw_kschedule SWCR_UN.SWCR_ENC.SW_kschedule
-#define sw_exf SWCR_UN.SWCR_ENC.SW_exf
-#define sw_size SWCR_UN.SWCR_COMP.SW_size
-#define sw_cxf SWCR_UN.SWCR_COMP.SW_cxf
-};
-
-struct swcr_session {
- struct mtx swcr_lock;
- struct swcr_data swcr_algorithms[2];
- unsigned swcr_nalgs;
-};
-
-#ifdef _KERNEL
-extern u_int8_t hmac_ipad_buffer[];
-extern u_int8_t hmac_opad_buffer[];
-#endif /* _KERNEL */
-
-#endif /* _CRYPTO_CRYPTO_H_ */
Index: sys/opencrypto/cryptosoft.c
===================================================================
--- sys/opencrypto/cryptosoft.c
+++ sys/opencrypto/cryptosoft.c
@@ -55,36 +55,60 @@
#include <sys/md5.h>
#include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
#include <opencrypto/xform.h>
#include <sys/kobj.h>
#include <sys/bus.h>
#include "cryptodev_if.h"
-_Static_assert(AES_CCM_IV_LEN == AES_GCM_IV_LEN,
- "AES_GCM_IV_LEN must currently be the same as AES_CCM_IV_LEN");
+struct swcr_auth {
+ void *sw_ictx;
+ void *sw_octx;
+ struct auth_hash *sw_axf;
+ uint16_t sw_mlen;
+ uint16_t sw_octx_len;
+};
+
+struct swcr_encdec {
+ uint8_t *sw_kschedule;
+ struct enc_xform *sw_exf;
+};
+
+struct swcr_compdec {
+ struct comp_algo *sw_cxf;
+};
+
+struct swcr_session {
+ struct mtx swcr_lock;
+ int (*swcr_process)(struct swcr_session *, struct cryptop *);
+
+ struct swcr_auth swcr_auth;
+ struct swcr_encdec swcr_encdec;
+ struct swcr_compdec swcr_compdec;
+};
static int32_t swcr_id;
-u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
-u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
-
-static int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
-static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
-static int swcr_authenc(struct cryptop *crp);
-static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
static void swcr_freesession(device_t dev, crypto_session_t cses);
+/* Used for CRYPTO_NULL_CBC. */
+static int
+swcr_null(struct swcr_session *ses, struct cryptop *crp)
+{
+
+ return (0);
+}
+
/*
* Apply a symmetric encryption/decryption algorithm.
*/
static int
-swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
- int flags)
+swcr_encdec(struct swcr_session *ses, struct cryptop *crp)
{
unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN];
unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
+ const struct crypto_session_params *csp;
+ struct swcr_encdec *sw;
struct enc_xform *exf;
int i, j, k, blks, ind, count, ivlen;
struct uio *uio, uiolcl;
@@ -92,51 +116,39 @@
struct iovec *iov;
int iovcnt, iovalloc;
int error;
+ bool encrypting;
error = 0;
+ sw = &ses->swcr_encdec;
exf = sw->sw_exf;
blks = exf->blocksize;
ivlen = exf->ivsize;
/* Check for non-padded data */
- if (crd->crd_len % blks)
+ if ((crp->crp_payload_length % blks) != 0)
return EINVAL;
- if (crd->crd_alg == CRYPTO_AES_ICM &&
- (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
+ if (exf == &enc_xform_aes_icm &&
+ (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
return (EINVAL);
- /* Initialize the IV */
- if (crd->crd_flags & CRD_F_ENCRYPT) {
- /* IV explicitly provided ? */
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, iv, ivlen);
- else
- arc4rand(iv, ivlen, 0);
-
- /* Do we need to write the IV */
- if (!(crd->crd_flags & CRD_F_IV_PRESENT))
- crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
-
- } else { /* Decryption */
- /* IV explicitly provided ? */
- if (crd->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crd->crd_iv, iv, ivlen);
- else {
- /* Get IV off buf */
- crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
- }
- }
-
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- int error;
-
+ /* IV explicitly provided ? */
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ bcopy(crp->crp_iv, iv, ivlen);
+ else if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+ arc4rand(iv, ivlen, 0);
+ crypto_copyback(crp, crp->crp_iv_start, ivlen, iv);
+ } else
+ crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
+
+ if (crp->crp_cipher_key != NULL) {
if (sw->sw_kschedule)
exf->zerokey(&(sw->sw_kschedule));
+ csp = crypto_get_params(crp->crp_session);
error = exf->setkey(&sw->sw_kschedule,
- crd->crd_key, crd->crd_klen / 8);
+ crp->crp_cipher_key, csp->csp_cipher_klen);
if (error)
return (error);
}
@@ -145,20 +157,24 @@
iovcnt = nitems(iovlcl);
iovalloc = 0;
uio = &uiolcl;
- if ((flags & CRYPTO_F_IMBUF) != 0) {
- error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt,
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ error = crypto_mbuftoiov(crp->crp_mbuf, &iov, &iovcnt,
&iovalloc);
if (error)
return (error);
uio->uio_iov = iov;
uio->uio_iovcnt = iovcnt;
- } else if ((flags & CRYPTO_F_IOV) != 0)
- uio = (struct uio *)buf;
- else {
- iov[0].iov_base = buf;
- iov[0].iov_len = crd->crd_skip + crd->crd_len;
+ break;
+ case CRYPTO_BUF_UIO:
+ uio = crp->crp_uio;
+ break;
+ case CRYPTO_BUF_CONTIG:
+ iov[0].iov_base = crp->crp_buf;
+ iov[0].iov_len = crp->crp_ilen;
uio->uio_iov = iov;
uio->uio_iovcnt = 1;
+ break;
}
ivp = iv;
@@ -171,14 +187,15 @@
exf->reinit(sw->sw_kschedule, iv);
}
- count = crd->crd_skip;
+ count = crp->crp_payload_start;
ind = cuio_getptr(uio, count, &k);
if (ind == -1) {
error = EINVAL;
goto out;
}
- i = crd->crd_len;
+ i = crp->crp_payload_length;
+ encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
while (i > 0) {
/*
@@ -191,14 +208,14 @@
/* Actual encryption/decryption */
if (exf->reinit) {
- if (crd->crd_flags & CRD_F_ENCRYPT) {
+ if (encrypting) {
exf->encrypt(sw->sw_kschedule,
blk);
} else {
exf->decrypt(sw->sw_kschedule,
blk);
}
- } else if (crd->crd_flags & CRD_F_ENCRYPT) {
+ } else if (encrypting) {
/* XOR with previous block */
for (j = 0; j < blks; j++)
blk[j] ^= ivp[j];
@@ -257,11 +274,10 @@
idat = (uint8_t *)uio->uio_iov[ind].iov_base + k;
if (exf->reinit) {
- if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 &&
- exf->encrypt_multi == NULL)
+ if (encrypting && exf->encrypt_multi == NULL)
exf->encrypt(sw->sw_kschedule,
idat);
- else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) {
+ else if (encrypting) {
nb = rounddown(rem, blks);
exf->encrypt_multi(sw->sw_kschedule,
idat, nb);
@@ -273,7 +289,7 @@
exf->decrypt_multi(sw->sw_kschedule,
idat, nb);
}
- } else if (crd->crd_flags & CRD_F_ENCRYPT) {
+ } else if (encrypting) {
/* XOR with previous block/IV */
for (j = 0; j < blks; j++)
idat[j] ^= ivp[j];
@@ -325,13 +341,10 @@
return (error);
}
-static int __result_use_check
-swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
- int klen)
+static void
+swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw,
+ const uint8_t *key, int klen)
{
- int k;
-
- klen /= 8;
switch (axf->type) {
case CRYPTO_MD5_HMAC:
@@ -342,22 +355,8 @@
case CRYPTO_SHA2_512_HMAC:
case CRYPTO_NULL_HMAC:
case CRYPTO_RIPEMD160_HMAC:
- for (k = 0; k < klen; k++)
- key[k] ^= HMAC_IPAD_VAL;
-
- axf->Init(sw->sw_ictx);
- axf->Update(sw->sw_ictx, key, klen);
- axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
- for (k = 0; k < klen; k++)
- key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
- axf->Init(sw->sw_octx);
- axf->Update(sw->sw_octx, key, klen);
- axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
-
- for (k = 0; k < klen; k++)
- key[k] ^= HMAC_OPAD_VAL;
+ hmac_init_ipad(axf, key, klen, sw->sw_ictx);
+ hmac_init_opad(axf, key, klen, sw->sw_octx);
break;
case CRYPTO_MD5_KPDK:
case CRYPTO_SHA1_KPDK:
@@ -374,7 +373,6 @@
*/
u_char buf[SHA1_RESULTLEN];
- sw->sw_klen = klen;
bcopy(key, sw->sw_octx, klen);
axf->Init(sw->sw_ictx);
axf->Update(sw->sw_ictx, key, klen);
@@ -382,55 +380,53 @@
break;
}
case CRYPTO_POLY1305:
- if (klen != POLY1305_KEY_LEN) {
- CRYPTDEB("bad poly1305 key size %d", klen);
- return EINVAL;
- }
- /* FALLTHROUGH */
case CRYPTO_BLAKE2B:
case CRYPTO_BLAKE2S:
axf->Setkey(sw->sw_ictx, key, klen);
axf->Init(sw->sw_ictx);
break;
default:
- printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
- "doesn't use keys.\n", __func__, axf->type);
- return EINVAL;
+ panic("%s: algorithm %d doesn't use keys", __func__, axf->type);
}
- return 0;
}
/*
- * Compute keyed-hash authenticator.
+ * Compute or verify hash.
*/
static int
-swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
- int flags)
+swcr_authcompute(struct swcr_session *ses, struct cryptop *crp)
{
- unsigned char aalg[HASH_MAX_LEN];
+ u_char aalg[HASH_MAX_LEN];
+ u_char uaalg[HASH_MAX_LEN];
+ const struct crypto_session_params *csp;
+ struct swcr_auth *sw;
struct auth_hash *axf;
union authctx ctx;
int err;
- if (sw->sw_ictx == 0)
- return EINVAL;
+ sw = &ses->swcr_auth;
axf = sw->sw_axf;
- if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- err = swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
- if (err != 0)
- return err;
+ if (crp->crp_auth_key != NULL) {
+ csp = crypto_get_params(crp->crp_session);
+ swcr_authprepare(axf, sw, crp->crp_auth_key,
+ csp->csp_auth_klen);
}
bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
- err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
- (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+ err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+ (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
if (err)
return err;
- switch (sw->sw_alg) {
+ err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length,
+ (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
+ if (err)
+ return err;
+
+ switch (axf->type) {
case CRYPTO_SHA1:
case CRYPTO_SHA2_224:
case CRYPTO_SHA2_256:
@@ -468,7 +464,7 @@
* and let Final() do the proper, natural "algofill"
* padding.
*/
- axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
+ axf->Update(&ctx, sw->sw_octx, sw->sw_octx_len);
axf->Final(aalg, &ctx);
break;
@@ -480,20 +476,22 @@
break;
}
- /* Inject the authentication data */
- crypto_copyback(flags, buf, crd->crd_inject,
- sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
- return 0;
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg);
+ if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0)
+ return (EBADMSG);
+ } else {
+ /* Inject the authentication data */
+ crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg);
+ }
+ return (0);
}
CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */
CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */
-/*
- * Apply a combined encryption-authentication transformation
- */
static int
-swcr_authenc(struct cryptop *crp)
+swcr_gmac(struct swcr_session *ses, struct cryptop *crp)
{
uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
u_char *blk = (u_char *)blkbuf;
@@ -501,288 +499,403 @@
u_char uaalg[AALG_MAX_RESULT_LEN];
u_char iv[EALG_MAX_BLOCK_LEN];
union authctx ctx;
- struct swcr_session *ses;
- struct cryptodesc *crd, *crda = NULL, *crde = NULL;
- struct swcr_data *sw, *swa, *swe = NULL;
- struct auth_hash *axf = NULL;
- struct enc_xform *exf = NULL;
- caddr_t buf = (caddr_t)crp->crp_buf;
+ struct swcr_auth *swa;
+ struct auth_hash *axf;
uint32_t *blkp;
- int aadlen, blksz, i, ivlen, len, iskip, oskip, r;
- int isccm = 0;
-
- ivlen = blksz = iskip = oskip = 0;
-
- ses = crypto_get_driver_session(crp->crp_session);
-
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- for (i = 0; i < nitems(ses->swcr_algorithms) &&
- ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
- ;
- if (i == nitems(ses->swcr_algorithms))
- return (EINVAL);
-
- sw = &ses->swcr_algorithms[i];
- switch (sw->sw_alg) {
- case CRYPTO_AES_CCM_16:
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_NIST_GMAC:
- swe = sw;
- crde = crd;
- exf = swe->sw_exf;
- /* AES_CCM_IV_LEN and AES_GCM_IV_LEN are both 12 */
- ivlen = AES_CCM_IV_LEN;
- break;
- case CRYPTO_AES_CCM_CBC_MAC:
- isccm = 1;
- /* FALLTHROUGH */
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- swa = sw;
- crda = crd;
- axf = swa->sw_axf;
- if (swa->sw_ictx == 0)
- return (EINVAL);
- bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
- blksz = axf->blocksize;
- break;
- default:
- return (EINVAL);
- }
+ int blksz, i, ivlen, len;
+
+ swa = &ses->swcr_auth;
+ axf = swa->sw_axf;
+
+ bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+ blksz = axf->blocksize;
+
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE)
+ return (EINVAL);
+
+ /* Initialize the IV */
+ ivlen = AES_GCM_IV_LEN;
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ bcopy(crp->crp_iv, iv, ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
+
+ axf->Reinit(&ctx, iv, ivlen);
+ for (i = 0; i < crp->crp_payload_length; i += blksz) {
+ len = MIN(crp->crp_payload_length - i, blksz);
+ crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+ bzero(blk + len, blksz - len);
+ axf->Update(&ctx, blk, blksz);
+ }
+
+ /* length block */
+ bzero(blk, blksz);
+ blkp = (uint32_t *)blk + 1;
+ *blkp = htobe32(crp->crp_payload_length * 8);
+ axf->Update(&ctx, blk, blksz);
+
+ /* Finalize MAC */
+ axf->Final(aalg, &ctx);
+
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+ uaalg);
+ if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0)
+ return (EBADMSG);
+ } else {
+ /* Inject the authentication data */
+ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg);
}
- if (crde == NULL || crda == NULL)
+ return (0);
+}
+
+static int
+swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
+{
+ uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+ u_char *blk = (u_char *)blkbuf;
+ u_char aalg[AALG_MAX_RESULT_LEN];
+ u_char uaalg[AALG_MAX_RESULT_LEN];
+ u_char iv[EALG_MAX_BLOCK_LEN];
+ union authctx ctx;
+ struct swcr_auth *swa;
+ struct swcr_encdec *swe;
+ struct auth_hash *axf;
+ struct enc_xform *exf;
+ uint32_t *blkp;
+ int blksz, i, ivlen, len, r;
+
+ swa = &ses->swcr_auth;
+ axf = swa->sw_axf;
+
+ bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+ blksz = axf->blocksize;
+
+ swe = &ses->swcr_encdec;
+ exf = swe->sw_exf;
+
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
return (EINVAL);
- /*
- * We need to make sure that the auth algorithm matches the
- * encr algorithm. Specifically, for AES-GCM must go with
- * AES NIST GMAC, and AES-CCM must go with CBC-MAC.
- */
- if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16) {
- switch (crda->crd_alg) {
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- break; /* Good! */
- default:
- return (EINVAL); /* Not good! */
+
+ /* Initialize the IV */
+ ivlen = AES_GCM_IV_LEN;
+ bcopy(crp->crp_iv, iv, ivlen);
+
+ /* Supply MAC with IV */
+ axf->Reinit(&ctx, iv, ivlen);
+
+ /* Supply MAC with AAD */
+ for (i = 0; i < crp->crp_aad_length; i += blksz) {
+ len = MIN(crp->crp_aad_length - i, blksz);
+ crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+ bzero(blk + len, blksz - len);
+ axf->Update(&ctx, blk, blksz);
+ }
+
+ exf->reinit(swe->sw_kschedule, iv);
+
+ /* Do encryption with MAC */
+ for (i = 0; i < crp->crp_payload_length; i += len) {
+ len = MIN(crp->crp_payload_length - i, blksz);
+ if (len < blksz)
+ bzero(blk, blksz);
+ crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ exf->encrypt(swe->sw_kschedule, blk);
+ axf->Update(&ctx, blk, len);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ blk);
+ } else {
+ axf->Update(&ctx, blk, len);
}
- } else if (crde->crd_alg == CRYPTO_AES_CCM_16 &&
- crda->crd_alg != CRYPTO_AES_CCM_CBC_MAC)
- return (EINVAL);
+ }
- if ((crde->crd_alg == CRYPTO_AES_NIST_GCM_16 ||
- crde->crd_alg == CRYPTO_AES_CCM_16) &&
- (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0)
- return (EINVAL);
+ /* length block */
+ bzero(blk, blksz);
+ blkp = (uint32_t *)blk + 1;
+ *blkp = htobe32(crp->crp_aad_length * 8);
+ blkp = (uint32_t *)blk + 3;
+ *blkp = htobe32(crp->crp_payload_length * 8);
+ axf->Update(&ctx, blk, blksz);
+
+ /* Finalize MAC */
+ axf->Final(aalg, &ctx);
+
+ /* Validate tag */
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+ uaalg);
- if (crde->crd_klen != crda->crd_klen)
+ r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen);
+ if (r != 0)
+ return (EBADMSG);
+
+ /* tag matches, decrypt data */
+ for (i = 0; i < crp->crp_payload_length; i += blksz) {
+ len = MIN(crp->crp_payload_length - i, blksz);
+ if (len < blksz)
+ bzero(blk, blksz);
+ crypto_copydata(crp, crp->crp_payload_start + i, len,
+ blk);
+ exf->decrypt(swe->sw_kschedule, blk);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ blk);
+ }
+ } else {
+ /* Inject the authentication data */
+ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
+ aalg);
+ }
+
+ return (0);
+}
+
+static int
+swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp)
+{
+ uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+ u_char *blk = (u_char *)blkbuf;
+ u_char aalg[AALG_MAX_RESULT_LEN];
+ u_char uaalg[AALG_MAX_RESULT_LEN];
+ u_char iv[EALG_MAX_BLOCK_LEN];
+ union authctx ctx;
+ struct swcr_auth *swa;
+ struct auth_hash *axf;
+ int blksz, i, ivlen, len;
+
+ swa = &ses->swcr_auth;
+ axf = swa->sw_axf;
+
+ bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+ blksz = axf->blocksize;
+
+ if (crp->crp_flags & CRYPTO_F_IV_GENERATE)
return (EINVAL);
/* Initialize the IV */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- /* IV explicitly provided ? */
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crde->crd_iv, iv, ivlen);
- else
- arc4rand(iv, ivlen, 0);
-
- /* Do we need to write the IV */
- if (!(crde->crd_flags & CRD_F_IV_PRESENT))
- crypto_copyback(crp->crp_flags, buf, crde->crd_inject,
- ivlen, iv);
-
- } else { /* Decryption */
- /* IV explicitly provided ? */
- if (crde->crd_flags & CRD_F_IV_EXPLICIT)
- bcopy(crde->crd_iv, iv, ivlen);
- else {
- /* Get IV off buf */
- crypto_copydata(crp->crp_flags, buf, crde->crd_inject,
- ivlen, iv);
- }
+ ivlen = AES_CCM_IV_LEN;
+ if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+ bcopy(crp->crp_iv, iv, ivlen);
+ else
+ crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
+
+ /*
+ * AES CCM-CBC-MAC needs to know the length of both the auth
+ * data and payload data before doing the auth computation.
+ */
+ ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length;
+ ctx.aes_cbc_mac_ctx.cryptDataLength = 0;
+
+ axf->Reinit(&ctx, iv, ivlen);
+ for (i = 0; i < crp->crp_payload_length; i += blksz) {
+ len = MIN(crp->crp_payload_length - i, blksz);
+ crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+ bzero(blk + len, blksz - len);
+ axf->Update(&ctx, blk, blksz);
}
- if (swa->sw_alg == CRYPTO_AES_CCM_CBC_MAC) {
- /*
- * AES CCM-CBC needs to know the length of
- * both the auth data, and payload data, before
- * doing the auth computation.
- */
- ctx.aes_cbc_mac_ctx.authDataLength = crda->crd_len;
- ctx.aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
+ /* Finalize MAC */
+ axf->Final(aalg, &ctx);
+
+ if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+ crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+ uaalg);
+ if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0)
+ return (EBADMSG);
+ } else {
+ /* Inject the authentication data */
+ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg);
}
+ return (0);
+}
+
+static int
+swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
+{
+ uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+ u_char *blk = (u_char *)blkbuf;
+ u_char aalg[AALG_MAX_RESULT_LEN];
+ u_char uaalg[AALG_MAX_RESULT_LEN];
+ u_char iv[EALG_MAX_BLOCK_LEN];
+ union authctx ctx;
+ struct swcr_auth *swa;
+ struct swcr_encdec *swe;
+ struct auth_hash *axf;
+ struct enc_xform *exf;
+ int blksz, i, ivlen, len, r;
+
+ swa = &ses->swcr_auth;
+ axf = swa->sw_axf;
+
+ bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+ blksz = axf->blocksize;
+
+ swe = &ses->swcr_encdec;
+ exf = swe->sw_exf;
+
+ if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+ return (EINVAL);
+
+ /* Initialize the IV */
+ ivlen = AES_CCM_IV_LEN;
+ bcopy(crp->crp_iv, iv, ivlen);
+
+ /*
+ * AES CCM-CBC-MAC needs to know the length of both the auth
+ * data and payload data before doing the auth computation.
+ */
+ ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
+ ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
+
/* Supply MAC with IV */
- if (axf->Reinit)
- axf->Reinit(&ctx, iv, ivlen);
+ axf->Reinit(&ctx, iv, ivlen);
/* Supply MAC with AAD */
- aadlen = crda->crd_len;
-
- for (i = iskip; i < crda->crd_len; i += blksz) {
- len = MIN(crda->crd_len - i, blksz - oskip);
- crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len,
- blk + oskip);
- bzero(blk + len + oskip, blksz - len - oskip);
+ for (i = 0; i < crp->crp_aad_length; i += blksz) {
+ len = MIN(crp->crp_aad_length - i, blksz);
+ crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+ bzero(blk + len, blksz - len);
axf->Update(&ctx, blk, blksz);
- oskip = 0; /* reset initial output offset */
}
- if (exf->reinit)
- exf->reinit(swe->sw_kschedule, iv);
+ exf->reinit(swe->sw_kschedule, iv);
/* Do encryption/decryption with MAC */
- for (i = 0; i < crde->crd_len; i += len) {
- if (exf->encrypt_multi != NULL) {
- len = rounddown(crde->crd_len - i, blksz);
- if (len == 0)
- len = blksz;
- else
- len = MIN(len, sizeof(blkbuf));
- } else
- len = blksz;
- len = MIN(crde->crd_len - i, len);
+ for (i = 0; i < crp->crp_payload_length; i += len) {
+ len = MIN(crp->crp_payload_length - i, blksz);
if (len < blksz)
bzero(blk, blksz);
- crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len,
- blk);
- /*
- * One of the problems with CCM+CBC is that the authentication
- * is done on the unecncrypted data. As a result, we have
- * to do the authentication update at different times,
- * depending on whether it's CCM or not.
- */
- if (crde->crd_flags & CRD_F_ENCRYPT) {
- if (isccm)
- axf->Update(&ctx, blk, len);
- if (exf->encrypt_multi != NULL)
- exf->encrypt_multi(swe->sw_kschedule, blk,
- len);
- else
- exf->encrypt(swe->sw_kschedule, blk);
- if (!isccm)
- axf->Update(&ctx, blk, len);
- crypto_copyback(crp->crp_flags, buf,
- crde->crd_skip + i, len, blk);
+ crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ axf->Update(&ctx, blk, len);
+ exf->encrypt(swe->sw_kschedule, blk);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ blk);
} else {
- if (isccm) {
- KASSERT(exf->encrypt_multi == NULL,
- ("assume CCM is single-block only"));
- exf->decrypt(swe->sw_kschedule, blk);
- }
+ /*
+ * One of the problems with CCM+CBC is that
+ * the authentication is done on the
+ * unecncrypted data. As a result, we have to
+ * decrypt the data twice: once to generate
+ * the tag and a second time after the tag is
+ * verified.
+ */
+ exf->decrypt(swe->sw_kschedule, blk);
axf->Update(&ctx, blk, len);
}
}
- /* Do any required special finalization */
- switch (crda->crd_alg) {
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- /* length block */
- bzero(blk, blksz);
- blkp = (uint32_t *)blk + 1;
- *blkp = htobe32(aadlen * 8);
- blkp = (uint32_t *)blk + 3;
- *blkp = htobe32(crde->crd_len * 8);
- axf->Update(&ctx, blk, blksz);
- break;
- }
-
/* Finalize MAC */
axf->Final(aalg, &ctx);
/* Validate tag */
- if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
- crypto_copydata(crp->crp_flags, buf, crda->crd_inject,
- axf->hashsize, uaalg);
+ if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+ uaalg);
- r = timingsafe_bcmp(aalg, uaalg, axf->hashsize);
- if (r == 0) {
- /* tag matches, decrypt data */
- if (isccm) {
- KASSERT(exf->reinit != NULL,
- ("AES-CCM reinit function must be set"));
- exf->reinit(swe->sw_kschedule, iv);
- }
- for (i = 0; i < crde->crd_len; i += blksz) {
- len = MIN(crde->crd_len - i, blksz);
- if (len < blksz)
- bzero(blk, blksz);
- crypto_copydata(crp->crp_flags, buf,
- crde->crd_skip + i, len, blk);
- exf->decrypt(swe->sw_kschedule, blk);
- crypto_copyback(crp->crp_flags, buf,
- crde->crd_skip + i, len, blk);
- }
- } else
+ r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen);
+ if (r != 0)
return (EBADMSG);
+
+ /* tag matches, decrypt data */
+ exf->reinit(swe->sw_kschedule, iv);
+ for (i = 0; i < crp->crp_payload_length; i += blksz) {
+ len = MIN(crp->crp_payload_length - i, blksz);
+ if (len < blksz)
+ bzero(blk, blksz);
+ crypto_copydata(crp, crp->crp_payload_start + i, len,
+ blk);
+ exf->decrypt(swe->sw_kschedule, blk);
+ crypto_copyback(crp, crp->crp_payload_start + i, len,
+ blk);
+ }
} else {
/* Inject the authentication data */
- crypto_copyback(crp->crp_flags, buf, crda->crd_inject,
- axf->hashsize, aalg);
+ crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
+ aalg);
}
return (0);
}
+/*
+ * Apply a cipher and a digest to perform EtA.
+ */
+static int
+swcr_eta(struct swcr_session *ses, struct cryptop *crp)
+{
+ int error;
+
+ if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+ error = swcr_encdec(ses, crp);
+ if (error == 0)
+ error = swcr_authcompute(ses, crp);
+ } else {
+ error = swcr_authcompute(ses, crp);
+ if (error == 0)
+ error = swcr_encdec(ses, crp);
+ }
+ return (error);
+}
+
/*
* Apply a compression/decompression algorithm
*/
static int
-swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
- caddr_t buf, int flags)
+swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
{
u_int8_t *data, *out;
struct comp_algo *cxf;
int adj;
u_int32_t result;
- cxf = sw->sw_cxf;
+ cxf = ses->swcr_compdec.sw_cxf;
/* We must handle the whole buffer of data in one time
* then if there is not all the data in the mbuf, we must
* copy in a buffer.
*/
- data = malloc(crd->crd_len, M_CRYPTO_DATA, M_NOWAIT);
+ data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT);
if (data == NULL)
return (EINVAL);
- crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
+ crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length,
+ data);
- if (crd->crd_flags & CRD_F_COMP)
- result = cxf->compress(data, crd->crd_len, &out);
+ if (CRYPTO_OP_IS_COMPRESS(crp->crp_op))
+ result = cxf->compress(data, crp->crp_payload_length, &out);
else
- result = cxf->decompress(data, crd->crd_len, &out);
+ result = cxf->decompress(data, crp->crp_payload_length, &out);
free(data, M_CRYPTO_DATA);
if (result == 0)
- return EINVAL;
+ return (EINVAL);
+ crp->crp_olen = result;
- /* Copy back the (de)compressed data. m_copyback is
- * extending the mbuf as necessary.
- */
- sw->sw_size = result;
/* Check the compressed size when doing compression */
- if (crd->crd_flags & CRD_F_COMP) {
- if (result >= crd->crd_len) {
+ if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) {
+ if (result >= crp->crp_payload_length) {
/* Compression was useless, we lost time */
free(out, M_CRYPTO_DATA);
- return 0;
+ return (0);
}
}
- crypto_copyback(flags, buf, crd->crd_skip, result, out);
- if (result < crd->crd_len) {
- adj = result - crd->crd_len;
- if (flags & CRYPTO_F_IMBUF) {
- adj = result - crd->crd_len;
- m_adj((struct mbuf *)buf, adj);
- } else if (flags & CRYPTO_F_IOV) {
- struct uio *uio = (struct uio *)buf;
+ /* Copy back the (de)compressed data. m_copyback is
+ * extending the mbuf as necessary.
+ */
+ crypto_copyback(crp, crp->crp_payload_start, result, out);
+ if (result < crp->crp_payload_length) {
+ switch (crp->crp_buf_type) {
+ case CRYPTO_BUF_MBUF:
+ adj = result - crp->crp_payload_length;
+ m_adj(crp->crp_mbuf, adj);
+ break;
+ case CRYPTO_BUF_UIO: {
+ struct uio *uio = crp->crp_uio;
int ind;
- adj = crd->crd_len - result;
+ adj = crp->crp_payload_length - result;
ind = uio->uio_iovcnt - 1;
while (adj > 0 && ind >= 0) {
@@ -796,391 +909,522 @@
ind--;
uio->uio_iovcnt--;
}
+ }
+ break;
}
}
free(out, M_CRYPTO_DATA);
return 0;
}
+static int
+swcr_setup_encdec(struct swcr_session *ses,
+ const struct crypto_session_params *csp)
+{
+ struct swcr_encdec *swe;
+ struct enc_xform *txf;
+ int error;
+
+ swe = &ses->swcr_encdec;
+ txf = crypto_cipher(csp);
+ MPASS(txf->ivsize == csp->csp_ivlen);
+ if (csp->csp_cipher_key != NULL) {
+ error = txf->setkey(&swe->sw_kschedule,
+ csp->csp_cipher_key, csp->csp_cipher_klen);
+ if (error)
+ return (error);
+ }
+ swe->sw_exf = txf;
+ return (0);
+}
+
+static int
+swcr_setup_auth(struct swcr_session *ses,
+ const struct crypto_session_params *csp)
+{
+ struct swcr_auth *swa;
+ struct auth_hash *axf;
+
+ swa = &ses->swcr_auth;
+
+ axf = crypto_auth_hash(csp);
+ swa->sw_axf = axf;
+ if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+ return (EINVAL);
+ if (csp->csp_auth_mlen == 0)
+ swa->sw_mlen = axf->hashsize;
+ else
+ swa->sw_mlen = csp->csp_auth_mlen;
+ swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+ if (swa->sw_ictx == NULL)
+ return (ENOBUFS);
+
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ case CRYPTO_NULL_HMAC:
+ case CRYPTO_RIPEMD160_HMAC:
+ swa->sw_octx_len = axf->ctxsize;
+ swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA,
+ M_NOWAIT);
+ if (swa->sw_octx == NULL)
+ return (ENOBUFS);
+
+ if (csp->csp_auth_key != NULL) {
+ swcr_authprepare(axf, swa, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ }
+
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_authcompute;
+ break;
+ case CRYPTO_MD5_KPDK:
+ case CRYPTO_SHA1_KPDK:
+ swa->sw_octx_len = csp->csp_auth_klen;
+ swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA,
+ M_NOWAIT);
+ if (swa->sw_octx == NULL)
+ return (ENOBUFS);
+
+ /* Store the key so we can "append" it to the payload */
+ if (csp->csp_auth_key != NULL) {
+ swcr_authprepare(axf, swa, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ }
+
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_authcompute;
+ break;
+#ifdef notdef
+ case CRYPTO_MD5:
+#endif
+ case CRYPTO_SHA1:
+ case CRYPTO_SHA2_224:
+ case CRYPTO_SHA2_256:
+ case CRYPTO_SHA2_384:
+ case CRYPTO_SHA2_512:
+ axf->Init(swa->sw_ictx);
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_authcompute;
+ break;
+ case CRYPTO_AES_NIST_GMAC:
+ axf->Init(swa->sw_ictx);
+ axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_gmac;
+ break;
+ case CRYPTO_POLY1305:
+ case CRYPTO_BLAKE2B:
+ case CRYPTO_BLAKE2S:
+ /*
+ * Blake2b and Blake2s support an optional key but do
+ * not require one.
+ */
+ if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL)
+ axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ axf->Init(swa->sw_ictx);
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_authcompute;
+ break;
+ case CRYPTO_AES_CCM_CBC_MAC:
+ axf->Init(swa->sw_ictx);
+ axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+ csp->csp_auth_klen);
+ if (csp->csp_mode == CSP_MODE_DIGEST)
+ ses->swcr_process = swcr_ccm_cbc_mac;
+ break;
+ }
+
+ return (0);
+}
+
+static int
+swcr_setup_gcm(struct swcr_session *ses,
+ const struct crypto_session_params *csp)
+{
+ struct swcr_encdec *swe;
+ struct swcr_auth *swa;
+ struct enc_xform *txf;
+ struct auth_hash *axf;
+ int error;
+
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (EINVAL);
+
+ /* First, setup the auth side. */
+ swa = &ses->swcr_auth;
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ axf = &auth_hash_nist_gmac_aes_128;
+ break;
+ case 192:
+ axf = &auth_hash_nist_gmac_aes_192;
+ break;
+ case 256:
+ axf = &auth_hash_nist_gmac_aes_256;
+ break;
+ default:
+ return (EINVAL);
+ }
+ swa->sw_axf = axf;
+ if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+ return (EINVAL);
+ if (csp->csp_auth_mlen == 0)
+ swa->sw_mlen = axf->hashsize;
+ else
+ swa->sw_mlen = csp->csp_auth_mlen;
+ swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+ if (swa->sw_ictx == NULL)
+ return (ENOBUFS);
+ axf->Init(swa->sw_ictx);
+ if (csp->csp_cipher_key != NULL)
+ axf->Setkey(swa->sw_ictx, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+
+ /* Second, setup the cipher side. */
+ swe = &ses->swcr_encdec;
+ txf = &enc_xform_aes_nist_gcm;
+ if (csp->csp_cipher_key != NULL) {
+ error = txf->setkey(&swe->sw_kschedule,
+ csp->csp_cipher_key, csp->csp_cipher_klen);
+ if (error)
+ return (error);
+ }
+ swe->sw_exf = txf;
+
+ return (0);
+}
+
+static int
+swcr_setup_ccm(struct swcr_session *ses,
+ const struct crypto_session_params *csp)
+{
+ struct swcr_encdec *swe;
+ struct swcr_auth *swa;
+ struct enc_xform *txf;
+ struct auth_hash *axf;
+ int error;
+
+ if (csp->csp_ivlen != AES_CCM_IV_LEN)
+ return (EINVAL);
+
+ /* First, setup the auth side. */
+ swa = &ses->swcr_auth;
+ switch (csp->csp_cipher_klen * 8) {
+ case 128:
+ axf = &auth_hash_ccm_cbc_mac_128;
+ break;
+ case 192:
+ axf = &auth_hash_ccm_cbc_mac_192;
+ break;
+ case 256:
+ axf = &auth_hash_ccm_cbc_mac_256;
+ break;
+ default:
+ return (EINVAL);
+ }
+ swa->sw_axf = axf;
+ if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+ return (EINVAL);
+ if (csp->csp_auth_mlen == 0)
+ swa->sw_mlen = axf->hashsize;
+ else
+ swa->sw_mlen = csp->csp_auth_mlen;
+ swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+ if (swa->sw_ictx == NULL)
+ return (ENOBUFS);
+ axf->Init(swa->sw_ictx);
+ if (csp->csp_cipher_key != NULL)
+ axf->Setkey(swa->sw_ictx, csp->csp_cipher_key,
+ csp->csp_cipher_klen);
+
+ /* Second, setup the cipher side. */
+ swe = &ses->swcr_encdec;
+ txf = &enc_xform_ccm;
+ if (csp->csp_cipher_key != NULL) {
+ error = txf->setkey(&swe->sw_kschedule,
+ csp->csp_cipher_key, csp->csp_cipher_klen);
+ if (error)
+ return (error);
+ }
+ swe->sw_exf = txf;
+
+ return (0);
+}
+
+static bool
+swcr_auth_supported(const struct crypto_session_params *csp)
+{
+ struct auth_hash *axf;
+
+ axf = crypto_auth_hash(csp);
+ if (axf == NULL)
+ return (false);
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_MD5_HMAC:
+ case CRYPTO_SHA1_HMAC:
+ case CRYPTO_SHA2_224_HMAC:
+ case CRYPTO_SHA2_256_HMAC:
+ case CRYPTO_SHA2_384_HMAC:
+ case CRYPTO_SHA2_512_HMAC:
+ case CRYPTO_NULL_HMAC:
+ case CRYPTO_RIPEMD160_HMAC:
+ case CRYPTO_MD5_KPDK:
+ case CRYPTO_SHA1_KPDK:
+ break;
+ case CRYPTO_AES_NIST_GMAC:
+ switch (csp->csp_auth_klen * 8) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (false);
+ }
+ if (csp->csp_auth_key == NULL)
+ return (false);
+ if (csp->csp_ivlen != AES_GCM_IV_LEN)
+ return (false);
+ break;
+ case CRYPTO_POLY1305:
+ if (csp->csp_auth_klen != POLY1305_KEY_LEN)
+ return (false);
+ break;
+ case CRYPTO_AES_CCM_CBC_MAC:
+ switch (csp->csp_auth_klen * 8) {
+ case 128:
+ case 192:
+ case 256:
+ break;
+ default:
+ return (false);
+ }
+ if (csp->csp_auth_key == NULL)
+ return (false);
+ if (csp->csp_ivlen != AES_CCM_IV_LEN)
+ return (false);
+ break;
+ }
+ return (true);
+}
+
+static bool
+swcr_cipher_supported(const struct crypto_session_params *csp)
+{
+ struct enc_xform *txf;
+
+ txf = crypto_cipher(csp);
+ if (txf == NULL)
+ return (false);
+ if (csp->csp_cipher_alg != CRYPTO_NULL_CBC &&
+ txf->ivsize != csp->csp_ivlen)
+ return (false);
+ return (true);
+}
+
+static int
+swcr_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+ if (csp->csp_flags != 0)
+ return (EINVAL);
+ switch (csp->csp_mode) {
+ case CSP_MODE_COMPRESS:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_DEFLATE_COMP:
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ return (EINVAL);
+ default:
+ if (!swcr_cipher_supported(csp))
+ return (EINVAL);
+ break;
+ }
+ break;
+ case CSP_MODE_DIGEST:
+ if (!swcr_auth_supported(csp))
+ return (EINVAL);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ break;
+ default:
+ return (EINVAL);
+ }
+ break;
+ case CSP_MODE_ETA:
+ /* AEAD algorithms cannot be used for EtA. */
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ return (EINVAL);
+ }
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_AES_NIST_GMAC:
+ case CRYPTO_AES_CCM_CBC_MAC:
+ return (EINVAL);
+ }
+
+ if (!swcr_cipher_supported(csp) ||
+ !swcr_auth_supported(csp))
+ return (EINVAL);
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ return (CRYPTODEV_PROBE_SOFTWARE);
+}
+
/*
* Generate a new software session.
*/
static int
-swcr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+swcr_newsession(device_t dev, crypto_session_t cses,
+ const struct crypto_session_params *csp)
{
struct swcr_session *ses;
- struct swcr_data *swd;
- struct auth_hash *axf;
- struct enc_xform *txf;
+ struct swcr_encdec *swe;
+ struct swcr_auth *swa;
struct comp_algo *cxf;
- size_t i;
- int len;
int error;
- if (cses == NULL || cri == NULL)
- return EINVAL;
-
ses = crypto_get_driver_session(cses);
mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF);
- for (i = 0; cri != NULL && i < nitems(ses->swcr_algorithms); i++) {
- swd = &ses->swcr_algorithms[i];
-
- switch (cri->cri_alg) {
- case CRYPTO_DES_CBC:
- txf = &enc_xform_des;
- goto enccommon;
- case CRYPTO_3DES_CBC:
- txf = &enc_xform_3des;
- goto enccommon;
- case CRYPTO_BLF_CBC:
- txf = &enc_xform_blf;
- goto enccommon;
- case CRYPTO_CAST_CBC:
- txf = &enc_xform_cast5;
- goto enccommon;
- case CRYPTO_SKIPJACK_CBC:
- txf = &enc_xform_skipjack;
- goto enccommon;
- case CRYPTO_RIJNDAEL128_CBC:
- txf = &enc_xform_rijndael128;
- goto enccommon;
- case CRYPTO_AES_XTS:
- txf = &enc_xform_aes_xts;
- goto enccommon;
- case CRYPTO_AES_ICM:
- txf = &enc_xform_aes_icm;
- goto enccommon;
- case CRYPTO_AES_NIST_GCM_16:
- txf = &enc_xform_aes_nist_gcm;
- goto enccommon;
- case CRYPTO_AES_CCM_16:
- txf = &enc_xform_ccm;
- goto enccommon;
- case CRYPTO_AES_NIST_GMAC:
- txf = &enc_xform_aes_nist_gmac;
- swd->sw_exf = txf;
- break;
- case CRYPTO_CAMELLIA_CBC:
- txf = &enc_xform_camellia;
- goto enccommon;
- case CRYPTO_NULL_CBC:
- txf = &enc_xform_null;
- goto enccommon;
- case CRYPTO_CHACHA20:
- txf = &enc_xform_chacha20;
- goto enccommon;
- enccommon:
- if (cri->cri_key != NULL) {
- error = txf->setkey(&swd->sw_kschedule,
- cri->cri_key, cri->cri_klen / 8);
- if (error) {
- swcr_freesession(dev, cses);
- return error;
- }
- }
- swd->sw_exf = txf;
- break;
-
- case CRYPTO_MD5_HMAC:
- axf = &auth_hash_hmac_md5;
- goto authcommon;
- case CRYPTO_SHA1_HMAC:
- axf = &auth_hash_hmac_sha1;
- goto authcommon;
- case CRYPTO_SHA2_224_HMAC:
- axf = &auth_hash_hmac_sha2_224;
- goto authcommon;
- case CRYPTO_SHA2_256_HMAC:
- axf = &auth_hash_hmac_sha2_256;
- goto authcommon;
- case CRYPTO_SHA2_384_HMAC:
- axf = &auth_hash_hmac_sha2_384;
- goto authcommon;
- case CRYPTO_SHA2_512_HMAC:
- axf = &auth_hash_hmac_sha2_512;
- goto authcommon;
- case CRYPTO_NULL_HMAC:
- axf = &auth_hash_null;
- goto authcommon;
- case CRYPTO_RIPEMD160_HMAC:
- axf = &auth_hash_hmac_ripemd_160;
- authcommon:
- swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_ictx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
-
- swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_octx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
-
- if (cri->cri_key != NULL) {
- error = swcr_authprepare(axf, swd,
- cri->cri_key, cri->cri_klen);
- if (error != 0) {
- swcr_freesession(dev, cses);
- return error;
- }
- }
-
- swd->sw_mlen = cri->cri_mlen;
- swd->sw_axf = axf;
- break;
-
- case CRYPTO_MD5_KPDK:
- axf = &auth_hash_key_md5;
- goto auth2common;
-
- case CRYPTO_SHA1_KPDK:
- axf = &auth_hash_key_sha1;
- auth2common:
- swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_ictx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
-
- swd->sw_octx = malloc(cri->cri_klen / 8,
- M_CRYPTO_DATA, M_NOWAIT);
- if (swd->sw_octx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
-
- /* Store the key so we can "append" it to the payload */
- if (cri->cri_key != NULL) {
- error = swcr_authprepare(axf, swd,
- cri->cri_key, cri->cri_klen);
- if (error != 0) {
- swcr_freesession(dev, cses);
- return error;
- }
- }
-
- swd->sw_mlen = cri->cri_mlen;
- swd->sw_axf = axf;
- break;
-#ifdef notdef
- case CRYPTO_MD5:
- axf = &auth_hash_md5;
- goto auth3common;
-#endif
-
- case CRYPTO_SHA1:
- axf = &auth_hash_sha1;
- goto auth3common;
- case CRYPTO_SHA2_224:
- axf = &auth_hash_sha2_224;
- goto auth3common;
- case CRYPTO_SHA2_256:
- axf = &auth_hash_sha2_256;
- goto auth3common;
- case CRYPTO_SHA2_384:
- axf = &auth_hash_sha2_384;
- goto auth3common;
- case CRYPTO_SHA2_512:
- axf = &auth_hash_sha2_512;
-
- auth3common:
- swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_ictx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
-
- axf->Init(swd->sw_ictx);
- swd->sw_mlen = cri->cri_mlen;
- swd->sw_axf = axf;
- break;
-
- case CRYPTO_AES_CCM_CBC_MAC:
- switch (cri->cri_klen) {
- case 128:
- axf = &auth_hash_ccm_cbc_mac_128;
- break;
- case 192:
- axf = &auth_hash_ccm_cbc_mac_192;
- break;
- case 256:
- axf = &auth_hash_ccm_cbc_mac_256;
- break;
- default:
- swcr_freesession(dev, cses);
- return EINVAL;
- }
- goto auth4common;
- case CRYPTO_AES_128_NIST_GMAC:
- axf = &auth_hash_nist_gmac_aes_128;
- goto auth4common;
-
- case CRYPTO_AES_192_NIST_GMAC:
- axf = &auth_hash_nist_gmac_aes_192;
- goto auth4common;
-
- case CRYPTO_AES_256_NIST_GMAC:
- axf = &auth_hash_nist_gmac_aes_256;
- auth4common:
- len = cri->cri_klen / 8;
- if (len != 16 && len != 24 && len != 32) {
- swcr_freesession(dev, cses);
- return EINVAL;
- }
-
- swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_ictx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
- axf->Init(swd->sw_ictx);
- axf->Setkey(swd->sw_ictx, cri->cri_key, len);
- swd->sw_axf = axf;
- break;
-
- case CRYPTO_BLAKE2B:
- axf = &auth_hash_blake2b;
- goto auth5common;
- case CRYPTO_BLAKE2S:
- axf = &auth_hash_blake2s;
- goto auth5common;
- case CRYPTO_POLY1305:
- axf = &auth_hash_poly1305;
- auth5common:
- swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
- if (swd->sw_ictx == NULL) {
- swcr_freesession(dev, cses);
- return ENOBUFS;
- }
- axf->Setkey(swd->sw_ictx, cri->cri_key,
- cri->cri_klen / 8);
- axf->Init(swd->sw_ictx);
- swd->sw_axf = axf;
- break;
-
+ error = 0;
+ swe = &ses->swcr_encdec;
+ swa = &ses->swcr_auth;
+ switch (csp->csp_mode) {
+ case CSP_MODE_COMPRESS:
+ switch (csp->csp_cipher_alg) {
case CRYPTO_DEFLATE_COMP:
cxf = &comp_algo_deflate;
- swd->sw_cxf = cxf;
break;
+#ifdef INVARIANTS
default:
- swcr_freesession(dev, cses);
- return EINVAL;
+ panic("bad compression algo");
+#endif
}
-
- swd->sw_alg = cri->cri_alg;
- cri = cri->cri_next;
- ses->swcr_nalgs++;
- }
+ ses->swcr_compdec.sw_cxf = cxf;
+ ses->swcr_process = swcr_compdec;
+ break;
+ case CSP_MODE_CIPHER:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_NULL_CBC:
+ ses->swcr_process = swcr_null;
+ break;
+#ifdef INVARIANTS
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ panic("bad cipher algo");
+#endif
+ default:
+ error = swcr_setup_encdec(ses, csp);
+ if (error == 0)
+ ses->swcr_process = swcr_encdec;
+ }
+ break;
+ case CSP_MODE_DIGEST:
+ error = swcr_setup_auth(ses, csp);
+ break;
+ case CSP_MODE_AEAD:
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ error = swcr_setup_gcm(ses, csp);
+ if (error == 0)
+ ses->swcr_process = swcr_gcm;
+ break;
+ case CRYPTO_AES_CCM_16:
+ error = swcr_setup_ccm(ses, csp);
+ if (error == 0)
+ ses->swcr_process = swcr_ccm;
+ break;
+#ifdef INVARIANTS
+ default:
+ panic("bad aead algo");
+#endif
+ }
+ break;
+ case CSP_MODE_ETA:
+#ifdef INVARIANTS
+ switch (csp->csp_cipher_alg) {
+ case CRYPTO_AES_NIST_GCM_16:
+ case CRYPTO_AES_CCM_16:
+ panic("bad eta cipher algo");
+ }
+ switch (csp->csp_auth_alg) {
+ case CRYPTO_AES_NIST_GMAC:
+ case CRYPTO_AES_CCM_CBC_MAC:
+ panic("bad eta auth algo");
+ }
+#endif
- if (cri != NULL) {
- CRYPTDEB("Bogus session request for three or more algorithms");
- return EINVAL;
+ error = swcr_setup_auth(ses, csp);
+ if (error)
+ break;
+ if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) {
+ /* Effectively degrade to digest mode. */
+ ses->swcr_process = swcr_authcompute;
+ break;
+ }
+
+ error = swcr_setup_encdec(ses, csp);
+ if (error == 0)
+ ses->swcr_process = swcr_eta;
+ break;
+ default:
+ error = EINVAL;
}
- return 0;
+
+ if (error)
+ swcr_freesession(dev, cses);
+ return (error);
}
static void
swcr_freesession(device_t dev, crypto_session_t cses)
{
struct swcr_session *ses;
- struct swcr_data *swd;
+ struct swcr_auth *swa;
struct enc_xform *txf;
struct auth_hash *axf;
- size_t i;
ses = crypto_get_driver_session(cses);
mtx_destroy(&ses->swcr_lock);
- for (i = 0; i < nitems(ses->swcr_algorithms); i++) {
- swd = &ses->swcr_algorithms[i];
- switch (swd->sw_alg) {
- case CRYPTO_DES_CBC:
- case CRYPTO_3DES_CBC:
- case CRYPTO_BLF_CBC:
- case CRYPTO_CAST_CBC:
- case CRYPTO_SKIPJACK_CBC:
- case CRYPTO_RIJNDAEL128_CBC:
- case CRYPTO_AES_XTS:
- case CRYPTO_AES_ICM:
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_NIST_GMAC:
- case CRYPTO_CAMELLIA_CBC:
- case CRYPTO_NULL_CBC:
- case CRYPTO_CHACHA20:
- case CRYPTO_AES_CCM_16:
- txf = swd->sw_exf;
-
- if (swd->sw_kschedule)
- txf->zerokey(&(swd->sw_kschedule));
- break;
-
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_NULL_HMAC:
- case CRYPTO_AES_CCM_CBC_MAC:
- axf = swd->sw_axf;
-
- if (swd->sw_ictx) {
- bzero(swd->sw_ictx, axf->ctxsize);
- free(swd->sw_ictx, M_CRYPTO_DATA);
- }
- if (swd->sw_octx) {
- bzero(swd->sw_octx, axf->ctxsize);
- free(swd->sw_octx, M_CRYPTO_DATA);
- }
- break;
-
- case CRYPTO_MD5_KPDK:
- case CRYPTO_SHA1_KPDK:
- axf = swd->sw_axf;
-
- if (swd->sw_ictx) {
- bzero(swd->sw_ictx, axf->ctxsize);
- free(swd->sw_ictx, M_CRYPTO_DATA);
- }
- if (swd->sw_octx) {
- bzero(swd->sw_octx, swd->sw_klen);
- free(swd->sw_octx, M_CRYPTO_DATA);
- }
- break;
-
- case CRYPTO_BLAKE2B:
- case CRYPTO_BLAKE2S:
- case CRYPTO_MD5:
- case CRYPTO_POLY1305:
- case CRYPTO_SHA1:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_512:
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- axf = swd->sw_axf;
-
- if (swd->sw_ictx) {
- explicit_bzero(swd->sw_ictx, axf->ctxsize);
- free(swd->sw_ictx, M_CRYPTO_DATA);
- }
- break;
-
- case CRYPTO_DEFLATE_COMP:
- /* Nothing to do */
- break;
+ txf = ses->swcr_encdec.sw_exf;
+ if (txf != NULL) {
+ if (ses->swcr_encdec.sw_kschedule != NULL)
+ txf->zerokey(&(ses->swcr_encdec.sw_kschedule));
+ }
+
+ axf = ses->swcr_auth.sw_axf;
+ if (axf != NULL) {
+ swa = &ses->swcr_auth;
+ if (swa->sw_ictx != NULL) {
+ explicit_bzero(swa->sw_ictx, axf->ctxsize);
+ free(swa->sw_ictx, M_CRYPTO_DATA);
+ }
+ if (swa->sw_octx != NULL) {
+ explicit_bzero(swa->sw_octx, swa->sw_octx_len);
+ free(swa->sw_octx, M_CRYPTO_DATA);
}
}
}
@@ -1191,117 +1435,16 @@
static int
swcr_process(device_t dev, struct cryptop *crp, int hint)
{
- struct swcr_session *ses = NULL;
- struct cryptodesc *crd;
- struct swcr_data *sw;
- size_t i;
-
- /* Sanity check */
- if (crp == NULL)
- return EINVAL;
-
- if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
- crp->crp_etype = EINVAL;
- goto done;
- }
+ struct swcr_session *ses;
ses = crypto_get_driver_session(crp->crp_session);
mtx_lock(&ses->swcr_lock);
- /* Go through crypto descriptors, processing as we go */
- for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
- /*
- * Find the crypto context.
- *
- * XXX Note that the logic here prevents us from having
- * XXX the same algorithm multiple times in a session
- * XXX (or rather, we can but it won't give us the right
- * XXX results). To do that, we'd need some way of differentiating
- * XXX between the various instances of an algorithm (so we can
- * XXX locate the correct crypto context).
- */
- for (i = 0; i < nitems(ses->swcr_algorithms) &&
- ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
- ;
+ crp->crp_etype = ses->swcr_process(ses, crp);
- /* No such context ? */
- if (i == nitems(ses->swcr_algorithms)) {
- crp->crp_etype = EINVAL;
- goto done;
- }
- sw = &ses->swcr_algorithms[i];
- switch (sw->sw_alg) {
- case CRYPTO_DES_CBC:
- case CRYPTO_3DES_CBC:
- case CRYPTO_BLF_CBC:
- case CRYPTO_CAST_CBC:
- case CRYPTO_SKIPJACK_CBC:
- case CRYPTO_RIJNDAEL128_CBC:
- case CRYPTO_AES_XTS:
- case CRYPTO_AES_ICM:
- case CRYPTO_CAMELLIA_CBC:
- case CRYPTO_CHACHA20:
- if ((crp->crp_etype = swcr_encdec(crd, sw,
- crp->crp_buf, crp->crp_flags)) != 0)
- goto done;
- break;
- case CRYPTO_NULL_CBC:
- crp->crp_etype = 0;
- break;
- case CRYPTO_MD5_HMAC:
- case CRYPTO_SHA1_HMAC:
- case CRYPTO_SHA2_224_HMAC:
- case CRYPTO_SHA2_256_HMAC:
- case CRYPTO_SHA2_384_HMAC:
- case CRYPTO_SHA2_512_HMAC:
- case CRYPTO_RIPEMD160_HMAC:
- case CRYPTO_NULL_HMAC:
- case CRYPTO_MD5_KPDK:
- case CRYPTO_SHA1_KPDK:
- case CRYPTO_MD5:
- case CRYPTO_SHA1:
- case CRYPTO_SHA2_224:
- case CRYPTO_SHA2_256:
- case CRYPTO_SHA2_384:
- case CRYPTO_SHA2_512:
- case CRYPTO_BLAKE2B:
- case CRYPTO_BLAKE2S:
- case CRYPTO_POLY1305:
- if ((crp->crp_etype = swcr_authcompute(crd, sw,
- crp->crp_buf, crp->crp_flags)) != 0)
- goto done;
- break;
-
- case CRYPTO_AES_NIST_GCM_16:
- case CRYPTO_AES_NIST_GMAC:
- case CRYPTO_AES_128_NIST_GMAC:
- case CRYPTO_AES_192_NIST_GMAC:
- case CRYPTO_AES_256_NIST_GMAC:
- case CRYPTO_AES_CCM_16:
- case CRYPTO_AES_CCM_CBC_MAC:
- crp->crp_etype = swcr_authenc(crp);
- goto done;
-
- case CRYPTO_DEFLATE_COMP:
- if ((crp->crp_etype = swcr_compdec(crd, sw,
- crp->crp_buf, crp->crp_flags)) != 0)
- goto done;
- else
- crp->crp_olen = (int)sw->sw_size;
- break;
-
- default:
- /* Unknown/unsupported algorithm */
- crp->crp_etype = EINVAL;
- goto done;
- }
- }
-
-done:
- if (ses)
- mtx_unlock(&ses->swcr_lock);
+ mtx_unlock(&ses->swcr_lock);
crypto_done(crp);
- return 0;
+ return (0);
}
static void
@@ -1323,58 +1466,15 @@
static int
swcr_attach(device_t dev)
{
- memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
- memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session),
CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
if (swcr_id < 0) {
device_printf(dev, "cannot initialize!");
- return ENOMEM;
+ return (ENXIO);
}
-#define REGISTER(alg) \
- crypto_register(swcr_id, alg, 0,0)
- REGISTER(CRYPTO_DES_CBC);
- REGISTER(CRYPTO_3DES_CBC);
- REGISTER(CRYPTO_BLF_CBC);
- REGISTER(CRYPTO_CAST_CBC);
- REGISTER(CRYPTO_SKIPJACK_CBC);
- REGISTER(CRYPTO_NULL_CBC);
- REGISTER(CRYPTO_MD5_HMAC);
- REGISTER(CRYPTO_SHA1_HMAC);
- REGISTER(CRYPTO_SHA2_224_HMAC);
- REGISTER(CRYPTO_SHA2_256_HMAC);
- REGISTER(CRYPTO_SHA2_384_HMAC);
- REGISTER(CRYPTO_SHA2_512_HMAC);
- REGISTER(CRYPTO_RIPEMD160_HMAC);
- REGISTER(CRYPTO_NULL_HMAC);
- REGISTER(CRYPTO_MD5_KPDK);
- REGISTER(CRYPTO_SHA1_KPDK);
- REGISTER(CRYPTO_MD5);
- REGISTER(CRYPTO_SHA1);
- REGISTER(CRYPTO_SHA2_224);
- REGISTER(CRYPTO_SHA2_256);
- REGISTER(CRYPTO_SHA2_384);
- REGISTER(CRYPTO_SHA2_512);
- REGISTER(CRYPTO_RIJNDAEL128_CBC);
- REGISTER(CRYPTO_AES_XTS);
- REGISTER(CRYPTO_AES_ICM);
- REGISTER(CRYPTO_AES_NIST_GCM_16);
- REGISTER(CRYPTO_AES_NIST_GMAC);
- REGISTER(CRYPTO_AES_128_NIST_GMAC);
- REGISTER(CRYPTO_AES_192_NIST_GMAC);
- REGISTER(CRYPTO_AES_256_NIST_GMAC);
- REGISTER(CRYPTO_CAMELLIA_CBC);
- REGISTER(CRYPTO_DEFLATE_COMP);
- REGISTER(CRYPTO_BLAKE2B);
- REGISTER(CRYPTO_BLAKE2S);
- REGISTER(CRYPTO_CHACHA20);
- REGISTER(CRYPTO_AES_CCM_16);
- REGISTER(CRYPTO_AES_CCM_CBC_MAC);
- REGISTER(CRYPTO_POLY1305);
-#undef REGISTER
- return 0;
+ return (0);
}
static int
@@ -1390,6 +1490,7 @@
DEVMETHOD(device_attach, swcr_attach),
DEVMETHOD(device_detach, swcr_detach),
+ DEVMETHOD(cryptodev_probesession, swcr_probesession),
DEVMETHOD(cryptodev_newsession, swcr_newsession),
DEVMETHOD(cryptodev_freesession,swcr_freesession),
DEVMETHOD(cryptodev_process, swcr_process),
Index: sys/opencrypto/ktls_ocf.c
===================================================================
--- sys/opencrypto/ktls_ocf.c
+++ sys/opencrypto/ktls_ocf.c
@@ -45,7 +45,6 @@
struct ocf_session {
crypto_session_t sid;
- int crda_alg;
struct mtx lock;
};
@@ -99,8 +98,6 @@
{
struct uio uio;
struct tls_aead_data ad;
- struct tls_nonce_data nd;
- struct cryptodesc *crde, *crda;
struct cryptop *crp;
struct ocf_session *os;
struct ocf_operation *oo;
@@ -115,19 +112,15 @@
oo->os = os;
iov = oo->iov;
- crp = crypto_getreq(2);
- if (crp == NULL) {
- free(oo, M_KTLS_OCF);
- return (ENOMEM);
- }
+ crp = crypto_getreq(os->sid, M_WAITOK);
/* Setup the IV. */
- memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);
- memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));
+ memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
+ memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
/* Setup the AAD. */
tls_comp_len = ntohs(hdr->tls_length) -
- (AES_GMAC_HASH_LEN + sizeof(nd.seq));
+ (AES_GMAC_HASH_LEN + sizeof(uint64_t));
ad.seq = htobe64(seqno);
ad.type = hdr->tls_type;
ad.tls_vmajor = hdr->tls_vmajor;
@@ -159,26 +152,20 @@
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_td = curthread;
- crp->crp_session = os->sid;
- crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
+ crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
+ crp->crp_buf_type = CRYPTO_BUF_UIO;
crp->crp_uio = &uio;
crp->crp_ilen = uio.uio_resid;
crp->crp_opaque = oo;
crp->crp_callback = ktls_ocf_callback;
- crde = crp->crp_desc;
- crda = crde->crd_next;
-
- crda->crd_alg = os->crda_alg;
- crda->crd_skip = 0;
- crda->crd_len = sizeof(ad);
- crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
-
- crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
- crde->crd_skip = sizeof(ad);
- crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
- crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- memcpy(crde->crd_iv, &nd, sizeof(nd));
+ crp->crp_aad_start = 0;
+ crp->crp_aad_length = sizeof(ad);
+ crp->crp_payload_start = sizeof(ad);
+ crp->crp_payload_length = crp->crp_ilen -
+ (sizeof(ad) + AES_GMAC_HASH_LEN);
+ crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
counter_u64_add(ocf_tls12_gcm_crypts, 1);
for (;;) {
@@ -215,7 +202,6 @@
struct uio uio;
struct tls_aead_data_13 ad;
char nonce[12];
- struct cryptodesc *crde, *crda;
struct cryptop *crp;
struct ocf_session *os;
struct ocf_operation *oo;
@@ -229,11 +215,7 @@
oo->os = os;
iov = oo->iov;
- crp = crypto_getreq(2);
- if (crp == NULL) {
- free(oo, M_KTLS_OCF);
- return (ENOMEM);
- }
+ crp = crypto_getreq(os->sid, M_WAITOK);
/* Setup the nonce. */
memcpy(nonce, tls->params.iv, tls->params.iv_len);
@@ -271,26 +253,21 @@
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_td = curthread;
- crp->crp_session = os->sid;
- crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
+ crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
+ crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
+ crp->crp_buf_type = CRYPTO_BUF_UIO;
crp->crp_uio = &uio;
crp->crp_ilen = uio.uio_resid;
crp->crp_opaque = oo;
crp->crp_callback = ktls_ocf_callback;
- crde = crp->crp_desc;
- crda = crde->crd_next;
-
- crda->crd_alg = os->crda_alg;
- crda->crd_skip = 0;
- crda->crd_len = sizeof(ad);
- crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
-
- crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
- crde->crd_skip = sizeof(ad);
- crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
- crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
- memcpy(crde->crd_iv, nonce, sizeof(nonce));
+ crp->crp_aad_start = 0;
+ crp->crp_aad_length = sizeof(ad);
+ crp->crp_payload_start = sizeof(ad);
+ crp->crp_payload_length = crp->crp_ilen -
+ (sizeof(ad) + AES_GMAC_HASH_LEN);
+ crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
+ memcpy(crp->crp_iv, nonce, sizeof(nonce));
counter_u64_add(ocf_tls13_gcm_crypts, 1);
for (;;) {
@@ -325,6 +302,7 @@
struct ocf_session *os;
os = tls->cipher;
+ crypto_freesession(os->sid);
mtx_destroy(&os->lock);
explicit_bzero(os, sizeof(*os));
free(os, M_KTLS_OCF);
@@ -333,27 +311,26 @@
static int
ktls_ocf_try(struct socket *so, struct ktls_session *tls)
{
- struct cryptoini cria, crie;
+ struct crypto_session_params csp;
struct ocf_session *os;
int error;
- memset(&cria, 0, sizeof(cria));
- memset(&crie, 0, sizeof(crie));
+ memset(&csp, 0, sizeof(csp));
switch (tls->params.cipher_algorithm) {
case CRYPTO_AES_NIST_GCM_16:
switch (tls->params.cipher_key_len) {
case 128 / 8:
- cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
- break;
case 256 / 8:
- cria.cri_alg = CRYPTO_AES_256_NIST_GMAC;
break;
default:
return (EINVAL);
}
- cria.cri_key = tls->params.cipher_key;
- cria.cri_klen = tls->params.cipher_key_len * 8;
+ csp.csp_mode = CSP_MODE_AEAD;
+ csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
+ csp.csp_cipher_key = tls->params.cipher_key;
+ csp.csp_cipher_klen = tls->params.cipher_key_len;
+ csp.csp_ivlen = AES_GCM_IV_LEN;
break;
default:
return (EPROTONOSUPPORT);
@@ -369,19 +346,13 @@
if (os == NULL)
return (ENOMEM);
- crie.cri_alg = tls->params.cipher_algorithm;
- crie.cri_key = tls->params.cipher_key;
- crie.cri_klen = tls->params.cipher_key_len * 8;
-
- crie.cri_next = &cria;
- error = crypto_newsession(&os->sid, &crie,
+ error = crypto_newsession(&os->sid, &csp,
CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
if (error) {
free(os, M_KTLS_OCF);
return (error);
}
- os->crda_alg = cria.cri_alg;
mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
tls->cipher = os;
if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
Index: sys/opencrypto/xform_gmac.c
===================================================================
--- sys/opencrypto/xform_gmac.c
+++ sys/opencrypto/xform_gmac.c
@@ -66,7 +66,7 @@
/* Authentication instances */
struct auth_hash auth_hash_nist_gmac_aes_128 = {
- CRYPTO_AES_128_NIST_GMAC, "GMAC-AES-128",
+ CRYPTO_AES_NIST_GMAC, "GMAC-AES-128",
AES_128_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
GMAC_BLOCK_LEN,
(void (*)(void *)) AES_GMAC_Init,
@@ -77,7 +77,7 @@
};
struct auth_hash auth_hash_nist_gmac_aes_192 = {
- CRYPTO_AES_192_NIST_GMAC, "GMAC-AES-192",
+ CRYPTO_AES_NIST_GMAC, "GMAC-AES-192",
AES_192_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
GMAC_BLOCK_LEN,
(void (*)(void *)) AES_GMAC_Init,
@@ -88,7 +88,7 @@
};
struct auth_hash auth_hash_nist_gmac_aes_256 = {
- CRYPTO_AES_256_NIST_GMAC, "GMAC-AES-256",
+ CRYPTO_AES_NIST_GMAC, "GMAC-AES-256",
AES_256_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
GMAC_BLOCK_LEN,
(void (*)(void *)) AES_GMAC_Init,
Index: sys/sys/bus_dma.h
===================================================================
--- sys/sys/bus_dma.h
+++ sys/sys/bus_dma.h
@@ -111,6 +111,7 @@
/* Forwards needed by prototypes below. */
union ccb;
struct bio;
+struct cryptop;
struct mbuf;
struct memdesc;
struct pmap;
@@ -263,6 +264,13 @@
bus_dmamap_callback_t *callback, void *callback_arg,
int flags);
+/*
+ * Like bus_dmamap_load but for crypto ops.
+ */
+int bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct cryptop *crp, bus_dmamap_callback_t *callback,
+ void *callback_arg, int flags);
+
/*
* Loads any memory descriptor.
*/
Index: tests/sys/opencrypto/cryptodev.py
===================================================================
--- tests/sys/opencrypto/cryptodev.py
+++ tests/sys/opencrypto/cryptodev.py
@@ -643,8 +643,7 @@
tag = _spdechex('0032a1dc85f1c9786925a2e71d8272dd')
tag = _spdechex('8d11a0929cb3fbe1fef01a4a38d5f8ea')
- c = Crypto(CRYPTO_AES_NIST_GCM_16, key,
- mac=CRYPTO_AES_128_NIST_GMAC, mackey=key)
+ c = Crypto(CRYPTO_AES_NIST_GCM_16, key)
enc, enctag = c.encrypt(pt, iv, aad=aad)
@@ -680,7 +679,7 @@
ct = _spdechex('93fe7d9e9bfd10348a5606e5cafa7354')
tag = _spdechex('0032a1dc85f1c9786925a2e71d8272dd')
- c = Crypto(CRYPTO_AES_GCM_16, key, mac=CRYPTO_AES_128_GMAC, mackey=key)
+ c = Crypto(CRYPTO_AES_GCM_16, key)
enc, enctag = c.encrypt(pt, iv, aad=aad)
Index: tests/sys/opencrypto/cryptodevh.py
===================================================================
--- tests/sys/opencrypto/cryptodevh.py
+++ tests/sys/opencrypto/cryptodevh.py
@@ -194,9 +194,6 @@
CRYPTO_AES_ICM = 23
CRYPTO_AES_NIST_GMAC = 24
CRYPTO_AES_NIST_GCM_16 = 25
-CRYPTO_AES_128_NIST_GMAC = 26
-CRYPTO_AES_192_NIST_GMAC = 27
-CRYPTO_AES_256_NIST_GMAC = 28
CRYPTO_BLAKE2B = 29
CRYPTO_BLAKE2S = 30
CRYPTO_CHACHA20 = 31
Index: tests/sys/opencrypto/cryptotest.py
===================================================================
--- tests/sys/opencrypto/cryptotest.py
+++ tests/sys/opencrypto/cryptotest.py
@@ -90,10 +90,6 @@
for i in katg('gcmtestvectors', 'gcmDecrypt*'):
self.runGCM(i, 'DECRYPT')
- _gmacsizes = { 32: cryptodev.CRYPTO_AES_256_NIST_GMAC,
- 24: cryptodev.CRYPTO_AES_192_NIST_GMAC,
- 16: cryptodev.CRYPTO_AES_128_NIST_GMAC,
- }
def runGCM(self, fname, mode):
curfun = None
if mode == 'ENCRYPT':
@@ -127,9 +123,7 @@
try:
c = Crypto(cryptodev.CRYPTO_AES_NIST_GCM_16,
- cipherkey,
- mac=self._gmacsizes[len(cipherkey)],
- mackey=cipherkey, crid=crid,
+ cipherkey, crid=crid,
maclen=16)
except EnvironmentError as e:
# Can't test algorithms the driver does not support.
Index: tools/tools/crypto/cryptocheck.c
===================================================================
--- tools/tools/crypto/cryptocheck.c
+++ tools/tools/crypto/cryptocheck.c
@@ -95,6 +95,9 @@
* sha256hmac 256-bit SHA-2 HMAC
* sha384hmac 384-bit SHA-2 HMAC
* sha512hmac 512-bit SHA-2 HMAC
+ * gmac 128-bit GMAC
+ * gmac192 192-bit GMAC
+ * gmac256 256-bit GMAC
*
* Ciphers:
* aes-cbc 128-bit AES-CBC
@@ -145,7 +148,7 @@
const char *name;
int cipher;
int mac;
- enum { T_HASH, T_HMAC, T_CIPHER, T_ETA, T_AEAD } type;
+ enum { T_HASH, T_HMAC, T_GMAC, T_CIPHER, T_ETA, T_AEAD } type;
const EVP_CIPHER *(*evp_cipher)(void);
const EVP_MD *(*evp_md)(void);
} algs[] = {
@@ -173,6 +176,12 @@
.evp_md = EVP_blake2b512 },
{ .name = "blake2s", .mac = CRYPTO_BLAKE2S, .type = T_HASH,
.evp_md = EVP_blake2s256 },
+ { .name = "gmac", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+ .evp_cipher = EVP_aes_128_gcm },
+ { .name = "gmac192", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+ .evp_cipher = EVP_aes_192_gcm },
+ { .name = "gmac256", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+ .evp_cipher = EVP_aes_256_gcm },
{ .name = "aes-cbc", .cipher = CRYPTO_AES_CBC, .type = T_CIPHER,
.evp_cipher = EVP_aes_128_cbc },
{ .name = "aes-cbc192", .cipher = CRYPTO_AES_CBC, .type = T_CIPHER,
@@ -191,23 +200,17 @@
.evp_cipher = EVP_aes_256_xts },
{ .name = "chacha20", .cipher = CRYPTO_CHACHA20, .type = T_CIPHER,
.evp_cipher = EVP_chacha20 },
- { .name = "aes-gcm", .cipher = CRYPTO_AES_NIST_GCM_16,
- .mac = CRYPTO_AES_128_NIST_GMAC, .type = T_AEAD,
+ { .name = "aes-gcm", .cipher = CRYPTO_AES_NIST_GCM_16, .type = T_AEAD,
.evp_cipher = EVP_aes_128_gcm },
{ .name = "aes-gcm192", .cipher = CRYPTO_AES_NIST_GCM_16,
- .mac = CRYPTO_AES_192_NIST_GMAC, .type = T_AEAD,
- .evp_cipher = EVP_aes_192_gcm },
+ .type = T_AEAD, .evp_cipher = EVP_aes_192_gcm },
{ .name = "aes-gcm256", .cipher = CRYPTO_AES_NIST_GCM_16,
- .mac = CRYPTO_AES_256_NIST_GMAC, .type = T_AEAD,
- .evp_cipher = EVP_aes_256_gcm },
- { .name = "aes-ccm", .cipher = CRYPTO_AES_CCM_16,
- .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+ .type = T_AEAD, .evp_cipher = EVP_aes_256_gcm },
+ { .name = "aes-ccm", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
.evp_cipher = EVP_aes_128_ccm },
- { .name = "aes-ccm192", .cipher = CRYPTO_AES_CCM_16,
- .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+ { .name = "aes-ccm192", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
.evp_cipher = EVP_aes_192_ccm },
- { .name = "aes-ccm256", .cipher = CRYPTO_AES_CCM_16,
- .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+ { .name = "aes-ccm256", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
.evp_cipher = EVP_aes_256_ccm },
};
@@ -832,7 +835,7 @@
return (ocf_init_session(&sop, "ETA", alg->name, ses));
}
-static bool
+static int
ocf_eta(const struct ocf_session *ses, const struct alg *alg, const char *iv,
size_t iv_len, const char *aad, size_t aad_len, const char *input,
char *output, size_t size, char *digest, int op)
@@ -844,7 +847,6 @@
ocf_init_caead(ses, &caead);
caead.op = op;
- caead.flags = op == COP_ENCRYPT ? COP_F_CIPHER_FIRST : 0;
caead.len = size;
caead.aadlen = aad_len;
caead.ivlen = iv_len;
@@ -860,7 +862,6 @@
ocf_init_cop(ses, &cop);
cop.op = op;
- cop.flags = op == COP_ENCRYPT ? COP_F_CIPHER_FIRST : 0;
cop.len = size;
cop.src = (char *)input;
cop.dst = output;
@@ -870,13 +871,9 @@
ret = ioctl(ses->fd, CIOCCRYPT, &cop);
}
- if (ret < 0) {
- warn("cryptodev %s (%zu) ETA failed for device %s",
- alg->name, size, crfind(crid));
- return (false);
- }
-
- return (true);
+ if (ret < 0)
+ return (errno);
+ return (0);
}
static void
@@ -887,7 +884,8 @@
const EVP_MD *md;
char *aad, *buffer, *cleartext, *ciphertext;
char *iv, *auth_key, *cipher_key;
- u_int i, iv_len, auth_key_len, cipher_key_len, digest_len;
+ u_int iv_len, auth_key_len, cipher_key_len, digest_len;
+ int error;
char control_digest[EVP_MAX_MD_SIZE], test_digest[EVP_MAX_MD_SIZE];
cipher = alg->evp_cipher();
@@ -935,10 +933,14 @@
goto out;
/* OCF encrypt + HMAC. */
- if (!ocf_eta(&ses, alg, iv, iv_len,
+ error = ocf_eta(&ses, alg, iv, iv_len,
aad_len != 0 ? cleartext : NULL, aad_len, cleartext + aad_len,
- buffer + aad_len, size, test_digest, COP_ENCRYPT))
+ buffer + aad_len, size, test_digest, COP_ENCRYPT);
+ if (error != 0) {
+ warnc(error, "cryptodev %s (%zu) ETA failed for device %s",
+ alg->name, size, crfind(ses.crid));
goto out;
+ }
if (memcmp(ciphertext + aad_len, buffer + aad_len, size) != 0) {
printf("%s (%zu) encryption mismatch:\n", alg->name, size);
printf("control:\n");
@@ -962,10 +964,14 @@
}
/* OCF HMAC + decrypt. */
- if (!ocf_eta(&ses, alg, iv, iv_len,
+ error = ocf_eta(&ses, alg, iv, iv_len,
aad_len != 0 ? ciphertext : NULL, aad_len, ciphertext + aad_len,
- buffer + aad_len, size, test_digest, COP_DECRYPT))
+ buffer + aad_len, size, test_digest, COP_DECRYPT);
+ if (error != 0) {
+ warnc(error, "cryptodev %s (%zu) ETA failed for device %s",
+ alg->name, size, crfind(ses.crid));
goto out;
+ }
if (memcmp(cleartext + aad_len, buffer + aad_len, size) != 0) {
printf("%s (%zu) decryption mismatch:\n", alg->name, size);
printf("control:\n");
@@ -975,6 +981,23 @@
goto out;
}
+ /* Verify OCF HMAC + decrypt fails with busted MAC. */
+ test_digest[0] ^= 0x1;
+ error = ocf_eta(&ses, alg, iv, iv_len,
+ aad_len != 0 ? ciphertext : NULL, aad_len, ciphertext + aad_len,
+ buffer + aad_len, size, test_digest, COP_DECRYPT);
+ if (error != EBADMSG) {
+ if (error != 0)
+ warnc(error,
+ "cryptodev %s (%zu) corrupt tag failed for device %s",
+ alg->name, size, crfind(ses.crid));
+ else
+ warnx(
+ "cryptodev %s (%zu) corrupt tag didn't fail for device %s",
+ alg->name, size, crfind(ses.crid));
+ goto out;
+ }
+
if (verbose)
printf("%s (%zu) matched (cryptodev device %s)\n",
alg->name, size, crfind(ses.crid));
@@ -989,6 +1012,115 @@
free(cipher_key);
}
+static void
+openssl_gmac(const struct alg *alg, const EVP_CIPHER *cipher, const char *key,
+ const char *iv, const char *input, size_t size, char *tag)
+{
+ EVP_CIPHER_CTX *ctx;
+ int outl;
+
+ ctx = EVP_CIPHER_CTX_new();
+ if (ctx == NULL)
+ errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
+ size, ERR_error_string(ERR_get_error(), NULL));
+ if (EVP_EncryptInit_ex(ctx, cipher, NULL, (const u_char *)key,
+ (const u_char *)iv) != 1)
+ errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
+ size, ERR_error_string(ERR_get_error(), NULL));
+ EVP_CIPHER_CTX_set_padding(ctx, 0);
+ if (EVP_EncryptUpdate(ctx, NULL, &outl, (const u_char *)input,
+ size) != 1)
+ errx(1, "OpenSSL %s (%zu) update failed: %s",
+ alg->name, size, ERR_error_string(ERR_get_error(), NULL));
+ if (EVP_EncryptFinal_ex(ctx, NULL, &outl) != 1)
+ errx(1, "OpenSSL %s (%zu) final failed: %s", alg->name,
+ size, ERR_error_string(ERR_get_error(), NULL));
+ if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, AES_GMAC_HASH_LEN,
+ tag) != 1)
+ errx(1, "OpenSSL %s (%zu) get tag failed: %s", alg->name,
+ size, ERR_error_string(ERR_get_error(), NULL));
+ EVP_CIPHER_CTX_free(ctx);
+}
+
+static bool
+ocf_gmac(const struct alg *alg, const char *input, size_t size, const char *key,
+ size_t key_len, const char *iv, char *tag, int *cridp)
+{
+ struct ocf_session ses;
+ struct session2_op sop;
+ struct crypt_op cop;
+
+ ocf_init_sop(&sop);
+ sop.mackeylen = key_len;
+ sop.mackey = (char *)key;
+ sop.mac = alg->mac;
+ if (!ocf_init_session(&sop, "GMAC", alg->name, &ses))
+ return (false);
+
+ ocf_init_cop(&ses, &cop);
+ cop.op = 0;
+ cop.len = size;
+ cop.src = (char *)input;
+ cop.mac = tag;
+ cop.iv = iv;
+
+ if (ioctl(ses.fd, CIOCCRYPT, &cop) < 0) {
+ warn("cryptodev %s (%zu) failed for device %s", alg->name,
+ size, crfind(crid));
+ ocf_destroy_session(&ses);
+ return (false);
+ }
+
+ *cridp = ses.crid;
+ ocf_destroy_session(&ses);
+ return (true);
+}
+
+static void
+run_gmac_test(const struct alg *alg, size_t size)
+{
+ const EVP_CIPHER *cipher;
+ char *iv, *key, *buffer;
+ u_int iv_len, key_len, digest_len;
+ int crid;
+ char control_tag[AES_GMAC_HASH_LEN], test_tag[AES_GMAC_HASH_LEN];
+
+ cipher = alg->evp_cipher();
+
+ memset(control_tag, 0x3c, sizeof(control_tag));
+ memset(test_tag, 0x3c, sizeof(test_tag));
+
+ key_len = EVP_CIPHER_key_length(cipher);
+ iv_len = EVP_CIPHER_iv_length(cipher);
+
+ key = alloc_buffer(key_len);
+ iv = generate_iv(iv_len, alg);
+ buffer = alloc_buffer(size);
+
+ /* OpenSSL GMAC. */
+ openssl_gmac(alg, cipher, key, iv, buffer, size, control_tag);
+
+ /* OCF GMAC. */
+ if (!ocf_gmac(alg, buffer, size, key, key_len, iv, test_tag, &crid))
+ goto out;
+ if (memcmp(control_tag, test_tag, sizeof(control_tag)) != 0) {
+ printf("%s (%zu) mismatch:\n", alg->name, size);
+ printf("control:\n");
+ hexdump(control_tag, sizeof(control_tag), NULL, 0);
+ printf("test (cryptodev device %s):\n", crfind(crid));
+ hexdump(test_tag, sizeof(test_tag), NULL, 0);
+ goto out;
+ }
+
+ if (verbose)
+ printf("%s (%zu) matched (cryptodev device %s)\n",
+ alg->name, size, crfind(crid));
+
+out:
+ free(buffer);
+ free(key);
+}
+
static void
openssl_gcm_encrypt(const struct alg *alg, const EVP_CIPHER *cipher,
const char *key, const char *iv, const char *aad, size_t aad_len,
@@ -1142,9 +1274,6 @@
sop.keylen = key_len;
sop.key = (char *)key;
sop.cipher = alg->cipher;
- sop.mackeylen = key_len;
- sop.mackey = (char *)key;
- sop.mac = alg->mac;
return (ocf_init_session(&sop, "AEAD", alg->name, ses));
}
@@ -1316,6 +1445,9 @@
case T_HMAC:
run_hmac_test(alg, size);
break;
+ case T_GMAC:
+ run_gmac_test(alg, size);
+ break;
case T_CIPHER:
run_cipher_test(alg, size);
break;
@@ -1353,7 +1485,7 @@
u_int i;
for (i = 0; i < nitems(algs); i++)
- if (algs[i].type == T_HMAC)
+ if (algs[i].type == T_HMAC || algs[i].type == T_GMAC)
run_test_sizes(&algs[i], sizes, nsizes);
}

File Metadata

Mime Type
text/plain
Expires
Mon, Nov 24, 5:25 PM (4 h, 43 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
26067539
Default Alt Text
D23677.id68298.diff (688 KB)

Event Timeline