Differential D25466 Diff 73930 head/sys/opencrypto/crypto.c

Changeset View

Standalone View

head/sys/opencrypto/crypto.c

Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines
* PURPOSE.		* PURPOSE.
*/		*/

#include "opt_compat.h"		#include "opt_compat.h"
#include "opt_ddb.h"		#include "opt_ddb.h"

#include <sys/param.h>		#include <sys/param.h>
#include <sys/systm.h>		#include <sys/systm.h>
#include <sys/eventhandler.h>		#include <sys/counter.h>
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/kthread.h>		#include <sys/kthread.h>
#include <sys/linker.h>		#include <sys/linker.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/module.h>		#include <sys/module.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/mbuf.h>		#include <sys/mbuf.h>
▲ Show 20 Lines • Show All 156 Lines • ▼ Show 20 Lines
static struct proc *cryptoproc;		static struct proc *cryptoproc;
static void crypto_ret_proc(struct crypto_ret_worker *ret_worker);		static void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
static void crypto_destroy(void);		static void crypto_destroy(void);
static int crypto_invoke(struct cryptocap cap, struct cryptop crp, int hint);		static int crypto_invoke(struct cryptocap cap, struct cryptop crp, int hint);
static int crypto_kinvoke(struct cryptkop *krp);		static int crypto_kinvoke(struct cryptkop *krp);
static void crypto_task_invoke(void *ctx, int pending);		static void crypto_task_invoke(void *ctx, int pending);
static void crypto_batch_enqueue(struct cryptop *crp);		static void crypto_batch_enqueue(struct cryptop *crp);

static struct cryptostats cryptostats;		static counter_u64_t cryptostats[sizeof(struct cryptostats) / sizeof(uint64_t)];
SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats,		SYSCTL_COUNTER_U64_ARRAY(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW,
cryptostats, "Crypto system statistics");		cryptostats, nitems(cryptostats),
		"Crypto system statistics");

		#define CRYPTOSTAT_INC(stat) do { \
		counter_u64_add( \
		cryptostats[offsetof(struct cryptostats, stat) / sizeof(uint64_t)],\
		1); \
		} while (0)

		static void
		cryptostats_init(void *arg __unused)
		{
		COUNTER_ARRAY_ALLOC(cryptostats, nitems(cryptostats), M_WAITOK);
		}
		SYSINIT(cryptostats_init, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_init, NULL);

		static void
		cryptostats_fini(void *arg __unused)
		{
		COUNTER_ARRAY_FREE(cryptostats, nitems(cryptostats));
		}
		SYSUNINIT(cryptostats_fini, SI_SUB_COUNTER, SI_ORDER_ANY, cryptostats_fini,
		NULL);

/* Try to avoid directly exposing the key buffer as a symbol */		/* Try to avoid directly exposing the key buffer as a symbol */
static struct keybuf *keybuf;		static struct keybuf *keybuf;

static struct keybuf empty_keybuf = {		static struct keybuf empty_keybuf = {
.kb_nents = 0		.kb_nents = 0
};		};

/* Obtain the key buffer from boot metadata */		/* Obtain the key buffer from boot metadata */
▲ Show 20 Lines • Show All 1,147 Lines • ▼ Show 20 Lines
{		{
struct cryptocap *cap;		struct cryptocap *cap;
int result;		int result;

#ifdef INVARIANTS		#ifdef INVARIANTS
crp_sanity(crp);		crp_sanity(crp);
#endif		#endif

cryptostats.cs_ops++;		CRYPTOSTAT_INC(cs_ops);

crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;		crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;

if (CRYPTOP_ASYNC(crp)) {		if (CRYPTOP_ASYNC(crp)) {
if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {		if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
struct crypto_ret_worker *ret_worker;		struct crypto_ret_worker *ret_worker;

ret_worker = CRYPTO_RETW(crp->crp_retw_id);		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines
* Add an asymetric crypto request to a queue,		* Add an asymetric crypto request to a queue,
* to be processed by the kernel thread.		* to be processed by the kernel thread.
*/		*/
int		int
crypto_kdispatch(struct cryptkop *krp)		crypto_kdispatch(struct cryptkop *krp)
{		{
int error;		int error;

cryptostats.cs_kops++;		CRYPTOSTAT_INC(cs_kops);

krp->krp_cap = NULL;		krp->krp_cap = NULL;
error = crypto_kinvoke(krp);		error = crypto_kinvoke(krp);
if (error == ERESTART) {		if (error == ERESTART) {
CRYPTO_Q_LOCK();		CRYPTO_Q_LOCK();
TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);		TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
if (crp_sleep)		if (crp_sleep)
wakeup_one(&crp_q);		wakeup_one(&crp_q);
▲ Show 20 Lines • Show All 290 Lines • ▼ Show 20 Lines
*/		*/
void		void
crypto_done(struct cryptop *crp)		crypto_done(struct cryptop *crp)
{		{
KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,		KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
("crypto_done: op already done, flags 0x%x", crp->crp_flags));		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
crp->crp_flags \|= CRYPTO_F_DONE;		crp->crp_flags \|= CRYPTO_F_DONE;
if (crp->crp_etype != 0)		if (crp->crp_etype != 0)
cryptostats.cs_errs++;		CRYPTOSTAT_INC(cs_errs);

/*		/*
* CBIMM means unconditionally do the callback immediately;		* CBIMM means unconditionally do the callback immediately;
* CBIFSYNC means do the callback immediately only if the		* CBIFSYNC means do the callback immediately only if the
* operation was done synchronously. Both are used to avoid		* operation was done synchronously. Both are used to avoid
* doing extraneous context switches; the latter is mostly		* doing extraneous context switches; the latter is mostly
* used with the software crypto driver.		* used with the software crypto driver.
*/		*/
▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines
*/		*/
void		void
crypto_kdone(struct cryptkop *krp)		crypto_kdone(struct cryptkop *krp)
{		{
struct crypto_ret_worker *ret_worker;		struct crypto_ret_worker *ret_worker;
struct cryptocap *cap;		struct cryptocap *cap;

if (krp->krp_status != 0)		if (krp->krp_status != 0)
cryptostats.cs_kerrs++;		CRYPTOSTAT_INC(cs_kerrs);
CRYPTO_DRIVER_LOCK();		CRYPTO_DRIVER_LOCK();
cap = krp->krp_cap;		cap = krp->krp_cap;
KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));		KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
cap->cc_koperations--;		cap->cc_koperations--;
if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)		if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
wakeup(cap);		wakeup(cap);
CRYPTO_DRIVER_UNLOCK();		CRYPTO_DRIVER_UNLOCK();
krp->krp_cap = NULL;		krp->krp_cap = NULL;
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Lines	if (submit != NULL) {
* the request back in the queue. It would		* the request back in the queue. It would
* best to put the request back where we got		* best to put the request back where we got
* it but that's hard so for now we put it		* it but that's hard so for now we put it
* at the front. This should be ok; putting		* at the front. This should be ok; putting
* it at the end does not work.		* it at the end does not work.
*/		*/
cap->cc_qblocked = 1;		cap->cc_qblocked = 1;
TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);		TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
cryptostats.cs_blocks++;		CRYPTOSTAT_INC(cs_blocks);
}		}
}		}

/* As above, but for key ops */		/* As above, but for key ops */
TAILQ_FOREACH(krp, &crp_kq, krp_next) {		TAILQ_FOREACH(krp, &crp_kq, krp_next) {
cap = krp->krp_cap;		cap = krp->krp_cap;
if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {		if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
/*		/*
Show All 20 Lines	if (krp != NULL) {
* the request back in the queue. It would		* the request back in the queue. It would
* best to put the request back where we got		* best to put the request back where we got
* it but that's hard so for now we put it		* it but that's hard so for now we put it
* at the front. This should be ok; putting		* at the front. This should be ok; putting
* it at the end does not work.		* it at the end does not work.
*/		*/
krp->krp_cap->cc_kqblocked = 1;		krp->krp_cap->cc_kqblocked = 1;
TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);		TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
cryptostats.cs_kblocks++;		CRYPTOSTAT_INC(cs_kblocks);
}		}
}		}

if (submit == NULL && krp == NULL) {		if (submit == NULL && krp == NULL) {
/*		/*
* Nothing more to be processed. Sleep until we're		* Nothing more to be processed. Sleep until we're
* woken because there are more ops to process.		* woken because there are more ops to process.
* This happens either by submission or by a driver		* This happens either by submission or by a driver
* becoming unblocked and notifying us through		* becoming unblocked and notifying us through
* crypto_unblock. Note that when we wakeup we		* crypto_unblock. Note that when we wakeup we
* start processing each queue again from the		* start processing each queue again from the
* front. It's not clear that it's important to		* front. It's not clear that it's important to
* preserve this ordering since ops may finish		* preserve this ordering since ops may finish
* out of order if dispatched to different devices		* out of order if dispatched to different devices
* and some become blocked while others do not.		* and some become blocked while others do not.
*/		*/
crp_sleep = 1;		crp_sleep = 1;
msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);		msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
crp_sleep = 0;		crp_sleep = 0;
if (cryptoproc == NULL)		if (cryptoproc == NULL)
break;		break;
cryptostats.cs_intrs++;		CRYPTOSTAT_INC(cs_intrs);
}		}
}		}
CRYPTO_Q_UNLOCK();		CRYPTO_Q_UNLOCK();

crypto_finis(&crp_q);		crypto_finis(&crp_q);
}		}

/*		/*
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	if (crpt != NULL \|\| krpt != NULL) {
/*		/*
* Nothing more to be processed. Sleep until we're		* Nothing more to be processed. Sleep until we're
* woken because there are more returns to process.		* woken because there are more returns to process.
*/		*/
msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,		msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
"crypto_ret_wait", 0);		"crypto_ret_wait", 0);
if (ret_worker->cryptoretproc == NULL)		if (ret_worker->cryptoretproc == NULL)
break;		break;
cryptostats.cs_rets++;		CRYPTOSTAT_INC(cs_rets);
}		}
}		}
CRYPTO_RETW_UNLOCK(ret_worker);		CRYPTO_RETW_UNLOCK(ret_worker);

crypto_finis(&ret_worker->crp_ret_q);		crypto_finis(&ret_worker->crp_ret_q);
}		}

#ifdef DDB		#ifdef DDB
▲ Show 20 Lines • Show All 133 Lines • Show Last 20 Lines