Page MenuHomeFreeBSD
Differential D25290 Diff 73486 head/sys/dev/cxgbe/crypto/t4_crypto.c

Changeset View

Standalone View

View Options

head/sys/dev/cxgbe/crypto/t4_crypto.c

Show First 20 Lines • Show All 381 Lines • ▼ Show 20 Linesccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs)
usgl->len0 = htobe32(ss->ss_len); usgl->len0 = htobe32(ss->ss_len);
usgl->addr0 = htobe64(ss->ss_paddr); usgl->addr0 = htobe64(ss->ss_paddr);
ss++; ss++;
for (i = 0; i < sg->sg_nseg - 1; i++) { for (i = 0; i < sg->sg_nseg - 1; i++) {
usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len); usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len);
usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr); usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr);
ss++; ss++;
} }
} }
static bool static bool
ccr_use_imm_data(u_int transhdr_len, u_int input_len) ccr_use_imm_data(u_int transhdr_len, u_int input_len)
{ {
if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN) if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN)
return (false); return (false);
▲ Show 20 Lines • Show All 515 Lines • ▼ Show 20 Linesccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len; imm_len = input_len;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(sc->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, if (crp->crp_aad != NULL)
crp->crp_aad_start, crp->crp_aad_length); error = sglist_append(sc->sg_ulptx,
crp->crp_aad, crp->crp_aad_length);
else
error = sglist_append_sglist(sc->sg_ulptx,
sc->sg_input, crp->crp_aad_start,
crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
crp->crp_digest_start, hash_size_in_response); crp->crp_digest_start, hash_size_in_response);
if (error) if (error)
return (error); return (error);
} }
sgl_nsegs = sc->sg_ulptx->sg_nseg; sgl_nsegs = sc->sg_ulptx->sg_nseg;
sgl_len = ccr_ulptx_sgl_len(sgl_nsegs); sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
} }
/* /* Any AAD comes after the IV. */
* Any auth-only data before the cipher region is marked as AAD.
* Auth-data that overlaps with the cipher region is placed in
* the auth section.
*/
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
aad_start = iv_len + 1; aad_start = iv_len + 1;
aad_stop = aad_start + crp->crp_aad_length - 1; aad_stop = aad_start + crp->crp_aad_length - 1;
} else { } else {
aad_start = 0; aad_start = 0;
aad_stop = 0; aad_stop = 0;
} }
cipher_start = iv_len + crp->crp_aad_length + 1; cipher_start = iv_len + crp->crp_aad_length + 1;
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Linesccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
if (imm_len != 0) { if (imm_len != 0) {
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL)
memcpy(dst, crp->crp_aad, crp->crp_aad_length);
else
crypto_copydata(crp, crp->crp_aad_start, crypto_copydata(crp, crp->crp_aad_start,
crp->crp_aad_length, dst); crp->crp_aad_length, dst);
dst += crp->crp_aad_length; dst += crp->crp_aad_length;
} }
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
dst += crp->crp_payload_length; dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
▲ Show 20 Lines • Show All 152 Lines • ▼ Show 20 Linesccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) { if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
imm_len = input_len; imm_len = input_len;
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
imm_len = 0; imm_len = 0;
sglist_reset(sc->sg_ulptx); sglist_reset(sc->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, if (crp->crp_aad != NULL)
crp->crp_aad_start, crp->crp_aad_length); error = sglist_append(sc->sg_ulptx,
crp->crp_aad, crp->crp_aad_length);
else
error = sglist_append_sglist(sc->sg_ulptx,
sc->sg_input, crp->crp_aad_start,
crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
▲ Show 20 Lines • Show All 95 Lines • ▼ Show 20 Linesccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
dst = (char *)(crwr + 1) + kctx_len; dst = (char *)(crwr + 1) + kctx_len;
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
if (imm_len != 0) { if (imm_len != 0) {
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL)
memcpy(dst, crp->crp_aad, crp->crp_aad_length);
else
crypto_copydata(crp, crp->crp_aad_start, crypto_copydata(crp, crp->crp_aad_start,
crp->crp_aad_length, dst); crp->crp_aad_length, dst);
dst += crp->crp_aad_length; dst += crp->crp_aad_length;
} }
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
dst += crp->crp_payload_length; dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
goto out; goto out;
} }
memcpy(iv, crp->crp_iv, 12); memcpy(iv, crp->crp_iv, 12);
*(uint32_t *)&iv[12] = htobe32(1); *(uint32_t *)&iv[12] = htobe32(1);
axf->Reinit(auth_ctx, iv, sizeof(iv)); axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */ /* MAC the AAD. */
if (crp->crp_aad != NULL) {
len = rounddown(crp->crp_aad_length, sizeof(block));
if (len != 0)
axf->Update(auth_ctx, crp->crp_aad, len);
if (crp->crp_aad_length != len) {
memset(block, 0, sizeof(block));
memcpy(block, (char *)crp->crp_aad + len,
crp->crp_aad_length - len);
axf->Update(auth_ctx, block, sizeof(block));
}
} else {
for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
len = imin(crp->crp_aad_length - i, sizeof(block)); len = imin(crp->crp_aad_length - i, sizeof(block));
crypto_copydata(crp, crp->crp_aad_start + i, len, block); crypto_copydata(crp, crp->crp_aad_start + i, len,
block);
bzero(block + len, sizeof(block) - len); bzero(block + len, sizeof(block) - len);
axf->Update(auth_ctx, block, sizeof(block)); axf->Update(auth_ctx, block, sizeof(block));
} }
}
exf->reinit(kschedule, iv); exf->reinit(kschedule, iv);
/* Do encryption with MAC */ /* Do encryption with MAC */
for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
len = imin(crp->crp_payload_length - i, sizeof(block)); len = imin(crp->crp_payload_length - i, sizeof(block));
crypto_copydata(crp, crp->crp_payload_start + i, len, block); crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len); bzero(block + len, sizeof(block) - len);
▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
sgl_nsegs = 0; sgl_nsegs = 0;
sgl_len = 0; sgl_len = 0;
} else { } else {
/* Block 0 is passed as immediate data. */ /* Block 0 is passed as immediate data. */
imm_len = b0_len; imm_len = b0_len;
sglist_reset(sc->sg_ulptx); sglist_reset(sc->sg_ulptx);
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, if (crp->crp_aad != NULL)
crp->crp_aad_start, crp->crp_aad_length); error = sglist_append(sc->sg_ulptx,
crp->crp_aad, crp->crp_aad_length);
else
error = sglist_append_sglist(sc->sg_ulptx,
sc->sg_input, crp->crp_aad_start,
crp->crp_aad_length);
if (error) if (error)
return (error); return (error);
} }
error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input, error = sglist_append_sglist(sc->sg_ulptx, sc->sg_input,
crp->crp_payload_start, crp->crp_payload_length); crp->crp_payload_start, crp->crp_payload_length);
if (error) if (error)
return (error); return (error);
if (op_type == CHCR_DECRYPT_OP) { if (op_type == CHCR_DECRYPT_OP) {
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Linesccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs); ccr_write_phys_dsgl(sc, s, dst, dsgl_nsegs);
dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len; dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
memcpy(dst, iv, iv_len); memcpy(dst, iv, iv_len);
dst += iv_len; dst += iv_len;
generate_ccm_b0(crp, hash_size_in_response, iv, dst); generate_ccm_b0(crp, hash_size_in_response, iv, dst);
if (sgl_nsegs == 0) { if (sgl_nsegs == 0) {
dst += b0_len; dst += b0_len;
if (crp->crp_aad_length != 0) { if (crp->crp_aad_length != 0) {
if (crp->crp_aad != NULL)
memcpy(dst, crp->crp_aad, crp->crp_aad_length);
else
crypto_copydata(crp, crp->crp_aad_start, crypto_copydata(crp, crp->crp_aad_start,
crp->crp_aad_length, dst); crp->crp_aad_length, dst);
dst += crp->crp_aad_length; dst += crp->crp_aad_length;
} }
crypto_copydata(crp, crp->crp_payload_start, crypto_copydata(crp, crp->crp_payload_start,
crp->crp_payload_length, dst); crp->crp_payload_length, dst);
dst += crp->crp_payload_length; dst += crp->crp_payload_length;
if (op_type == CHCR_DECRYPT_OP) if (op_type == CHCR_DECRYPT_OP)
crypto_copydata(crp, crp->crp_digest_start, crypto_copydata(crp, crp->crp_digest_start,
hash_size_in_response, dst); hash_size_in_response, dst);
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Linesccr_ccm_soft(struct ccr_session *s, struct cryptop *crp)
} }
memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN); memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN);
auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length; auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length; auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
axf->Reinit(auth_ctx, iv, sizeof(iv)); axf->Reinit(auth_ctx, iv, sizeof(iv));
/* MAC the AAD. */ /* MAC the AAD. */
for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) { if (crp->crp_aad != NULL)
len = imin(crp->crp_aad_length - i, sizeof(block)); error = axf->Update(auth_ctx, crp->crp_aad,
crypto_copydata(crp, crp->crp_aad_start + i, len, block); crp->crp_aad_length);
bzero(block + len, sizeof(block) - len); else
axf->Update(auth_ctx, block, sizeof(block)); error = crypto_apply(crp, crp->crp_aad_start,
} crp->crp_aad_length, axf->Update, auth_ctx);
if (error)
goto out;
exf->reinit(kschedule, iv); exf->reinit(kschedule, iv);
/* Do encryption/decryption with MAC */ /* Do encryption/decryption with MAC */
for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) { for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
len = imin(crp->crp_payload_length - i, sizeof(block)); len = imin(crp->crp_payload_length - i, sizeof(block));
crypto_copydata(crp, crp->crp_payload_start + i, len, block); crypto_copydata(crp, crp->crp_payload_start + i, len, block);
bzero(block + len, sizeof(block) - len); bzero(block + len, sizeof(block) - len);
▲ Show 20 Lines • Show All 412 Lines • ▼ Show 20 Linesccr_cipher_mode(const struct crypto_session_params *csp)
} }
} }
static int static int
ccr_probesession(device_t dev, const struct crypto_session_params *csp) ccr_probesession(device_t dev, const struct crypto_session_params *csp)
{ {
unsigned int cipher_mode; unsigned int cipher_mode;
if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT)) != 0) if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
0)
return (EINVAL); return (EINVAL);
switch (csp->csp_mode) { switch (csp->csp_mode) {
case CSP_MODE_DIGEST: case CSP_MODE_DIGEST:
if (!ccr_auth_supported(csp)) if (!ccr_auth_supported(csp))
return (EINVAL); return (EINVAL);
break; break;
case CSP_MODE_CIPHER: case CSP_MODE_CIPHER:
if (!ccr_cipher_supported(csp)) if (!ccr_cipher_supported(csp))
▲ Show 20 Lines • Show All 490 Lines • Show Last 20 Lines