diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c index 8fa0a798304e..01e9eca1f10a 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c @@ -1,545 +1,545 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2022 Intel Corporation */ /* $FreeBSD$ */ /** *************************************************************************** * @file lac_sym_cb.c Callback handler functions for symmetric components * * @ingroup LacSym * ***************************************************************************/ /* ******************************************************************************* * Include public/global header files ******************************************************************************* */ #include "cpa.h" #include "cpa_cy_sym.h" #include "icp_accel_devices.h" #include "icp_adf_init.h" #include "icp_qat_fw_la.h" #include "icp_adf_transport.h" #include "icp_adf_debug.h" #include "lac_sym.h" #include "lac_sym_cipher.h" #include "lac_common.h" #include "lac_list.h" #include "lac_sal_types_crypto.h" #include "lac_sal.h" #include "lac_sal_ctrl.h" #include "lac_session.h" #include "lac_sym_stats.h" #include "lac_log.h" #include "lac_sym_cb.h" #include "lac_sym_hash.h" #include "lac_sym_qat_cipher.h" #include "lac_sym_qat.h" #define DEQUEUE_MSGPUT_MAX_RETRIES 10000 /* ******************************************************************************* * Define static function definitions ******************************************************************************* */ /** ***************************************************************************** * @ingroup LacSymCb * Function to clean computed data. * * @description * This function cleans GCM or CCM data in the case of a failure. * * @param[in] pSessionDesc pointer to the session descriptor * @param[out] pBufferList pointer to the bufferlist to clean * @param[in] pOpData pointer to operation data * @param[in] isCCM is it a CCM operation boolean * * @return None *****************************************************************************/ static void LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc, CpaBufferList *pBufferList, const CpaCySymOpData *pOpData, CpaBoolean isCCM) { Cpa8U authTagLen = 0; /* Retrieve authTagLen */ authTagLen = pSessionDesc->hashResultSize; /* Cleaning */ if (isCCM) { /* for CCM the digest is inside the buffer list */ LacBuffDesc_BufferListZeroFromOffset( pBufferList, pOpData->cryptoStartSrcOffsetInBytes, pOpData->messageLenToCipherInBytes + authTagLen); } else { /* clean buffer list */ LacBuffDesc_BufferListZeroFromOffset( pBufferList, pOpData->cryptoStartSrcOffsetInBytes, pOpData->messageLenToCipherInBytes); } if ((CPA_TRUE != pSessionDesc->digestIsAppended) && (NULL != pOpData->pDigestResult)) { /* clean digest */ memset(pOpData->pDigestResult, 0, authTagLen); } } /** ***************************************************************************** * @ingroup LacSymCb * Definition of callback function for processing symmetric responses * * @description * This callback is invoked to process symmetric response messages from * the QAT. It will extract some details from the message and invoke * the user's callback to complete a symmetric operation. * * @param[in] pCookie Pointer to cookie associated with this request * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT * @param[in] status Status variable indicating an error occurred * in sending the message (e.g. when dequeueing) * @param[in] pSessionDesc Session descriptor * * @return None *****************************************************************************/ static void LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, CpaBoolean qatRespStatusOkFlag, CpaStatus status, lac_session_desc_t *pSessionDesc) { CpaCySymCbFunc pSymCb = NULL; void *pCallbackTag = NULL; CpaCySymOpData *pOpData = NULL; CpaBufferList *pDstBuffer = NULL; CpaCySymOp operationType = CPA_CY_SYM_OP_NONE; CpaStatus dequeueStatus = CPA_STATUS_SUCCESS; CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE; /* NOTE: cookie pointer validated in previous function */ instanceHandle = pCookie->instanceHandle; pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData); operationType = pSessionDesc->symOperation; /* Set the destination pointer to the one supplied in the cookie. */ pDstBuffer = pCookie->pDstBuffer; /* For a digest verify operation - for full packet and final partial * only, perform a comparison with the digest generated and with the one * supplied in the packet. */ if (((pSessionDesc->isSinglePass && (CPA_CY_SYM_CIPHER_AES_GCM == pSessionDesc->cipherAlgorithm)) || (CPA_CY_SYM_OP_CIPHER != operationType)) && (CPA_TRUE == pSessionDesc->digestVerify) && ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) || (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) { if (CPA_FALSE == qatRespStatusOkFlag) { LAC_SYM_STAT_INC(numSymOpVerifyFailures, instanceHandle); /* The comparison has failed at this point (status is * fail), * need to clean any sensitive calculated data up to * this point. * The data calculated is no longer useful to the end * result and * does not need to be returned to the user so setting * buffers to * zero. */ if (pSessionDesc->cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_CCM) { LacSymCb_CleanUserData(pSessionDesc, pDstBuffer, pOpData, CPA_TRUE); } else if (pSessionDesc->cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_GCM) { LacSymCb_CleanUserData(pSessionDesc, pDstBuffer, pOpData, CPA_FALSE); } } } else { /* Most commands have no point of failure and always return * success. This is the default response from the QAT. * If status is already set to an error value, don't overwrite * it */ if ((CPA_STATUS_SUCCESS == status) && (CPA_TRUE != qatRespStatusOkFlag)) { LAC_LOG_ERROR("Response status value not as expected"); status = CPA_STATUS_FAIL; } } pSymCb = pSessionDesc->pSymCb; pCallbackTag = pCookie->pCallbackTag; /* State returned to the client for intermediate partials packets * for hash only and cipher only partial packets. Cipher update * allow next partial through */ if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) { if ((CPA_CY_SYM_OP_CIPHER == operationType) || (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { if (CPA_TRUE == pCookie->updateUserIvOnRecieve) { /* Update the user's IV buffer * Very important to do this BEFORE dequeuing * subsequent partial requests, as the state * buffer * may get overwritten */ memcpy(pCookie->pOpData->pIv, pSessionDesc->cipherPartialOpState, pCookie->pOpData->ivLenInBytes); } if (CPA_TRUE == pCookie->updateKeySizeOnRecieve && LAC_CIPHER_IS_XTS_MODE( pSessionDesc->cipherAlgorithm)) { LacSymQat_CipherXTSModeUpdateKeyLen( pSessionDesc, pSessionDesc->cipherKeyLenInBytes / 2); } } } else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) { if ((CPA_CY_SYM_OP_CIPHER == operationType) || (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { if (CPA_TRUE == LAC_CIPHER_IS_XTS_MODE( pSessionDesc->cipherAlgorithm)) { /* * For XTS mode, we replace the updated key with * the original key - for subsequent partial * requests * */ LacSymQat_CipherXTSModeUpdateKeyLen( pSessionDesc, pSessionDesc->cipherKeyLenInBytes); } } } if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) && (qatRespStatusOkFlag != CPA_FALSE)) { /* There may be requests blocked pending the completion of this * operation */ dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc); if (CPA_STATUS_SUCCESS != dequeueStatus) { LAC_SYM_STAT_INC(numSymOpCompletedErrors, instanceHandle); qatRespStatusOkFlag = CPA_FALSE; if (CPA_STATUS_SUCCESS == status) { status = dequeueStatus; } } } if (CPA_STATUS_SUCCESS == status) { /* update stats */ if (pSessionDesc->internalSession == CPA_FALSE) { LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle); if (CPA_STATUS_SUCCESS != status) { LAC_SYM_STAT_INC(numSymOpCompletedErrors, instanceHandle); } } } qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount)); /* deallocate the memory for the internal callback cookie */ Lac_MemPoolEntryFree(pCookie); /* user callback function is the last thing to be called */ pSymCb(pCallbackTag, status, operationType, pOpData, pDstBuffer, qatRespStatusOkFlag); } /** ****************************************************************************** * @ingroup LacSymCb * Definition of callback function for processing symmetric Data Plane * responses * * @description * This callback checks the status, decrements the number of operations * pending and calls the user callback * * @param[in/out] pResponse pointer to the response structure * @param[in] qatRespStatusOkFlag status * @param[in] pSessionDesc pointer to the session descriptor * * @return None ******************************************************************************/ static void LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse, CpaBoolean qatRespStatusOkFlag, lac_session_desc_t *pSessionDesc) { CpaStatus status = CPA_STATUS_SUCCESS; /* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be * cleaned as stated in RFC 3610; in DP mode, it is the user * responsability * to do so */ if (CPA_FALSE == pSessionDesc->isSinglePass) { if ((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) || (CPA_FALSE == pSessionDesc->digestVerify)) { /* If not doing digest compare and qatRespStatusOkFlag != CPA_TRUE then there is something very wrong */ if (CPA_FALSE == qatRespStatusOkFlag) { LAC_LOG_ERROR( "Response status value not as expected"); status = CPA_STATUS_FAIL; } } } ((sal_crypto_service_t *)pResponse->instanceHandle) ->pSymDpCb(pResponse, status, qatRespStatusOkFlag); /* * Decrement the number of pending CB. * * If the @pendingDpCbCount becomes zero, we may remove the session, * please * read more information in the cpaCySymRemoveSession(). * * But there is a field in the @pResponse to store the session, * the "sessionCtx". In another word, in the above @->pSymDpCb() * callback, * it may use the session again. If we decrease the @pendingDpCbCount * before * the @->pSymDpCb(), there is a _risk_ the @->pSymDpCb() may reference * to * a deleted session. * * So in order to avoid the risk, we decrease the @pendingDpCbCount * after * the @->pSymDpCb() callback. */ qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount); } /** ****************************************************************************** * @ingroup LacSymCb * Definition of callback function for processing symmetric responses * * @description * This callback, which is registered with the common symmetric response * message handler, is invoked to process symmetric response messages from * the QAT. It will extract the response status from the cmnRespFlags set * by the QAT, and then will pass it to @ref * LacSymCb_ProcessCallbackInternal to complete the response processing. * * @param[in] lacCmdId ID of the symmetric QAT command of the request * message * @param[in] pOpaqueData pointer to opaque data in the request message * @param[in] cmnRespFlags Flags set by QAT to indicate response status * * @return None ******************************************************************************/ static void LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId, void *pOpaqueData, icp_qat_fw_comn_flags cmnRespFlags) { CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData; lac_session_desc_t *pSessionDesc = LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx); CpaBoolean qatRespStatusOkFlag = (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK == ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags)); if (CPA_TRUE == pSessionDesc->isDPSession) { /* DP session */ LacSymCb_ProcessDpCallback(pDpOpData, qatRespStatusOkFlag, pSessionDesc); } else { /* Trad session */ LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *) pOpaqueData, qatRespStatusOkFlag, CPA_STATUS_SUCCESS, pSessionDesc); } } /* ******************************************************************************* * Define public/global function definitions ******************************************************************************* */ /** * @ingroup LacSymCb * * @return CpaStatus * value returned will be the result of icp_adf_transPutMsg */ CpaStatus LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) { CpaStatus status = CPA_STATUS_SUCCESS; sal_crypto_service_t *pService = NULL; Cpa32U retries = 0; pService = (sal_crypto_service_t *)pSessionDesc->pInstance; /* Need to protect access to queue head and tail pointers, which may * be accessed by multiple contexts simultaneously for enqueue and * dequeue operations */ if (CPA_STATUS_SUCCESS != LAC_SPINLOCK(&pSessionDesc->requestQueueLock)) { LAC_LOG_ERROR("Failed to lock request queue"); return CPA_STATUS_RESOURCE; } /* Clear the blocking flag in the session descriptor */ pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE; while ((NULL != pSessionDesc->pRequestQueueHead) && (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) { /* If we send a partial packet request, set the * blockingOpsInProgress * flag for the session to indicate that subsequent requests * must be * queued up until this request completes */ if (CPA_CY_SYM_PACKET_TYPE_FULL != pSessionDesc->pRequestQueueHead->pOpData->packetType) { pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE; } /* At this point, we're clear to send the request. For cipher * requests, * we need to check if the session IV needs to be updated. This * can * only be done when no other partials are in flight for this * session, * to ensure the cipherPartialOpState buffer in the session * descriptor * is not currently in use */ if (CPA_TRUE == pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) { if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { memcpy(pSessionDesc->cipherPartialOpState, pSessionDesc->cipherARC4InitialState, LAC_CIPHER_ARC4_STATE_LEN_BYTES); } else { memcpy(pSessionDesc->cipherPartialOpState, pSessionDesc->pRequestQueueHead->pOpData ->pIv, pSessionDesc->pRequestQueueHead->pOpData ->ivLenInBytes); } } /* * Now we'll attempt to send the message directly to QAT. We'll * keep * looing until it succeeds (or at least a very high number of * retries), * as the failure only happens when the ring is full, and this * is only * a temporary situation. After a few retries, space will become * availble, allowing the putMsg to succeed. */ retries = 0; do { /* Send to QAT */ status = icp_adf_transPutMsg( pService->trans_handle_sym_tx, (void *)&(pSessionDesc->pRequestQueueHead->qatMsg), LAC_QAT_SYM_REQ_SZ_LW); retries++; /* * Yield to allow other threads that may be on this * session to poll * and make some space on the ring */ if (CPA_STATUS_SUCCESS != status) { qatUtilsYield(); } } while ((CPA_STATUS_SUCCESS != status) && (retries < DEQUEUE_MSGPUT_MAX_RETRIES)); if ((CPA_STATUS_SUCCESS != status) || (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) { LAC_LOG_ERROR( "Failed to SalQatMsg_transPutMsg, maximum retries exceeded."); goto cleanup; } pSessionDesc->pRequestQueueHead = pSessionDesc->pRequestQueueHead->pNext; } /* If we've drained the queue, ensure the tail pointer is set to NULL */ if (NULL == pSessionDesc->pRequestQueueHead) { pSessionDesc->pRequestQueueTail = NULL; } cleanup: if (CPA_STATUS_SUCCESS != LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock)) { LAC_LOG_ERROR("Failed to unlock request queue"); } return status; } /** * @ingroup LacSymCb */ void -LacSymCb_CallbacksRegister() +LacSymCb_CallbacksRegister(void) { /*** HASH ***/ LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH, LacSymCb_ProcessCallback); /*** ALGORITHM-CHAINING CIPHER_HASH***/ LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH, LacSymCb_ProcessCallback); /*** ALGORITHM-CHAINING HASH_CIPHER***/ LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER, LacSymCb_ProcessCallback); /*** CIPHER ***/ LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER, LacSymCb_ProcessCallback); /* Call compile time param check function to ensure it is included in the build by the compiler - this compile time check ensures callbacks run as expected */ LacSym_CompileTimeAssertions(); }