diff --git a/share/man/man4/qat.4 b/share/man/man4/qat.4 --- a/share/man/man4/qat.4 +++ b/share/man/man4/qat.4 @@ -64,8 +64,8 @@ A complete API for offloading these operations is exposed in the kernel and may be used by any other entity directly. For details of usage and supported operations and algorithms refer to the -following documentation available from -.Lk 01.org : +following documentation available from Intel Download Center +.Lk https://downloadcenter.intel.com : .Bl -bullet -compact .It .Rs diff --git a/sys/dev/qat/qat_api/common/compression/dc_dp.c b/sys/dev/qat/qat_api/common/compression/dc_dp.c --- a/sys/dev/qat/qat_api/common/compression/dc_dp.c +++ b/sys/dev/qat/qat_api/common/compression/dc_dp.c @@ -217,6 +217,89 @@ return CPA_STATUS_SUCCESS; } +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Partial-read parameters validation utility. + * + * @description + * Basic check that all partial-read related parameters provided by + * caller are valid. + * + * @param[in] pOpData Pointer to a structure containing the + * request parameters + * @param[in] pPartReadData Pointer to a structure containing the + * partial-read request parameters. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in + * + *****************************************************************************/ +static CpaStatus +dcDataPlanePartReadCheck(CpaDcDpOpData *pOpData, + CpaDcDpPartialReadData *pPartReadData) +{ + sal_compression_service_t *pService = NULL; + + LAC_CHECK_NULL_PARAM(pPartReadData); + + pService = (sal_compression_service_t *)(pOpData->dcInstance); + + if (!isDcGen4x(pService)) { + /* Extended features are not supported prior Gen4 */ + return CPA_STATUS_UNSUPPORTED; + } + + if (pOpData->sessDirection == CPA_DC_DIR_COMPRESS) { + /* Decompression specific feature */ + return CPA_STATUS_INVALID_PARAM; + } + + if (pPartReadData->length > pOpData->bufferLenForData) { + QAT_UTILS_LOG( + "Partial read data length can not be greater than the destination buffer size\n"); + return CPA_STATUS_INVALID_PARAM; + } + + return CPA_STATUS_SUCCESS; +} + +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Zero-padding parameters validation utility. + * + * @description + * Basic check that all zero-padding related parameters provided by + * caller are valid. + * + * @param[in] pOpData Pointer to a structure containing the + * request parameters. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in + * @retval CPA_STATUS_NOT_SUPPORTED Feature not supported + * + *****************************************************************************/ +static CpaStatus +dcDataPlaneZeroPadCheck(CpaDcDpOpData *pOpData) +{ + sal_compression_service_t *pService = NULL; + + pService = (sal_compression_service_t *)(pOpData->dcInstance); + + if (!isDcGen4x(pService)) { + /* Extended features are not supported prior Gen4 */ + return CPA_STATUS_UNSUPPORTED; + } + + if (pOpData->sessDirection == CPA_DC_DIR_DECOMPRESS) { + /* Compression specific feature */ + return CPA_STATUS_INVALID_PARAM; + } + + return CPA_STATUS_SUCCESS; +} CpaStatus cpaDcDpGetSessionSize(CpaInstanceHandle dcInstance, CpaDcSessionSetupData *pSessionData, @@ -379,8 +462,60 @@ pCurrentQatMsg->comp_pars.out_buffer_sz = pOpData->bufferLenForData; } -CpaStatus -cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) +/** + ***************************************************************************** + * @ingroup cpaDcDp + * + * @description + * Updates the request decryptor with optional parameters: + * - partial read specific fields + * - zero-padding specific field + * + * @param[in] pOpData Pointer to a structure containing the + * request parameters. + * @param[in] pPartReadData Pointer to a structure containing the + * partial-read request parameters. + * @param[in] zeroPadFlag Boolean indicator containing the + * zero-padding enablement flag. + * @param[in] pCurrentQatMsg Pointer to current QAT message on the ring. + * + *****************************************************************************/ +static void +dcDpUpdateRingMsg(CpaDcDpOpData *pOpData, + CpaDcDpPartialReadData *pPartReadData, + CpaBoolean zeroPadFlag, + icp_qat_fw_comp_req_t *pCurrentQatMsg) +{ + sal_compression_service_t *pService = NULL; + + pService = (sal_compression_service_t *)(pOpData->dcInstance); + if (!isDcGen4x(pService)) { + return; + } + + /* Partial read settings */ + if (NULL != pPartReadData) { + pCurrentQatMsg->u1.partial_decompress + .partial_decompress_offset = pPartReadData->dataOffset; + pCurrentQatMsg->u1.partial_decompress + .partial_decompress_length = pPartReadData->length; + ICP_QAT_FW_COMP_PART_DECOMP_SET( + pCurrentQatMsg->comp_pars.req_par_flags, + ICP_QAT_FW_COMP_PART_DECOMP); + } + /* Zero padding settings */ + if (CPA_TRUE == zeroPadFlag) { + ICP_QAT_FW_COMP_ZEROPAD_SET( + pCurrentQatMsg->comp_pars.req_par_flags, + ICP_QAT_FW_COMP_ZEROPAD); + } +} + +static CpaStatus +dcDpEnqueueOpBase(CpaDcDpOpData *pOpData, + CpaDcDpPartialReadData *pPartReadData, + CpaBoolean zeroPadFlag, + const CpaBoolean performOpNow) { icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; icp_comms_trans_handle trans_handle = NULL; @@ -392,6 +527,20 @@ return status; } + if (NULL != pPartReadData) { + status = dcDataPlanePartReadCheck(pOpData, pPartReadData); + if (CPA_STATUS_SUCCESS != status) { + return status; + } + } + + if (CPA_TRUE == zeroPadFlag) { + status = dcDataPlaneZeroPadCheck(pOpData); + if (CPA_STATUS_SUCCESS != status) { + return status; + } + } + if ((CPA_FALSE == pOpData->compressAndVerify) && (CPA_DC_DIR_COMPRESS == pOpData->sessDirection)) { return CPA_STATUS_UNSUPPORTED; @@ -422,6 +571,13 @@ } dcDpWriteRingMsg(pOpData, pCurrentQatMsg); + if (NULL != pPartReadData || CPA_TRUE == zeroPadFlag) { + dcDpUpdateRingMsg(pOpData, + pPartReadData, + zeroPadFlag, + pCurrentQatMsg); + } + pSessionDesc->pendingDpStatelessCbCount++; if (CPA_TRUE == performOpNow) { @@ -432,9 +588,36 @@ } CpaStatus -cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, - CpaDcDpOpData *pOpData[], - const CpaBoolean performOpNow) +cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow) +{ + + return dcDpEnqueueOpBase(pOpData, NULL, CPA_FALSE, performOpNow); +} + +CpaStatus +cpaDcDpEnqueueOpWithPartRead(CpaDcDpOpData *pOpData, + CpaDcDpPartialReadData *pPartReadData, + const CpaBoolean performOpNow) +{ + return dcDpEnqueueOpBase(pOpData, + pPartReadData, + CPA_FALSE, + performOpNow); +} + +CpaStatus +cpaDcDpEnqueueOpWithZeroPad(CpaDcDpOpData *pOpData, + const CpaBoolean performOpNow) +{ + return dcDpEnqueueOpBase(pOpData, NULL, CPA_TRUE, performOpNow); +} + +static CpaStatus +dcDpEnqueueOpBatchBase(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + CpaDcDpPartialReadData *pPartData[], + CpaBoolean zeroPadFlag, + const CpaBoolean performOpNow) { icp_qat_fw_comp_req_t *pCurrentQatMsg = NULL; icp_comms_trans_handle trans_handle = NULL; @@ -462,6 +645,21 @@ return status; } + if (NULL != pPartData) { + status = + dcDataPlanePartReadCheck(pOpData[i], pPartData[i]); + if (CPA_STATUS_SUCCESS != status) { + return status; + } + } + + if (CPA_TRUE == zeroPadFlag) { + status = dcDataPlaneZeroPadCheck(pOpData[i]); + if (CPA_STATUS_SUCCESS != status) { + return status; + } + } + /* Check that all instance handles and session handles are the * same */ if (pOpData[i]->dcInstance != pOpData[0]->dcInstance) { @@ -516,6 +714,18 @@ for (i = 0; i < numberRequests; i++) { dcDpWriteRingMsg(pOpData[i], pCurrentQatMsg); + if (pPartData) { + dcDpUpdateRingMsg(pOpData[i], + pPartData[i], + CPA_FALSE, + pCurrentQatMsg); + } + if (CPA_TRUE == zeroPadFlag) { + dcDpUpdateRingMsg(pOpData[i], + NULL, + CPA_TRUE, + pCurrentQatMsg); + } icp_adf_getQueueNext(trans_handle, (void **)&pCurrentQatMsg); } @@ -529,6 +739,34 @@ } CpaStatus +cpaDcDpEnqueueOpBatch(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + const CpaBoolean performOpNow) +{ + return dcDpEnqueueOpBatchBase( + numberRequests, pOpData, NULL, CPA_FALSE, performOpNow); +} + +CpaStatus +cpaDcDpEnqueueOpWithPartReadBatch(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + CpaDcDpPartialReadData *pPartReadData[], + const CpaBoolean performOpNow) +{ + return dcDpEnqueueOpBatchBase( + numberRequests, pOpData, pPartReadData, CPA_FALSE, performOpNow); +} + +CpaStatus +cpaDcDpEnqueueOpWithZeroPadBatch(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + const CpaBoolean performOpNow) +{ + return dcDpEnqueueOpBatchBase( + numberRequests, pOpData, NULL, CPA_TRUE, performOpNow); +} + +CpaStatus icp_sal_DcPollDpInstance(CpaInstanceHandle dcInstance, Cpa32U responseQuota) { icp_comms_trans_handle trans_handle = NULL; @@ -565,3 +803,51 @@ return CPA_STATUS_SUCCESS; } + +CpaStatus +cpaDcDpIsPartReadSupported(const CpaInstanceHandle instanceHandle, + CpaBoolean *flag) +{ + sal_compression_service_t *pService = NULL; + dc_extd_ftrs_t *pExtendedFtrs = NULL; + + LAC_CHECK_NULL_PARAM(instanceHandle); + SAL_CHECK_INSTANCE_TYPE(instanceHandle, SAL_SERVICE_TYPE_COMPRESSION); + + pService = (sal_compression_service_t *)instanceHandle; + if (!isDcGen4x(pService)) { + *flag = CPA_FALSE; + return CPA_STATUS_SUCCESS; + } + + pExtendedFtrs = (dc_extd_ftrs_t *)&( + ((sal_service_t *)instanceHandle)->dcExtendedFeatures); + + *flag = (CpaBoolean)pExtendedFtrs->is_part_read; + + return CPA_STATUS_SUCCESS; +} + +CpaStatus +cpaDcDpIsZeroPadSupported(const CpaInstanceHandle instanceHandle, + CpaBoolean *flag) +{ + sal_compression_service_t *pService = NULL; + dc_extd_ftrs_t *pExtendedFtrs = NULL; + + LAC_CHECK_NULL_PARAM(instanceHandle); + SAL_CHECK_INSTANCE_TYPE(instanceHandle, SAL_SERVICE_TYPE_COMPRESSION); + + pService = (sal_compression_service_t *)instanceHandle; + if (!isDcGen4x(pService)) { + *flag = CPA_FALSE; + return CPA_STATUS_SUCCESS; + } + + pExtendedFtrs = (dc_extd_ftrs_t *)&( + ((sal_service_t *)instanceHandle)->dcExtendedFeatures); + + *flag = (CpaBoolean)pExtendedFtrs->is_zero_pad; + + return CPA_STATUS_SUCCESS; +} diff --git a/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h b/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h --- a/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h +++ b/sys/dev/qat/qat_api/common/compression/include/dc_datapath.h @@ -196,4 +196,15 @@ /* CNV = TRUE, CNVNR = TRUE */ } dc_cnv_mode_t; +/* Type to access extended features bit fields */ +typedef struct dc_extended_features_s { + unsigned is_cnv : 1; /* Bit<0> */ + unsigned padding : 7; + unsigned is_cnvnr : 1; /* Bit<8> */ + unsigned reserved : 2; + unsigned is_part_read : 1; /* Bit<11> */ + unsigned is_zero_pad : 1; /* Bit<12> */ + unsigned not_used : 19; +} dc_extd_ftrs_t; + #endif /* DC_DATAPATH_H_ */ diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_compression.c b/sys/dev/qat/qat_api/common/ctrl/sal_compression.c --- a/sys/dev/qat/qat_api/common/ctrl/sal_compression.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_compression.c @@ -52,14 +52,6 @@ /* C string null terminator size */ #define SAL_NULL_TERM_SIZE 1 -/* Type to access extended features bit fields */ -typedef struct dc_extended_features_s { - unsigned is_cnv : 1; /* Bit<0> */ - unsigned padding : 7; - unsigned is_cnvnr : 1; /* Bit<8> */ - unsigned not_used : 23; -} dc_extd_ftrs_t; - /* * Prints statistics for a compression instance */ diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw.h @@ -1386,21 +1386,9 @@ /**< Error Code constant value for submission of empty dynamic stored block to * slice */ -#define ERR_CODE_EXCEED_MAX_REQ_TIME -24 -/**< Error Code constant for exceeding max request time */ - -#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_HANDLE -20 -/**< Error Code constant for invalid handle in kpt crypto service */ - -#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_HMAC_FAILED -21 -/**< Error Code constant for failed hmac in kpt crypto service */ - -#define ERR_CODE_KPT_CRYPTO_SERVICE_FAIL_INVALID_WRAPPING_ALGO -22 -/**< Error Code constant for invalid wrapping algo in kpt crypto service */ - -#define ERR_CODE_KPT_DRNG_SEED_NOT_LOAD -23 -/**< Error Code constant for no drng seed is not loaded in kpt ecdsa signrs -/service */ +#define ERR_CODE_REGION_OUT_OF_BOUNDS -21 +/**< Error returned when decompression ends before the specified partial + * decompression region was produced */ #define ERR_CODE_MISC_ERROR -50 /**< Error Code constant for error detected but the source diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_fw_comp.h @@ -47,6 +47,7 @@ } icp_qat_fw_comp_cmd_id_t; + /* * REQUEST FLAGS IN COMMON COMPRESSION * In common message it is named as SERVICE SPECIFIC FLAGS. @@ -64,7 +65,8 @@ * are don't care. i.e., these features are removed from QAT 2.0. */ -/** Flag usage */ + +/**< Flag usage */ #define ICP_QAT_FW_COMP_STATELESS_SESSION 0 /**< @ingroup icp_qat_fw_comp @@ -108,7 +110,7 @@ * Flag representing secure RAM from being used as * an intermediate buffer is ENABLED. */ -/** Flag mask & bit position */ +/**< Flag mask & bit position */ #define ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS 2 /**< @ingroup icp_qat_fw_comp @@ -148,7 +150,7 @@ #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS 7 /**< @ingroup icp_qat_fw_comp * Starting bit position for flag used to disable secure ram from - * being used as an intermediate buffer. */ + being used as an intermediate buffer. */ #define ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK 0x1 /**< @ingroup icp_qat_fw_comp @@ -170,7 +172,7 @@ * @ret_uncomp RetUnCompressed * @secure_ram Secure Ram usage * - *********************************************************************************/ + ******************************************************************************/ #define ICP_QAT_FW_COMP_FLAGS_BUILD( \ sesstype, autoselect, enhanced_asb, ret_uncomp, secure_ram) \ (((sesstype & ICP_QAT_FW_COMP_SESSION_TYPE_MASK) \ @@ -215,7 +217,7 @@ * * @param flags Flags to extract the session type bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_SESSION_TYPE_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_SESSION_TYPE_BITPOS, \ @@ -230,7 +232,7 @@ * * @param flags Flags to extract the autoSelectBest bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_AUTO_SELECT_BEST_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_AUTO_SELECT_BEST_BITPOS, \ @@ -245,7 +247,7 @@ * * @param flags Flags to extract the enhanced asb bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_EN_ASB_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_ENHANCED_AUTO_SELECT_BEST_BITPOS, \ @@ -260,7 +262,7 @@ * * @param flags Flags to extract the Ret Uncomp bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_RET_UNCOMP_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_RET_DISABLE_TYPE0_HEADER_DATA_BITPOS, \ @@ -275,21 +277,21 @@ * * @param flags Flags to extract the Secure Ram usage from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_SECURE_RAM_USE_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_BITPOS, \ ICP_QAT_FW_COMP_DISABLE_SECURE_RAM_AS_INTMD_BUF_MASK) /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp * Definition of the compression header cd pars block * @description * Definition of the compression processing cd pars block. * The structure is a service-specific implementation of the common * structure. - ******************************************************************************/ + *****************************************************************************/ typedef union icp_qat_fw_comp_req_hdr_cd_pars_s { /**< LWs 2-5 */ struct { @@ -301,15 +303,10 @@ uint8_t content_desc_params_sz; /**< Size of the content descriptor parameters in quad words. - * These - * parameters describe the session setup configuration info for - * the - * slices that this request relies upon i.e. the configuration - * word and - * cipher key needed by the cipher slice if there is a request - * for - * cipher - * processing. */ + * These parameters describe the session setup configuration + * info for the slices that this request relies upon i.e. the + * configuration word and cipher key needed by the cipher slice + * if there is a request for cipher processing. */ uint8_t content_desc_hdr_resrvd2; /**< Content descriptor reserved field */ @@ -324,28 +321,28 @@ uint32_t content_desc_resrvd4; /**< Content descriptor reserved field */ + } sl; } icp_qat_fw_comp_req_hdr_cd_pars_t; /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp * Definition of the compression request parameters block * @description * Definition of the compression processing request parameters block. * The structure below forms part of the Compression + Translation - * Parameters block spanning LWs 14-21, thus differing from the common + * Parameters block spanning LWs 14-23, thus differing from the common * base Parameters block structure. Unused fields must be set to 0. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_comp_req_params_s { /**< LW 14 */ uint32_t comp_len; /**< Size of input to process in bytes Note: Only EOP requests can be - * odd - * for decompression. IA must set LSB to zero for odd sized intermediate - * inputs */ + * odd for decompression. IA must set LSB to zero for odd sized + * intermediate inputs */ /**< LW 15 */ uint32_t out_buffer_sz; @@ -368,10 +365,10 @@ /**< CRC data structure pointer */ } crc; - /** LW 18 */ + /**< LW 18 */ uint32_t req_par_flags; - /** LW 19 */ + /**< LW 19 */ uint32_t rsrvd; } icp_qat_fw_comp_req_params_t; @@ -394,8 +391,10 @@ * @param cnvnr Whether internal CNV recovery is to be performed * * ICP_QAT_FW_COMP_NO_CNV_RECOVERY * * ICP_QAT_FW_COMP_CNV_RECOVERY + * @param cnvdfx Whether CNV error injection is to be performed + * * ICP_QAT_FW_COMP_NO_CNV_DFX + * * ICP_QAT_FW_COMP_CNV_DFX * @param crc CRC Mode Flag - 0 legacy, 1 crc data struct - * *****************************************************************************/ #define ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD( \ sop, eop, bfinal, cnv, cnvnr, cnvdfx, crc) \ @@ -411,21 +410,64 @@ ((crc & ICP_QAT_FW_COMP_CRC_MODE_MASK) \ << ICP_QAT_FW_COMP_CRC_MODE_BITPOS)) + +/* + * REQUEST FLAGS IN REQUEST PARAMETERS COMPRESSION + * + * +=====+-----+----- + --- + --- +-----+ --- + ----- + --- + ---- + -- + -- + + * | Bit |31-24| 20 | 19 | 18 | 17 | 16 | 15-7 | 6 | 5-2 | 1 | 0 | + * +=====+-----+----- + --- + ----+-----+ --- + ----- + --- + ---- + -- + -- + + * |Flags|Resvd|xxHash| CRC | CNV |CNVNR| CNV | Resvd |BFin | Resvd|EOP |SOP | + * | |=0 |acc | MODE| DFX | | | =0 | | =0 | | | + * | | | | | | | | | | | | | + * +=====+-----+----- + --- + ----+-----+ --- + ----- + --- + ---- + -- + -- + + */ + + +/** +***************************************************************************** +* @ingroup icp_qat_fw_comp +* Definition of the additional QAT2.0 Compression command types +* @description +* Enumeration which is used to indicate the ids of functions +* that are exposed by the Compression QAT FW service +* +*****************************************************************************/ +typedef enum { + ICP_QAT_FW_COMP_20_CMD_LZ4_COMPRESS = 3, + /*!< LZ4 Compress Request */ + + ICP_QAT_FW_COMP_20_CMD_LZ4_DECOMPRESS = 4, + /*!< LZ4 Decompress Request */ + + ICP_QAT_FW_COMP_20_CMD_LZ4S_COMPRESS = 5, + /*!< LZ4S Compress Request */ + + ICP_QAT_FW_COMP_20_CMD_LZ4S_DECOMPRESS = 6, + /*!< LZ4S Decompress Request */ + + ICP_QAT_FW_COMP_20_CMD_XP10_COMPRESS = 7, + /*!< XP10 Compress Request -- Placeholder */ + + ICP_QAT_FW_COMP_20_CMD_XP10_DECOMPRESS = 8, + /*!< XP10 Decompress Request -- Placeholder */ + + ICP_QAT_FW_COMP_20_CMD_DELIMITER + /**< Delimiter type */ + +} icp_qat_fw_comp_20_cmd_id_t; + + /* * REQUEST FLAGS IN REQUEST PARAMETERS COMPRESSION * - * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- + ---- + --- + - * --- + - * | Bit | 31-20 | 19 | 18 | 17 | 16 | 15 - 7 | 6 | 5-2 | 1 | 0 - * | - * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- | ---- + --- + - * --- + - * | Flags | Resvd | CRC |Resvd| CNVNR | CNV |Resvd Bits|BFin |Resvd | EOP | - * SOP | - * | | =0 | Mode| =0 | | | =0 | | =0 | | | - * | | | | | | | | | | | | - * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- | ---- + --- + - * --- + + * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- + ---- + --- + --- + + * | Bit | 31-20 | 19 | 18 | 17 | 16 | 15 - 7 | 6 | 5-2 | 1 | 0 | + * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- | ---- + --- + --- + + * | Flags | Resvd | CRC | CNV | CNVNR | CNV |Resvd Bits|BFin |Resvd | EOP | SOP | + * | | =0 | Mode| DFX | | | =0 | | =0 | | | + * | | | | | | | | | | | | + * + ===== + ----- + --- +-----+-------+ --- + ---------+ --- | ---- + --- + --- + */ #define ICP_QAT_FW_COMP_NOT_SOP 0 @@ -434,15 +476,15 @@ #define ICP_QAT_FW_COMP_SOP 1 /**< @ingroup icp_qat_fw_comp - * Flag representing that a request IS Start of Packet */ + * * Flag representing that a request IS Start of Packet */ #define ICP_QAT_FW_COMP_NOT_EOP 0 /**< @ingroup icp_qat_fw_comp - * Flag representing that a request is NOT Start of Packet */ + * Flag representing that a request is NOT Start of Packet */ #define ICP_QAT_FW_COMP_EOP 1 /**< @ingroup icp_qat_fw_comp - * Flag representing that a request IS End of Packet */ + * Flag representing that a request IS End of Packet */ #define ICP_QAT_FW_COMP_NOT_BFINAL 0 /**< @ingroup icp_qat_fw_comp @@ -484,6 +526,30 @@ /**< @ingroup icp_qat_fw_comp * Flag representing to use the external CRC data struct */ +#define ICP_QAT_FW_COMP_NO_XXHASH_ACC 0 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating that xxHash will NOT be accumulated across requests */ + +#define ICP_QAT_FW_COMP_XXHASH_ACC 1 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating that xxHash WILL be accumulated across requests */ + +#define ICP_QAT_FW_COMP_PART_DECOMP 1 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating to perform partial de-compressing */ + +#define ICP_QAT_FW_COMP_NO_PART_DECOMP 1 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating to not perform partial de-compressing */ + +#define ICP_QAT_FW_COMP_ZEROPAD 1 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating to perform zero-padding in compression request */ + +#define ICP_QAT_FW_COMP_NO_ZEROPAD 0 +/**< @ingroup icp_qat_fw_comp + * * Flag indicating to not perform zero-padding in compression request */ + #define ICP_QAT_FW_COMP_SOP_BITPOS 0 /**< @ingroup icp_qat_fw_comp * Starting bit position for SOP */ @@ -516,14 +582,6 @@ /**< @ingroup icp_qat_fw_comp * Starting bit position for the CNV bit */ -#define ICP_QAT_FW_COMP_CNV_RECOVERY_MASK 0x1 -/**< @ingroup icp_qat_fw_comp - * One bit mask for the CNV Recovery bit */ - -#define ICP_QAT_FW_COMP_CNV_RECOVERY_BITPOS 17 -/**< @ingroup icp_qat_fw_comp - * Starting bit position for the CNV Recovery bit */ - #define ICP_QAT_FW_COMP_CNVNR_MASK 0x1 /**< @ingroup icp_qat_fw_comp * One bit mask for the CNV Recovery bit */ @@ -556,6 +614,22 @@ /**< @ingroup icp_qat_fw_comp * One bit mask used to determine xxHash accumulate mode */ +#define ICP_QAT_FW_COMP_PART_DECOMP_BITPOS 27 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the partial de-compress bit */ + +#define ICP_QAT_FW_COMP_PART_DECOMP_MASK 0x1 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the partial de-compress mask */ + +#define ICP_QAT_FW_COMP_ZEROPAD_BITPOS 26 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the partial zero-pad bit */ + +#define ICP_QAT_FW_COMP_ZEROPAD_MASK 0x1 +/**< @ingroup icp_qat_fw_comp + * Starting bit position for the partial zero-pad mask */ + /** ****************************************************************************** * @ingroup icp_qat_fw_comp @@ -565,7 +639,7 @@ * * @param flags Flags to extract the SOP bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_SOP_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_SOP_BITPOS, \ @@ -585,8 +659,9 @@ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_EOP_BITPOS, \ ICP_QAT_FW_COMP_EOP_MASK) - /** + + ****************************************************************************** * @ingroup icp_qat_fw_comp * @@ -595,7 +670,7 @@ * * @param flags Flags to extract the bfinal bit from * - ******************************************************************************/ + *****************************************************************************/ #define ICP_QAT_FW_COMP_BFINAL_GET(flags) \ QAT_FIELD_GET(flags, \ ICP_QAT_FW_COMP_BFINAL_BITPOS, \ @@ -666,27 +741,91 @@ /** ****************************************************************************** * @ingroup icp_qat_fw_comp + * + * @description + * Macro for extraction of the partial de-compress on/off bit + * + * @param flags Flags to extract the partial de-compress on/off bit from + * + ******************************************************************************/ +#define ICP_QAT_FW_COMP_PART_DECOMP_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMP_PART_DECOMP_BITPOS, \ + ICP_QAT_FW_COMP_PART_DECOMP_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp + * + * @description + * Macro for setting of the partial de-compress on/off bit + * + * @param flags Flags to set the partial de-compress on/off bit to + * @param val partial de-compress on/off bit + * + *****************************************************************************/ +#define ICP_QAT_FW_COMP_PART_DECOMP_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + ICP_QAT_FW_COMP_PART_DECOMP_BITPOS, \ + ICP_QAT_FW_COMP_PART_DECOMP_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp + * + * @description + * Macro for extraction of the zero padding on/off bit + * + * @param flags Flags to extract the zero padding on/off bit from + * + ******************************************************************************/ +#define ICP_QAT_FW_COMP_ZEROPAD_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMP_ZEROPAD_BITPOS, \ + ICP_QAT_FW_COMP_ZEROPAD_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp + * + * @description + * Macro for setting of the zero-padding on/off bit + * + * @param flags Flags to set the zero-padding on/off bit to + * @param val zero-padding on/off bit + * + *****************************************************************************/ +#define ICP_QAT_FW_COMP_ZEROPAD_SET(flags, val) \ + QAT_FIELD_SET(flags, \ + val, \ + ICP_QAT_FW_COMP_ZEROPAD_BITPOS, \ + ICP_QAT_FW_COMP_ZEROPAD_MASK) + +/** + ****************************************************************************** + * @ingroup icp_qat_fw_comp * Definition of the translator request parameters block * @description * Definition of the translator processing request parameters block * The structure below forms part of the Compression + Translation - * Parameters block spanning LWs 20-21, thus differing from the common + * Parameters block spanning LWs 14-23, thus differing from the common * base Parameters block structure. Unused fields must be set to 0. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_xlt_req_params_s { /**< LWs 20-21 */ uint64_t inter_buff_ptr; /**< This field specifies the physical address of an intermediate - * buffer SGL array. The array contains a pair of 64-bit - * intermediate buffer pointers to SGL buffer descriptors, one pair - * per CPM. Please refer to the CPM1.6 Firmware Interface HLD - * specification for more details. - * Placeholder for QAT2.0. */ + * buffer SGL array. The array contains a pair of 64-bit + * intermediate buffer pointers to SGL buffer descriptors, one pair + * per CPM. Please refer to the CPM1.6 Firmware Interface HLD + * specification for more details. */ + } icp_qat_fw_xlt_req_params_t; /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp * Compression header of the content descriptor block * @description @@ -697,7 +836,7 @@ * cd block, thus differing from the common base content descriptor * structure. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_comp_cd_hdr_s { /**< LW 24 */ uint16_t ram_bank_flags; @@ -705,16 +844,15 @@ uint8_t comp_cfg_offset; /**< Quad word offset from the content descriptor parameters address to - * the - * parameters for the compression processing */ + * the parameters for the compression processing */ uint8_t next_curr_id; /**< This field combines the next and current id (each four bits) - - * the next id is the most significant nibble. - * Next Id: Set to the next slice to pass the compressed data through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * anymore slices after compression - * Current Id: Initialised with the compression slice type */ + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the compressed data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * anymore slices after compression + * Current Id: Initialised with the compression slice type */ /**< LW 25 */ uint32_t resrvd; @@ -733,7 +871,7 @@ #define COMP_CPR_INITIAL_ADLER 1 /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp * Translator content descriptor header block * @description @@ -743,7 +881,7 @@ * spans LWs 30-31, forming part of the compression + translation cd block, * thus differing from the common base content descriptor structure. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_xlt_cd_hdr_s { /**< LW 30 */ uint16_t resrvd1; @@ -754,27 +892,28 @@ uint8_t next_curr_id; /**< This field combines the next and current id (each four bits) - - * the next id is the most significant nibble. - * Next Id: Set to the next slice to pass the translated data through. - * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through - * any more slices after compression - * Current Id: Initialised with the translation slice type */ + * the next id is the most significant nibble. + * Next Id: Set to the next slice to pass the translated data through. + * Set to ICP_QAT_FW_SLICE_DRAM_WR if the data is not to go through + * any more slices after compression + * Current Id: Initialised with the translation slice type */ /**< LW 31 */ uint32_t resrvd3; /**< Reserved and should be set to zero, needed for quadword alignment */ + } icp_qat_fw_xlt_cd_hdr_t; /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp * Definition of the common Compression QAT FW request * @description * This is a definition of the full request structure for * compression and translation. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_comp_req_s { /**< LWs 0-1 */ icp_qat_fw_comn_req_hdr_t comn_hdr; @@ -803,6 +942,18 @@ uint32_t resrvd1[ICP_QAT_FW_NUM_LONGWORDS_2]; /**< Reserved if not used for translation */ + + struct { + uint32_t partial_decompress_length; + /**< LW 20 \n Length of the decompressed data to return + */ + + uint32_t partial_decompress_offset; + /**< LW 21 \n Offset of the decompressed data at which + * to return */ + + } partial_decompress; + } u1; /**< LWs 22-23 */ @@ -810,10 +961,8 @@ uint32_t resrvd2[ICP_QAT_FW_NUM_LONGWORDS_2]; /**< Reserved - not used if Batch and Pack is disabled.*/ - uint64_t bnp_res_table_addr; - /**< A generic pointer to the unbounded list of - * icp_qat_fw_resp_comp_pars_t members. This pointer is only - * used when the Batch and Pack is enabled. */ + uint64_t resrvd3; + /**< Reserved - not used if Batch and Pack is disabled.*/ } u3; /**< LWs 24-29 */ @@ -829,19 +978,19 @@ uint32_t resrvd3[ICP_QAT_FW_NUM_LONGWORDS_2]; /**< Reserved if not used for translation */ + } u2; } icp_qat_fw_comp_req_t; /** - ****************************************************************************** + ***************************************************************************** * @ingroup icp_qat_fw_comp - * Definition of the compression QAT FW response descriptor - * parameters + * Definition of the compression QAT FW response descriptor parameters * @description * This part of the response is specific to the compression response. * - ******************************************************************************/ + *****************************************************************************/ typedef struct icp_qat_fw_resp_comp_pars_s { /**< LW 4 */ uint32_t input_byte_counter; @@ -872,163 +1021,12 @@ /** ***************************************************************************** * @ingroup icp_qat_fw_comp - * Definition of a single result metadata structure inside Batch and Pack - * results table array. It describes the output if single job in the - * batch and pack jobs. - * Total number of entries in BNP Out table shall be equal to total - * number of requests in the 'batch'. - * @description - * This structure is specific to the compression output. - * - *****************************************************************************/ -typedef struct icp_qat_fw_comp_bnp_out_tbl_entry_s { - /**< LWs 0-3 */ - icp_qat_fw_resp_comp_pars_t comp_out_pars; - /**< Common output params (checksums and byte counts) */ - - /**< LW 4 */ - icp_qat_fw_comn_error_t comn_error; - /**< This field is overloaded to allow for one 8 bit common error field - * or two 8 bit error fields from compression and translator */ - - uint8_t comn_status; - /**< Status field which specifies which slice(s) report an error */ - - uint8_t reserved0; - /**< Reserved, shall be set to zero */ - - uint32_t reserved1; - /**< Reserved, shall be set to zero, - added for aligning entries to quadword boundary */ -} icp_qat_fw_comp_bnp_out_tbl_entry_t; - -/** -***************************************************************************** -* @ingroup icp_qat_fw_comp -* Supported modes for skipping regions of input or output buffers. -* -* @description -* This enumeration lists the supported modes for skipping regions of -* input or output buffers. -* -*****************************************************************************/ -typedef enum icp_qat_fw_comp_bnp_skip_mode_s { - ICP_QAT_FW_SKIP_DISABLED = 0, - /**< Skip mode is disabled */ - ICP_QAT_FW_SKIP_AT_START = 1, - /**< Skip region is at the start of the buffer. */ - ICP_QAT_FW_SKIP_AT_END = 2, - /**< Skip region is at the end of the buffer. */ - ICP_QAT_FW_SKIP_STRIDE = 3 - /**< Skip region occurs at regular intervals within the buffer. - specifies the number of bytes between each - skip region. */ -} icp_qat_fw_comp_bnp_skip_mode_t; - -/** - ***************************************************************************** - * @ingroup icp_qat_fw_comn - * Flags describing the skip and compression job bahaviour. refer to flag - * definitions on skip mode and reset/flush types. - * Note: compression behaviour flags are ignored for destination skip info. - * @description - * Definition of the common request flags. - * - *****************************************************************************/ -typedef uint8_t icp_qat_fw_comp_bnp_flags_t; - -/** - ***************************************************************************** - * @ingroup icp_qat_fw_comn - * Skip Region Data. - * @description - * This structure contains data relating to configuring skip region - * behaviour. A skip region is a region of an input buffer that - * should be omitted from processing or a region that should be inserted - * into the output buffer. - * - *****************************************************************************/ -typedef struct icp_qat_fw_comp_bnp_skip_info_s { - /**< LW 0 */ - uint16_t skip_length; - /** // built in support for _byteswap_ulong -#define BYTE_SWAP_32 _byteswap_ulong -#else #define BYTE_SWAP_32 __builtin_bswap32 -#endif /** ***************************************************************************** @@ -34,16 +30,17 @@ * *****************************************************************************/ typedef struct icp_qat_hw_comp_20_config_csr_lower_s { - // Fields programmable directly by the SW. + /* Fields programmable directly by the SW. */ icp_qat_hw_comp_20_extended_delay_match_mode_t edmm; icp_qat_hw_comp_20_hw_comp_format_t algo; icp_qat_hw_comp_20_search_depth_t sd; icp_qat_hw_comp_20_hbs_control_t hbs; - // Fields programmable directly by the FW. - // Block Drop enable. (Set by FW) + /* Fields programmable directly by the FW. */ + /* Block Drop enable. (Set by FW) */ icp_qat_hw_comp_20_abd_t abd; icp_qat_hw_comp_20_lllbd_ctrl_t lllbd; - // Advanced HW control (Set to default vals) + /* Advanced HW control (Set to default vals) */ + icp_qat_hw_comp_20_min_match_control_t mmctrl; icp_qat_hw_comp_20_skip_hash_collision_t hash_col; icp_qat_hw_comp_20_skip_hash_update_t hash_update; icp_qat_hw_comp_20_byte_skip_t skip_ctrl; @@ -62,7 +59,7 @@ ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(icp_qat_hw_comp_20_config_csr_lower_t csr) { uint32_t val32 = 0; - // Programmable values + /* Programmable values */ QAT_FIELD_SET(val32, csr.algo, ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_BITPOS, @@ -85,9 +82,9 @@ ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_MASK); QAT_FIELD_SET(val32, - csr.lllbd, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_BITPOS, - ICP_QAT_HW_COMP_20_CONFIG_CSR_LLLBD_CTRL_MASK); + csr.mmctrl, + ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); QAT_FIELD_SET(val32, csr.hash_col, @@ -103,7 +100,7 @@ csr.skip_ctrl, ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_BITPOS, ICP_QAT_HW_COMP_20_CONFIG_CSR_BYTE_SKIP_MASK); - // Default values. + /* Default values. */ QAT_FIELD_SET(val32, csr.abd, @@ -135,6 +132,7 @@ icp_qat_hw_comp_20_scb_unload_control_t scb_unload_ctrl; icp_qat_hw_comp_20_disable_token_fusion_control_t disable_token_fusion_ctrl; + icp_qat_hw_comp_20_lbms_t lbms; icp_qat_hw_comp_20_scb_mode_reset_mask_t scb_mode_reset; uint16_t lazy; uint16_t nice; @@ -185,6 +183,11 @@ ICP_QAT_HW_COMP_20_CONFIG_CSR_DISABLE_TOKEN_FUSION_CONTROL_MASK); QAT_FIELD_SET(val32, + csr.lbms, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS, + ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK); + + QAT_FIELD_SET(val32, csr.scb_mode_reset, ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS, ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK); @@ -214,8 +217,11 @@ typedef struct icp_qat_hw_decomp_20_config_csr_lower_s { /* Fields programmable directly by the SW. */ icp_qat_hw_decomp_20_hbs_control_t hbs; + icp_qat_hw_decomp_20_lbms_t lbms; /* Advanced HW control (Set to default vals) */ icp_qat_hw_decomp_20_hw_comp_format_t algo; + icp_qat_hw_decomp_20_min_match_control_t mmctrl; + icp_qat_hw_decomp_20_lz4_block_checksum_present_t lbc; } icp_qat_hw_decomp_20_config_csr_lower_t; /** @@ -238,10 +244,26 @@ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HBS_CONTROL_MASK); QAT_FIELD_SET(val32, + csr.lbms, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK); + + QAT_FIELD_SET(val32, csr.algo, ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS, ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_MASK); + QAT_FIELD_SET(val32, + csr.mmctrl, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK); + + QAT_FIELD_SET( + val32, + csr.lbc, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK); + return BYTE_SWAP_32(val32); } @@ -257,7 +279,7 @@ typedef struct icp_qat_hw_decomp_20_config_csr_upper_s { /* Advanced HW control (Set to default vals) */ icp_qat_hw_decomp_20_speculative_decoder_control_t sdc; - icp_qat_hw_decomp_20_mini_cam_control_t mcc; + icp_qat_hw_decomp_20_reserved4_control_t res4; } icp_qat_hw_decomp_20_config_csr_upper_t; /** @@ -281,11 +303,10 @@ ICP_QAT_HW_DECOMP_20_CONFIG_CSR_SPECULATIVE_DECODER_CONTROL_MASK); QAT_FIELD_SET(val32, - csr.mcc, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS, - ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK); + csr.res4, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_RESERVED4_CONTROL_BITPOS, + ICP_QAT_HW_DECOMP_20_CONFIG_CSR_RESERVED4_CONTROL_MASK); return BYTE_SWAP_32(val32); } - #endif /* ICP_QAT_HW__2X_COMP_H_ */ diff --git a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h --- a/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h +++ b/sys/dev/qat/qat_api/firmware/include/icp_qat_hw_20_comp_defs.h @@ -139,6 +139,30 @@ ICP_QAT_HW_COMP_20_DISABLE_TOKEN_FUSION_CONTROL_ENABLE /*****************************************************************************/ +/* LZ4 Block Maximum Size (LBMS). Set by FW , located in upper 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_BITPOS 19 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_MASK 0x3 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible LBMS field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB = 0x0, + /* LZ4 Block Maximum Size (LBMS) == 64 KB */ + ICP_QAT_HW_COMP_20_LBMS_LBMS_256KB = 0x1, + /* LZ4 Block Maximum Size (LBMS) == 256 KB */ + ICP_QAT_HW_COMP_20_LBMS_LBMS_1MB = 0x2, + /* LZ4 Block Maximum Size (LBMS) == 1 MB */ + ICP_QAT_HW_COMP_20_LBMS_LBMS_4MB = 0x3, + /* LZ4 Block Maximum Size (LBMS) == 4 MB */ +} icp_qat_hw_comp_20_lbms_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_LBMS_LBMS_64KB + +/*****************************************************************************/ /* SCB Mode Reset Mask (Set By FW) , located in upper 32bit */ #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_BITPOS 18 #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_MASK 0x1 @@ -150,23 +174,24 @@ *****************************************************************************/ typedef enum { ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS = 0x0, - /* iLZ77 mode: Reset LFCT, OBC */ + /* LZ4 mode: Reset LIBC, LOBC, In iLZ77 mode: Reset LFCT, OBC */ ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS_AND_HISTORY = 0x1, - /* iLZ77 mode: Reset LFCT, OBC, HB, HT */ + /* LZ4 mode: Reset LIBC, LOBC, HB, HT, In iLZ77 mode: Reset LFCT, OBC, + HB, HT */ } icp_qat_hw_comp_20_scb_mode_reset_mask_t; #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SCB_MODE_RESET_MASK_DEFAULT_VAL \ ICP_QAT_HW_COMP_20_SCB_MODE_RESET_MASK_RESET_COUNTERS /*****************************************************************************/ -/* Lazy - For iLZ77 and Static DEFLATE, Lazy = 102h , located in upper +/* Lazy - For iLZ77, LZ4, and Static DEFLATE, Lazy = 102h , located in upper * 32bit */ #define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_BITPOS 9 #define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_MASK 0x1ff #define ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL 258 /*****************************************************************************/ -/* Nice - For iLZ77 and Static DEFLATE, Nice = 103h , located in upper +/* Nice - For iLZ77, LZ4, and Static DEFLATE, Nice = 103h , located in upper * 32bit */ #define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS 0 #define ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK 0x1ff @@ -186,8 +211,6 @@ typedef enum { ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_32KB = 0x0, /* 000b - 32KB */ - ICP_QAT_HW_COMP_20_HBS_CONTROL_HBS_IS_64KB = 0x1, - /* 001b - 64KB */ } icp_qat_hw_comp_20_hbs_control_t; #define ICP_QAT_HW_COMP_20_CONFIG_CSR_HBS_CONTROL_DEFAULT_VAL \ @@ -248,9 +271,11 @@ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1 = 0x1, /* 0001b - Level 1 (search depth = 2^1 = 2) */ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_6 = 0x3, - /* 0011b - Level 6 (search depth = 2^3 = 8) */ + /* 0001b - Level 6 (search depth = 2^3 = 8) */ ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9 = 0x4, - /* 0100b - Level 9 (search depth = 2^4 = 16) */ + /* 0001b - Level 9 (search depth = 2^4 = 16) */ + ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_9P = 0x12, + /* 0001b - Level 9P (search depth = 2^12 = 4096) */ } icp_qat_hw_comp_20_search_depth_t; #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SEARCH_DEPTH_DEFAULT_VAL \ @@ -272,13 +297,38 @@ /* 000 - iLZ77. (Must set Min_Match = 3 bytes and HB size = 32KB.) */ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE = 0x1, /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size = - 32KB.) */ + * 32KB.) + */ + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4 = 0x2, + /* 010 - LZ4. (Must set Min Match = 4 bytes and HB size = 64KB.) */ + ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_LZ4S = 0x3, + /* 011 - LZ4s. (Min_Match and HBSize must be set accordingly.) */ } icp_qat_hw_comp_20_hw_comp_format_t; #define ICP_QAT_HW_COMP_20_CONFIG_CSR_HW_COMP_FORMAT_DEFAULT_VAL \ ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_DEFLATE /*****************************************************************************/ +/* Min Match (Set By FW to default value), located in lower 32bit */ +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible MIN_MATCH_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + /* 0 - Match 3 B */ + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, + /* 1 - Match 4 B */ +} icp_qat_hw_comp_20_min_match_control_t; + +#define ICP_QAT_HW_COMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_COMP_20_MIN_MATCH_CONTROL_MATCH_3B + +/*****************************************************************************/ /* Skip Hash Collision (Set By FW to default value), located in lower 32bit */ #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_BITPOS 3 #define ICP_QAT_HW_COMP_20_CONFIG_CSR_SKIP_HASH_COLLISION_MASK 0x1 @@ -382,23 +432,23 @@ /*****************************************************************************/ /* Mini CAM Disable (Set By the Driver/ Application), located in upper 32bit */ -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS 30 -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK 0x1 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_RESERVED4_CONTROL_BITPOS 30 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_RESERVED4_CONTROL_MASK 0x1 /* **************************************************************************** * @ingroup icp_qat_hw_defs * @description - * Enumeration of possible MINI_CAM_CONTROL field values + * Enumeration of possible RESERVED4 field values *****************************************************************************/ typedef enum { - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE = 0x0, + ICP_QAT_HW_DECOMP_20_RESERVED4_CONTROL_ENABLE = 0x0, /* 0b - Enabled */ - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_DISABLE = 0x1, + ICP_QAT_HW_DECOMP_20_RESERVED4_CONTROL_DISABLE = 0x1, /* 1b - Disabled */ -} icp_qat_hw_decomp_20_mini_cam_control_t; +} icp_qat_hw_decomp_20_reserved4_control_t; -#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_DEFAULT_VAL \ - ICP_QAT_HW_DECOMP_20_MINI_CAM_CONTROL_ENABLE +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_RESERVED4_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_RESERVED4_CONTROL_ENABLE /*****************************************************************************/ /* History Buffer Size (Set By the Driver/ Application), located in lower 32bit @@ -420,6 +470,30 @@ ICP_QAT_HW_DECOMP_20_HBS_CONTROL_HBS_IS_32KB /*****************************************************************************/ +/* LZ4 Block Maximum Size (LBMS). Set by FW , located in lower 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_BITPOS 8 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_MASK 0x3 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible LBMS field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB = 0x0, + /* LZ4 Block Maximum Size (LBMS) == 64 KB */ + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_256KB = 0x1, + /* LZ4 Block Maximum Size (LBMS) == 256 KB */ + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_1MB = 0x2, + /* LZ4 Block Maximum Size (LBMS) == 1 MB */ + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_4MB = 0x3, + /* LZ4 Block Maximum Size (LBMS) == 4 MB */ +} icp_qat_hw_decomp_20_lbms_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LBMS_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_LBMS_LBMS_64KB + +/*****************************************************************************/ /* Decompression Format (Set By Driver/Application. Also See CMD ID), located in * lower 32bit */ #define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_BITPOS 5 @@ -433,10 +507,57 @@ typedef enum { ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE = 0x1, /* 001 - Static DEFLATE. (Must set Min_Match = 3 bytes and HB size = - 32KB.) */ + * 32KB.) + */ + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4 = 0x2, + /* 010 - LZ4. (Must set Min Match = 4 bytes and HB size = 32KB.) */ + ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_LZ4S = 0x3, + /* 011 - LZ4s. (Min_Match and HBSize must be set accordingly.) */ } icp_qat_hw_decomp_20_hw_comp_format_t; #define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_HW_DECOMP_FORMAT_DEFAULT_VAL \ ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE -#endif //_ICP_QAT_HW_20_COMP_DEFS_H +/*****************************************************************************/ +/* Decompression Format (Set By Driver/Application. Also See CMD ID), located in + * lower 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_BITPOS 4 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_MASK 0x1 + +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible MIN_MATCH_CONTROL field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B = 0x0, + /* 0 - Match 3 B */ + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_4B = 0x1, + /* 1 - Match 4 B */ +} icp_qat_hw_decomp_20_min_match_control_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MIN_MATCH_CONTROL_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_MIN_MATCH_CONTROL_MATCH_3B + +/*****************************************************************************/ +/* LZ4 Block Checksum Present, located in lower 32bit */ +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS 3 +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK 0x1 +/* + **************************************************************************** + * @ingroup icp_qat_hw_defs + * @description + * Enumeration of possible LZ4_CHECKSUM_PRESENT field values + *****************************************************************************/ +typedef enum { + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT = 0x0, + /* the LZ4 Block does not contain the 4-byte checksum */ + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_PRESENT = 0x1, + /* LZ4 Block contains a 4-byte checksum. */ +} icp_qat_hw_decomp_20_lz4_block_checksum_present_t; + +#define ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_DEFAULT_VAL \ + ICP_QAT_HW_DECOMP_20_LZ4_BLOCK_CHKSUM_ABSENT + +#endif /* _ICP_QAT_HW_20_COMP_DEFS_H */ diff --git a/sys/dev/qat/qat_api/include/dc/cpa_dc.h b/sys/dev/qat/qat_api/include/dc/cpa_dc.h --- a/sys/dev/qat/qat_api/include/dc/cpa_dc.h +++ b/sys/dev/qat/qat_api/include/dc/cpa_dc.h @@ -482,6 +482,9 @@ * (not supported) */ CPA_DC_CRC_INTEG_ERR = -20, /**< A data integrity CRC error was detected */ + CPA_DC_REGION_OUT_OF_BOUNDS = -21, + /**< Error returned when decompression ends before the specified partial + * decompression region was produced */ CPA_DC_LZ4_MAX_BLOCK_SIZE_EXCEEDED = -93, /**< LZ4 max block size exceeded */ CPA_DC_LZ4_BLOCK_OVERFLOW_ERR = -95, @@ -898,7 +901,11 @@ CpaDcReqStatus status; /**< Additional status details from accelerator */ Cpa32U produced; - /**< Octets produced by the operation */ + /**< Octets produced by the operation. + * For Data Plane "partial read" operations, the size of the produced + * data should be equal to the sum of the data offset and length of + * the requested decompressed data chunk. + * See ref @CpaDcDpPartialReadData. */ Cpa32U consumed; /**< Octets consumed by the operation */ Cpa32U checksum; @@ -911,7 +918,11 @@ * The checksum algorithm CPA_DC_XXHASH32 does not support passing an * input value in this parameter. Any initial value passed will be * ignored by the compression/decompression operation when this - * checksum algorithm is used. */ + * checksum algorithm is used. + * + * For Data Plane "partial read" operations, the checksum is computed + * from the beginning of the decompressed data to the end of the + * requested chunk. See ref @CpaDcDpPartialReadData. */ CpaBoolean endOfLastBlock; /**< Decompression operation has stopped at the end of the last * block in a deflate stream. */ diff --git a/sys/dev/qat/qat_api/include/dc/cpa_dc_dp.h b/sys/dev/qat/qat_api/include/dc/cpa_dc_dp.h --- a/sys/dev/qat/qat_api/include/dc/cpa_dc_dp.h +++ b/sys/dev/qat/qat_api/include/dc/cpa_dc_dp.h @@ -94,6 +94,31 @@ /** ***************************************************************************** * @ingroup cpaDcDp + * Decompression partial read data. + * @description + * This structure contains configuration related to requesting + * specific chunk of decompression data. + * + ****************************************************************************/ +typedef struct _CpaDcDpPartialReadData { + Cpa32U bufferOffset; + /**< Number of bytes to skip in a destination buffer (or buffers list) + * before writing. At this point only zero is supported. + */ + Cpa32U dataOffset; + /**< The offset in the decompressed data of the first byte written to + * the destination buffer. The data offset length should be an integer + * multiple of 4KB in order to achieve the best performance. + */ + Cpa32U length; + /**< Size of requested decompressed data chunk. The length should be + * an integer multiple of 4KB in order to achieve the best performance. + */ +} CpaDcDpPartialReadData; + +/** + ***************************************************************************** + * @ingroup cpaDcDp * Operation Data for compression data plane API. * * @description @@ -662,6 +687,152 @@ cpaDcDpEnqueueOp(CpaDcDpOpData *pOpData, const CpaBoolean performOpNow); +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Enqueue a single decompression request with partial read configuration. + * See @CpaDcDpPartialReadData for more details. + * + * @description + * This function enqueues a single request to perform a decompression + * operation and allows to specify particular region of decompressed + * data to be placed in to the destination buffer (or buffer list). + * + * The function is asynchronous; control is returned to the user once + * the request has been submitted. On completion of the request, the + * application may poll for responses, which will cause a callback + * function (registered via @ref cpaDcDpRegCbFunc) to be invoked. + * Callbacks within a session are guaranteed to be in the same order + * in which they were submitted. + * + * The following restrictions apply to the pOpData parameter: + * + * - The memory MUST be aligned on an 8-byte boundary. + * - The reserved fields of the structure MUST NOT be written to + * or read from. + * - The structure MUST reside in physically contiguous memory. + * + * @context + * This function will not sleep, and hence can be executed in a context + * that does not permit sleeping. + * + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * No + * + * @param[in,out] pOpData See @ref cpaDcDpEnqueueOp pOpData description. + * + * @param[in] pPartReadData Pointer to a structure containing the partial + * read configuration parameters. + * See @CpaDcDpPartialReadData for more details. + * + * @param[in] performOpNow See @ref cpaDcDpEnqueueOp performOpNow input + * parameter. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_RETRY Resubmit the request. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_RESTARTING API implementation is restarting. Resubmit + * the request. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * @pre + * The session identified by pOpData->pSessionHandle was setup using + * @ref cpaDcDpInitSession. The instance identified by pOpData->dcInstance + * has had a callback function registered via @ref cpaDcDpRegCbFunc. + * + * @post + * None + * + * @note + * A callback of type @ref CpaDcDpCallbackFn is generated in + * response to this function call. Any errors generated during + * processing are reported as part of the callback status code. + * + * @see + * @ref cpaDcDpPerformOpNow + *****************************************************************************/ +CpaStatus +cpaDcDpEnqueueOpWithPartRead(CpaDcDpOpData *pOpData, + CpaDcDpPartialReadData *pPartReadData, + const CpaBoolean performOpNow); + +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Enqueue a single compression request with an option set to zero-fill + * data after the compression output in the leftover bytes. + * + * @description + * This function enqueues a single request to perform a compression + * operation with zero-filling leftover bytes with 4KB alignment + * in the destination buffer (or buffer list). + * + * The function is asynchronous; control is returned to the user once + * the request has been submitted. On completion of the request, the + * application may poll for responses, which will cause a callback + * function (registered via @ref cpaDcDpRegCbFunc) to be invoked. + * Callbacks within a session are guaranteed to be in the same order + * in which they were submitted. + * + * The following restrictions apply to the pOpData parameter: + * + * - The memory MUST be aligned on an 8-byte boundary. + * - The reserved fields of the structure MUST NOT be written to + * or read from. + * - The structure MUST reside in physically contiguous memory. + * + * @context + * This function will not sleep, and hence can be executed in a context + * that does not permit sleeping. + * + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * No + * + * @param[in,out] pOpData See @ref cpaDcDpEnqueueOp pOpData description. + * + * @param[in] performOpNow See @ref cpaDcDpEnqueueOp performOpNow input + * parameter. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_RETRY Resubmit the request. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_RESTARTING API implementation is restarting. Resubmit + * the request. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * @pre + * The session identified by pOpData->pSessionHandle was setup using + * @ref cpaDcDpInitSession. The instance identified by pOpData->dcInstance + * has had a callback function registered via @ref cpaDcDpRegCbFunc. + * + * @post + * None + * + * @note + * A callback of type @ref CpaDcDpCallbackFn is generated in + * response to this function call. Any errors generated during + * processing are reported as part of the callback status code. + * + * @see + * @ref cpaDcDpPerformOpNow + *****************************************************************************/ +CpaStatus +cpaDcDpEnqueueOpWithZeroPad(CpaDcDpOpData *pOpData, + const CpaBoolean performOpNow); /** ***************************************************************************** @@ -759,6 +930,177 @@ CpaDcDpOpData *pOpData[], const CpaBoolean performOpNow); +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Enqueue multiple decompression request with partial read configuration. + * See @CpaDcDpPartialReadData for more details. + * + * @description + * This function enqueues multiple requests to perform decompression + * operations and allows to specify particular region of decompressed + * data to be placed in to the destination buffer (or buffer list) for + * each individual request. + * + * The function is asynchronous; control is returned to the user once + * the request has been submitted. On completion of the request, the + * application may poll for responses, which will cause a callback + * function (registered via @ref cpaDcDpRegCbFunc) to be invoked. + * Separate callbacks will be invoked for each request. + * Callbacks within a session and at the same priority are guaranteed + * to be in the same order in which they were submitted. + * + * The following restrictions apply to each element of the pOpData + * array: + * + * - The memory MUST be aligned on an 8-byte boundary. + * - The reserved fields of the structure MUST be set to zero. + * - The structure MUST reside in physically contiguous memory. + * + * @context + * See @ref cpaDcDpEnqueueOpBatch context. + * + * @assumptions + * See @ref cpaDcDpEnqueueOpBatch assumptions. + * + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * No + * + * @param[in] numberRequests The number of requests in the array of + * CpaDcDpOpData structures. + * + * @param[in,out] pOpData See @ref cpaDcDpEnqueueOpBatch pOpData for more + * details. + * + * @param[in] pPartReadData An array of pointers to a structures containing + * the partial read configuration parameters. + * See @CpaDcDpPartialReadData for more details. + * + * @param[in] performOpNow See @ref cpaDcDpEnqueueOpBatch performOpNow + * input parameter. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_RETRY Resubmit the request. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_RESTARTING API implementation is restarting. Resubmit + * the request. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * + * @pre + * The session identified by pOpData[i]->pSessionHandle was setup using + * @ref cpaDcDpInitSession. The instance identified by + * pOpData[i]->dcInstance has had a callback function registered via + * @ref cpaDcDpRegCbFunc. + * + * @post + * None + * + * @note + * Multiple callbacks of type @ref CpaDcDpCallbackFn are generated in + * response to this function call (one per request). Any errors + * generated during processing are reported as part of the callback + * status code. + * + * @see + * @ref cpaDcDpEnqueueOp + *****************************************************************************/ +CpaStatus +cpaDcDpEnqueueOpWithPartReadBatch(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + CpaDcDpPartialReadData *pPartReadData[], + const CpaBoolean performOpNow); + +/** + ***************************************************************************** + * @ingroup cpaDcDp + * Enqueue multiple compression requests with an option set to zero-fill + * data after the compression output in the leftover bytes. + * + * @description + * This function enqueues multiple requests to perform compression + * operations with an option set to zero-fill leftover bytes in the + * destination buffer (of buffer list) for each individual request. + * Please note that optional zero-filling leftover output buffer bytes + * is aligned to 4KB. + * + * The function is asynchronous; control is returned to the user once + * the request has been submitted. On completion of the request, the + * application may poll for responses, which will cause a callback + * function (registered via @ref cpaDcDpRegCbFunc) to be invoked. + * Separate callbacks will be invoked for each request. + * Callbacks within a session and at the same priority are guaranteed + * to be in the same order in which they were submitted. + * + * The following restrictions apply to each element of the pOpData + * array: + * + * - The memory MUST be aligned on an 8-byte boundary. + * - The reserved fields of the structure MUST be set to zero. + * - The structure MUST reside in physically contiguous memory. + * + * @context + * See @ref cpaDcDpEnqueueOpBatch context. + * + * @assumptions + * See @ref cpaDcDpEnqueueOpBatch assumptions. + * + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * No + * + * @param[in] numberRequests The number of requests in the array of + * CpaDcDpOpData structures. + * + * @param[in,out] pOpData See @ref cpaDcDpEnqueueOpBatch pOpData for more + * details. + * + * @param[in] performOpNow See @ref cpaDcDpEnqueueOpBatch performOpNow + * input parameter. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_RETRY Resubmit the request. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_RESTARTING API implementation is restarting. Resubmit + * the request. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * + * @pre + * The session identified by pOpData[i]->pSessionHandle was setup using + * @ref cpaDcDpInitSession. The instance identified by + * pOpData[i]->dcInstance has had a callback function registered via + * @ref cpaDcDpRegCbFunc. + * + * @post + * None + * + * @note + * Multiple callbacks of type @ref CpaDcDpCallbackFn are generated in + * response to this function call (one per request). Any errors + * generated during processing are reported as part of the callback + * status code. + * + * @see + * @ref cpaDcDpEnqueueOp + *****************************************************************************/ +CpaStatus +cpaDcDpEnqueueOpWithZeroPadBatch(const Cpa32U numberRequests, + CpaDcDpOpData *pOpData[], + const CpaBoolean performOpNow); /** ***************************************************************************** @@ -809,6 +1151,95 @@ CpaStatus cpaDcDpPerformOpNow(CpaInstanceHandle dcInstance); +/** + ***************************************************************************** + * @ingroup cpaDc + * Function to return the "partial read" feature support. + * + * @description + * This function is used to determine if given instance supports + * "partial read" feature. + * + * @context + * This function may be called from any context. + * @assumptions + * None + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * Yes + * + * @param[in] instanceHandle Handle to an instance of this API. + * @param[out] pFlag Pointer to boolean flag which indicates + * whether a feature is supported. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * @pre + * None + * @post + * None + * @note + * None + * @see + * cpaDcQueryCapabilities() + * + *****************************************************************************/ +CpaStatus +cpaDcDpIsPartReadSupported(const CpaInstanceHandle instanceHandle, + CpaBoolean *pFlag); + +/** + ***************************************************************************** + * @ingroup cpaDc + * Function to return the "zero pad" feature support. + * + * @description + * This function is used to determine if given instance supports + * "zero pad" feature. + * + * @context + * This function may be called from any context. + * @assumptions + * None + * @sideEffects + * None + * @blocking + * No + * @reentrant + * No + * @threadSafe + * Yes + * + * @param[in] instanceHandle Handle to an instance of this API. + * @param[out] pFlag Pointer to boolean flag which indicates + * whether a feature is supported. + * + * @retval CPA_STATUS_SUCCESS Function executed successfully. + * @retval CPA_STATUS_FAIL Function failed. + * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in. + * @retval CPA_STATUS_UNSUPPORTED Function is not supported. + * + * @pre + * None + * @post + * None + * @note + * None + * @see + * cpaDcQueryCapabilities() + * + *****************************************************************************/ +CpaStatus +cpaDcDpIsZeroPadSupported(const CpaInstanceHandle instanceHandle, + CpaBoolean *pFlag); #ifdef __cplusplus diff --git a/sys/dev/qat/qat_api/include/icp_sal_versions.h b/sys/dev/qat/qat_api/include/icp_sal_versions.h --- a/sys/dev/qat/qat_api/include/icp_sal_versions.h +++ b/sys/dev/qat/qat_api/include/icp_sal_versions.h @@ -26,7 +26,7 @@ /* Part name and number of the accelerator device */ #define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3 -#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 13 +#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 14 #define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0 /** diff --git a/sys/dev/qat/qat_common/adf_ctl_drv.c b/sys/dev/qat/qat_common/adf_ctl_drv.c --- a/sys/dev/qat/qat_common/adf_ctl_drv.c +++ b/sys/dev/qat/qat_common/adf_ctl_drv.c @@ -278,7 +278,6 @@ mutex_unlock(&bundle->lock); return -EINVAL; } - mutex_unlock(&bundle->lock); /* Find the list entry for this process */ mutex_lock(&bundle->list_lock); @@ -292,11 +291,11 @@ if (!pid_entry_found) { pr_err("QAT: process %d not found\n", curproc->p_pid); + mutex_unlock(&bundle->lock); return -EINVAL; } instance_rings->ring_mask |= reserve.ring_mask; - mutex_lock(&bundle->lock); bundle->rings_used |= reserve.ring_mask; mutex_unlock(&bundle->lock); diff --git a/sys/dev/qat/qat_common/adf_freebsd_dev_processes.c b/sys/dev/qat/qat_common/adf_freebsd_dev_processes.c --- a/sys/dev/qat/qat_common/adf_freebsd_dev_processes.c +++ b/sys/dev/qat/qat_common/adf_freebsd_dev_processes.c @@ -20,7 +20,8 @@ #define ADF_STATE_CALLOUT_TIME 10 -static const char *mtx_name = "state_callout_mtx"; +static const char *mtx_name = "state_mtx"; +static const char *mtx_callout_name = "callout_mtx"; static d_open_t adf_processes_open; static void adf_processes_release(void *data); @@ -36,6 +37,7 @@ static struct callout callout; static struct mtx mtx; +static struct mtx callout_mtx; static struct service_hndl adf_state_hndl; struct entry_proc_events { @@ -422,8 +424,8 @@ STAILQ_INSERT_TAIL(head, state, entries_state); } } - callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); mtx_unlock(&mtx); + callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); } static int @@ -484,7 +486,7 @@ case EVFILT_READ: kn->kn_fop = &adf_state_read_filterops; kn->kn_hook = priv; - knlist_add(&priv->rsel.si_note, kn, 0); + knlist_add(&priv->rsel.si_note, kn, 1); mtx_unlock(&mtx); return 0; default: @@ -530,17 +532,16 @@ ADF_DEV_STATE_NAME); SLIST_INIT(&proc_events_head); mtx_init(&mtx, mtx_name, NULL, MTX_DEF); - callout_init_mtx(&callout, &mtx, 0); + mtx_init(&callout_mtx, mtx_callout_name, NULL, MTX_DEF); + callout_init_mtx(&callout, &callout_mtx, 0); explicit_bzero(&adf_state_hndl, sizeof(adf_state_hndl)); adf_state_hndl.event_hld = adf_state_event_handler; adf_state_hndl.name = "adf_state_event_handler"; - mtx_lock(&mtx); adf_service_register(&adf_state_hndl); callout_reset(&callout, ADF_STATE_CALLOUT_TIME, adf_state_callout_notify_ev, NULL); - mtx_unlock(&mtx); } void @@ -548,17 +549,20 @@ { struct entry_proc_events *proc_events = NULL; - mtx_lock(&mtx); adf_service_unregister(&adf_state_hndl); + mtx_lock(&callout_mtx); callout_stop(&callout); + mtx_unlock(&callout_mtx); + mtx_destroy(&callout_mtx); + mtx_lock(&mtx); while (!SLIST_EMPTY(&proc_events_head)) { proc_events = SLIST_FIRST(&proc_events_head); SLIST_REMOVE_HEAD(&proc_events_head, entries_proc_events); free(proc_events, M_QAT); } - destroy_dev(adf_state_dev); mtx_unlock(&mtx); mtx_destroy(&mtx); + destroy_dev(adf_state_dev); } static int @@ -586,6 +590,7 @@ SLIST_INSERT_HEAD(&proc_events_head, entry_proc_events, entries_proc_events); + mtx_unlock(&mtx); ret = devfs_set_cdevpriv(prv_data, adf_state_release); if (ret) { SLIST_REMOVE(&proc_events_head, @@ -596,7 +601,6 @@ free(prv_data, M_QAT); } callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); - mtx_unlock(&mtx); return ret; } @@ -636,8 +640,8 @@ KNOTE_UNLOCKED(&prv_data->rsel.si_note, 0); } } - callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); mtx_unlock(&mtx); + callout_schedule(&callout, ADF_STATE_CALLOUT_TIME); return ret; } diff --git a/sys/dev/qat/qat_common/adf_freebsd_uio_cleanup.c b/sys/dev/qat/qat_common/adf_freebsd_uio_cleanup.c --- a/sys/dev/qat/qat_common/adf_freebsd_uio_cleanup.c +++ b/sys/dev/qat/qat_common/adf_freebsd_uio_cleanup.c @@ -323,7 +323,7 @@ void adf_uio_do_cleanup_orphan(int bank, struct adf_uio_control_accel *accel) { - int ret, pid_found; + int ret; struct adf_uio_instance_rings *instance_rings, *tmp; struct adf_uio_control_bundle *bundle; /* orphan is local pointer allocated and deallocated in this function */ @@ -347,7 +347,6 @@ if (hw_data->ring_pair_reset) { hw_data->ring_pair_reset( accel_dev, orphan->bundle->hardware_bundle_number); - mutex_lock(&orphan->bundle->lock); /* * If processes exit normally, rx_mask, tx_mask * and rings_enabled are all 0, below expression @@ -357,7 +356,6 @@ */ orphan->bundle->rings_enabled &= ~(orphan->rx_mask | orphan->tx_mask); - mutex_unlock(&orphan->bundle->lock); goto out; } @@ -385,19 +383,12 @@ * then force a release here. */ mutex_lock(&bundle->list_lock); - pid_found = 0; list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list) { if (instance_rings->user_pid == curproc->p_pid) { - pid_found = 1; + bundle->rings_used &= ~instance_rings->ring_mask; break; } } mutex_unlock(&bundle->list_lock); - - if (pid_found) { - mutex_lock(&bundle->lock); - bundle->rings_used &= ~instance_rings->ring_mask; - mutex_unlock(&bundle->lock); - } } diff --git a/sys/dev/qat/qat_common/adf_pfvf_vf_proto.c b/sys/dev/qat/qat_common/adf_pfvf_vf_proto.c --- a/sys/dev/qat/qat_common/adf_pfvf_vf_proto.c +++ b/sys/dev/qat/qat_common/adf_pfvf_vf_proto.c @@ -389,6 +389,11 @@ struct adf_hw_device_data *hw_data = accel_dev->hw_device; int ret; + /* init workqueue for VF */ + ret = adf_init_vf_wq(); + if (ret) + return ret; + hw_data->enable_pf2vf_interrupt(accel_dev); ret = adf_vf2pf_request_version(accel_dev); diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c --- a/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c @@ -86,7 +86,7 @@ struct adf_accel_dev *accel_dev; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; - unsigned int i, bar_nr; + unsigned int bar_nr; int ret, rid; struct adf_cfg_device *cfg_dev = NULL; @@ -178,15 +178,19 @@ } /* Find and map all the device's BARS */ - i = 0; - for (bar_nr = 0; i < ADF_PCI_MAX_BARS && bar_nr < PCIR_MAX_BAR_0; - bar_nr++) { + /* Logical BARs configuration for 64bit BARs: + bar 0 and 1 - logical BAR0 + bar 2 and 3 - logical BAR1 + bar 4 and 5 - logical BAR3 + */ + for (bar_nr = 0; + bar_nr < (ADF_PCI_MAX_BARS * 2) && bar_nr < PCIR_MAX_BAR_0; + bar_nr += 2) { struct adf_bar *bar; rid = PCIR_BAR(bar_nr); - if (bus_get_resource(dev, SYS_RES_MEMORY, rid, NULL, NULL) != 0) - continue; - bar = &accel_pci_dev->pci_bars[i++]; + bar = &accel_pci_dev->pci_bars[bar_nr / 2]; + bar->virt_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, diff --git a/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c b/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c --- a/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c @@ -93,7 +93,7 @@ struct adf_accel_dev *pf; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; - unsigned int i, bar_nr; + unsigned int bar_nr; int ret = 0; int rid; struct adf_cfg_device *cfg_dev = NULL; @@ -159,40 +159,30 @@ hw_data->accel_capabilities_mask = adf_4xxxvf_get_hw_cap(accel_dev); /* Find and map all the device's BARS */ - i = 0; - for (bar_nr = 0; i < ADF_PCI_MAX_BARS && bar_nr < PCIR_MAX_BAR_0; - bar_nr++) { + /* Logical BARs configuration for 64bit BARs: + bar 0 and 1 - logical BAR0 + bar 2 and 3 - logical BAR1 + bar 4 and 5 - logical BAR3 + */ + for (bar_nr = 0; + bar_nr < (ADF_PCI_MAX_BARS * 2) && bar_nr < PCIR_MAX_BAR_0; + bar_nr += 2) { struct adf_bar *bar; rid = PCIR_BAR(bar_nr); - if (bus_get_resource(dev, SYS_RES_MEMORY, rid, NULL, NULL) != - 0) { - continue; - } - bar = &accel_pci_dev->pci_bars[i++]; + bar = &accel_pci_dev->pci_bars[bar_nr / 2]; bar->virt_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!bar->virt_addr) { - device_printf(GET_DEV(accel_dev), - "Failed to map BAR %d\n", - bar_nr); + device_printf(dev, "Failed to map BAR %d\n", bar_nr); ret = ENXIO; goto out_err; } bar->base_addr = rman_get_start(bar->virt_addr); bar->size = rman_get_size(bar->virt_addr); } - - if (i == 0) { - device_printf( - GET_DEV(accel_dev), - "No BARs mapped. Please check if PCI BARs are mapped correctly for device\n"); - ret = ENXIO; - goto out_err; - } - pci_enable_busmaster(dev); /* Completion for VF2PF request/response message exchange */