diff --git a/sys/contrib/dev/qat/qat_402xx.bin b/sys/contrib/dev/qat/qat_402xx.bin new file mode 100644 index 000000000000..74151547edce Binary files /dev/null and b/sys/contrib/dev/qat/qat_402xx.bin differ diff --git a/sys/contrib/dev/qat/qat_402xx_mmp.bin b/sys/contrib/dev/qat/qat_402xx_mmp.bin new file mode 100644 index 000000000000..6404eb009d2f Binary files /dev/null and b/sys/contrib/dev/qat/qat_402xx_mmp.bin differ diff --git a/sys/dev/qat/include/common/adf_accel_devices.h b/sys/dev/qat/include/common/adf_accel_devices.h index c09aee8ea4bd..eeffc6a9132c 100644 --- a/sys/dev/qat/include/common/adf_accel_devices.h +++ b/sys/dev/qat/include/common/adf_accel_devices.h @@ -1,727 +1,731 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ #include "qat_freebsd.h" #include "adf_cfg_common.h" #include "adf_pfvf_msg.h" #include "opt_qat.h" #define ADF_CFG_NUM_SERVICES 4 #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" #define ADF_C62X_DEVICE_NAME "c6xx" #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_200XX_DEVICE_NAME "200xx" #define ADF_200XXVF_DEVICE_NAME "200xxvf" #define ADF_C4XXX_DEVICE_NAME "c4xxx" #define ADF_C4XXXVF_DEVICE_NAME "c4xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_4XXXVF_DEVICE_NAME "4xxxvf" #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 #define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 #define ADF_C62X_PCI_DEVICE_ID 0x37c8 #define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9 #define ADF_C3XXX_PCI_DEVICE_ID 0x19e2 #define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3 #define ADF_200XX_PCI_DEVICE_ID 0x18ee #define ADF_200XXIOV_PCI_DEVICE_ID 0x18ef #define ADF_D15XX_PCI_DEVICE_ID 0x6f54 #define ADF_D15XXIOV_PCI_DEVICE_ID 0x6f55 #define ADF_C4XXX_PCI_DEVICE_ID 0x18a0 #define ADF_C4XXXIOV_PCI_DEVICE_ID 0x18a1 #define ADF_4XXX_PCI_DEVICE_ID 0x4940 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_401XX_PCI_DEVICE_ID 0x4942 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 +#define ADF_402XX_PCI_DEVICE_ID 0x4944 +#define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 #define IS_QAT_GEN3(ID) ({ (ID == ADF_C4XXX_PCI_DEVICE_ID); }) static inline bool IS_QAT_GEN4(const unsigned int id) { return (id == ADF_4XXX_PCI_DEVICE_ID || id == ADF_401XX_PCI_DEVICE_ID || + id == ADF_402XX_PCI_DEVICE_ID || + id == ADF_402XXIOV_PCI_DEVICE_ID || id == ADF_4XXXIOV_PCI_DEVICE_ID || id == ADF_401XXIOV_PCI_DEVICE_ID); } #define IS_QAT_GEN3_OR_GEN4(ID) (IS_QAT_GEN3(ID) || IS_QAT_GEN4(ID)) #define ADF_VF2PF_SET_SIZE 32 #define ADF_MAX_VF2PF_SET 4 #define ADF_VF2PF_SET_OFFSET(set_nr) ((set_nr)*ADF_VF2PF_SET_SIZE) #define ADF_VF2PF_VFNR_TO_SET(vf_nr) ((vf_nr) / ADF_VF2PF_SET_SIZE) #define ADF_VF2PF_VFNR_TO_MASK(vf_nr) \ ({ \ u32 vf_nr_ = (vf_nr); \ BIT((vf_nr_)-ADF_VF2PF_SET_SIZE *ADF_VF2PF_VFNR_TO_SET( \ vf_nr_)); \ }) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 #define ADF_PCI_MAX_BARS 3 #define ADF_DEVICE_NAME_LENGTH 32 #define ADF_ETR_MAX_RINGS_PER_BANK 16 #define ADF_MAX_MSIX_VECTOR_NAME 32 #define ADF_DEVICE_NAME_PREFIX "qat_" #define ADF_STOP_RETRY 50 #define ADF_NUM_THREADS_PER_AE (8) #define ADF_AE_ADMIN_THREAD (7) #define ADF_NUM_PKE_STRAND (2) #define ADF_AE_STRAND0_THREAD (8) #define ADF_AE_STRAND1_THREAD (9) #define ADF_CFG_NUM_SERVICES 4 #define ADF_SRV_TYPE_BIT_LEN 3 #define ADF_SRV_TYPE_MASK 0x7 #define ADF_RINGS_PER_SRV_TYPE 2 #define ADF_THRD_ABILITY_BIT_LEN 4 #define ADF_THRD_ABILITY_MASK 0xf #define ADF_VF_OFFSET 0x8 #define ADF_MAX_FUNC_PER_DEV 0x7 #define ADF_PCI_DEV_OFFSET 0x3 #define ADF_SRV_TYPE_BIT_LEN 3 #define ADF_SRV_TYPE_MASK 0x7 #define GET_SRV_TYPE(ena_srv_mask, srv) \ (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.csr_ops) #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->csr_info.pfvf_ops) #define ADF_DEFAULT_RING_TO_SRV_MAP \ (CRYPTO | CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) enum adf_accel_capabilities { ADF_ACCEL_CAPABILITIES_NULL = 0, ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, ADF_ACCEL_CAPABILITIES_CIPHER = 4, ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, ADF_ACCEL_CAPABILITIES_DEPRECATED = 64, ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 }; struct adf_bar { rman_res_t base_addr; struct resource *virt_addr; rman_res_t size; } __packed; struct adf_accel_msix { struct msix_entry *entries; u32 num_entries; } __packed; struct adf_accel_pci { device_t pci_dev; struct adf_accel_msix msix_entries; struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; uint8_t revid; uint8_t sku; int node; } __packed; enum dev_state { DEV_DOWN = 0, DEV_UP }; enum dev_sku_info { DEV_SKU_1 = 0, DEV_SKU_2, DEV_SKU_3, DEV_SKU_4, DEV_SKU_VF, DEV_SKU_1_CY, DEV_SKU_2_CY, DEV_SKU_3_CY, DEV_SKU_UNKNOWN }; static inline const char * get_sku_info(enum dev_sku_info info) { switch (info) { case DEV_SKU_1: return "SKU1"; case DEV_SKU_1_CY: return "SKU1CY"; case DEV_SKU_2: return "SKU2"; case DEV_SKU_2_CY: return "SKU2CY"; case DEV_SKU_3: return "SKU3"; case DEV_SKU_3_CY: return "SKU3CY"; case DEV_SKU_4: return "SKU4"; case DEV_SKU_VF: return "SKUVF"; case DEV_SKU_UNKNOWN: default: break; } return "Unknown SKU"; } enum adf_accel_unit_services { ADF_ACCEL_SERVICE_NULL = 0, ADF_ACCEL_INLINE_CRYPTO = 1, ADF_ACCEL_CRYPTO = 2, ADF_ACCEL_COMPRESSION = 4, ADF_ACCEL_ASYM = 8, ADF_ACCEL_ADMIN = 16 }; struct adf_ae_info { u32 num_asym_thd; u32 num_sym_thd; u32 num_dc_thd; } __packed; struct adf_accel_unit { u8 au_mask; u32 accel_mask; u64 ae_mask; u64 comp_ae_mask; u32 num_ae; enum adf_accel_unit_services services; } __packed; struct adf_accel_unit_info { u32 inline_ingress_msk; u32 inline_egress_msk; u32 sym_ae_msk; u32 asym_ae_msk; u32 dc_ae_msk; u8 num_cy_au; u8 num_dc_au; u8 num_asym_au; u8 num_inline_au; struct adf_accel_unit *au; const struct adf_ae_info *ae_info; } __packed; struct adf_hw_aram_info { /* Inline Egress mask. "1" = AE is working with egress traffic */ u32 inline_direction_egress_mask; /* Inline congestion managmenet profiles set in config file */ u32 inline_congest_mngt_profile; /* Initialise CY AE mask, "1" = AE is used for CY operations */ u32 cy_ae_mask; /* Initialise DC AE mask, "1" = AE is used for DC operations */ u32 dc_ae_mask; /* Number of long words used to define the ARAM regions */ u32 num_aram_lw_entries; /* ARAM region definitions */ u32 mmp_region_size; u32 mmp_region_offset; u32 skm_region_size; u32 skm_region_offset; /* * Defines size and offset of compression intermediate buffers stored * in ARAM (device's on-chip memory). */ u32 inter_buff_aram_region_size; u32 inter_buff_aram_region_offset; u32 sadb_region_size; u32 sadb_region_offset; } __packed; struct adf_hw_device_class { const char *name; const enum adf_device_type type; uint32_t instances; } __packed; struct arb_info { u32 arbiter_offset; u32 wrk_thd_2_srv_arb_map; u32 wrk_cfg_offset; } __packed; struct admin_info { u32 admin_msg_ur; u32 admin_msg_lr; u32 mailbox_offset; } __packed; struct adf_hw_csr_ops { u64 (*build_csr_ring_base_addr)(bus_addr_t addr, u32 size); u32 (*read_csr_ring_head)(struct resource *csr_base_addr, u32 bank, u32 ring); void (*write_csr_ring_head)(struct resource *csr_base_addr, u32 bank, u32 ring, u32 value); u32 (*read_csr_ring_tail)(struct resource *csr_base_addr, u32 bank, u32 ring); void (*write_csr_ring_tail)(struct resource *csr_base_addr, u32 bank, u32 ring, u32 value); u32 (*read_csr_e_stat)(struct resource *csr_base_addr, u32 bank); void (*write_csr_ring_config)(struct resource *csr_base_addr, u32 bank, u32 ring, u32 value); bus_addr_t (*read_csr_ring_base)(struct resource *csr_base_addr, u32 bank, u32 ring); void (*write_csr_ring_base)(struct resource *csr_base_addr, u32 bank, u32 ring, bus_addr_t addr); void (*write_csr_int_flag)(struct resource *csr_base_addr, u32 bank, u32 value); void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank); void (*write_csr_int_col_en)(struct resource *csr_base_addr, u32 bank, u32 value); void (*write_csr_int_col_ctl)(struct resource *csr_base_addr, u32 bank, u32 value); void (*write_csr_int_flag_and_col)(struct resource *csr_base_addr, u32 bank, u32 value); u32 (*read_csr_ring_srv_arb_en)(struct resource *csr_base_addr, u32 bank); void (*write_csr_ring_srv_arb_en)(struct resource *csr_base_addr, u32 bank, u32 value); u32 (*get_src_sel_mask)(void); u32 (*get_int_col_ctl_enable_mask)(void); u32 (*get_bank_irq_mask)(u32 irq_mask); }; struct adf_cfg_device_data; struct adf_accel_dev; struct adf_etr_data; struct adf_etr_ring_data; struct adf_pfvf_ops { int (*enable_comms)(struct adf_accel_dev *accel_dev); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vf2pf_offset)(u32 i); void (*enable_vf2pf_interrupts)(struct resource *pmisc_addr, u32 vf_mask); void (*disable_all_vf2pf_interrupts)(struct resource *pmisc_addr); u32 (*disable_pending_vf2pf_interrupts)(struct resource *pmisc_addr); int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, u32 pfvf_offset, struct mutex *csr_lock); struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, u32 pfvf_offset, u8 compat_ver); }; struct adf_hw_csr_info { struct adf_hw_csr_ops csr_ops; struct adf_pfvf_ops pfvf_ops; u32 csr_addr_offset; u32 ring_bundle_size; u32 bank_int_flag_clear_mask; u32 num_rings_per_int_srcsel; u32 arb_enable_mask; }; struct adf_hw_device_data { struct adf_hw_device_class *dev_class; uint32_t (*get_accel_mask)(struct adf_accel_dev *accel_dev); uint32_t (*get_ae_mask)(struct adf_accel_dev *accel_dev); uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_num_aes)(struct adf_hw_device_data *self); uint32_t (*get_num_accels)(struct adf_hw_device_data *self); void (*notify_and_wait_ethernet)(struct adf_accel_dev *accel_dev); bool (*get_eth_doorbell_msg)(struct adf_accel_dev *accel_dev); void (*get_arb_info)(struct arb_info *arb_csrs_info); void (*get_admin_info)(struct admin_info *admin_csrs_info); void (*get_errsou_offset)(u32 *errsou3, u32 *errsou5); uint32_t (*get_num_accel_units)(struct adf_hw_device_data *self); int (*init_accel_units)(struct adf_accel_dev *accel_dev); void (*exit_accel_units)(struct adf_accel_dev *accel_dev); uint32_t (*get_clock_speed)(struct adf_hw_device_data *self); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); bool (*check_prod_sku)(struct adf_accel_dev *accel_dev); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); void (*enable_error_correction)(struct adf_accel_dev *accel_dev); int (*check_uncorrectable_error)(struct adf_accel_dev *accel_dev); void (*print_err_registers)(struct adf_accel_dev *accel_dev); void (*disable_error_interrupts)(struct adf_accel_dev *accel_dev); int (*init_ras)(struct adf_accel_dev *accel_dev); void (*exit_ras)(struct adf_accel_dev *accel_dev); void (*disable_arb)(struct adf_accel_dev *accel_dev); void (*update_ras_errors)(struct adf_accel_dev *accel_dev, int error); bool (*ras_interrupts)(struct adf_accel_dev *accel_dev, bool *reset_required); int (*init_admin_comms)(struct adf_accel_dev *accel_dev); void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); int (*send_admin_init)(struct adf_accel_dev *accel_dev); void (*set_asym_rings_mask)(struct adf_accel_dev *accel_dev); int (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map); uint32_t (*get_accel_cap)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, const uint32_t **cfg); int (*init_device)(struct adf_accel_dev *accel_dev); int (*get_heartbeat_status)(struct adf_accel_dev *accel_dev); int (*int_timer_init)(struct adf_accel_dev *accel_dev); void (*int_timer_exit)(struct adf_accel_dev *accel_dev); uint32_t (*get_ae_clock)(struct adf_hw_device_data *self); uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); bool (*check_slice_hang)(struct adf_accel_dev *accel_dev); int (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); void (*enable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev); void (*disable_pf2vf_interrupt)(struct adf_accel_dev *accel_dev); int (*interrupt_active_pf2vf)(struct adf_accel_dev *accel_dev); int (*get_int_active_bundles)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); void (*reset_hw_units)(struct adf_accel_dev *accel_dev); int (*measure_clock)(struct adf_accel_dev *accel_dev); void (*restore_device)(struct adf_accel_dev *accel_dev); uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev, enum adf_accel_unit_services services); enum adf_accel_unit_services ( *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num); int (*add_pke_stats)(struct adf_accel_dev *accel_dev); void (*remove_pke_stats)(struct adf_accel_dev *accel_dev); int (*add_misc_error)(struct adf_accel_dev *accel_dev); int (*count_ras_event)(struct adf_accel_dev *accel_dev, u32 *ras_event, char *aeidstr); void (*remove_misc_error)(struct adf_accel_dev *accel_dev); int (*configure_accel_units)(struct adf_accel_dev *accel_dev); int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_number); void (*config_ring_irq)(struct adf_accel_dev *accel_dev, u32 bank_number, u16 ring_mask); uint32_t (*get_objs_num)(struct adf_accel_dev *accel_dev); const char *(*get_obj_name)(struct adf_accel_dev *accel_dev, enum adf_accel_unit_services services); void (*pre_reset)(struct adf_accel_dev *accel_dev); void (*post_reset)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); void (*get_ring_svc_map_data)(int ring_pair_index, u16 ring_to_svc_map, u8 *serv_type, int *ring_index, int *num_rings_per_srv, int bundle_num); struct adf_hw_csr_info csr_info; const char *fw_name; const char *fw_mmp_name; bool reset_ack; uint32_t fuses; uint32_t accel_capabilities_mask; uint32_t instance_id; uint16_t accel_mask; u32 aerucm_mask; u32 ae_mask; u32 admin_ae_mask; u32 service_mask; u32 service_to_load_mask; u32 heartbeat_ctr_num; uint16_t tx_rings_mask; uint8_t tx_rx_gap; uint8_t num_banks; u8 num_rings_per_bank; uint8_t num_accel; uint8_t num_logical_accel; uint8_t num_engines; bool get_ring_to_svc_done; int (*get_storage_enabled)(struct adf_accel_dev *accel_dev, uint32_t *storage_enabled); u8 query_storage_cap; u32 clock_frequency; u8 storage_enable; u32 extended_dc_capabilities; int (*config_device)(struct adf_accel_dev *accel_dev); u32 asym_ae_active_thd_mask; u16 asym_rings_mask; int (*get_fw_image_type)(struct adf_accel_dev *accel_dev, enum adf_cfg_fw_image_type *fw_image_type); u16 ring_to_svc_map; } __packed; /* helper enum for performing CSR operations */ enum operation { AND, OR, }; /* 32-bit CSR write macro */ #define ADF_CSR_WR(csr_base, csr_offset, val) \ bus_write_4(csr_base, csr_offset, val) /* 64-bit CSR write macro */ #ifdef __x86_64__ #define ADF_CSR_WR64(csr_base, csr_offset, val) \ bus_write_8(csr_base, csr_offset, val) #else static __inline void adf_csr_wr64(struct resource *csr_base, bus_size_t offset, uint64_t value) { bus_write_4(csr_base, offset, (uint32_t)value); bus_write_4(csr_base, offset + 4, (uint32_t)(value >> 32)); } #define ADF_CSR_WR64(csr_base, csr_offset, val) \ adf_csr_wr64(csr_base, csr_offset, val) #endif /* 32-bit CSR read macro */ #define ADF_CSR_RD(csr_base, csr_offset) bus_read_4(csr_base, csr_offset) /* 64-bit CSR read macro */ #ifdef __x86_64__ #define ADF_CSR_RD64(csr_base, csr_offset) bus_read_8(csr_base, csr_offset) #else static __inline uint64_t adf_csr_rd64(struct resource *csr_base, bus_size_t offset) { return (((uint64_t)bus_read_4(csr_base, offset)) | (((uint64_t)bus_read_4(csr_base, offset + 4)) << 32)); } #define ADF_CSR_RD64(csr_base, csr_offset) adf_csr_rd64(csr_base, csr_offset) #endif #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev) #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) #define GET_DEV_SKU(accel_dev) (accel_dev->accel_pci_dev.sku) #define GET_NUM_RINGS_PER_BANK(accel_dev) \ (GET_HW_DATA(accel_dev)->num_rings_per_bank) #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev #define GET_SRV_TYPE(ena_srv_mask, srv) \ (((ena_srv_mask) >> (ADF_SRV_TYPE_BIT_LEN * (srv))) & ADF_SRV_TYPE_MASK) #define SET_ASYM_MASK(asym_mask, srv) \ ({ \ typeof(srv) srv_ = (srv); \ (asym_mask) |= ((1 << (srv_)*ADF_RINGS_PER_SRV_TYPE) | \ (1 << ((srv_)*ADF_RINGS_PER_SRV_TYPE + 1))); \ }) #define GET_NUM_RINGS_PER_BANK(accel_dev) \ (GET_HW_DATA(accel_dev)->num_rings_per_bank) #define GET_MAX_PROCESSES(accel_dev) \ ({ \ typeof(accel_dev) dev = (accel_dev); \ (GET_MAX_BANKS(dev) * (GET_NUM_RINGS_PER_BANK(dev) / 2)); \ }) #define GET_DU_TABLE(accel_dev) (accel_dev->du_table) static inline void adf_csr_fetch_and_and(struct resource *csr, size_t offs, unsigned long mask) { unsigned int val = ADF_CSR_RD(csr, offs); val &= mask; ADF_CSR_WR(csr, offs, val); } static inline void adf_csr_fetch_and_or(struct resource *csr, size_t offs, unsigned long mask) { unsigned int val = ADF_CSR_RD(csr, offs); val |= mask; ADF_CSR_WR(csr, offs, val); } static inline void adf_csr_fetch_and_update(enum operation op, struct resource *csr, size_t offs, unsigned long mask) { switch (op) { case AND: adf_csr_fetch_and_and(csr, offs, mask); break; case OR: adf_csr_fetch_and_or(csr, offs, mask); break; } } struct pfvf_stats { struct dentry *stats_file; /* Messages put in CSR */ unsigned int tx; /* Messages read from CSR */ unsigned int rx; /* Interrupt fired but int bit was clear */ unsigned int spurious; /* Block messages sent */ unsigned int blk_tx; /* Block messages received */ unsigned int blk_rx; /* Blocks received with CRC errors */ unsigned int crc_err; /* CSR in use by other side */ unsigned int busy; /* Receiver did not acknowledge */ unsigned int no_ack; /* Collision detected */ unsigned int collision; /* Couldn't send a response */ unsigned int tx_timeout; /* Didn't receive a response */ unsigned int rx_timeout; /* Responses received */ unsigned int rx_rsp; /* Messages re-transmitted */ unsigned int retry; /* Event put timeout */ unsigned int event_timeout; }; #define NUM_PFVF_COUNTERS 14 void adf_get_admin_info(struct admin_info *admin_csrs_info); struct adf_admin_comms { bus_addr_t phy_addr; bus_addr_t const_tbl_addr; bus_addr_t aram_map_phys_addr; bus_addr_t phy_hb_addr; bus_dmamap_t aram_map; bus_dmamap_t const_tbl_map; bus_dmamap_t hb_map; char *virt_addr; char *virt_hb_addr; struct resource *mailbox_addr; struct sx lock; struct bus_dmamem dma_mem; struct bus_dmamem dma_hb; }; struct icp_qat_fw_loader_handle; struct adf_fw_loader_data { struct icp_qat_fw_loader_handle *fw_loader; const struct firmware *uof_fw; const struct firmware *mmp_fw; }; struct adf_accel_vf_info { struct adf_accel_dev *accel_dev; struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ u32 vf_nr; bool init; u8 compat_ver; struct pfvf_stats pfvf_counters; }; struct adf_fw_versions { u8 fw_version_major; u8 fw_version_minor; u8 fw_version_patch; u8 mmp_version_major; u8 mmp_version_minor; u8 mmp_version_patch; }; struct adf_int_timer { struct adf_accel_dev *accel_dev; struct workqueue_struct *timer_irq_wq; struct timer_list timer; u32 timeout_val; u32 int_cnt; bool enabled; }; #define ADF_COMPAT_CHECKER_MAX 8 typedef int (*adf_iov_compat_checker_t)(struct adf_accel_dev *accel_dev, u8 vf_compat_ver); struct adf_accel_compat_manager { u8 num_chker; adf_iov_compat_checker_t iov_compat_checkers[ADF_COMPAT_CHECKER_MAX]; }; struct adf_heartbeat; struct adf_accel_dev { struct adf_hw_aram_info *aram_info; struct adf_accel_unit_info *au_info; struct adf_etr_data *transport; struct adf_hw_device_data *hw_device; struct adf_cfg_device_data *cfg; struct adf_fw_loader_data *fw_loader; struct adf_admin_comms *admin; struct adf_uio_control_accel *accel; struct adf_heartbeat *heartbeat; struct adf_int_timer *int_timer; struct adf_fw_versions fw_versions; unsigned int autoreset_on_error; struct adf_fw_counters_data *fw_counters_data; struct sysctl_oid *debugfs_ae_config; struct list_head crypto_list; atomic_t *ras_counters; unsigned long status; atomic_t ref_count; bus_dma_tag_t dma_tag; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *ras_correctable; struct sysctl_oid *ras_uncorrectable; struct sysctl_oid *ras_fatal; struct sysctl_oid *ras_reset; struct sysctl_oid *pke_replay_dbgfile; struct sysctl_oid *misc_error_dbgfile; struct sysctl_oid *fw_version_oid; struct sysctl_oid *mmp_version_oid; struct sysctl_oid *hw_version_oid; struct sysctl_oid *cnv_error_oid; struct list_head list; struct adf_accel_pci accel_pci_dev; struct adf_accel_compat_manager *cm; u8 compat_ver; #ifdef QAT_DISABLE_SAFE_DC_MODE struct sysctl_oid *safe_dc_mode; u8 disable_safe_dc_mode; #endif /* QAT_DISABLE_SAFE_DC_MODE */ union { struct { /* vf_info is non-zero when SR-IOV is init'ed */ struct adf_accel_vf_info *vf_info; int num_vfs; } pf; struct { bool irq_enabled; struct resource *irq; void *cookie; struct task pf2vf_bh_tasklet; struct mutex vf2pf_lock; /* protect CSR access */ struct completion msg_received; struct pfvf_message response; /* temp field holding pf2vf response */ enum ring_reset_result rpreset_sts; struct mutex rpreset_lock; /* protect rpreset_sts */ struct pfvf_stats pfvf_counters; u8 pf_compat_ver; } vf; } u1; bool is_vf; u32 accel_id; void *lac_dev; struct mutex lock; /* protect accel_dev during start/stop e.t.c */ }; #endif diff --git a/sys/dev/qat/qat_api/include/icp_sal_versions.h b/sys/dev/qat/qat_api/include/icp_sal_versions.h index 03bcef4fcbbb..0eb227ade09c 100644 --- a/sys/dev/qat/qat_api/include/icp_sal_versions.h +++ b/sys/dev/qat/qat_api/include/icp_sal_versions.h @@ -1,96 +1,96 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ /** *************************************************************************** * @file icp_sal_versions.h * * @defgroup SalVersions * * @ingroup SalVersions * * API and structures definition for obtaining software and hardware versions * ***************************************************************************/ #ifndef _ICP_SAL_VERSIONS_H_ #define _ICP_SAL_VERSIONS_H_ #define ICP_SAL_VERSIONS_FW_VERSION_SIZE 16 /**< Max length of firmware version string */ #define ICP_SAL_VERSIONS_SW_VERSION_SIZE 16 /**< Max length of software version string */ #define ICP_SAL_VERSIONS_MMP_VERSION_SIZE 16 /**< Max length of MMP binary version string */ #define ICP_SAL_VERSIONS_HW_VERSION_SIZE 4 /**< Max length of hardware version string */ /* Part name and number of the accelerator device */ #define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3 -#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 15 +#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 16 #define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0 /** ******************************************************************************* * @ingroup SalVersions * Structure holding versions information * * @description * This structure stores information about versions of software * and hardware being run on a particular device. *****************************************************************************/ typedef struct icp_sal_dev_version_info_s { Cpa32U devId; /**< Number of acceleration device for which this structure holds * version * information */ Cpa8U firmwareVersion[ICP_SAL_VERSIONS_FW_VERSION_SIZE]; /**< String identifying the version of the firmware associated with * the device. */ Cpa8U mmpVersion[ICP_SAL_VERSIONS_MMP_VERSION_SIZE]; /**< String identifying the version of the MMP binary associated with * the device. */ Cpa8U softwareVersion[ICP_SAL_VERSIONS_SW_VERSION_SIZE]; /**< String identifying the version of the software associated with * the device. */ Cpa8U hardwareVersion[ICP_SAL_VERSIONS_HW_VERSION_SIZE]; /**< String identifying the version of the hardware (stepping and * revision ID) associated with the device. */ } icp_sal_dev_version_info_t; /** ******************************************************************************* * @ingroup SalVersions * Obtains the version information for a given device * @description * This function obtains hardware and software version information * associated with a given device. * * @param[in] accelId ID of the acceleration device for which version * information is to be obtained. * @param[out] pVerInfo Pointer to a structure that will hold version * information * * @context * This function might sleep. It cannot be executed in a context that * does not permit sleeping. * @assumptions * The system has been started * @sideEffects * None * @blocking * No * @reentrant * No * @threadSafe * Yes * * @return CPA_STATUS_SUCCESS Operation finished successfully * @return CPA_STATUS_INVALID_PARAM Invalid parameter passed to the function * @return CPA_STATUS_RESOURCE System resources problem * @return CPA_STATUS_FAIL Operation failed * *****************************************************************************/ CpaStatus icp_sal_getDevVersionInfo(Cpa32U accelId, icp_sal_dev_version_info_t *pVerInfo); #endif diff --git a/sys/dev/qat/qat_common/qat_uclo.c b/sys/dev/qat/qat_common/qat_uclo.c index 54e8e8eb7421..b17020286d24 100644 --- a/sys/dev/qat/qat_common/qat_uclo.c +++ b/sys/dev/qat/qat_common/qat_uclo.c @@ -1,2414 +1,2415 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #include "qat_freebsd.h" #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_accel_devices.h" #include "icp_qat_uclo.h" #include "icp_qat_fw.h" #include "icp_qat_fw_init_admin.h" #include "adf_cfg_strings.h" #include "adf_transport_access_macros.h" #include "adf_transport_internal.h" #include #include #include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" #include "icp_qat_hal.h" #include "icp_qat_fw_loader_handle.h" #define UWORD_CPYBUF_SIZE 1024 #define INVLD_UWORD 0xffffffffffull #define PID_MINOR_REV 0xf #define PID_MAJOR_REV (0xf << 4) #define MAX_UINT32_VAL 0xfffffffful static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, unsigned int ae, unsigned int image_num) { struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_encapme *encap_image; struct icp_qat_uclo_page *page = NULL; struct icp_qat_uclo_aeslice *ae_slice = NULL; ae_data = &obj_handle->ae_data[ae]; encap_image = &obj_handle->ae_uimage[image_num]; ae_slice = &ae_data->ae_slices[ae_data->slice_num]; ae_slice->encap_image = encap_image; if (encap_image->img_ptr) { ae_slice->ctx_mask_assigned = encap_image->img_ptr->ctx_assigned; ae_data->shareable_ustore = ICP_QAT_SHARED_USTORE_MODE(encap_image->img_ptr->ae_mode); if (obj_handle->prod_type == ICP_QAT_AC_4XXX_A_DEV_TYPE) ae_data->eff_ustore_size = obj_handle->ustore_phy_size; else { ae_data->eff_ustore_size = ae_data->shareable_ustore ? (obj_handle->ustore_phy_size << 1) : obj_handle->ustore_phy_size; } } else { ae_slice->ctx_mask_assigned = 0; } ae_slice->region = malloc(sizeof(*ae_slice->region), M_QAT, M_WAITOK | M_ZERO); ae_slice->page = malloc(sizeof(*ae_slice->page), M_QAT, M_WAITOK | M_ZERO); page = ae_slice->page; page->encap_page = encap_image->page; ae_slice->page->region = ae_slice->region; ae_data->slice_num++; return 0; } static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) { unsigned int i; if (!ae_data) { pr_err("QAT: bad argument, ae_data is NULL\n "); return EINVAL; } for (i = 0; i < ae_data->slice_num; i++) { free(ae_data->ae_slices[i].region, M_QAT); ae_data->ae_slices[i].region = NULL; free(ae_data->ae_slices[i].page, M_QAT); ae_data->ae_slices[i].page = NULL; } return 0; } static char * qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, unsigned int str_offset) { if (!str_table->table_len || str_offset > str_table->table_len) return NULL; return (char *)(((uintptr_t)(str_table->strings)) + str_offset); } static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) { int maj = hdr->maj_ver & 0xff; int min = hdr->min_ver & 0xff; if (hdr->file_id != ICP_QAT_UOF_FID) { pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); return EINVAL; } if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", maj, min); return EINVAL; } return 0; } static int qat_uclo_check_suof_format(const struct icp_qat_suof_filehdr *suof_hdr) { int maj = suof_hdr->maj_ver & 0xff; int min = suof_hdr->min_ver & 0xff; if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); return EINVAL; } if (suof_hdr->fw_type != 0) { pr_err("QAT: unsupported firmware type\n"); return EINVAL; } if (suof_hdr->num_chunks <= 0x1) { pr_err("QAT: SUOF chunk amount is incorrect\n"); return EINVAL; } if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", maj, min); return EINVAL; } return 0; } static int qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, unsigned int addr, const unsigned int *val, unsigned int num_in_bytes) { unsigned int outval; const unsigned char *ptr = (const unsigned char *)val; if (num_in_bytes > handle->hal_sram_size) { pr_err("QAT: error, mmp size overflow %d\n", num_in_bytes); return EINVAL; } while (num_in_bytes) { memcpy(&outval, ptr, 4); SRAM_WRITE(handle, addr, outval); num_in_bytes -= 4; ptr += 4; addr += 4; } return 0; } static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int addr, unsigned int *val, unsigned int num_in_bytes) { unsigned int outval; unsigned char *ptr = (unsigned char *)val; addr >>= 0x2; /* convert to uword address */ while (num_in_bytes) { memcpy(&outval, ptr, 4); qat_hal_wr_umem(handle, ae, addr++, 1, &outval); num_in_bytes -= 4; ptr += 4; } } static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, unsigned char ae, struct icp_qat_uof_batch_init *umem_init_header) { struct icp_qat_uof_batch_init *umem_init; if (!umem_init_header) return; umem_init = umem_init_header->next; while (umem_init) { unsigned int addr, *value, size; ae = umem_init->ae; addr = umem_init->addr; value = umem_init->value; size = umem_init->size; qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); umem_init = umem_init->next; } } static void qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_batch_init **base) { struct icp_qat_uof_batch_init *umem_init; umem_init = *base; while (umem_init) { struct icp_qat_uof_batch_init *pre; pre = umem_init; umem_init = umem_init->next; free(pre, M_QAT); } *base = NULL; } static int qat_uclo_parse_num(char *str, unsigned int *num) { char buf[16] = { 0 }; unsigned long ae = 0; int i; strncpy(buf, str, 15); for (i = 0; i < 16; i++) { if (!isdigit(buf[i])) { buf[i] = '\0'; break; } } if ((compat_strtoul(buf, 10, &ae))) return EFAULT; if (ae > MAX_UINT32_VAL) return EFAULT; *num = (unsigned int)ae; return 0; } static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem, unsigned int size_range, unsigned int *ae) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; char *str; if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { pr_err("QAT: initmem is out of range"); return EINVAL; } if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { pr_err("QAT: Memory scope for init_mem error\n"); return EINVAL; } str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); if (!str) { pr_err("QAT: AE name assigned in UOF init table is NULL\n"); return EINVAL; } if (qat_uclo_parse_num(str, ae)) { pr_err("QAT: Parse num for AE number failed\n"); return EINVAL; } if (*ae >= ICP_QAT_UCLO_MAX_AE) { pr_err("QAT: ae %d out of range\n", *ae); return EINVAL; } return 0; } static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem, unsigned int ae, struct icp_qat_uof_batch_init **init_tab_base) { struct icp_qat_uof_batch_init *init_header, *tail; struct icp_qat_uof_batch_init *mem_init, *tail_old; struct icp_qat_uof_memvar_attr *mem_val_attr; unsigned int i = 0; mem_val_attr = (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + sizeof( struct icp_qat_uof_initmem)); init_header = *init_tab_base; if (!init_header) { init_header = malloc(sizeof(*init_header), M_QAT, M_WAITOK | M_ZERO); init_header->size = 1; *init_tab_base = init_header; } tail_old = init_header; while (tail_old->next) tail_old = tail_old->next; tail = tail_old; for (i = 0; i < init_mem->val_attr_num; i++) { mem_init = malloc(sizeof(*mem_init), M_QAT, M_WAITOK | M_ZERO); mem_init->ae = ae; mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; mem_init->value = &mem_val_attr->value; mem_init->size = 4; mem_init->next = NULL; tail->next = mem_init; tail = mem_init; init_header->size += qat_hal_get_ins_num(); mem_val_attr++; } return 0; } static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae; unsigned int lmem; lmem = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))) ? ICP_QAT_UCLO_MAX_LMEM_REG_2X : ICP_QAT_UCLO_MAX_LMEM_REG; if (qat_uclo_fetch_initmem_ae(handle, init_mem, lmem, &ae)) return EINVAL; if (qat_uclo_create_batch_init_list( handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) return EINVAL; return 0; } static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae, ustore_size, uaddr, i; struct icp_qat_uclo_aedata *aed; ustore_size = obj_handle->ustore_phy_size; if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) return EINVAL; if (qat_uclo_create_batch_init_list( handle, init_mem, ae, &obj_handle->umem_init_tab[ae])) return EINVAL; /* set the highest ustore address referenced */ uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; aed = &obj_handle->ae_data[ae]; for (i = 0; i < aed->slice_num; i++) { if (aed->ae_slices[i].encap_image->uwords_num < uaddr) aed->ae_slices[i].encap_image->uwords_num = uaddr; } return 0; } #define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { switch (init_mem->region) { case ICP_QAT_UOF_LMEM_REGION: if (qat_uclo_init_lmem_seg(handle, init_mem)) return EINVAL; break; case ICP_QAT_UOF_UMEM_REGION: if (qat_uclo_init_umem_seg(handle, init_mem)) return EINVAL; break; default: pr_err("QAT: initmem region error. region type=0x%x\n", init_mem->region); return EINVAL; } return 0; } static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uclo_encapme *image) { unsigned int i; struct icp_qat_uclo_encap_page *page; struct icp_qat_uof_image *uof_image; unsigned char ae = 0; unsigned char neigh_ae; unsigned int ustore_size; unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; uint64_t *fill_data; static unsigned int init[32] = { 0 }; unsigned long ae_mask = handle->hal_handle->ae_mask; uof_image = image->img_ptr; /*if shared CS mode, the ustore size should be 2*ustore_phy_size*/ fill_data = malloc(obj_handle->ustore_phy_size * 2 * sizeof(uint64_t), M_QAT, M_WAITOK | M_ZERO); for (i = 0; i < obj_handle->ustore_phy_size * 2; i++) memcpy(&fill_data[i], &uof_image->fill_pattern, sizeof(uint64_t)); page = image->page; for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned long ae_assigned = uof_image->ae_assigned; const bool gen4 = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))); if (!test_bit(ae, &cfg_ae_mask)) continue; if (!test_bit(ae, &ae_assigned)) continue; if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1) && !gen4) { qat_hal_get_scs_neigh_ae(ae, &neigh_ae); if (test_bit(neigh_ae, &ae_assigned)) continue; } ustore_size = obj_handle->ae_data[ae].eff_ustore_size; patt_pos = page->beg_addr_p + page->micro_words_num; if (obj_handle->ae_data[ae].shareable_ustore && !gen4) { qat_hal_get_scs_neigh_ae(ae, &neigh_ae); if (init[ae] == 0 && page->beg_addr_p != 0) { qat_hal_wr_coalesce_uwords(handle, (unsigned char)ae, 0, page->beg_addr_p, &fill_data[0]); } qat_hal_wr_coalesce_uwords( handle, (unsigned char)ae, patt_pos, ustore_size - patt_pos, &fill_data[page->beg_addr_p]); init[ae] = 1; init[neigh_ae] = 1; } else { if (gen4 && (ae % 4 != 0)) continue; qat_hal_wr_uwords(handle, (unsigned char)ae, 0, page->beg_addr_p, &fill_data[0]); qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, ustore_size - patt_pos + 1, &fill_data[page->beg_addr_p]); } } free(fill_data, M_QAT); return 0; } static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) { int i; int ae = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; unsigned long ae_mask = handle->hal_handle->ae_mask; for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { if (initmem->num_in_bytes) { if (qat_uclo_init_ae_memory(handle, initmem)) return EINVAL; } initmem = (struct icp_qat_uof_initmem *)((uintptr_t)((uintptr_t)initmem + sizeof(struct icp_qat_uof_initmem)) + (sizeof(struct icp_qat_uof_memvar_attr) * initmem->val_attr_num)); } for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { pr_err("QAT: fail to batch init lmem for AE %d\n", ae); return EINVAL; } qat_uclo_cleanup_batch_init_list(handle, &obj_handle->lm_init_tab[ae]); qat_uclo_batch_wr_umem(handle, ae, obj_handle->umem_init_tab[ae]); qat_uclo_cleanup_batch_init_list( handle, &obj_handle->umem_init_tab[ae]); } return 0; } static void * qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, char *chunk_id, void *cur) { int i; struct icp_qat_uof_chunkhdr *chunk_hdr = (struct icp_qat_uof_chunkhdr *)((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); for (i = 0; i < obj_hdr->num_chunks; i++) { if ((cur < (void *)&chunk_hdr[i]) && !strncmp(chunk_hdr[i].chunk_id, chunk_id, ICP_QAT_UOF_OBJID_LEN)) { return &chunk_hdr[i]; } } return NULL; } static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) { int i; unsigned int topbit = 1 << 0xF; unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); reg ^= inbyte << 0x8; for (i = 0; i < 0x8; i++) { if (reg & topbit) reg = (reg << 1) ^ 0x1021; else reg <<= 1; } return reg & 0xFFFF; } static unsigned int qat_uclo_calc_str_checksum(const char *ptr, int num) { unsigned int chksum = 0; if (ptr) while (num--) chksum = qat_uclo_calc_checksum(chksum, *ptr++); return chksum; } static struct icp_qat_uclo_objhdr * qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, char *chunk_id) { struct icp_qat_uof_filechunkhdr *file_chunk; struct icp_qat_uclo_objhdr *obj_hdr; char *chunk; int i; file_chunk = (struct icp_qat_uof_filechunkhdr *)(buf + sizeof(struct icp_qat_uof_filehdr)); for (i = 0; i < file_hdr->num_chunks; i++) { if (!strncmp(file_chunk->chunk_id, chunk_id, ICP_QAT_UOF_OBJID_LEN)) { chunk = buf + file_chunk->offset; if (file_chunk->checksum != qat_uclo_calc_str_checksum(chunk, file_chunk->size)) break; obj_hdr = malloc(sizeof(*obj_hdr), M_QAT, M_WAITOK | M_ZERO); obj_hdr->file_buff = chunk; obj_hdr->checksum = file_chunk->checksum; obj_hdr->size = file_chunk->size; return obj_hdr; } file_chunk++; } return NULL; } static unsigned int qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, struct icp_qat_uof_image *image) { struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; struct icp_qat_uof_objtable *neigh_reg_tab; struct icp_qat_uof_code_page *code_page; code_page = (struct icp_qat_uof_code_page *)((char *)image + sizeof(struct icp_qat_uof_image)); uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + code_page->uc_var_tab_offset); imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + code_page->imp_var_tab_offset); imp_expr_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { pr_err("QAT: UOF can't contain imported variable to be parsed"); return EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { pr_err("QAT: UOF can't contain neighbor register table\n"); return EINVAL; } if (image->numpages > 1) { pr_err("QAT: UOF can't contain multiple pages\n"); return EINVAL; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { pr_err("QAT: UOF can't use reloadable feature\n"); return EFAULT; } return 0; } static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj *encap_uof_obj, struct icp_qat_uof_image *img, struct icp_qat_uclo_encap_page *page) { struct icp_qat_uof_code_page *code_page; struct icp_qat_uof_code_area *code_area; struct icp_qat_uof_objtable *uword_block_tab; struct icp_qat_uof_uword_block *uwblock; int i; code_page = (struct icp_qat_uof_code_page *)((char *)img + sizeof(struct icp_qat_uof_image)); page->def_page = code_page->def_page; page->page_region = code_page->page_region; page->beg_addr_v = code_page->beg_addr_v; page->beg_addr_p = code_page->beg_addr_p; code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + code_page->code_area_offset); page->micro_words_num = code_area->micro_words_num; uword_block_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + code_area->uword_block_tab); page->uwblock_num = uword_block_tab->entry_num; uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + sizeof(struct icp_qat_uof_objtable)); page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; for (i = 0; i < uword_block_tab->entry_num; i++) page->uwblock[i].micro_words = (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; } static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, struct icp_qat_uclo_encapme *ae_uimage, int max_image) { int i, j; struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; struct icp_qat_uof_image *image; struct icp_qat_uof_objtable *ae_regtab; struct icp_qat_uof_objtable *init_reg_sym_tab; struct icp_qat_uof_objtable *sbreak_tab; struct icp_qat_uof_encap_obj *encap_uof_obj = &obj_handle->encap_uof_obj; for (j = 0; j < max_image; j++) { chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, ICP_QAT_UOF_IMAG, chunk_hdr); if (!chunk_hdr) break; image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + chunk_hdr->offset); ae_regtab = (struct icp_qat_uof_objtable *)(image->reg_tab_offset + obj_handle->obj_hdr ->file_buff); ae_uimage[j].ae_reg_num = ae_regtab->entry_num; ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)(((char *)ae_regtab) + sizeof(struct icp_qat_uof_objtable)); init_reg_sym_tab = (struct icp_qat_uof_objtable *)(image->init_reg_sym_tab + obj_handle->obj_hdr ->file_buff); ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)(((char *)init_reg_sym_tab) + sizeof(struct icp_qat_uof_objtable)); sbreak_tab = (struct icp_qat_uof_objtable *)(image->sbreak_tab + obj_handle->obj_hdr ->file_buff); ae_uimage[j].sbreak_num = sbreak_tab->entry_num; ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)(((char *)sbreak_tab) + sizeof(struct icp_qat_uof_objtable)); ae_uimage[j].img_ptr = image; if (qat_uclo_check_image_compat(encap_uof_obj, image)) goto out_err; ae_uimage[j].page = malloc(sizeof(struct icp_qat_uclo_encap_page), M_QAT, M_WAITOK | M_ZERO); qat_uclo_map_image_page(encap_uof_obj, image, ae_uimage[j].page); } return j; out_err: for (i = 0; i < j; i++) free(ae_uimage[i].page, M_QAT); return 0; } static int UcLo_checkTGroupList2X(struct icp_qat_fw_loader_handle *handle) { int i; unsigned int swAe = 0; unsigned int ii, jj; struct icp_qat_uclo_aedata *ae_data0, *ae_datax; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; for (i = 0; i < obj_handle->uimage_num; i++) { struct icp_qat_uof_image *image = obj_handle->ae_uimage[i].img_ptr; if (image->numpages > 1) { pr_err( "Only 1 page is allowed in a UOF for CPM2X; We found %d in %s\n", image->numpages, qat_uclo_get_string(&obj_handle->str_table, image->img_name)); return EINVAL; } } for (swAe = 0; (swAe < obj_handle->ae_num) && (swAe < ICP_QAT_UCLO_MAX_AE); swAe += AE_TG_NUM_CPM2X) { if (!qat_hal_check_ae_active(handle, swAe)) { continue; } for (ii = swAe; ii < (swAe + AE_TG_NUM_CPM2X); ii++) { ae_data0 = &obj_handle->ae_data[ii]; if (ae_data0->slice_num != 1) // not assigned continue; for (jj = ii + 1; jj < (swAe + AE_TG_NUM_CPM2X); jj++) { ae_datax = &obj_handle->ae_data[jj]; if (ae_datax->slice_num != 1) // not assigned continue; if (ae_data0->ae_slices[0] .encap_image->img_ptr != ae_datax->ae_slices[0] .encap_image->img_ptr) { pr_err("Only 1 list is allowed in a "); pr_err("Tgroup for CPM2X;\n"); pr_err("ME%d, %d is assigned", ii, jj); pr_err(" different list files\n"); return EINVAL; } } } } return 0; } static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) { int i; int ae = 0; unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned long cfg_ae_mask = handle->cfg_ae_mask; int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; for_each_set_bit(ae, &ae_mask, max_ae) { if (!test_bit(ae, &cfg_ae_mask)) continue; for (i = 0; i < obj_handle->uimage_num; i++) { unsigned long ae_assigned = obj_handle->ae_uimage[i].img_ptr->ae_assigned; if (!test_bit(ae, &ae_assigned)) continue; mflag = 1; if (qat_uclo_init_ae_data(obj_handle, ae, i)) return EINVAL; } } if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { if (UcLo_checkTGroupList2X(handle)) { return EINVAL; } } if (!mflag) { pr_err("QAT: uimage uses AE not set"); return EINVAL; } return 0; } static struct icp_qat_uof_strtable * qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, char *tab_name, struct icp_qat_uof_strtable *str_table) { struct icp_qat_uof_chunkhdr *chunk_hdr; chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)obj_hdr->file_buff, tab_name, NULL); if (chunk_hdr) { int hdr_size; memcpy(&str_table->table_len, obj_hdr->file_buff + chunk_hdr->offset, sizeof(str_table->table_len)); hdr_size = (char *)&str_table->strings - (char *)str_table; str_table->strings = (uintptr_t)obj_hdr->file_buff + chunk_hdr->offset + hdr_size; return str_table; } return NULL; } static void qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, struct icp_qat_uclo_init_mem_table *init_mem_tab) { struct icp_qat_uof_chunkhdr *chunk_hdr; chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, ICP_QAT_UOF_IMEM, NULL); if (chunk_hdr) { memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + chunk_hdr->offset, sizeof(unsigned int)); init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)(encap_uof_obj->beg_uof + chunk_hdr->offset + sizeof(unsigned int)); } } static unsigned int qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) { switch (pci_get_device(GET_DEV(handle->accel_dev))) { case ADF_DH895XCC_PCI_DEVICE_ID: return ICP_QAT_AC_895XCC_DEV_TYPE; case ADF_C62X_PCI_DEVICE_ID: return ICP_QAT_AC_C62X_DEV_TYPE; case ADF_C3XXX_PCI_DEVICE_ID: return ICP_QAT_AC_C3XXX_DEV_TYPE; case ADF_200XX_PCI_DEVICE_ID: return ICP_QAT_AC_200XX_DEV_TYPE; case ADF_C4XXX_PCI_DEVICE_ID: return ICP_QAT_AC_C4XXX_DEV_TYPE; case ADF_4XXX_PCI_DEVICE_ID: case ADF_401XX_PCI_DEVICE_ID: + case ADF_402XX_PCI_DEVICE_ID: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", pci_get_device(GET_DEV(handle->accel_dev))); return 0; } } static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) { unsigned int maj_ver, prod_type = obj_handle->prod_type; if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, prod_type); return EINVAL; } maj_ver = obj_handle->prod_rev & 0xff; if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { pr_err("QAT: UOF maj_ver 0x%x out of range\n", maj_ver); return EINVAL; } return 0; } static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_addr, unsigned int value) { switch (reg_type) { case ICP_GPA_ABS: case ICP_GPB_ABS: ctx_mask = 0; return qat_hal_init_gpr( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_GPA_REL: case ICP_GPB_REL: return qat_hal_init_gpr( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_SR_ABS: case ICP_DR_ABS: case ICP_SR_RD_ABS: case ICP_DR_RD_ABS: ctx_mask = 0; return qat_hal_init_rd_xfer( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_SR_REL: case ICP_DR_REL: case ICP_SR_RD_REL: case ICP_DR_RD_REL: return qat_hal_init_rd_xfer( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_SR_WR_ABS: case ICP_DR_WR_ABS: ctx_mask = 0; return qat_hal_init_wr_xfer( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_SR_WR_REL: case ICP_DR_WR_REL: return qat_hal_init_wr_xfer( handle, ae, ctx_mask, reg_type, reg_addr, value); case ICP_NEIGH_REL: return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); default: pr_err("QAT: UOF uses unsupported reg type 0x%x\n", reg_type); return EFAULT; } return 0; } static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, unsigned int ae, struct icp_qat_uclo_encapme *encap_ae) { unsigned int i; unsigned char ctx_mask; struct icp_qat_uof_init_regsym *init_regsym; if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == ICP_QAT_UCLO_MAX_CTX) ctx_mask = 0xff; else ctx_mask = 0x55; for (i = 0; i < encap_ae->init_regsym_num; i++) { unsigned int exp_res; init_regsym = &encap_ae->init_regsym[i]; exp_res = init_regsym->value; switch (init_regsym->init_type) { case ICP_QAT_UOF_INIT_REG: qat_uclo_init_reg(handle, ae, ctx_mask, (enum icp_qat_uof_regtype) init_regsym->reg_type, (unsigned short)init_regsym->reg_addr, exp_res); break; case ICP_QAT_UOF_INIT_REG_CTX: /* check if ctx is appropriate for the ctxMode */ if (!((1 << init_regsym->ctx) & ctx_mask)) { pr_err("QAT: invalid ctx num = 0x%x\n", init_regsym->ctx); return EINVAL; } qat_uclo_init_reg( handle, ae, (unsigned char)(1 << init_regsym->ctx), (enum icp_qat_uof_regtype)init_regsym->reg_type, (unsigned short)init_regsym->reg_addr, exp_res); break; case ICP_QAT_UOF_INIT_EXPR: pr_err("QAT: INIT_EXPR feature not supported\n"); return EINVAL; case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: pr_err("QAT: INIT_EXPR_ENDIAN_SWAP not supported\n"); return EINVAL; default: break; } } return 0; } static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int s; unsigned int ae = 0; struct icp_qat_uclo_aedata *aed; unsigned long ae_mask = handle->hal_handle->ae_mask; if (obj_handle->global_inited) return 0; if (obj_handle->init_mem_tab.entry_num) { if (qat_uclo_init_memory(handle)) { pr_err("QAT: initialize memory failed\n"); return EINVAL; } } for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { aed = &obj_handle->ae_data[ae]; for (s = 0; s < aed->slice_num; s++) { if (!aed->ae_slices[s].encap_image) continue; if (qat_uclo_init_reg_sym( handle, ae, aed->ae_slices[s].encap_image)) return EINVAL; } } obj_handle->global_inited = 1; return 0; } static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uclo_objhandle *obj_handle, unsigned char ae, struct icp_qat_uof_image *uof_image) { unsigned char nn_mode; char ae_mode = 0; ae_mode = (char)ICP_QAT_CTX_MODE(uof_image->ae_mode); if (qat_hal_set_ae_ctx_mode(handle, ae, ae_mode)) { pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); return EFAULT; } ae_mode = (char)ICP_QAT_SHARED_USTORE_MODE(uof_image->ae_mode); qat_hal_set_ae_scs_mode(handle, ae, ae_mode); if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); return EFAULT; } } ae_mode = (char)ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, ae_mode)) { pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); return EFAULT; } ae_mode = (char)ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, ae_mode)) { pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); return EFAULT; } if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { ae_mode = (char)ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, ae_mode)) { pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); return EFAULT; } ae_mode = (char)ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, ae_mode)) { pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); return EFAULT; } ae_mode = (char)ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); qat_hal_set_ae_tindex_mode(handle, ae, ae_mode); } return 0; } static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) { int error; unsigned char s; unsigned char ae = 0; struct icp_qat_uof_image *uof_image; struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { unsigned long cfg_ae_mask = handle->cfg_ae_mask; if (!test_bit(ae, &cfg_ae_mask)) continue; ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) continue; uof_image = ae_data->ae_slices[s].encap_image->img_ptr; error = qat_hal_set_modes(handle, obj_handle, ae, uof_image); if (error) return error; } } return 0; } static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; struct icp_qat_uclo_encapme *image; int a; for (a = 0; a < obj_handle->uimage_num; a++) { image = &obj_handle->ae_uimage[a]; image->uwords_num = image->page->beg_addr_p + image->page->micro_words_num; } } static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae; obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)obj_handle->obj_hdr->file_buff; obj_handle->uword_in_bytes = 6; obj_handle->prod_type = qat_uclo_get_dev_type(handle); obj_handle->prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (qat_uclo_check_uof_compat(obj_handle)) { pr_err("QAT: UOF incompatible\n"); return EINVAL; } obj_handle->uword_buf = malloc(UWORD_CPYBUF_SIZE * sizeof(uint64_t), M_QAT, M_WAITOK | M_ZERO); obj_handle->ustore_phy_size = (obj_handle->prod_type == ICP_QAT_AC_C4XXX_DEV_TYPE) ? 0x2000 : 0x4000; if (!obj_handle->obj_hdr->file_buff || !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, &obj_handle->str_table)) { pr_err("QAT: UOF doesn't have effective images\n"); goto out_err; } obj_handle->uimage_num = qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); if (!obj_handle->uimage_num) goto out_err; if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { pr_err("QAT: Bad object\n"); goto out_check_uof_aemask_err; } qat_uclo_init_uword_num(handle); qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, &obj_handle->init_mem_tab); if (qat_uclo_set_ae_mode(handle)) goto out_check_uof_aemask_err; return 0; out_check_uof_aemask_err: for (ae = 0; ae < obj_handle->uimage_num; ae++) free(obj_handle->ae_uimage[ae].page, M_QAT); out_err: free(obj_handle->uword_buf, M_QAT); obj_handle->uword_buf = NULL; return EFAULT; } static int qat_uclo_map_suof_file_hdr(const struct icp_qat_fw_loader_handle *handle, const struct icp_qat_suof_filehdr *suof_ptr, int suof_size) { unsigned int check_sum = 0; unsigned int min_ver_offset = 0; struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; suof_handle->file_id = ICP_QAT_SUOF_FID; suof_handle->suof_buf = (const char *)suof_ptr; suof_handle->suof_size = suof_size; min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, min_ver); check_sum = qat_uclo_calc_str_checksum((const char *)&suof_ptr->min_ver, min_ver_offset); if (check_sum != suof_ptr->check_sum) { pr_err("QAT: incorrect SUOF checksum\n"); return EINVAL; } suof_handle->check_sum = suof_ptr->check_sum; suof_handle->min_ver = suof_ptr->min_ver; suof_handle->maj_ver = suof_ptr->maj_ver; suof_handle->fw_type = suof_ptr->fw_type; return 0; } static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *suof_img_hdr, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; const struct icp_qat_simg_ae_mode *ae_mode; struct icp_qat_suof_objhdr *suof_objhdr; unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); suof_img_hdr->simg_buf = (suof_handle->suof_buf + suof_chunk_hdr->offset + sizeof(*suof_objhdr)); suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)(suof_handle->suof_buf + suof_chunk_hdr->offset)) ->img_length; suof_img_hdr->css_header = suof_img_hdr->simg_buf; suof_img_hdr->css_key = (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); suof_img_hdr->css_signature = suof_img_hdr->css_key + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id); suof_img_hdr->css_simg = suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN(device_id); ae_mode = (const struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; suof_img_hdr->fw_type = ae_mode->fw_type; } static void qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { char **sym_str = (char **)&suof_handle->sym_str; unsigned int *sym_size = &suof_handle->sym_size; struct icp_qat_suof_strtable *str_table_obj; *sym_size = *(unsigned int *)(uintptr_t)(suof_chunk_hdr->offset + suof_handle->suof_buf); *sym_str = (char *)(uintptr_t)(suof_handle->suof_buf + suof_chunk_hdr->offset + sizeof(str_table_obj->tab_length)); } static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *img_hdr) { const struct icp_qat_simg_ae_mode *img_ae_mode = NULL; unsigned int prod_rev, maj_ver, prod_type; prod_type = qat_uclo_get_dev_type(handle); img_ae_mode = (const struct icp_qat_simg_ae_mode *)img_hdr->css_simg; prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (img_ae_mode->dev_type != prod_type) { pr_err("QAT: incompatible product type %x\n", img_ae_mode->dev_type); return EINVAL; } maj_ver = prod_rev & 0xff; if (maj_ver > img_ae_mode->devmax_ver || maj_ver < img_ae_mode->devmin_ver) { pr_err("QAT: incompatible device maj_ver 0x%x\n", maj_ver); return EINVAL; } return 0; } static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; free(sobj_handle->img_table.simg_hdr, M_QAT); sobj_handle->img_table.simg_hdr = NULL; free(handle->sobj_handle, M_QAT); handle->sobj_handle = NULL; } static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, unsigned int img_id, unsigned int num_simgs) { struct icp_qat_suof_img_hdr img_header; if ((img_id != num_simgs - 1) && img_id != ICP_QAT_UCLO_MAX_AE) { memcpy(&img_header, &suof_img_hdr[num_simgs - 1], sizeof(*suof_img_hdr)); memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], sizeof(*suof_img_hdr)); memcpy(&suof_img_hdr[img_id], &img_header, sizeof(*suof_img_hdr)); } } static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, const struct icp_qat_suof_filehdr *suof_ptr, int suof_size) { struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE, aeMax_img = ICP_QAT_UCLO_MAX_AE; unsigned int i = 0; struct icp_qat_suof_img_hdr img_header; if (!suof_ptr || suof_size == 0) { pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); return EINVAL; } if (qat_uclo_check_suof_format(suof_ptr)) return EINVAL; ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); if (ret) return ret; suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)((uintptr_t)suof_ptr + sizeof(*suof_ptr)); qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; if (suof_handle->img_table.num_simgs != 0) { suof_img_hdr = malloc(suof_handle->img_table.num_simgs * sizeof(img_header), M_QAT, M_WAITOK | M_ZERO); suof_handle->img_table.simg_hdr = suof_img_hdr; } for (i = 0; i < suof_handle->img_table.num_simgs; i++) { qat_uclo_map_simg(handle, &suof_img_hdr[i], &suof_chunk_hdr[1 + i]); ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]); if (ret) return ret; suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; if ((suof_img_hdr[i].ae_mask & 0x1) != 0) ae0_img = i; } if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { qat_uclo_tail_img(suof_img_hdr, ae0_img, suof_handle->img_table.num_simgs); } else { if (suof_handle->img_table.num_simgs == 1) return 0; qat_uclo_tail_img(suof_img_hdr, ae0_img, suof_handle->img_table.num_simgs - 1); for (i = 0; i < suof_handle->img_table.num_simgs; i++) { if ((suof_img_hdr[i].ae_mask & (0x1 << (handle->hal_handle->ae_max_num - 1))) != 0) { aeMax_img = i; break; } } qat_uclo_tail_img(suof_img_hdr, aeMax_img, suof_handle->img_table.num_simgs); } return 0; } #define ADD_ADDR(high, low) ((((uint64_t)high) << 32) + (low)) #define BITS_IN_DWORD 32 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { unsigned int fcu_sts, mem_cfg_err, retry = 0; unsigned int fcu_ctl_csr, fcu_sts_csr; unsigned int fcu_dram_hi_csr, fcu_dram_lo_csr; u64 bus_addr; bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - sizeof(struct icp_qat_auth_chunk); if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { fcu_ctl_csr = FCU_CONTROL_C4XXX; fcu_sts_csr = FCU_STATUS_C4XXX; fcu_dram_hi_csr = FCU_DRAM_ADDR_HI_C4XXX; fcu_dram_lo_csr = FCU_DRAM_ADDR_LO_C4XXX; } else { fcu_ctl_csr = FCU_CONTROL; fcu_sts_csr = FCU_STATUS; fcu_dram_hi_csr = FCU_DRAM_ADDR_HI; fcu_dram_lo_csr = FCU_DRAM_ADDR_LO; } SET_FCU_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); SET_FCU_CSR(handle, fcu_dram_lo_csr, bus_addr); SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); do { pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) goto auth_fail; if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) return 0; } while (retry++ < FW_AUTH_MAX_RETRY); auth_fail: pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", fcu_sts & FCU_AUTH_STS_MASK, retry); if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { mem_cfg_err = (GET_FCU_CSR(handle, FCU_STATUS1_C4XXX) & MEM_CFG_ERR_BIT); if (mem_cfg_err) pr_err("QAT: MEM_CFG_ERR\n"); } return EINVAL; } static int qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, int imgid) { struct icp_qat_suof_handle *sobj_handle; if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) return 0; sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; if (handle->hal_handle->admin_ae_mask & sobj_handle->img_table.simg_hdr[imgid].ae_mask) return 0; return 1; } static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { unsigned int i = 0; unsigned int fcuSts = 0, fcuAeBroadcastMask = 0; unsigned int retry = 0; unsigned int fcuStsCsr = 0; unsigned int fcuCtlCsr = 0; unsigned int loadedAes = 0; unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); if (IS_QAT_GEN4(device_id)) { fcuCtlCsr = FCU_CONTROL_4XXX; fcuStsCsr = FCU_STATUS_4XXX; } else { pr_err("Uclo_BroadcastLoadFW only applicable for CPM20\n"); return EINVAL; } for (i = 0; i < ICP_QAT_UCLO_MAX_AE; i++) { if (!test_bit(i, (unsigned long *)&handle->hal_handle->ae_mask)) continue; if (qat_hal_check_ae_active(handle, (unsigned char)i)) { pr_err( "Uclo_BroadcastLoadFW error (invalid AE status)\n"); return EINVAL; } if ((desc->ae_mask >> i) & 0x1) { fcuAeBroadcastMask |= 1 << i; } } if (fcuAeBroadcastMask) { retry = 0; SET_FCU_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, fcuAeBroadcastMask); SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD); do { msleep(FW_AUTH_WAIT_PERIOD); fcuSts = GET_FCU_CSR(handle, fcuStsCsr); if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_FAIL) { pr_err( "Uclo_BroadcastLoadFW fail (fcu_status = 0x%x)\n", fcuSts & FCU_AUTH_STS_MASK); return EINVAL; } else if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_DONE) { if (IS_QAT_GEN4(device_id)) loadedAes = GET_FCU_CSR(handle, FCU_AE_LOADED_4XXX); else loadedAes = (fcuSts >> FCU_LOADED_AE_POS); if ((loadedAes & fcuAeBroadcastMask) == fcuAeBroadcastMask) break; } else if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) { SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD); } } while (retry++ < FW_BROADCAST_MAX_RETRY); if (retry > FW_BROADCAST_MAX_RETRY) { pr_err( "Uclo_BroadcastLoadFW fail(fcu_status = 0x%x),retry = %d\n", fcuSts & FCU_AUTH_STS_MASK, retry); return EINVAL; } } return 0; } static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc, unsigned int size) { int ret; ret = bus_dma_mem_create(&dram_desc->dram_mem, handle->accel_dev->dma_tag, 1, BUS_SPACE_MAXADDR, size, 0); if (ret != 0) return ret; dram_desc->dram_base_addr_v = dram_desc->dram_mem.dma_vaddr; dram_desc->dram_bus_addr = dram_desc->dram_mem.dma_baddr; dram_desc->dram_size = size; return 0; } static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc) { if (handle && dram_desc && dram_desc->dram_base_addr_v) bus_dma_mem_free(&dram_desc->dram_mem); if (dram_desc) explicit_bzero(dram_desc, sizeof(*dram_desc)); } static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, const char *image, unsigned int size, struct icp_firml_dram_desc *img_desc, struct icp_qat_fw_auth_desc **desc) { const struct icp_qat_css_hdr *css_hdr = (const struct icp_qat_css_hdr *)image; struct icp_qat_fw_auth_desc *auth_desc; struct icp_qat_auth_chunk *auth_chunk; u64 virt_addr, bus_addr, virt_base; unsigned int length, simg_offset = sizeof(*auth_chunk); unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev)); if (size <= ICP_QAT_AE_IMG_OFFSET(device_id)) { pr_err("QAT: error, input image size too small %d\n", size); return EINVAL; } if (size > (ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return EINVAL; } length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? ICP_QAT_CSS_AE_SIMG_LEN(device_id) + simg_offset : size + ICP_QAT_CSS_FWSK_PAD_LEN(device_id) + simg_offset; if (qat_uclo_simg_alloc(handle, img_desc, length)) { pr_err("QAT: error, allocate continuous dram fail\n"); return -ENOMEM; } auth_chunk = img_desc->dram_base_addr_v; auth_chunk->chunk_size = img_desc->dram_size; auth_chunk->chunk_bus_addr = img_desc->dram_bus_addr; virt_base = (uintptr_t)img_desc->dram_base_addr_v + simg_offset; bus_addr = img_desc->dram_bus_addr + simg_offset; auth_desc = img_desc->dram_base_addr_v; auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->css_hdr_low = (unsigned int)bus_addr; virt_addr = virt_base; memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); /* pub key */ bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->fwsk_pub_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (const void *)(image + sizeof(*css_hdr)), ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)); /* padding */ explicit_bzero((void *)(uintptr_t)( virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), ICP_QAT_CSS_FWSK_PAD_LEN(device_id)); /* exponent */ memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + ICP_QAT_CSS_FWSK_PAD_LEN(device_id)), (const void *)(image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(device_id); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(device_id); auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (const void *)(image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) + ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id)), ICP_QAT_CSS_SIGNATURE_LEN(device_id)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + ICP_QAT_CSS_SIGNATURE_LEN(device_id); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(device_id); auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_low = (unsigned int)bus_addr; auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(device_id); memcpy((void *)(uintptr_t)virt_addr, (const void *)(image + ICP_QAT_AE_IMG_OFFSET(device_id)), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == CSS_AE_FIRMWARE) { auth_desc->img_ae_mode_data_high = auth_desc->img_high; auth_desc->img_ae_mode_data_low = auth_desc->img_low; bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); auth_desc->img_ae_init_data_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; auth_desc->img_ae_insts_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; virt_addr += sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN(device_id) + ICP_QAT_CSS_SIGNATURE_LEN(device_id); auth_desc->ae_mask = ((struct icp_qat_simg_ae_mode *)virt_addr)->ae_mask & handle->cfg_ae_mask; } else { auth_desc->img_ae_insts_high = auth_desc->img_high; auth_desc->img_ae_insts_low = auth_desc->img_low; } *desc = auth_desc; return 0; } static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { unsigned int i = 0; unsigned int fcu_sts; unsigned int fcu_sts_csr, fcu_ctl_csr; unsigned int loaded_aes = FCU_LOADED_AE_POS; unsigned long ae_mask = handle->hal_handle->ae_mask; if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { fcu_ctl_csr = FCU_CONTROL_C4XXX; fcu_sts_csr = FCU_STATUS_C4XXX; } else { fcu_ctl_csr = FCU_CONTROL; fcu_sts_csr = FCU_STATUS; } for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { pr_err("QAT: AE %d is active\n", i); return EINVAL; } SET_FCU_CSR(handle, fcu_ctl_csr, (FCU_CTRL_CMD_LOAD | (IS_QAT_GEN4( pci_get_device(GET_DEV(handle->accel_dev))) ? (1 << FCU_CTRL_BROADCAST_POS) : 0) | (i << FCU_CTRL_AE_POS))); do { pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_DONE) { loaded_aes = IS_QAT_GEN3_OR_GEN4(pci_get_device( GET_DEV(handle->accel_dev))) ? GET_FCU_CSR(handle, FCU_AE_LOADED_C4XXX) : (fcu_sts >> FCU_LOADED_AE_POS); if (loaded_aes & (1 << i)) break; } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { pr_err("QAT: firmware load failed timeout %x\n", retry); return EINVAL; } } return 0; } static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, const void *addr_ptr, int mem_size) { struct icp_qat_suof_handle *suof_handle; suof_handle = malloc(sizeof(*suof_handle), M_QAT, M_WAITOK | M_ZERO); handle->sobj_handle = suof_handle; if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { qat_uclo_del_suof(handle); pr_err("QAT: map SUOF failed\n"); return EINVAL; } return 0; } int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, const void *addr_ptr, int mem_size) { struct icp_qat_fw_auth_desc *desc = NULL; struct icp_firml_dram_desc img_desc; int status = 0; if (handle->fw_auth) { status = qat_uclo_map_auth_fw( handle, addr_ptr, mem_size, &img_desc, &desc); if (!status) status = qat_uclo_auth_fw(handle, desc); qat_uclo_simg_free(handle, &img_desc); } else { if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { device_printf( NULL, "QAT: PKE service is not allowed because "); device_printf(NULL, "MMP fw will not be loaded for "); device_printf(NULL, "device 0x%x", pci_get_device( GET_DEV(handle->accel_dev))); return status; } status = qat_uclo_wr_sram_by_words(handle, handle->hal_sram_offset, addr_ptr, mem_size); } return status; } static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, const void *addr_ptr, int mem_size) { struct icp_qat_uof_filehdr *filehdr; struct icp_qat_uclo_objhandle *objhdl; objhdl = malloc(sizeof(*objhdl), M_QAT, M_WAITOK | M_ZERO); objhdl->obj_buf = malloc(mem_size, M_QAT, M_WAITOK); bcopy(addr_ptr, objhdl->obj_buf, mem_size); filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; if (qat_uclo_check_uof_format(filehdr)) goto out_objhdr_err; objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, ICP_QAT_UOF_OBJS); if (!objhdl->obj_hdr) { pr_err("QAT: object file chunk is null\n"); goto out_objhdr_err; } handle->obj_handle = objhdl; if (qat_uclo_parse_uof_obj(handle)) goto out_overlay_obj_err; return 0; out_overlay_obj_err: handle->obj_handle = NULL; free(objhdl->obj_hdr, M_QAT); out_objhdr_err: free(objhdl->obj_buf, M_QAT); free(objhdl, M_QAT); return ENOMEM; } static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, const struct icp_qat_mof_file_hdr *mof_ptr, u32 mof_size) { unsigned int checksum = 0; unsigned int min_ver_offset = 0; struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; mobj_handle->file_id = ICP_QAT_MOF_FID; mobj_handle->mof_buf = (const char *)mof_ptr; mobj_handle->mof_size = mof_size; min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, min_ver); checksum = qat_uclo_calc_str_checksum((const char *)&mof_ptr->min_ver, min_ver_offset); if (checksum != mof_ptr->checksum) { pr_err("QAT: incorrect MOF checksum\n"); return EINVAL; } mobj_handle->checksum = mof_ptr->checksum; mobj_handle->min_ver = mof_ptr->min_ver; mobj_handle->maj_ver = mof_ptr->maj_ver; return 0; } void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; free(mobj_handle->obj_table.obj_hdr, M_QAT); mobj_handle->obj_table.obj_hdr = NULL; free(handle->mobj_handle, M_QAT); handle->mobj_handle = NULL; } static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, const char *obj_name, const char **obj_ptr, unsigned int *obj_size) { unsigned int i; struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { if (!strncmp(obj_hdr[i].obj_name, obj_name, ICP_QAT_SUOF_OBJ_NAME_LEN)) { *obj_ptr = obj_hdr[i].obj_buf; *obj_size = obj_hdr[i].obj_size; break; } } if (i >= mobj_handle->obj_table.num_objs) { pr_err("QAT: object %s is not found inside MOF\n", obj_name); return EFAULT; } return 0; } static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, struct icp_qat_mof_objhdr *mobj_hdr, struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) { if ((strncmp((char *)obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) { mobj_hdr->obj_buf = (const char *)((unsigned long)obj_chunkhdr->offset + mobj_handle->uobjs_hdr); } else if ((strncmp((char *)(obj_chunkhdr->chunk_id), ICP_QAT_SUOF_IMAG, ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) { mobj_hdr->obj_buf = (const char *)((unsigned long)obj_chunkhdr->offset + mobj_handle->sobjs_hdr); } else { pr_err("QAT: unsupported chunk id\n"); return EINVAL; } mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; mobj_hdr->obj_name = (char *)(obj_chunkhdr->name + mobj_handle->sym_str); return 0; } static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) { struct icp_qat_mof_objhdr *mof_obj_hdr; const struct icp_qat_mof_obj_hdr *uobj_hdr; const struct icp_qat_mof_obj_hdr *sobj_hdr; struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; unsigned int uobj_chunk_num = 0, sobj_chunk_num = 0; unsigned int *valid_chunks = 0; int ret, i; uobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; sobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; if (uobj_hdr) uobj_chunk_num = uobj_hdr->num_chunks; if (sobj_hdr) sobj_chunk_num = sobj_hdr->num_chunks; mof_obj_hdr = (struct icp_qat_mof_objhdr *) malloc((uobj_chunk_num + sobj_chunk_num) * sizeof(*mof_obj_hdr), M_QAT, M_WAITOK | M_ZERO); mobj_handle->obj_table.obj_hdr = mof_obj_hdr; valid_chunks = &mobj_handle->obj_table.num_objs; uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); /* map uof objects */ for (i = 0; i < uobj_chunk_num; i++) { ret = qat_uclo_map_obj_from_mof(mobj_handle, &mof_obj_hdr[*valid_chunks], &uobj_chunkhdr[i]); if (ret) return ret; (*valid_chunks)++; } /* map suof objects */ for (i = 0; i < sobj_chunk_num; i++) { ret = qat_uclo_map_obj_from_mof(mobj_handle, &mof_obj_hdr[*valid_chunks], &sobj_chunkhdr[i]); if (ret) return ret; (*valid_chunks)++; } if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunks) { pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); return EINVAL; } return 0; } static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, struct icp_qat_mof_chunkhdr *mof_chunkhdr) { char **sym_str = (char **)&mobj_handle->sym_str; unsigned int *sym_size = &mobj_handle->sym_size; struct icp_qat_mof_str_table *str_table_obj; *sym_size = *(unsigned int *)(uintptr_t)(mof_chunkhdr->offset + mobj_handle->mof_buf); *sym_str = (char *)(uintptr_t)(mobj_handle->mof_buf + mof_chunkhdr->offset + sizeof(str_table_obj->tab_len)); } static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, struct icp_qat_mof_chunkhdr *mof_chunkhdr) { if (!strncmp(mof_chunkhdr->chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); else if (!strncmp(mof_chunkhdr->chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) mobj_handle->uobjs_hdr = mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset; else if (!strncmp(mof_chunkhdr->chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) mobj_handle->sobjs_hdr = mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset; } static int qat_uclo_check_mof_format(const struct icp_qat_mof_file_hdr *mof_hdr) { int maj = mof_hdr->maj_ver & 0xff; int min = mof_hdr->min_ver & 0xff; if (mof_hdr->file_id != ICP_QAT_MOF_FID) { pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); return EINVAL; } if (mof_hdr->num_chunks <= 0x1) { pr_err("QAT: MOF chunk amount is incorrect\n"); return EINVAL; } if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", maj, min); return EINVAL; } return 0; } static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, const struct icp_qat_mof_file_hdr *mof_ptr, u32 mof_size, const char *obj_name, const char **obj_ptr, unsigned int *obj_size) { struct icp_qat_mof_handle *mobj_handle; struct icp_qat_mof_chunkhdr *mof_chunkhdr; unsigned short chunks_num; int ret; unsigned int i; if (mof_ptr->file_id == ICP_QAT_UOF_FID || mof_ptr->file_id == ICP_QAT_SUOF_FID) { if (obj_ptr) *obj_ptr = (const char *)mof_ptr; if (obj_size) *obj_size = (unsigned int)mof_size; return 0; } if (qat_uclo_check_mof_format(mof_ptr)) return EINVAL; mobj_handle = malloc(sizeof(*mobj_handle), M_QAT, M_WAITOK | M_ZERO); handle->mobj_handle = mobj_handle; ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); if (ret) return ret; mof_chunkhdr = (struct icp_qat_mof_chunkhdr *)((uintptr_t)mof_ptr + sizeof(*mof_ptr)); chunks_num = mof_ptr->num_chunks; /*Parse MOF file chunks*/ for (i = 0; i < chunks_num; i++) qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); /*All sym_objs uobjs and sobjs should be available*/ if (!mobj_handle->sym_str || (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) return EINVAL; ret = qat_uclo_map_objs_from_mof(mobj_handle); if (ret) return ret; /*Seek specified uof object in MOF*/ ret = qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, obj_ptr, obj_size); if (ret) return ret; return 0; } int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, const void *addr_ptr, u32 mem_size, const char *obj_name) { const char *obj_addr; u32 obj_size; int ret; BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE > (sizeof(handle->hal_handle->ae_mask) * 8)); if (!handle || !addr_ptr || mem_size < 24) return EINVAL; if (obj_name) { ret = qat_uclo_map_mof_obj( handle, addr_ptr, mem_size, obj_name, &obj_addr, &obj_size); if (ret) return ret; } else { obj_addr = addr_ptr; obj_size = mem_size; } return (handle->fw_auth) ? qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; unsigned long ae_mask = handle->hal_handle->ae_mask; if (handle->mobj_handle) qat_uclo_del_mof(handle); if (handle->sobj_handle) qat_uclo_del_suof(handle); if (!obj_handle) return; free(obj_handle->uword_buf, M_QAT); for (a = 0; a < obj_handle->uimage_num; a++) free(obj_handle->ae_uimage[a].page, M_QAT); for_each_set_bit(a, &ae_mask, handle->hal_handle->ae_max_num) { qat_uclo_free_ae_data(&obj_handle->ae_data[a]); } free(obj_handle->obj_hdr, M_QAT); free(obj_handle->obj_buf, M_QAT); free(obj_handle, M_QAT); handle->obj_handle = NULL; } static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, struct icp_qat_uclo_encap_page *encap_page, uint64_t *uword, unsigned int addr_p, unsigned int raddr, uint64_t fill) { uint64_t uwrd = 0; unsigned int i, addr; if (!encap_page) { *uword = fill; return; } addr = (encap_page->page_region) ? raddr : addr_p; for (i = 0; i < encap_page->uwblock_num; i++) { if (addr >= encap_page->uwblock[i].start_addr && addr <= encap_page->uwblock[i].start_addr + encap_page->uwblock[i].words_num - 1) { addr -= encap_page->uwblock[i].start_addr; addr *= obj_handle->uword_in_bytes; memcpy(&uwrd, (void *)(((uintptr_t)encap_page->uwblock[i] .micro_words) + addr), obj_handle->uword_in_bytes); uwrd = uwrd & 0xbffffffffffull; } } *uword = uwrd; if (*uword == INVLD_UWORD) *uword = fill; } static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uclo_encap_page *encap_page, unsigned int ae) { unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; uint64_t fill_pat; /* load the page starting at appropriate ustore address */ /* get fill-pattern from an image -- they are all the same */ memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, sizeof(uint64_t)); uw_physical_addr = encap_page->beg_addr_p; uw_relative_addr = 0; words_num = encap_page->micro_words_num; while (words_num) { if (words_num < UWORD_CPYBUF_SIZE) cpylen = words_num; else cpylen = UWORD_CPYBUF_SIZE; /* load the buffer */ for (i = 0; i < cpylen; i++) qat_uclo_fill_uwords(obj_handle, encap_page, &obj_handle->uword_buf[i], uw_physical_addr + i, uw_relative_addr + i, fill_pat); if (obj_handle->ae_data[ae].shareable_ustore && !IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) /* copy the buffer to ustore */ qat_hal_wr_coalesce_uwords(handle, (unsigned char)ae, uw_physical_addr, cpylen, obj_handle->uword_buf); else /* copy the buffer to ustore */ qat_hal_wr_uwords(handle, (unsigned char)ae, uw_physical_addr, cpylen, obj_handle->uword_buf); uw_physical_addr += cpylen; uw_relative_addr += cpylen; words_num -= cpylen; } } static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ctx_mask, s; struct icp_qat_uclo_page *page; unsigned char ae = 0; int ctx; struct icp_qat_uclo_aedata *aed; unsigned long ae_mask = handle->hal_handle->ae_mask; if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) ctx_mask = 0xff; else ctx_mask = 0x55; /* load the default page and set assigned CTX PC * to the entrypoint address */ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned long ae_assigned = image->ae_assigned; if (!test_bit(ae, &cfg_ae_mask)) continue; if (!test_bit(ae, &ae_assigned)) continue; aed = &obj_handle->ae_data[ae]; /* find the slice to which this image is assigned */ for (s = 0; s < aed->slice_num; s++) { if (image->ctx_assigned & aed->ae_slices[s].ctx_mask_assigned) break; } if (s >= aed->slice_num) continue; page = aed->ae_slices[s].page; if (!page->encap_page->def_page) continue; qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); page = aed->ae_slices[s].page; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) aed->ae_slices[s].cur_page[ctx] = (ctx_mask & (1 << ctx)) ? page : NULL; qat_hal_set_live_ctx(handle, (unsigned char)ae, image->ctx_assigned); qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, image->entry_address); } } static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) { unsigned int i; struct icp_qat_fw_auth_desc *desc = NULL; struct icp_firml_dram_desc img_desc; struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { if (qat_uclo_map_auth_fw(handle, (const char *)simg_hdr[i].simg_buf, (unsigned int)(simg_hdr[i].simg_len), &img_desc, &desc)) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) goto wr_err; if (qat_uclo_is_broadcast(handle, i)) { if (qat_uclo_broadcast_load_fw(handle, desc)) goto wr_err; } else { if (qat_uclo_load_fw(handle, desc)) goto wr_err; } qat_uclo_simg_free(handle, &img_desc); } return 0; wr_err: qat_uclo_simg_free(handle, &img_desc); return -EINVAL; } static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int i; if (qat_uclo_init_globals(handle)) return EINVAL; for (i = 0; i < obj_handle->uimage_num; i++) { if (!obj_handle->ae_uimage[i].img_ptr) return EINVAL; if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) return EINVAL; qat_uclo_wr_uimage_page(handle, obj_handle->ae_uimage[i].img_ptr); } return 0; } int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) { return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, unsigned int cfg_ae_mask) { if (!cfg_ae_mask) return EINVAL; handle->cfg_ae_mask = cfg_ae_mask; return 0; } diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c index d730efd5952b..49e1e1859e78 100644 --- a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c @@ -1,1042 +1,1072 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #include #include #include #include #include #include #include #include #include #include "adf_4xxx_hw_data.h" #include "adf_heartbeat.h" #include "icp_qat_fw_init_admin.h" #include "icp_qat_hw.h" #define ADF_CONST_TABLE_SIZE 1024 struct adf_fw_config { u32 ae_mask; char *obj_name; }; /* Accel unit information */ static const struct adf_accel_unit adf_4xxx_au_a_ae[] = { { 0x1, 0x1, 0xF, 0x1B, 4, ADF_ACCEL_SERVICE_NULL }, { 0x2, 0x1, 0xF0, 0x6C0, 4, ADF_ACCEL_SERVICE_NULL }, { 0x4, 0x1, 0x100, 0xF000, 1, ADF_ACCEL_ADMIN }, }; /* Worker thread to service arbiter mappings */ static u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555, 0x5555555, 0x5555555, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0x0 }; /* Masks representing ME thread-service mappings. * Thread 7 carries out Admin work and is thus * left out. */ static u8 default_active_thd_mask = 0x7F; static u8 dc_me_active_thd_mask = 0x03; static u32 thrd_to_arb_map_gen[ADF_4XXX_MAX_ACCELENGINES] = { 0 }; #define ADF_4XXX_ASYM_SYM \ (ASYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_DC \ (COMP | COMP << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_SYM \ (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ SYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_ASYM \ (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ ASYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_ASYM_DC \ (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_SYM_DC \ (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_NA \ (NA | NA << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \ NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \ NA << ADF_CFG_SERV_RING_PAIR_3_SHIFT) #define ADF_4XXX_DEFAULT_RING_TO_SRV_MAP ADF_4XXX_ASYM_SYM struct adf_enabled_services { const char svcs_enabled[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; u16 rng_to_svc_msk; }; static struct adf_enabled_services adf_4xxx_svcs[] = { { "dc", ADF_4XXX_DC }, { "sym", ADF_4XXX_SYM }, { "asym", ADF_4XXX_ASYM }, { "dc;asym", ADF_4XXX_ASYM_DC }, { "asym;dc", ADF_4XXX_ASYM_DC }, { "sym;dc", ADF_4XXX_SYM_DC }, { "dc;sym", ADF_4XXX_SYM_DC }, { "asym;sym", ADF_4XXX_ASYM_SYM }, { "sym;asym", ADF_4XXX_ASYM_SYM }, { "cy", ADF_4XXX_ASYM_SYM } }; static struct adf_hw_device_class adf_4xxx_class = { .name = ADF_4XXX_DEVICE_NAME, .type = DEV_4XXX, .instances = 0, }; static u32 get_accel_mask(struct adf_accel_dev *accel_dev) { return ADF_4XXX_ACCELERATORS_MASK; } static u32 get_ae_mask(struct adf_accel_dev *accel_dev) { u32 fusectl4 = accel_dev->hw_device->fuses; return ~fusectl4 & ADF_4XXX_ACCELENGINES_MASK; } static void adf_set_asym_rings_mask(struct adf_accel_dev *accel_dev) { accel_dev->hw_device->asym_rings_mask = ADF_4XXX_DEF_ASYM_MASK; } static int get_ring_to_svc_map(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; u32 i = 0; *ring_to_svc_map = 0; /* Get the services enabled by user */ snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) return EFAULT; for (i = 0; i < ARRAY_SIZE(adf_4xxx_svcs); i++) { if (!strncmp(val, adf_4xxx_svcs[i].svcs_enabled, ADF_CFG_MAX_KEY_LEN_IN_BYTES)) { *ring_to_svc_map = adf_4xxx_svcs[i].rng_to_svc_msk; return 0; } } device_printf(GET_DEV(accel_dev), "Invalid services enabled: %s\n", val); return EFAULT; } static u32 get_num_accels(struct adf_hw_device_data *self) { return ADF_4XXX_MAX_ACCELERATORS; } static u32 get_num_aes(struct adf_hw_device_data *self) { if (!self || !self->ae_mask) return 0; return hweight32(self->ae_mask); } static u32 get_misc_bar_id(struct adf_hw_device_data *self) { return ADF_4XXX_PMISC_BAR; } static u32 get_etr_bar_id(struct adf_hw_device_data *self) { return ADF_4XXX_ETR_BAR; } static u32 get_sram_bar_id(struct adf_hw_device_data *self) { return ADF_4XXX_SRAM_BAR; } /* * The vector routing table is used to select the MSI-X entry to use for each * interrupt source. * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. * The final entry corresponds to VF2PF or error interrupts. * This vector table could be used to configure one MSI-X entry to be shared * between multiple interrupt sources. * * The default routing is set to have a one to one correspondence between the * interrupt source and the MSI-X entry used. */ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) { struct resource *csr; int i; csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); } static u32 adf_4xxx_get_hw_cap(struct adf_accel_dev *accel_dev) { device_t pdev = accel_dev->accel_pci_dev.pci_dev; u32 fusectl1; u32 capabilities_sym, capabilities_sym_cipher, capabilities_sym_auth, capabilities_asym, capabilities_dc, capabilities_other; capabilities_other = ICP_ACCEL_CAPABILITIES_RL; /* Read accelerator capabilities mask */ fusectl1 = pci_read_config(pdev, ADF_4XXX_FUSECTL1_OFFSET, 4); capabilities_sym_cipher = ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SM4 | ICP_ACCEL_CAPABILITIES_CHACHA_POLY | ICP_ACCEL_CAPABILITIES_AESGCM_SPC | ICP_ACCEL_CAPABILITIES_AES_V2; capabilities_sym_auth = ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SHA3 | ICP_ACCEL_CAPABILITIES_SHA3_EXT; /* A set bit in fusectl1 means the feature is OFF in this SKU */ if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_HKDF; capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_SM4; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) { capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_AES_V2; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) { capabilities_sym_auth &= ~ICP_ACCEL_CAPABILITIES_SM3; capabilities_sym_auth &= ~ICP_ACCEL_CAPABILITIES_SHA3; capabilities_sym_auth &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; } if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) { capabilities_sym_cipher &= ~ICP_ACCEL_CAPABILITIES_SM4; capabilities_sym_auth &= ~ICP_ACCEL_CAPABILITIES_SM3; } if (capabilities_sym_cipher) capabilities_sym_cipher |= ICP_ACCEL_CAPABILITIES_CIPHER; if (capabilities_sym_auth) capabilities_sym_auth |= ICP_ACCEL_CAPABILITIES_AUTHENTICATION; capabilities_sym = capabilities_sym_cipher | capabilities_sym_auth; if (capabilities_sym) capabilities_sym |= ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_SM2 | ICP_ACCEL_CAPABILITIES_ECEDMONT; if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; } capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) { capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; } return capabilities_sym | capabilities_dc | capabilities_asym | capabilities_other; } static u32 get_hb_clock(struct adf_hw_device_data *self) { /* * 4XXX uses KPT counter for HB */ return ADF_4XXX_KPT_COUNTER_FREQ; } static u32 get_ae_clock(struct adf_hw_device_data *self) { /* * Clock update interval is <16> ticks for qat_4xxx. */ return self->clock_frequency / 16; } static int measure_clock(struct adf_accel_dev *accel_dev) { u32 frequency; int ret = 0; ret = adf_dev_measure_clock(accel_dev, &frequency, ADF_4XXX_MIN_AE_FREQ, ADF_4XXX_MAX_AE_FREQ); if (ret) return ret; accel_dev->hw_device->clock_frequency = frequency; return 0; } static int adf_4xxx_configure_accel_units(struct adf_accel_dev *accel_dev) { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 }; char val_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; if (adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC)) goto err; snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); snprintf(val_str, sizeof(val_str), ADF_CFG_ASYM ADF_SERVICES_SEPARATOR ADF_CFG_SYM); if (adf_cfg_add_key_value_param( accel_dev, ADF_GENERAL_SEC, key, (void *)val_str, ADF_STR)) goto err; return 0; err: device_printf(GET_DEV(accel_dev), "Failed to configure accel units\n"); return EINVAL; } static u32 get_num_accel_units(struct adf_hw_device_data *self) { return ADF_4XXX_MAX_ACCELUNITS; } static void get_accel_unit(struct adf_hw_device_data *self, struct adf_accel_unit **accel_unit) { memcpy(*accel_unit, adf_4xxx_au_a_ae, sizeof(adf_4xxx_au_a_ae)); } static void adf_exit_accel_unit_services(struct adf_accel_dev *accel_dev) { if (accel_dev->au_info) { kfree(accel_dev->au_info->au); accel_dev->au_info->au = NULL; kfree(accel_dev->au_info); accel_dev->au_info = NULL; } } static int get_accel_unit_config(struct adf_accel_dev *accel_dev, u8 *num_sym_au, u8 *num_dc_au, u8 *num_asym_au) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; u32 num_au = hw_data->get_num_accel_units(hw_data); /* One AU will be allocated by default if a service enabled */ u32 alloc_au = 1; /* There's always one AU that is used for Admin AE */ u32 service_mask = ADF_ACCEL_ADMIN; char *token, *cur_str; u32 disabled_caps = 0; /* Get the services enabled by user */ snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) return EFAULT; cur_str = val; token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); while (token) { if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM))) service_mask |= ADF_ACCEL_CRYPTO; if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM))) service_mask |= ADF_ACCEL_ASYM; /* cy means both asym & crypto should be enabled * Hardware resources allocation check will be done later */ if (!strncmp(token, ADF_CFG_CY, strlen(ADF_CFG_CY))) service_mask |= ADF_ACCEL_ASYM | ADF_ACCEL_CRYPTO; if (!strncmp(token, ADF_SERVICE_DC, strlen(ADF_SERVICE_DC))) service_mask |= ADF_ACCEL_COMPRESSION; token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); } /* Ensure the user won't enable more services than it can support */ if (hweight32(service_mask) > num_au) { device_printf(GET_DEV(accel_dev), "Can't enable more services than "); device_printf(GET_DEV(accel_dev), "%d!\n", num_au); return EFAULT; } else if (hweight32(service_mask) == 2) { /* Due to limitation, besides AU for Admin AE * only 2 more AUs can be allocated */ alloc_au = 2; } if (service_mask & ADF_ACCEL_CRYPTO) *num_sym_au = alloc_au; if (service_mask & ADF_ACCEL_ASYM) *num_asym_au = alloc_au; if (service_mask & ADF_ACCEL_COMPRESSION) *num_dc_au = alloc_au; /*update capability*/ if (!*num_sym_au || !(service_mask & ADF_ACCEL_CRYPTO)) { disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CIPHER | ICP_ACCEL_CAPABILITIES_SHA3 | ICP_ACCEL_CAPABILITIES_SHA3_EXT | ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | ICP_ACCEL_CAPABILITIES_CHACHA_POLY | ICP_ACCEL_CAPABILITIES_AESGCM_SPC | ICP_ACCEL_CAPABILITIES_AES_V2 | ICP_ACCEL_CAPABILITIES_AUTHENTICATION; } if (!*num_asym_au || !(service_mask & ADF_ACCEL_ASYM)) { disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | ICP_ACCEL_CAPABILITIES_ECEDMONT; } if (!*num_dc_au || !(service_mask & ADF_ACCEL_COMPRESSION)) { disabled_caps |= ICP_ACCEL_CAPABILITIES_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; accel_dev->hw_device->extended_dc_capabilities = 0; } accel_dev->hw_device->accel_capabilities_mask = adf_4xxx_get_hw_cap(accel_dev) & ~disabled_caps; hw_data->service_mask = service_mask; hw_data->service_to_load_mask = service_mask; return 0; } static int adf_init_accel_unit_services(struct adf_accel_dev *accel_dev) { u8 num_sym_au = 0, num_dc_au = 0, num_asym_au = 0; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 num_au = hw_data->get_num_accel_units(hw_data); u32 au_size = num_au * sizeof(struct adf_accel_unit); u8 i; if (get_accel_unit_config( accel_dev, &num_sym_au, &num_dc_au, &num_asym_au)) return EFAULT; accel_dev->au_info = kzalloc(sizeof(*accel_dev->au_info), GFP_KERNEL); if (!accel_dev->au_info) return ENOMEM; accel_dev->au_info->au = kzalloc(au_size, GFP_KERNEL); if (!accel_dev->au_info->au) { kfree(accel_dev->au_info); accel_dev->au_info = NULL; return ENOMEM; } accel_dev->au_info->num_cy_au = num_sym_au; accel_dev->au_info->num_dc_au = num_dc_au; accel_dev->au_info->num_asym_au = num_asym_au; get_accel_unit(hw_data, &accel_dev->au_info->au); /* Enable ASYM accel units */ for (i = 0; i < num_au && num_asym_au > 0; i++) { if (accel_dev->au_info->au[i].services == ADF_ACCEL_SERVICE_NULL) { accel_dev->au_info->au[i].services = ADF_ACCEL_ASYM; num_asym_au--; } } /* Enable SYM accel units */ for (i = 0; i < num_au && num_sym_au > 0; i++) { if (accel_dev->au_info->au[i].services == ADF_ACCEL_SERVICE_NULL) { accel_dev->au_info->au[i].services = ADF_ACCEL_CRYPTO; num_sym_au--; } } /* Enable compression accel units */ for (i = 0; i < num_au && num_dc_au > 0; i++) { if (accel_dev->au_info->au[i].services == ADF_ACCEL_SERVICE_NULL) { accel_dev->au_info->au[i].services = ADF_ACCEL_COMPRESSION; num_dc_au--; } } accel_dev->au_info->dc_ae_msk |= hw_data->get_obj_cfg_ae_mask(accel_dev, ADF_ACCEL_COMPRESSION); return 0; } static int adf_init_accel_units(struct adf_accel_dev *accel_dev) { return adf_init_accel_unit_services(accel_dev); } static void adf_exit_accel_units(struct adf_accel_dev *accel_dev) { /* reset the AU service */ adf_exit_accel_unit_services(accel_dev); } static const char * -get_obj_name(struct adf_accel_dev *accel_dev, - enum adf_accel_unit_services service) +get_obj_name_4xxx(struct adf_accel_dev *accel_dev, + enum adf_accel_unit_services service) { switch (service) { case ADF_ACCEL_ASYM: return ADF_4XXX_ASYM_OBJ; case ADF_ACCEL_CRYPTO: return ADF_4XXX_SYM_OBJ; case ADF_ACCEL_COMPRESSION: return ADF_4XXX_DC_OBJ; case ADF_ACCEL_ADMIN: return ADF_4XXX_ADMIN_OBJ; default: return NULL; } } +static const char * +get_obj_name_402xx(struct adf_accel_dev *accel_dev, + enum adf_accel_unit_services service) +{ + switch (service) { + case ADF_ACCEL_ASYM: + return ADF_402XX_ASYM_OBJ; + case ADF_ACCEL_CRYPTO: + return ADF_402XX_SYM_OBJ; + case ADF_ACCEL_COMPRESSION: + return ADF_402XX_DC_OBJ; + case ADF_ACCEL_ADMIN: + return ADF_402XX_ADMIN_OBJ; + default: + return NULL; + } +} + static uint32_t get_objs_num(struct adf_accel_dev *accel_dev) { return ADF_4XXX_MAX_OBJ; } static uint32_t get_obj_cfg_ae_mask(struct adf_accel_dev *accel_dev, enum adf_accel_unit_services service) { u32 ae_mask = 0; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 num_au = hw_data->get_num_accel_units(hw_data); struct adf_accel_unit *accel_unit = accel_dev->au_info->au; u32 i = 0; if (service == ADF_ACCEL_SERVICE_NULL) return 0; for (i = 0; i < num_au; i++) { if (accel_unit[i].services == service) ae_mask |= accel_unit[i].ae_mask; } return ae_mask; } static enum adf_accel_unit_services adf_4xxx_get_service_type(struct adf_accel_dev *accel_dev, s32 obj_num) { struct adf_accel_unit *accel_unit; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_au = hw_data->get_num_accel_units(hw_data); int i; if (!hw_data->service_to_load_mask) return ADF_ACCEL_SERVICE_NULL; if (accel_dev->au_info && accel_dev->au_info->au) accel_unit = accel_dev->au_info->au; else return ADF_ACCEL_SERVICE_NULL; for (i = num_au - 2; i >= 0; i--) { if (hw_data->service_to_load_mask & accel_unit[i].services) { hw_data->service_to_load_mask &= ~accel_unit[i].services; return accel_unit[i].services; } } /* admin AE should be loaded last */ if (hw_data->service_to_load_mask & accel_unit[num_au - 1].services) { hw_data->service_to_load_mask &= ~accel_unit[num_au - 1].services; return accel_unit[num_au - 1].services; } return ADF_ACCEL_SERVICE_NULL; } static void get_ring_svc_map_data(int ring_pair_index, u16 ring_to_svc_map, u8 *serv_type, int *ring_index, int *num_rings_per_srv, int bundle_num) { *serv_type = GET_SRV_TYPE(ring_to_svc_map, bundle_num % ADF_CFG_NUM_SERVICES); *ring_index = 0; *num_rings_per_srv = ADF_4XXX_NUM_RINGS_PER_BANK / 2; } static int adf_get_dc_extcapabilities(struct adf_accel_dev *accel_dev, u32 *capabilities) { struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; u8 i; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_au = hw_data->get_num_accel_units(hw_data); u32 first_dc_ae = 0; for (i = 0; i < num_au; i++) { if (accel_dev->au_info->au[i].services & ADF_ACCEL_COMPRESSION) { first_dc_ae = accel_dev->au_info->au[i].ae_mask; first_dc_ae &= ~(first_dc_ae - 1); } } memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; if (likely(first_dc_ae)) { if (adf_send_admin(accel_dev, &req, &resp, first_dc_ae) || resp.status) { *capabilities = 0; return EFAULT; } *capabilities = resp.extended_features; } return 0; } static int adf_get_fw_status(struct adf_accel_dev *accel_dev, u8 *major, u8 *minor, u8 *patch) { struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; u32 ae_mask = 1; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd_id = ICP_QAT_FW_STATUS_GET; if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) return EFAULT; *major = resp.version_major_num; *minor = resp.version_minor_num; *patch = resp.version_patch_num; return 0; } static int adf_4xxx_send_admin_init(struct adf_accel_dev *accel_dev) { int ret = 0; struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 ae_mask = hw_data->ae_mask; u32 admin_ae_mask = hw_data->admin_ae_mask; u8 num_au = hw_data->get_num_accel_units(hw_data); u8 i; u32 dc_capabilities = 0; for (i = 0; i < num_au; i++) { if (accel_dev->au_info->au[i].services == ADF_ACCEL_SERVICE_NULL) ae_mask &= ~accel_dev->au_info->au[i].ae_mask; if (accel_dev->au_info->au[i].services != ADF_ACCEL_ADMIN) admin_ae_mask &= ~accel_dev->au_info->au[i].ae_mask; } if (!accel_dev->admin) { device_printf(GET_DEV(accel_dev), "adf_admin not available\n"); return EFAULT; } memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; req.init_cfg_sz = ADF_CONST_TABLE_SIZE; req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; if (adf_send_admin(accel_dev, &req, &resp, admin_ae_mask)) { device_printf(GET_DEV(accel_dev), "Error sending constants config message\n"); return EFAULT; } memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd_id = ICP_QAT_FW_INIT_ME; #ifdef QAT_DISABLE_SAFE_DC_MODE if (accel_dev->disable_safe_dc_mode) req.fw_flags = ICP_QAT_FW_INIT_DISABLE_SAFE_DC_MODE_FLAG; #endif /* QAT_DISABLE_SAFE_DC_MODE */ if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) { device_printf(GET_DEV(accel_dev), "Error sending init message\n"); return EFAULT; } memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; req.init_cfg_ptr = accel_dev->admin->phy_hb_addr; if (adf_get_hb_timer(accel_dev, &req.heartbeat_ticks)) return EINVAL; if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) device_printf(GET_DEV(accel_dev), "Heartbeat is not supported\n"); ret = adf_get_dc_extcapabilities(accel_dev, &dc_capabilities); if (unlikely(ret)) { device_printf(GET_DEV(accel_dev), "Could not get FW ext. capabilities\n"); } accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; adf_get_fw_status(accel_dev, &accel_dev->fw_versions.fw_version_major, &accel_dev->fw_versions.fw_version_minor, &accel_dev->fw_versions.fw_version_patch); device_printf(GET_DEV(accel_dev), "FW version: %d.%d.%d\n", accel_dev->fw_versions.fw_version_major, accel_dev->fw_versions.fw_version_minor, accel_dev->fw_versions.fw_version_patch); return ret; } static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { return DEV_SKU_1; } static struct adf_accel_unit * get_au_by_ae(struct adf_accel_dev *accel_dev, int ae_num) { int i = 0; struct adf_accel_unit *accel_unit = accel_dev->au_info->au; if (!accel_unit) return NULL; for (i = 0; i < ADF_4XXX_MAX_ACCELUNITS; i++) if (accel_unit[i].ae_mask & BIT(ae_num)) return &accel_unit[i]; return NULL; } static bool check_accel_unit_service(enum adf_accel_unit_services au_srv, enum adf_cfg_service_type ring_srv) { if ((ADF_ACCEL_SERVICE_NULL == au_srv) && ring_srv == NA) return true; if ((au_srv & ADF_ACCEL_COMPRESSION) && ring_srv == COMP) return true; if ((au_srv & ADF_ACCEL_ASYM) && ring_srv == ASYM) return true; if ((au_srv & ADF_ACCEL_CRYPTO) && ring_srv == SYM) return true; return false; } static void adf_4xxx_cfg_gen_dispatch_arbiter(struct adf_accel_dev *accel_dev, u32 *thrd_to_arb_map_gen) { struct adf_accel_unit *au = NULL; int engine = 0; int thread = 0; int service; u16 ena_srv_mask; u16 service_type; u32 service_mask; unsigned long thd_srv_mask = default_active_thd_mask; struct adf_hw_device_data *hw_data = accel_dev->hw_device; ena_srv_mask = accel_dev->hw_device->ring_to_svc_map; /* If ring_to_svc_map is not changed, return default arbiter value */ if (ena_srv_mask == ADF_4XXX_DEFAULT_RING_TO_SRV_MAP) { memcpy(thrd_to_arb_map_gen, thrd_to_arb_map, sizeof(thrd_to_arb_map_gen[0]) * ADF_4XXX_MAX_ACCELENGINES); return; } for (engine = 0; engine < ADF_4XXX_MAX_ACCELENGINES - 1; engine++) { thrd_to_arb_map_gen[engine] = 0; service_mask = 0; au = get_au_by_ae(accel_dev, engine); if (!au) continue; for (service = 0; service < ADF_CFG_MAX_SERVICES; service++) { service_type = GET_SRV_TYPE(ena_srv_mask, service); if (check_accel_unit_service(au->services, service_type)) service_mask |= BIT(service); } if (au->services == ADF_ACCEL_COMPRESSION) thd_srv_mask = dc_me_active_thd_mask; else if (au->services == ADF_ACCEL_ASYM) thd_srv_mask = hw_data->asym_ae_active_thd_mask; else thd_srv_mask = default_active_thd_mask; for_each_set_bit(thread, &thd_srv_mask, 8) { thrd_to_arb_map_gen[engine] |= (service_mask << (ADF_CFG_MAX_SERVICES * thread)); } } } static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, u32 const **arb_map_config) { int i; struct adf_hw_device_data *hw_device = accel_dev->hw_device; for (i = 1; i < ADF_4XXX_MAX_ACCELENGINES; i++) { if (~hw_device->ae_mask & (1 << i)) thrd_to_arb_map[i] = 0; } adf_4xxx_cfg_gen_dispatch_arbiter(accel_dev, thrd_to_arb_map_gen); *arb_map_config = thrd_to_arb_map_gen; } static void get_arb_info(struct arb_info *arb_info) { arb_info->wrk_cfg_offset = ADF_4XXX_ARB_CONFIG; arb_info->arbiter_offset = ADF_4XXX_ARB_OFFSET; arb_info->wrk_thd_2_srv_arb_map = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; } static void get_admin_info(struct admin_info *admin_csrs_info) { admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; } static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; struct resource *csr = misc_bar->virt_addr; /* Enable all in errsou3 except VFLR notification on host */ ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY); } static void adf_enable_ints(struct adf_accel_dev *accel_dev) { struct resource *addr; addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; /* Enable bundle interrupts */ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); /* Enable misc interrupts */ ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); } static int adf_init_device(struct adf_accel_dev *accel_dev) { struct resource *addr; u32 status; u32 csr; int ret; addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; /* Temporarily mask PM interrupt */ csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2); csr |= ADF_4XXX_PM_SOU; ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr); /* Set DRV_ACTIVE bit to power up the device */ ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE); /* Poll status register to make sure the device is powered up */ status = 0; ret = read_poll_timeout(ADF_CSR_RD, status, status & ADF_4XXX_PM_INIT_STATE, ADF_4XXX_PM_POLL_DELAY_US, ADF_4XXX_PM_POLL_TIMEOUT_US, true, addr, ADF_4XXX_PM_STATUS); if (ret) device_printf(GET_DEV(accel_dev), "Failed to power up the device\n"); return ret; } void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id) { hw_data->dev_class = &adf_4xxx_class; hw_data->instance_id = adf_4xxx_class.instances++; hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; hw_data->num_logical_accel = 1; hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_arb_info = get_arb_info; hw_data->get_admin_info = get_admin_info; hw_data->get_accel_cap = adf_4xxx_get_hw_cap; hw_data->clock_frequency = ADF_4XXX_AE_FREQ; hw_data->get_sku = get_sku; hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE; - hw_data->fw_name = ADF_4XXX_FW; - hw_data->fw_mmp_name = ADF_4XXX_MMP; + switch (id) { + case ADF_402XX_PCI_DEVICE_ID: + hw_data->fw_name = ADF_402XX_FW; + hw_data->fw_mmp_name = ADF_402XX_MMP; + hw_data->asym_ae_active_thd_mask = DEFAULT_4XXX_ASYM_AE_MASK; + break; + case ADF_401XX_PCI_DEVICE_ID: + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->asym_ae_active_thd_mask = DEFAULT_401XX_ASYM_AE_MASK; + break; + + default: + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->asym_ae_active_thd_mask = DEFAULT_4XXX_ASYM_AE_MASK; + } hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; hw_data->send_admin_init = adf_4xxx_send_admin_init; hw_data->init_arb = adf_init_gen2_arb; hw_data->exit_arb = adf_exit_arb; hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; hw_data->init_device = adf_init_device; hw_data->reset_device = adf_reset_flr; hw_data->restore_device = adf_dev_restore; hw_data->init_accel_units = adf_init_accel_units; hw_data->exit_accel_units = adf_exit_accel_units; hw_data->get_num_accel_units = get_num_accel_units; hw_data->configure_accel_units = adf_4xxx_configure_accel_units; hw_data->get_ring_to_svc_map = get_ring_to_svc_map; hw_data->get_ring_svc_map_data = get_ring_svc_map_data; hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; hw_data->get_objs_num = get_objs_num; - hw_data->get_obj_name = get_obj_name; + switch (id) { + case ADF_402XX_PCI_DEVICE_ID: + hw_data->get_obj_name = get_obj_name_402xx; + break; + default: + hw_data->get_obj_name = get_obj_name_4xxx; + } hw_data->get_obj_cfg_ae_mask = get_obj_cfg_ae_mask; hw_data->get_service_type = adf_4xxx_get_service_type; hw_data->set_msix_rttable = set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; hw_data->disable_iov = adf_disable_sriov; hw_data->config_device = adf_config_device; hw_data->set_asym_rings_mask = adf_set_asym_rings_mask; hw_data->get_hb_clock = get_hb_clock; hw_data->int_timer_init = adf_int_timer_init; hw_data->int_timer_exit = adf_int_timer_exit; hw_data->pre_reset = adf_dev_pre_reset; hw_data->post_reset = adf_dev_post_reset; hw_data->disable_arb = adf_disable_arb; hw_data->get_heartbeat_status = adf_get_heartbeat_status; hw_data->get_ae_clock = get_ae_clock; hw_data->measure_clock = measure_clock; hw_data->query_storage_cap = 1; hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; - switch (id) { - case ADF_401XX_PCI_DEVICE_ID: - hw_data->asym_ae_active_thd_mask = DEFAULT_401XX_ASYM_AE_MASK; - break; - case ADF_4XXX_PCI_DEVICE_ID: - default: - hw_data->asym_ae_active_thd_mask = DEFAULT_4XXX_ASYM_AE_MASK; - } - adf_gen4_init_hw_csr_info(&hw_data->csr_info); adf_gen4_init_pf_pfvf_ops(&hw_data->csr_info.pfvf_ops); } void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class->instances--; } diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h index c35ebbcadcd7..fa7249dca596 100644 --- a/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.h @@ -1,114 +1,120 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #ifndef ADF_4XXX_HW_DATA_H_ #define ADF_4XXX_HW_DATA_H_ #include #define DEFAULT_4XXX_ASYM_AE_MASK 0x03 #define DEFAULT_401XX_ASYM_AE_MASK 0x3F /* PCIe configuration space */ #define ADF_4XXX_SRAM_BAR 0 #define ADF_4XXX_PMISC_BAR 1 #define ADF_4XXX_ETR_BAR 2 #define ADF_4XXX_RX_RINGS_OFFSET 1 #define ADF_4XXX_TX_RINGS_MASK 0x1 #define ADF_4XXX_MAX_ACCELERATORS 1 #define ADF_4XXX_MAX_ACCELENGINES 9 #define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) /* 2 Accel units dedicated to services and */ /* 1 Accel unit dedicated to Admin AE */ #define ADF_4XXX_MAX_ACCELUNITS 3 /* Physical function fuses */ #define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) #define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) #define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) #define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) #define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) #define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) #define ADF_4XXX_ACCELERATORS_MASK (0x1) #define ADF_4XXX_ACCELENGINES_MASK (0x1FF) #define ADF_4XXX_ADMIN_AE_MASK (0x100) #define ADF_4XXX_ETR_MAX_BANKS 64 /* MSIX interrupt */ #define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) #define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) #define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) #define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i)*0x04)) /* Bank and ring configuration */ #define ADF_4XXX_NUM_RINGS_PER_BANK 2 /* Error source registers */ #define ADF_4XXX_ERRSOU0 (0x41A200) #define ADF_4XXX_ERRSOU1 (0x41A204) #define ADF_4XXX_ERRSOU2 (0x41A208) #define ADF_4XXX_ERRSOU3 (0x41A20C) /* Error source mask registers */ #define ADF_4XXX_ERRMSK0 (0x41A210) #define ADF_4XXX_ERRMSK1 (0x41A214) #define ADF_4XXX_ERRMSK2 (0x41A218) #define ADF_4XXX_ERRMSK3 (0x41A21C) #define ADF_4XXX_VFLNOTIFY BIT(7) #define ADF_4XXX_DEF_ASYM_MASK 0x1 /* Arbiter configuration */ #define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) #define ADF_4XXX_ARB_OFFSET (0x0) #define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) /* Admin Interface Reg Offset */ #define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) #define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) #define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) /* Power management */ #define ADF_4XXX_PM_POLL_DELAY_US 20 #define ADF_4XXX_PM_POLL_TIMEOUT_US USEC_PER_SEC #define ADF_4XXX_PM_STATUS (0x50A00C) #define ADF_4XXX_PM_INTERRUPT (0x50A028) #define ADF_4XXX_PM_DRV_ACTIVE BIT(20) #define ADF_4XXX_PM_INIT_STATE BIT(21) /* Power management source in ERRSOU2 and ERRMSK2 */ #define ADF_4XXX_PM_SOU BIT(18) /* Firmware Binaries */ #define ADF_4XXX_FW "qat_4xxx_fw" #define ADF_4XXX_MMP "qat_4xxx_mmp_fw" #define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin" #define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" #define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" #define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" +#define ADF_402XX_FW "qat_402xx_fw" +#define ADF_402XX_MMP "qat_402xx_mmp_fw" +#define ADF_402XX_DC_OBJ "qat_402xx_dc.bin" +#define ADF_402XX_SYM_OBJ "qat_402xx_sym.bin" +#define ADF_402XX_ASYM_OBJ "qat_402xx_asym.bin" +#define ADF_402XX_ADMIN_OBJ "qat_402xx_admin.bin" /* Only 3 types of images can be loaded including the admin image */ #define ADF_4XXX_MAX_OBJ 3 #define ADF_4XXX_AE_FREQ (1000 * 1000000) #define ADF_4XXX_KPT_COUNTER_FREQ (100 * 1000000) #define ADF_4XXX_MIN_AE_FREQ (9 * 1000000) #define ADF_4XXX_MAX_AE_FREQ (1100 * 1000000) /* qat_4xxx fuse bits are different from old GENs, redefine them */ enum icp_qat_4xxx_slice_mask { ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(7), }; void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 id); void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); #endif diff --git a/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c index cb534dd03b86..f9ad39fa45f0 100644 --- a/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c @@ -1,353 +1,356 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #include "qat_freebsd.h" #include "adf_cfg.h" #include "adf_common_drv.h" #include "adf_accel_devices.h" #include "adf_4xxx_hw_data.h" #include "adf_gen4_hw_data.h" #include "adf_fw_counters.h" #include "adf_cfg_device.h" #include "adf_dbgfs.h" #include #include #include #include #include static MALLOC_DEFINE(M_QAT_4XXX, "qat_4xxx", "qat_4xxx"); #define ADF_SYSTEM_DEVICE(device_id) \ { \ PCI_VENDOR_ID_INTEL, device_id \ } -static const struct pci_device_id adf_pci_tbl[] = - { ADF_SYSTEM_DEVICE(ADF_4XXX_PCI_DEVICE_ID), - ADF_SYSTEM_DEVICE(ADF_401XX_PCI_DEVICE_ID), - { - 0, - } }; +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_4XXX_PCI_DEVICE_ID), + ADF_SYSTEM_DEVICE(ADF_401XX_PCI_DEVICE_ID), + ADF_SYSTEM_DEVICE(ADF_402XX_PCI_DEVICE_ID), + { + 0, + } +}; static int adf_probe(device_t dev) { const struct pci_device_id *id; for (id = adf_pci_tbl; id->vendor != 0; id++) { if (pci_get_vendor(dev) == id->vendor && pci_get_device(dev) == id->device) { device_set_desc(dev, "Intel " ADF_4XXX_DEVICE_NAME " QuickAssist"); return BUS_PROBE_GENERIC; } } return ENXIO; } #ifdef QAT_DISABLE_SAFE_DC_MODE static int adf_4xxx_sysctl_disable_safe_dc_mode(SYSCTL_HANDLER_ARGS) { struct adf_accel_dev *accel_dev = arg1; int error, value = accel_dev->disable_safe_dc_mode; error = sysctl_handle_int(oidp, &value, 0, req); if (error || !req->newptr) return error; if (value != 1 && value != 0) return EINVAL; if (adf_dev_started(accel_dev)) { device_printf( GET_DEV(accel_dev), "QAT: configuration can only be changed in \"down\" device state\n"); return EBUSY; } accel_dev->disable_safe_dc_mode = (u8)value; return 0; } static void adf_4xxx_disable_safe_dc_sysctl_add(struct adf_accel_dev *accel_dev) { struct sysctl_ctx_list *qat_sysctl_ctx; struct sysctl_oid *qat_sysctl_tree; qat_sysctl_ctx = device_get_sysctl_ctx(accel_dev->accel_pci_dev.pci_dev); qat_sysctl_tree = device_get_sysctl_tree(accel_dev->accel_pci_dev.pci_dev); accel_dev->safe_dc_mode = SYSCTL_ADD_OID(qat_sysctl_ctx, SYSCTL_CHILDREN(qat_sysctl_tree), OID_AUTO, "disable_safe_dc_mode", CTLTYPE_INT | CTLFLAG_WR | CTLFLAG_TUN | CTLFLAG_SKIP, accel_dev, 0, adf_4xxx_sysctl_disable_safe_dc_mode, "LU", "Disable QAT safe data compression mode"); } static void adf_4xxx_disable_safe_dc_sysctl_remove(struct adf_accel_dev *accel_dev) { int ret; struct sysctl_ctx_list *qat_sysctl_ctx = device_get_sysctl_ctx(accel_dev->accel_pci_dev.pci_dev); ret = sysctl_ctx_entry_del(qat_sysctl_ctx, accel_dev->safe_dc_mode); if (ret) { device_printf(GET_DEV(accel_dev), "Failed to delete entry\n"); } else { ret = sysctl_remove_oid(accel_dev->safe_dc_mode, 1, 1); if (ret) device_printf(GET_DEV(accel_dev), "Failed to delete oid\n"); } } #endif /* QAT_DISABLE_SAFE_DC_MODE */ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; if (accel_dev->dma_tag) bus_dma_tag_destroy(accel_dev->dma_tag); for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; if (bar->virt_addr) bus_free_resource(accel_pci_dev->pci_dev, SYS_RES_MEMORY, bar->virt_addr); } if (accel_dev->hw_device) { switch (pci_get_device(accel_pci_dev->pci_dev)) { case ADF_4XXX_PCI_DEVICE_ID: case ADF_401XX_PCI_DEVICE_ID: + case ADF_402XX_PCI_DEVICE_ID: adf_clean_hw_data_4xxx(accel_dev->hw_device); break; default: break; } free(accel_dev->hw_device, M_QAT_4XXX); accel_dev->hw_device = NULL; } #ifdef QAT_DISABLE_SAFE_DC_MODE adf_4xxx_disable_safe_dc_sysctl_remove(accel_dev); #endif /* QAT_DISABLE_SAFE_DC_MODE */ adf_dbgfs_exit(accel_dev); adf_cfg_dev_remove(accel_dev); adf_devmgr_rm_dev(accel_dev, NULL); } static int adf_attach(device_t dev) { struct adf_accel_dev *accel_dev; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; unsigned int bar_nr; int ret = 0, rid; struct adf_cfg_device *cfg_dev = NULL; /* Set pci MaxPayLoad to 512. Implemented to avoid the issue of * Pci-passthrough causing Maxpayload to be reset to 128 bytes * when the device is reset. */ if (pci_get_max_payload(dev) != 512) pci_set_max_payload(dev, 512); accel_dev = device_get_softc(dev); mutex_init(&accel_dev->lock); INIT_LIST_HEAD(&accel_dev->crypto_list); accel_pci_dev = &accel_dev->accel_pci_dev; accel_pci_dev->pci_dev = dev; if (bus_get_domain(dev, &accel_pci_dev->node) != 0) accel_pci_dev->node = 0; /* Add accel device to accel table. * This should be called before adf_cleanup_accel is called */ ret = adf_devmgr_add_dev(accel_dev, NULL); if (ret) { device_printf(dev, "Failed to add new accelerator device.\n"); goto out_err_lock; } /* Allocate and configure device configuration structure */ hw_data = malloc(sizeof(*hw_data), M_QAT_4XXX, M_WAITOK | M_ZERO); accel_dev->hw_device = hw_data; adf_init_hw_data_4xxx(accel_dev->hw_device, pci_get_device(dev)); accel_pci_dev->revid = pci_get_revid(dev); hw_data->fuses = pci_read_config(dev, ADF_4XXX_FUSECTL4_OFFSET, 4); /* Get PPAERUCM values and store */ ret = adf_aer_store_ppaerucm_reg(dev, hw_data); if (ret) goto out_err; /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || (~hw_data->ae_mask & 0x01)) { device_printf(dev, "No acceleration units found\n"); ret = ENXIO; goto out_err; } /* Create device configuration table */ ret = adf_cfg_dev_add(accel_dev); if (ret) goto out_err; ret = adf_clock_debugfs_add(accel_dev); if (ret) goto out_err; #ifdef QAT_DISABLE_SAFE_DC_MODE adf_4xxx_disable_safe_dc_sysctl_add(accel_dev); #endif /* QAT_DISABLE_SAFE_DC_MODE */ pci_set_max_read_req(dev, 4096); ret = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, /* BUS_SPACE_UNRESTRICTED */ 1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &accel_dev->dma_tag); if (ret) goto out_err; if (hw_data->get_accel_cap) { hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); } /* Find and map all the device's BARS */ /* Logical BARs configuration for 64bit BARs: bar 0 and 1 - logical BAR0 bar 2 and 3 - logical BAR1 bar 4 and 5 - logical BAR3 */ for (bar_nr = 0; bar_nr < (ADF_PCI_MAX_BARS * 2) && bar_nr < PCIR_MAX_BAR_0; bar_nr += 2) { struct adf_bar *bar; rid = PCIR_BAR(bar_nr); bar = &accel_pci_dev->pci_bars[bar_nr / 2]; bar->virt_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!bar->virt_addr) { device_printf(dev, "Failed to map BAR %d\n", bar_nr); ret = ENXIO; goto out_err; } bar->base_addr = rman_get_start(bar->virt_addr); bar->size = rman_get_size(bar->virt_addr); } ret = pci_enable_busmaster(dev); if (ret) goto out_err; adf_dbgfs_init(accel_dev); if (!accel_dev->hw_device->config_device) { ret = EFAULT; goto out_err_disable; } ret = accel_dev->hw_device->config_device(accel_dev); if (ret) goto out_err_disable; ret = adf_dev_init(accel_dev); if (ret) goto out_dev_shutdown; ret = adf_dev_start(accel_dev); if (ret) goto out_dev_stop; cfg_dev = accel_dev->cfg->dev; adf_cfg_device_clear(cfg_dev, accel_dev); free(cfg_dev, M_QAT); accel_dev->cfg->dev = NULL; return ret; out_dev_stop: adf_dev_stop(accel_dev); out_dev_shutdown: adf_dev_shutdown(accel_dev); out_err_disable: pci_disable_busmaster(dev); out_err: adf_cleanup_accel(accel_dev); out_err_lock: mutex_destroy(&accel_dev->lock); return ret; } static int adf_detach(device_t dev) { struct adf_accel_dev *accel_dev = device_get_softc(dev); if (adf_dev_stop(accel_dev)) { device_printf(dev, "Failed to stop QAT accel dev\n"); return EBUSY; } adf_dev_shutdown(accel_dev); pci_disable_busmaster(dev); adf_cleanup_accel(accel_dev); mutex_destroy(&accel_dev->lock); return 0; } static device_method_t adf_methods[] = { DEVMETHOD(device_probe, adf_probe), DEVMETHOD(device_attach, adf_attach), DEVMETHOD(device_detach, adf_detach), DEVMETHOD_END }; static driver_t adf_driver = { "qat", adf_methods, sizeof(struct adf_accel_dev) }; DRIVER_MODULE_ORDERED(qat_4xxx, pci, adf_driver, NULL, NULL, SI_ORDER_THIRD); MODULE_VERSION(qat_4xxx, 1); MODULE_DEPEND(qat_4xxx, qat_common, 1, 1, 1); MODULE_DEPEND(qat_4xxx, qat_api, 1, 1, 1); MODULE_DEPEND(qat_4xxx, linuxkpi, 1, 1, 1); diff --git a/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c b/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c index 2bbccb4d6b17..dbe40835ccbf 100644 --- a/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c +++ b/sys/dev/qat/qat_hw/qat_4xxxvf/adf_drv.c @@ -1,284 +1,287 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright(c) 2007-2025 Intel Corporation */ #include "qat_freebsd.h" #include #include #include #include "adf_4xxxvf_hw_data.h" #include "adf_gen4_hw_data.h" #include "adf_fw_counters.h" #include "adf_cfg_device.h" #include "adf_dbgfs.h" #include #include #include #include #include static MALLOC_DEFINE(M_QAT_4XXXVF, "qat_4xxxvf", "qat_4xxxvf"); #define ADF_SYSTEM_DEVICE(device_id) \ { \ PCI_VENDOR_ID_INTEL, device_id \ } -static const struct pci_device_id adf_pci_tbl[] = - { ADF_SYSTEM_DEVICE(ADF_4XXXIOV_PCI_DEVICE_ID), - ADF_SYSTEM_DEVICE(ADF_401XXIOV_PCI_DEVICE_ID), - { - 0, - } }; +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_4XXXIOV_PCI_DEVICE_ID), + ADF_SYSTEM_DEVICE(ADF_401XXIOV_PCI_DEVICE_ID), + ADF_SYSTEM_DEVICE(ADF_402XXIOV_PCI_DEVICE_ID), + { + 0, + } +}; static int adf_probe(device_t dev) { const struct pci_device_id *id; for (id = adf_pci_tbl; id->vendor != 0; id++) { if (pci_get_vendor(dev) == id->vendor && pci_get_device(dev) == id->device) { device_set_desc(dev, "Intel " ADF_4XXXVF_DEVICE_NAME " QuickAssist"); return BUS_PROBE_GENERIC; } } return ENXIO; } static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; struct adf_accel_dev *pf; int i; if (accel_dev->dma_tag) bus_dma_tag_destroy(accel_dev->dma_tag); for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; if (bar->virt_addr) bus_free_resource(accel_pci_dev->pci_dev, SYS_RES_MEMORY, bar->virt_addr); } /* * As adf_clean_hw_data_4xxxiov() will update class index, before * index is updated, vf must be remove from accel_table. */ pf = adf_devmgr_pci_to_accel_dev(pci_find_pf(accel_pci_dev->pci_dev)); adf_devmgr_rm_dev(accel_dev, pf); if (accel_dev->hw_device) { switch (pci_get_device(accel_pci_dev->pci_dev)) { case ADF_4XXXIOV_PCI_DEVICE_ID: case ADF_401XXIOV_PCI_DEVICE_ID: + case ADF_402XXIOV_PCI_DEVICE_ID: adf_clean_hw_data_4xxxiov(accel_dev->hw_device); break; default: break; } free(accel_dev->hw_device, M_QAT_4XXXVF); accel_dev->hw_device = NULL; } adf_dbgfs_exit(accel_dev); adf_cfg_dev_remove(accel_dev); } static int adf_attach(device_t dev) { struct adf_accel_dev *accel_dev; struct adf_accel_dev *pf; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; unsigned int bar_nr; int ret = 0; int rid; struct adf_cfg_device *cfg_dev = NULL; accel_dev = device_get_softc(dev); mutex_init(&accel_dev->lock); accel_dev->is_vf = true; pf = adf_devmgr_pci_to_accel_dev(pci_find_pf(dev)); INIT_LIST_HEAD(&accel_dev->crypto_list); accel_pci_dev = &accel_dev->accel_pci_dev; accel_pci_dev->pci_dev = dev; if (bus_get_domain(dev, &accel_pci_dev->node) != 0) accel_pci_dev->node = 0; /* Add accel device to accel table */ ret = adf_devmgr_add_dev(accel_dev, pf); if (ret) { device_printf(GET_DEV(accel_dev), "Failed to add new accelerator device.\n"); goto out_err_lock; } /* Allocate and configure device configuration structure */ hw_data = malloc(sizeof(*hw_data), M_QAT_4XXXVF, M_WAITOK | M_ZERO); accel_dev->hw_device = hw_data; adf_init_hw_data_4xxxiov(accel_dev->hw_device); accel_pci_dev->revid = pci_get_revid(dev); hw_data->fuses = pci_read_config(dev, ADF_4XXXIOV_VFFUSECTL4_OFFSET, 4); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(accel_dev); hw_data->ae_mask = hw_data->get_ae_mask(accel_dev); hw_data->admin_ae_mask = hw_data->ae_mask; accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create device configuration table */ ret = adf_cfg_dev_add(accel_dev); if (ret) goto out_err; pci_set_max_read_req(dev, 1024); ret = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, /* BUS_SPACE_UNRESTRICTED */ 1, BUS_SPACE_MAXSIZE, 0, NULL, NULL, &accel_dev->dma_tag); if (ret) goto out_err; hw_data->accel_capabilities_mask = adf_4xxxvf_get_hw_cap(accel_dev); /* Find and map all the device's BARS */ /* Logical BARs configuration for 64bit BARs: bar 0 and 1 - logical BAR0 bar 2 and 3 - logical BAR1 bar 4 and 5 - logical BAR3 */ for (bar_nr = 0; bar_nr < (ADF_PCI_MAX_BARS * 2) && bar_nr < PCIR_MAX_BAR_0; bar_nr += 2) { struct adf_bar *bar; rid = PCIR_BAR(bar_nr); bar = &accel_pci_dev->pci_bars[bar_nr / 2]; bar->virt_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!bar->virt_addr) { device_printf(dev, "Failed to map BAR %d\n", bar_nr); ret = ENXIO; goto out_err; } bar->base_addr = rman_get_start(bar->virt_addr); bar->size = rman_get_size(bar->virt_addr); } ret = pci_enable_busmaster(dev); if (ret) goto out_err; adf_dbgfs_init(accel_dev); /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->u1.vf.msg_received); mutex_init(&accel_dev->u1.vf.rpreset_lock); ret = hw_data->config_device(accel_dev); if (ret) goto out_err_disable; ret = adf_dev_init(accel_dev); if (!ret) ret = adf_dev_start(accel_dev); if (ret) { device_printf( GET_DEV(accel_dev), "Failed to start - make sure PF enabled services match VF configuration.\n"); adf_dev_stop(accel_dev); adf_dev_shutdown(accel_dev); return 0; } cfg_dev = accel_dev->cfg->dev; adf_cfg_device_clear(cfg_dev, accel_dev); free(cfg_dev, M_QAT); accel_dev->cfg->dev = NULL; return ret; out_err_disable: pci_disable_busmaster(dev); out_err: adf_cleanup_accel(accel_dev); out_err_lock: mutex_destroy(&accel_dev->lock); return ret; } static int adf_detach(device_t dev) { struct adf_accel_dev *accel_dev = device_get_softc(dev); if (!accel_dev) { printf("QAT: Driver removal failed\n"); return EFAULT; } adf_flush_vf_wq(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); adf_dev_stop(accel_dev); adf_dev_shutdown(accel_dev); pci_disable_busmaster(dev); adf_cleanup_accel(accel_dev); mutex_destroy(&accel_dev->lock); return 0; } static int adf_modevent(module_t mod, int type, void *data) { switch (type) { case MOD_UNLOAD: adf_clean_vf_map(true); return 0; default: return EOPNOTSUPP; } } static device_method_t adf_methods[] = { DEVMETHOD(device_probe, adf_probe), DEVMETHOD(device_attach, adf_attach), DEVMETHOD(device_detach, adf_detach), DEVMETHOD_END }; static driver_t adf_driver = { "qat", adf_methods, sizeof(struct adf_accel_dev) }; DRIVER_MODULE_ORDERED(qat_4xxxvf, pci, adf_driver, adf_modevent, NULL, SI_ORDER_THIRD); MODULE_VERSION(qat_4xxxvf, 1); MODULE_DEPEND(qat_4xxxvf, qat_common, 1, 1, 1); MODULE_DEPEND(qat_4xxxvf, qat_api, 1, 1, 1); MODULE_DEPEND(qat_4xxxvf, linuxkpi, 1, 1, 1); diff --git a/sys/modules/qatfw/qat_4xxx/Makefile b/sys/modules/qatfw/qat_4xxx/Makefile index fb7171bcaf45..f6f19d6cbe32 100644 --- a/sys/modules/qatfw/qat_4xxx/Makefile +++ b/sys/modules/qatfw/qat_4xxx/Makefile @@ -1,9 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2007-2022 Intel Corporation .PATH: ${SRCTOP}/sys/contrib/dev/qat KMOD= qat_4xxx_fw -FIRMWS= qat_4xxx.bin:qat_4xxx_fw:111 qat_4xxx_mmp.bin:qat_4xxx_mmp_fw:111 +FIRMWS= qat_4xxx.bin:qat_4xxx_fw:111 \ + qat_4xxx_mmp.bin:qat_4xxx_mmp_fw:111 \ + qat_402xx.bin:qat_402xx_fw:111 \ + qat_402xx_mmp.bin:qat_402xx_mmp_fw:111 .include