Differential D36254 Diff 115177 sys/dev/qat/qat_common/adf_transport.c

Changeset View

Standalone View

sys/dev/qat/qat_common/adf_transport.c

Show First 20 Lines • Show All 67 Lines • ▼ Show 20 Lines	adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring)
mtx_lock(&bank->lock);		mtx_lock(&bank->lock);
bank->ring_mask &= ~(1 << ring);		bank->ring_mask &= ~(1 << ring);
mtx_unlock(&bank->lock);		mtx_unlock(&bank->lock);
}		}

static void		static void
adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)		adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);

mtx_lock(&bank->lock);		mtx_lock(&bank->lock);
bank->irq_mask \|= (1 << ring);		bank->irq_mask \|= (1 << ring);
mtx_unlock(&bank->lock);		mtx_unlock(&bank->lock);
WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask);		csr_ops->write_csr_int_col_en(bank->csr_addr,
WRITE_CSR_INT_COL_CTL(bank->csr_addr,
bank->bank_number,		bank->bank_number,
		bank->irq_mask);
		csr_ops->write_csr_int_col_ctl(bank->csr_addr,
		bank->bank_number,
bank->irq_coalesc_timer);		bank->irq_coalesc_timer);
}		}

static void		static void
adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)		adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);

mtx_lock(&bank->lock);		mtx_lock(&bank->lock);
bank->irq_mask &= ~(1 << ring);		bank->irq_mask &= ~(1 << ring);
mtx_unlock(&bank->lock);		mtx_unlock(&bank->lock);
WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask);		csr_ops->write_csr_int_col_en(bank->csr_addr,
		bank->bank_number,
		bank->irq_mask);
}		}

int		int
adf_send_message(struct adf_etr_ring_data ring, u32 msg)		adf_send_message(struct adf_etr_ring_data ring, u32 msg)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 msg_size = 0;		u32 msg_size = 0;

if (atomic_add_return(1, ring->inflights) > ring->max_inflights) {		if (atomic_add_return(1, ring->inflights) > ring->max_inflights) {
atomic_dec(ring->inflights);		atomic_dec(ring->inflights);
return EAGAIN;		return EAGAIN;
}		}

msg_size = ADF_MSG_SIZE_TO_BYTES(ring->msg_size);		msg_size = ADF_MSG_SIZE_TO_BYTES(ring->msg_size);
mtx_lock(&ring->lock);		mtx_lock(&ring->lock);
memcpy((void *)((uintptr_t)ring->base_addr + ring->tail),		memcpy((void *)((uintptr_t)ring->base_addr + ring->tail),
msg,		msg,
msg_size);		msg_size);

ring->tail = adf_modulo(ring->tail + msg_size,		ring->tail = adf_modulo(ring->tail + msg_size,
ADF_RING_SIZE_MODULO(ring->ring_size));		ADF_RING_SIZE_MODULO(ring->ring_size));

WRITE_CSR_RING_TAIL(ring->bank->csr_addr,		csr_ops->write_csr_ring_tail(ring->bank->csr_addr,
ring->bank->bank_number,		ring->bank->bank_number,
ring->ring_number,		ring->ring_number,
ring->tail);		ring->tail);
ring->csr_tail_offset = ring->tail;		ring->csr_tail_offset = ring->tail;
mtx_unlock(&ring->lock);		mtx_unlock(&ring->lock);
return 0;		return 0;
}		}

int		int
adf_handle_response(struct adf_etr_ring_data *ring, u32 quota)		adf_handle_response(struct adf_etr_ring_data *ring, u32 quota)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 msg_counter = 0;		u32 msg_counter = 0;
u32 msg = (u32 )((uintptr_t)ring->base_addr + ring->head);		u32 msg = (u32 )((uintptr_t)ring->base_addr + ring->head);

if (!quota)		if (!quota)
quota = ADF_NO_RESPONSE_QUOTA;		quota = ADF_NO_RESPONSE_QUOTA;

while ((*msg != ADF_RING_EMPTY_SIG) && (msg_counter < quota)) {		while ((*msg != ADF_RING_EMPTY_SIG) && (msg_counter < quota)) {
ring->callback((u32 *)msg);		ring->callback((u32 *)msg);
atomic_dec(ring->inflights);		atomic_dec(ring->inflights);
*msg = ADF_RING_EMPTY_SIG;		*msg = ADF_RING_EMPTY_SIG;
ring->head = adf_modulo(ring->head + ADF_MSG_SIZE_TO_BYTES(		ring->head = adf_modulo(ring->head + ADF_MSG_SIZE_TO_BYTES(
ring->msg_size),		ring->msg_size),
ADF_RING_SIZE_MODULO(ring->ring_size));		ADF_RING_SIZE_MODULO(ring->ring_size));
msg_counter++;		msg_counter++;
msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);		msg = (u32 *)((uintptr_t)ring->base_addr + ring->head);
}		}
if (msg_counter > 0)		if (msg_counter > 0)
WRITE_CSR_RING_HEAD(ring->bank->csr_addr,		csr_ops->write_csr_ring_head(ring->bank->csr_addr,
ring->bank->bank_number,		ring->bank->bank_number,
ring->ring_number,		ring->ring_number,
ring->head);		ring->head);
return msg_counter;		return msg_counter;
}		}

int		int
adf_poll_bank(u32 accel_id, u32 bank_num, u32 quota)		adf_poll_bank(u32 accel_id, u32 bank_num, u32 quota)
{		{
int num_resp;		int num_resp;
struct adf_accel_dev *accel_dev;		struct adf_accel_dev *accel_dev;
struct adf_etr_data *trans_data;		struct adf_etr_data *trans_data;
struct adf_etr_bank_data *bank;		struct adf_etr_bank_data *bank;
struct adf_etr_ring_data *ring;		struct adf_etr_ring_data *ring;
		struct adf_hw_csr_ops *csr_ops;
u32 rings_not_empty;		u32 rings_not_empty;
u32 ring_num;		u32 ring_num;
u32 resp_total = 0;		u32 resp_total = 0;
u32 num_rings_per_bank;		u32 num_rings_per_bank;

/* Find the accel device associated with the accelId		/* Find the accel device associated with the accelId
* passed in.		* passed in.
*/		*/
accel_dev = adf_devmgr_get_dev_by_id(accel_id);		accel_dev = adf_devmgr_get_dev_by_id(accel_id);
if (!accel_dev) {		if (!accel_dev) {
pr_err("There is no device with id: %d\n", accel_id);		pr_err("There is no device with id: %d\n", accel_id);
return EINVAL;		return EINVAL;
}		}

		csr_ops = GET_CSR_OPS(accel_dev);
trans_data = accel_dev->transport;		trans_data = accel_dev->transport;
bank = &trans_data->banks[bank_num];		bank = &trans_data->banks[bank_num];
mtx_lock(&bank->lock);		mtx_lock(&bank->lock);

/* Read the ring status CSR to determine which rings are empty. */		/* Read the ring status CSR to determine which rings are empty. */
rings_not_empty = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number);		rings_not_empty =
		csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number);
/* Complement to find which rings have data to be processed. */		/* Complement to find which rings have data to be processed. */
rings_not_empty = (~rings_not_empty) & bank->ring_mask;		rings_not_empty = (~rings_not_empty) & bank->ring_mask;

/* Return RETRY if the bank polling rings		/* Return RETRY if the bank polling rings
* are all empty.		* are all empty.
*/		*/
if (!(rings_not_empty & bank->ring_mask)) {		if (!(rings_not_empty & bank->ring_mask)) {
mtx_unlock(&bank->lock);		mtx_unlock(&bank->lock);
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Lines	if (stat_total)
return 0;		return 0;

return EAGAIN;		return EAGAIN;
}		}

static void		static void
adf_configure_tx_ring(struct adf_etr_ring_data *ring)		adf_configure_tx_ring(struct adf_etr_ring_data *ring)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);		u32 ring_config = BUILD_RING_CONFIG(ring->ring_size);

WRITE_CSR_RING_CONFIG(ring->bank->csr_addr,		csr_ops->write_csr_ring_config(ring->bank->csr_addr,
ring->bank->bank_number,		ring->bank->bank_number,
ring->ring_number,		ring->ring_number,
ring_config);		ring_config);
}		}

static void		static void
adf_configure_rx_ring(struct adf_etr_ring_data *ring)		adf_configure_rx_ring(struct adf_etr_ring_data *ring)
{		{
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size,		u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size,
ADF_RING_NEAR_WATERMARK_512,		ADF_RING_NEAR_WATERMARK_512,
ADF_RING_NEAR_WATERMARK_0);		ADF_RING_NEAR_WATERMARK_0);

WRITE_CSR_RING_CONFIG(ring->bank->csr_addr,		csr_ops->write_csr_ring_config(ring->bank->csr_addr,
ring->bank->bank_number,		ring->bank->bank_number,
ring->ring_number,		ring->ring_number,
ring_config);		ring_config);
}		}

static int		static int
adf_init_ring(struct adf_etr_ring_data *ring)		adf_init_ring(struct adf_etr_ring_data *ring)
{		{
struct adf_etr_bank_data *bank = ring->bank;		struct adf_etr_bank_data *bank = ring->bank;
struct adf_accel_dev *accel_dev = bank->accel_dev;		struct adf_accel_dev *accel_dev = bank->accel_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;		struct adf_hw_device_data *hw_data = accel_dev->hw_device;
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
u64 ring_base;		u64 ring_base;
u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);		u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);

ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);		ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes);
int ret;		int ret;

ret = bus_dma_mem_create(&ring->dma_mem,		ret = bus_dma_mem_create(&ring->dma_mem,
accel_dev->dma_tag,		accel_dev->dma_tag,
Show All 15 Lines	if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) {
return EFAULT;		return EFAULT;
}		}

if (hw_data->tx_rings_mask & (1 << ring->ring_number))		if (hw_data->tx_rings_mask & (1 << ring->ring_number))
adf_configure_tx_ring(ring);		adf_configure_tx_ring(ring);
else		else
adf_configure_rx_ring(ring);		adf_configure_rx_ring(ring);

ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size);		ring_base =
WRITE_CSR_RING_BASE(ring->bank->csr_addr,		csr_ops->build_csr_ring_base_addr(ring->dma_addr, ring->ring_size);
		csr_ops->write_csr_ring_base(ring->bank->csr_addr,
ring->bank->bank_number,		ring->bank->bank_number,
ring->ring_number,		ring->ring_number,
ring_base);		ring_base);
mtx_init(&ring->lock, "adf bank", NULL, MTX_DEF);		mtx_init(&ring->lock, "adf bank", NULL, MTX_DEF);
return 0;		return 0;
}		}

static void		static void
adf_cleanup_ring(struct adf_etr_ring_data *ring)		adf_cleanup_ring(struct adf_etr_ring_data *ring)
{		{
u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);		u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size);
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines	err:
adf_update_ring_arb(ring);		adf_update_ring_arb(ring);
return ret;		return ret;
}		}

void		void
adf_remove_ring(struct adf_etr_ring_data *ring)		adf_remove_ring(struct adf_etr_ring_data *ring)
{		{
struct adf_etr_bank_data *bank = ring->bank;		struct adf_etr_bank_data *bank = ring->bank;
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);

/* Disable interrupts for the given ring */		/* Disable interrupts for the given ring */
adf_disable_ring_irq(bank, ring->ring_number);		adf_disable_ring_irq(bank, ring->ring_number);

/* Clear PCI config space */		/* Clear PCI config space */
WRITE_CSR_RING_CONFIG(bank->csr_addr,		csr_ops->write_csr_ring_config(bank->csr_addr,
bank->bank_number,		bank->bank_number,
ring->ring_number,		ring->ring_number,
0);		0);
WRITE_CSR_RING_BASE(bank->csr_addr,		csr_ops->write_csr_ring_base(bank->csr_addr,
bank->bank_number,		bank->bank_number,
ring->ring_number,		ring->ring_number,
0);		0);
adf_ring_debugfs_rm(ring);		adf_ring_debugfs_rm(ring);
adf_unreserve_ring(bank, ring->ring_number);		adf_unreserve_ring(bank, ring->ring_number);
/* Disable HW arbitration for the given ring */		/* Disable HW arbitration for the given ring */
adf_update_ring_arb(ring);		adf_update_ring_arb(ring);
adf_cleanup_ring(ring);		adf_cleanup_ring(ring);
}		}

static void		static void
adf_ring_response_handler(struct adf_etr_bank_data *bank)		adf_ring_response_handler(struct adf_etr_bank_data *bank)
{		{
struct adf_accel_dev *accel_dev = bank->accel_dev;		struct adf_accel_dev *accel_dev = bank->accel_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;		struct adf_hw_device_data *hw_data = accel_dev->hw_device;
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
u8 num_rings_per_bank = hw_data->num_rings_per_bank;		u8 num_rings_per_bank = hw_data->num_rings_per_bank;
u32 empty_rings, i;		u32 empty_rings, i;

empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number);		empty_rings =
		csr_ops->read_csr_e_stat(bank->csr_addr, bank->bank_number);
empty_rings = ~empty_rings & bank->irq_mask;		empty_rings = ~empty_rings & bank->irq_mask;

for (i = 0; i < num_rings_per_bank; ++i) {		for (i = 0; i < num_rings_per_bank; ++i) {
if (empty_rings & (1 << i))		if (empty_rings & (1 << i))
adf_handle_response(&bank->rings[i], 0);		adf_handle_response(&bank->rings[i], 0);
}		}
}		}

void		void
adf_response_handler(uintptr_t bank_addr)		adf_response_handler(uintptr_t bank_addr)
{		{
struct adf_etr_bank_data bank = (void )bank_addr;		struct adf_etr_bank_data bank = (void )bank_addr;
		struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);

/* Handle all the responses and re-enable IRQs */		/* Handle all the responses and re-enable IRQs */
adf_ring_response_handler(bank);		adf_ring_response_handler(bank);
WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr,		csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
bank->bank_number,		bank->bank_number,
bank->irq_mask);		bank->irq_mask);
}		}

static inline int		static inline int
adf_get_cfg_int(struct adf_accel_dev *accel_dev,		adf_get_cfg_int(struct adf_accel_dev *accel_dev,
const char *section,		const char *section,
const char *format,		const char *format,
u32 key,		u32 key,
u32 *value)		u32 *value)
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines

static int		static int
adf_init_bank(struct adf_accel_dev *accel_dev,		adf_init_bank(struct adf_accel_dev *accel_dev,
struct adf_etr_bank_data *bank,		struct adf_etr_bank_data *bank,
u32 bank_num,		u32 bank_num,
struct resource *csr_addr)		struct resource *csr_addr)
{		{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;		struct adf_hw_device_data *hw_data = accel_dev->hw_device;
		struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops;
struct adf_etr_ring_data *ring;		struct adf_etr_ring_data *ring;
struct adf_etr_ring_data *tx_ring;		struct adf_etr_ring_data *tx_ring;
u32 i, coalesc_enabled = 0;		u32 i, coalesc_enabled = 0;
u8 num_rings_per_bank = hw_data->num_rings_per_bank;		u8 num_rings_per_bank = hw_data->num_rings_per_bank;
		u32 irq_mask = BIT(num_rings_per_bank) - 1;
u32 size = 0;		u32 size = 0;

explicit_bzero(bank, sizeof(*bank));		explicit_bzero(bank, sizeof(*bank));
bank->bank_number = bank_num;		bank->bank_number = bank_num;
bank->csr_addr = csr_addr;		bank->csr_addr = csr_addr;
bank->accel_dev = accel_dev;		bank->accel_dev = accel_dev;
mtx_init(&bank->lock, "adf bank", NULL, MTX_DEF);		mtx_init(&bank->lock, "adf bank", NULL, MTX_DEF);

Show All 12 Lines	if ((adf_get_cfg_int(accel_dev,
bank_num,		bank_num,
&coalesc_enabled) == 0) &&		&coalesc_enabled) == 0) &&
coalesc_enabled)		coalesc_enabled)
adf_get_coalesc_timer(bank, "Accelerator0", bank_num);		adf_get_coalesc_timer(bank, "Accelerator0", bank_num);
else		else
bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;		bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME;

for (i = 0; i < num_rings_per_bank; i++) {		for (i = 0; i < num_rings_per_bank; i++) {
WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0);		csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0);
WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0);		csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0);
ring = &bank->rings[i];		ring = &bank->rings[i];
if (hw_data->tx_rings_mask & (1 << i)) {		if (hw_data->tx_rings_mask & (1 << i)) {
ring->inflights =		ring->inflights =
kzalloc_node(sizeof(atomic_t),		kzalloc_node(sizeof(atomic_t),
M_WAITOK \| M_ZERO,		M_WAITOK \| M_ZERO,
dev_to_node(GET_DEV(accel_dev)));		dev_to_node(GET_DEV(accel_dev)));
} else {		} else {
if (i < hw_data->tx_rx_gap) {		if (i < hw_data->tx_rx_gap) {
device_printf(GET_DEV(accel_dev),		device_printf(GET_DEV(accel_dev),
"Invalid tx rings mask config\n");		"Invalid tx rings mask config\n");
goto err;		goto err;
}		}
tx_ring = &bank->rings[i - hw_data->tx_rx_gap];		tx_ring = &bank->rings[i - hw_data->tx_rx_gap];
ring->inflights = tx_ring->inflights;		ring->inflights = tx_ring->inflights;
}		}
}		}

if (adf_bank_debugfs_add(bank)) {		if (adf_bank_debugfs_add(bank)) {
device_printf(GET_DEV(accel_dev),		device_printf(GET_DEV(accel_dev),
"Failed to add bank debugfs entry\n");		"Failed to add bank debugfs entry\n");
goto err;		goto err;
}		}

WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK);		csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask);
WRITE_CSR_INT_SRCSEL(csr_addr, bank_num);		csr_ops->write_csr_int_srcsel(csr_addr, bank_num);
return 0;		return 0;
err:		err:
for (i = 0; i < num_rings_per_bank; i++) {		for (i = 0; i < num_rings_per_bank; i++) {
ring = &bank->rings[i];		ring = &bank->rings[i];
if (hw_data->tx_rings_mask & (1 << i)) {		if (hw_data->tx_rings_mask & (1 << i)) {
kfree(ring->inflights);		kfree(ring->inflights);
ring->inflights = NULL;		ring->inflights = NULL;
}		}
▲ Show 20 Lines • Show All 130 Lines • Show Last 20 Lines