diff --git a/sys/dev/nxge/xgehal/xgehal-fifo.c b/sys/dev/nxge/xgehal/xgehal-fifo.c index 303afafbf7ce..78c0923dee0f 100644 --- a/sys/dev/nxge/xgehal/xgehal-fifo.c +++ b/sys/dev/nxge/xgehal/xgehal-fifo.c @@ -1,560 +1,560 @@ /*- * Copyright (c) 2002-2007 Neterion, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include static xge_hal_status_e __hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh, void *memblock, int memblock_index, xge_hal_mempool_dma_t *dma_object, void *item, int index, int is_last, void *userdata) { int memblock_item_idx; xge_hal_fifo_txdl_priv_t *txdl_priv; xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)item; xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata; xge_assert(item); txdl_priv = (xge_hal_fifo_txdl_priv_t *) \ __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh, memblock_index, item, &memblock_item_idx); xge_assert(txdl_priv); /* pre-format HAL's TxDL's private */ txdl_priv->dma_offset = (char*)item - (char*)memblock; txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset; txdl_priv->dma_handle = dma_object->handle; txdl_priv->memblock = memblock; txdl_priv->first_txdp = (xge_hal_fifo_txd_t *)item; txdl_priv->next_txdl_priv = NULL; txdl_priv->dang_txdl = NULL; txdl_priv->dang_frags = 0; txdl_priv->alloc_frags = 0; #ifdef XGE_DEBUG_ASSERT txdl_priv->dma_object = dma_object; #endif txdp->host_control = (u64)(ulong_t)txdl_priv; #ifdef XGE_HAL_ALIGN_XMIT txdl_priv->align_vaddr = NULL; txdl_priv->align_dma_addr = (dma_addr_t)0; #ifndef XGE_HAL_ALIGN_XMIT_ALLOC_RT { xge_hal_status_e status; if (fifo->config->alignment_size) { status =__hal_fifo_dtr_align_alloc_map(fifo, txdp); if (status != XGE_HAL_OK) { xge_debug_mm(XGE_ERR, "align buffer[%d] %d bytes, status %d", index, fifo->align_size, status); return status; } } } #endif #endif if (fifo->channel.dtr_init) { fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index, fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL); } return XGE_HAL_OK; } static xge_hal_status_e __hal_fifo_mempool_item_free(xge_hal_mempool_h mempoolh, void *memblock, int memblock_index, xge_hal_mempool_dma_t *dma_object, void *item, int index, int is_last, void *userdata) { int memblock_item_idx; xge_hal_fifo_txdl_priv_t *txdl_priv; #ifdef XGE_HAL_ALIGN_XMIT xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata; #endif xge_assert(item); txdl_priv = (xge_hal_fifo_txdl_priv_t *) \ __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh, memblock_index, item, &memblock_item_idx); xge_assert(txdl_priv); #ifdef XGE_HAL_ALIGN_XMIT if (fifo->config->alignment_size) { if (txdl_priv->align_dma_addr != 0) { xge_os_dma_unmap(fifo->channel.pdev, txdl_priv->align_dma_handle, txdl_priv->align_dma_addr, fifo->align_size, XGE_OS_DMA_DIR_TODEVICE); txdl_priv->align_dma_addr = 0; } if (txdl_priv->align_vaddr != NULL) { xge_os_dma_free(fifo->channel.pdev, txdl_priv->align_vaddr, fifo->align_size, &txdl_priv->align_dma_acch, &txdl_priv->align_dma_handle); txdl_priv->align_vaddr = NULL; } } #endif return XGE_HAL_OK; } xge_hal_status_e __hal_fifo_open(xge_hal_channel_h channelh, xge_hal_channel_attr_t *attr) { xge_hal_device_t *hldev; xge_hal_status_e status; xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_hal_fifo_queue_t *queue; int i, txdl_size, max_arr_index, mid_point; xge_hal_dtr_h dtrh; hldev = (xge_hal_device_t *)fifo->channel.devh; fifo->config = &hldev->config.fifo; queue = &fifo->config->queue[attr->post_qid]; #if defined(XGE_HAL_TX_MULTI_RESERVE) xge_os_spin_lock_init(&fifo->channel.reserve_lock, hldev->pdev); #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) xge_os_spin_lock_init_irq(&fifo->channel.reserve_lock, hldev->irqh); #endif #if defined(XGE_HAL_TX_MULTI_POST) if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA) { fifo->post_lock_ptr = &hldev->xena_post_lock; } else { xge_os_spin_lock_init(&fifo->channel.post_lock, hldev->pdev); fifo->post_lock_ptr = &fifo->channel.post_lock; } #elif defined(XGE_HAL_TX_MULTI_POST_IRQ) if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA) { fifo->post_lock_ptr = &hldev->xena_post_lock; } else { xge_os_spin_lock_init_irq(&fifo->channel.post_lock, hldev->irqh); fifo->post_lock_ptr = &fifo->channel.post_lock; } #endif fifo->align_size = fifo->config->alignment_size * fifo->config->max_aligned_frags; /* Initializing the BAR1 address as the start of * the FIFO queue pointer and as a location of FIFO control * word. */ fifo->hw_pair = (xge_hal_fifo_hw_pair_t *) (void *)(hldev->bar1 + (attr->post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET)); /* apply "interrupts per txdl" attribute */ fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_UTILZ; if (queue->intr) { fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_PER_LIST; } fifo->no_snoop_bits = (int)(XGE_HAL_TX_FIFO_NO_SNOOP(queue->no_snoop_bits)); /* * FIFO memory management strategy: * * TxDL splitted into three independent parts: * - set of TxD's * - TxD HAL private part * - upper layer private part * * Adaptative memory allocation used. i.e. Memory allocated on * demand with the size which will fit into one memory block. * One memory block may contain more than one TxDL. In simple case * memory block size can be equal to CPU page size. On more * sophisticated OS's memory block can be contigious across * several pages. * * During "reserve" operations more memory can be allocated on demand * for example due to FIFO full condition. * * Pool of memory memblocks never shrinks except __hal_fifo_close * routine which will essentially stop channel and free the resources. */ /* TxDL common private size == TxDL private + ULD private */ fifo->priv_size = sizeof(xge_hal_fifo_txdl_priv_t) + attr->per_dtr_space; fifo->priv_size = ((fifo->priv_size + __xge_os_cacheline_size -1) / __xge_os_cacheline_size) * __xge_os_cacheline_size; /* recompute txdl size to be cacheline aligned */ fifo->txdl_size = fifo->config->max_frags * sizeof(xge_hal_fifo_txd_t); txdl_size = ((fifo->txdl_size + __xge_os_cacheline_size - 1) / __xge_os_cacheline_size) * __xge_os_cacheline_size; if (fifo->txdl_size != txdl_size) xge_debug_fifo(XGE_ERR, "cacheline > 128 ( ?? ): %d, %d, %d, %d", fifo->config->max_frags, fifo->txdl_size, txdl_size, __xge_os_cacheline_size); fifo->txdl_size = txdl_size; /* since dtr_init() callback will be called from item_alloc(), * the same way channels userdata might be used prior to * channel_initialize() */ fifo->channel.dtr_init = attr->dtr_init; fifo->channel.userdata = attr->userdata; fifo->txdl_per_memblock = fifo->config->memblock_size / fifo->txdl_size; fifo->mempool = __hal_mempool_create(hldev->pdev, fifo->config->memblock_size, fifo->txdl_size, fifo->priv_size, queue->initial, queue->max, __hal_fifo_mempool_item_alloc, __hal_fifo_mempool_item_free, fifo); if (fifo->mempool == NULL) { return XGE_HAL_ERR_OUT_OF_MEMORY; } status = __hal_channel_initialize(channelh, attr, (void **) __hal_mempool_items_arr(fifo->mempool), queue->initial, queue->max, fifo->config->reserve_threshold); if (status != XGE_HAL_OK) { __hal_fifo_close(channelh); return status; } xge_debug_fifo(XGE_TRACE, "DTR reserve_length:%d reserve_top:%d\n" "max_frags:%d reserve_threshold:%d\n" "memblock_size:%d alignment_size:%d max_aligned_frags:%d", fifo->channel.reserve_length, fifo->channel.reserve_top, fifo->config->max_frags, fifo->config->reserve_threshold, fifo->config->memblock_size, fifo->config->alignment_size, fifo->config->max_aligned_frags); #ifdef XGE_DEBUG_ASSERT for ( i = 0; i < fifo->channel.reserve_length; i++) { xge_debug_fifo(XGE_TRACE, "DTR before reversing index:%d" " handle:%p", i, fifo->channel.reserve_arr[i]); } #endif xge_assert(fifo->channel.reserve_length); /* reverse the FIFO dtr array */ max_arr_index = fifo->channel.reserve_length - 1; max_arr_index -=fifo->channel.reserve_top; xge_assert(max_arr_index); mid_point = (fifo->channel.reserve_length - fifo->channel.reserve_top)/2; for (i = 0; i < mid_point; i++) { dtrh = fifo->channel.reserve_arr[i]; fifo->channel.reserve_arr[i] = fifo->channel.reserve_arr[max_arr_index - i]; fifo->channel.reserve_arr[max_arr_index - i] = dtrh; } #ifdef XGE_DEBUG_ASSERT for ( i = 0; i < fifo->channel.reserve_length; i++) { xge_debug_fifo(XGE_TRACE, "DTR after reversing index:%d" " handle:%p", i, fifo->channel.reserve_arr[i]); } #endif return XGE_HAL_OK; } void __hal_fifo_close(xge_hal_channel_h channelh) { xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_hal_device_t *hldev = (xge_hal_device_t *)fifo->channel.devh; if (fifo->mempool) { __hal_mempool_destroy(fifo->mempool); } __hal_channel_terminate(channelh); #if defined(XGE_HAL_TX_MULTI_RESERVE) xge_os_spin_lock_destroy(&fifo->channel.reserve_lock, hldev->pdev); #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) xge_os_spin_lock_destroy_irq(&fifo->channel.reserve_lock, hldev->pdev); #endif if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) { #if defined(XGE_HAL_TX_MULTI_POST) xge_os_spin_lock_destroy(&fifo->channel.post_lock, hldev->pdev); #elif defined(XGE_HAL_TX_MULTI_POST_IRQ) xge_os_spin_lock_destroy_irq(&fifo->channel.post_lock, hldev->pdev); #endif } } void __hal_fifo_hw_initialize(xge_hal_device_h devh) { xge_hal_device_t *hldev = (xge_hal_device_t *)devh; xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void *)hldev->bar0; u64* tx_fifo_partitions[4]; u64* tx_fifo_wrr[5]; u64 tx_fifo_wrr_value[5]; u64 val64, part0; int i; /* Tx DMA Initialization */ tx_fifo_partitions[0] = &bar0->tx_fifo_partition_0; tx_fifo_partitions[1] = &bar0->tx_fifo_partition_1; tx_fifo_partitions[2] = &bar0->tx_fifo_partition_2; tx_fifo_partitions[3] = &bar0->tx_fifo_partition_3; tx_fifo_wrr[0] = &bar0->tx_w_round_robin_0; tx_fifo_wrr[1] = &bar0->tx_w_round_robin_1; tx_fifo_wrr[2] = &bar0->tx_w_round_robin_2; tx_fifo_wrr[3] = &bar0->tx_w_round_robin_3; tx_fifo_wrr[4] = &bar0->tx_w_round_robin_4; tx_fifo_wrr_value[0] = XGE_HAL_FIFO_WRR_0; tx_fifo_wrr_value[1] = XGE_HAL_FIFO_WRR_1; tx_fifo_wrr_value[2] = XGE_HAL_FIFO_WRR_2; tx_fifo_wrr_value[3] = XGE_HAL_FIFO_WRR_3; tx_fifo_wrr_value[4] = XGE_HAL_FIFO_WRR_4; /* Note: WRR calendar must be configured before the transmit * FIFOs are enabled! page 6-77 user guide */ if (!hldev->config.rts_qos_en) { /* all zeroes for Round-Robin */ for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0, tx_fifo_wrr[i]); } /* reset all of them but '0' */ for (i=1; i < XGE_HAL_FIFO_MAX_PARTITION; i++) { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0ULL, tx_fifo_partitions[i]); } } else { /* Change the default settings */ for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, tx_fifo_wrr_value[i], tx_fifo_wrr[i]); } } /* configure only configured FIFOs */ val64 = 0; part0 = 0; for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) { int reg_half = i % 2; int reg_num = i / 2; if (hldev->config.fifo.queue[i].configured) { int priority = hldev->config.fifo.queue[i].priority; val64 |= vBIT((hldev->config.fifo.queue[i].max-1), (((reg_half) * 32) + 19), 13) | vBIT(priority, (((reg_half)*32) + 5), 3); } /* NOTE: do write operation for each second u64 half * or force for first one if configured number * is even */ if (reg_half) { if (reg_num == 0) { /* skip partition '0', must write it once at * the end */ part0 = val64; } else { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, tx_fifo_partitions[reg_num]); xge_debug_fifo(XGE_TRACE, "fifo partition_%d at: " "0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT, reg_num, (unsigned long long)(ulong_t) tx_fifo_partitions[reg_num], (unsigned long long)val64); } val64 = 0; } } part0 |= BIT(0); /* to enable the FIFO partition. */ __hal_pio_mem_write32_lower(hldev->pdev, hldev->regh0, (u32)part0, tx_fifo_partitions[0]); xge_os_wmb(); __hal_pio_mem_write32_upper(hldev->pdev, hldev->regh0, (u32)(part0>>32), tx_fifo_partitions[0]); xge_debug_fifo(XGE_TRACE, "fifo partition_0 at: " "0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT, (unsigned long long)(ulong_t) tx_fifo_partitions[0], (unsigned long long) part0); /* * Initialization of Tx_PA_CONFIG register to ignore packet * integrity checking. */ val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, &bar0->tx_pa_cfg); val64 |= XGE_HAL_TX_PA_CFG_IGNORE_FRM_ERR | XGE_HAL_TX_PA_CFG_IGNORE_SNAP_OUI | XGE_HAL_TX_PA_CFG_IGNORE_LLC_CTRL | XGE_HAL_TX_PA_CFG_IGNORE_L2_ERR; xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->tx_pa_cfg); /* * Assign MSI-X vectors */ for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) { xge_list_t *item; xge_hal_channel_t *channel = NULL; if (!hldev->config.fifo.queue[i].configured || !hldev->config.fifo.queue[i].intr_vector || - !hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX) + hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX) continue; /* find channel */ xge_list_for_each(item, &hldev->free_channels) { xge_hal_channel_t *tmp; tmp = xge_container_of(item, xge_hal_channel_t, item); if (tmp->type == XGE_HAL_CHANNEL_TYPE_FIFO && tmp->post_qid == i) { channel = tmp; break; } } if (channel) { xge_hal_channel_msix_set(channel, hldev->config.fifo.queue[i].intr_vector); } } xge_debug_fifo(XGE_TRACE, "%s", "fifo channels initialized"); } #ifdef XGE_HAL_ALIGN_XMIT void __hal_fifo_dtr_align_free_unmap(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh) { xge_hal_fifo_txdl_priv_t *txdl_priv; xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; txdl_priv = __hal_fifo_txdl_priv(txdp); if (txdl_priv->align_dma_addr != 0) { xge_os_dma_unmap(fifo->channel.pdev, txdl_priv->align_dma_handle, txdl_priv->align_dma_addr, fifo->align_size, XGE_OS_DMA_DIR_TODEVICE); txdl_priv->align_dma_addr = 0; } if (txdl_priv->align_vaddr != NULL) { xge_os_dma_free(fifo->channel.pdev, txdl_priv->align_vaddr, fifo->align_size, &txdl_priv->align_dma_acch, &txdl_priv->align_dma_handle); txdl_priv->align_vaddr = NULL; } } xge_hal_status_e __hal_fifo_dtr_align_alloc_map(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh) { xge_hal_fifo_txdl_priv_t *txdl_priv; xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; xge_assert(txdp); txdl_priv = __hal_fifo_txdl_priv(txdp); /* allocate alignment DMA-buffer */ txdl_priv->align_vaddr = (char *)xge_os_dma_malloc(fifo->channel.pdev, fifo->align_size, XGE_OS_DMA_CACHELINE_ALIGNED | XGE_OS_DMA_STREAMING, &txdl_priv->align_dma_handle, &txdl_priv->align_dma_acch); if (txdl_priv->align_vaddr == NULL) { return XGE_HAL_ERR_OUT_OF_MEMORY; } /* map it */ txdl_priv->align_dma_addr = xge_os_dma_map(fifo->channel.pdev, txdl_priv->align_dma_handle, txdl_priv->align_vaddr, fifo->align_size, XGE_OS_DMA_DIR_TODEVICE, XGE_OS_DMA_STREAMING); if (txdl_priv->align_dma_addr == XGE_OS_INVALID_DMA_ADDR) { __hal_fifo_dtr_align_free_unmap(channelh, dtrh); return XGE_HAL_ERR_OUT_OF_MAPPING; } return XGE_HAL_OK; } #endif diff --git a/sys/dev/nxge/xgehal/xgehal-ring.c b/sys/dev/nxge/xgehal/xgehal-ring.c index 444e10c4f06d..061b77384213 100644 --- a/sys/dev/nxge/xgehal/xgehal-ring.c +++ b/sys/dev/nxge/xgehal/xgehal-ring.c @@ -1,661 +1,661 @@ /*- * Copyright (c) 2002-2007 Neterion, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) static ptrdiff_t __hal_ring_item_dma_offset(xge_hal_mempool_h mempoolh, void *item) { int memblock_idx; void *memblock; /* get owner memblock index */ memblock_idx = __hal_ring_block_memblock_idx(item); /* get owner memblock by memblock index */ memblock = __hal_mempool_memblock(mempoolh, memblock_idx); return (char*)item - (char*)memblock; } #endif static dma_addr_t __hal_ring_item_dma_addr(xge_hal_mempool_h mempoolh, void *item, pci_dma_h *dma_handle) { int memblock_idx; void *memblock; xge_hal_mempool_dma_t *memblock_dma_object; ptrdiff_t dma_item_offset; /* get owner memblock index */ memblock_idx = __hal_ring_block_memblock_idx((xge_hal_ring_block_t *) item); /* get owner memblock by memblock index */ memblock = __hal_mempool_memblock((xge_hal_mempool_t *) mempoolh, memblock_idx); /* get memblock DMA object by memblock index */ memblock_dma_object = __hal_mempool_memblock_dma((xge_hal_mempool_t *) mempoolh, memblock_idx); /* calculate offset in the memblock of this item */ dma_item_offset = (char*)item - (char*)memblock; *dma_handle = memblock_dma_object->handle; return memblock_dma_object->addr + dma_item_offset; } static void __hal_ring_rxdblock_link(xge_hal_mempool_h mempoolh, xge_hal_ring_t *ring, int from, int to) { xge_hal_ring_block_t *to_item, *from_item; dma_addr_t to_dma, from_dma; pci_dma_h to_dma_handle, from_dma_handle; /* get "from" RxD block */ from_item = (xge_hal_ring_block_t *) __hal_mempool_item((xge_hal_mempool_t *) mempoolh, from); xge_assert(from_item); /* get "to" RxD block */ to_item = (xge_hal_ring_block_t *) __hal_mempool_item((xge_hal_mempool_t *) mempoolh, to); xge_assert(to_item); /* return address of the beginning of previous RxD block */ to_dma = __hal_ring_item_dma_addr(mempoolh, to_item, &to_dma_handle); /* set next pointer for this RxD block to point on * previous item's DMA start address */ __hal_ring_block_next_pointer_set(from_item, to_dma); /* return "from" RxD block's DMA start address */ from_dma = __hal_ring_item_dma_addr(mempoolh, from_item, &from_dma_handle); #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING) /* we must sync "from" RxD block, so hardware will see it */ xge_os_dma_sync(ring->channel.pdev, from_dma_handle, from_dma + XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET, __hal_ring_item_dma_offset(mempoolh, from_item) + XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET, sizeof(u64), XGE_OS_DMA_DIR_TODEVICE); #endif xge_debug_ring(XGE_TRACE, "block%d:0x"XGE_OS_LLXFMT" => block%d:0x"XGE_OS_LLXFMT, from, (unsigned long long)from_dma, to, (unsigned long long)to_dma); } static xge_hal_status_e __hal_ring_mempool_item_alloc(xge_hal_mempool_h mempoolh, void *memblock, int memblock_index, xge_hal_mempool_dma_t *dma_object, void *item, int index, int is_last, void *userdata) { int i; xge_hal_ring_t *ring = (xge_hal_ring_t *)userdata; xge_assert(item); xge_assert(ring); /* format rxds array */ for (i=ring->rxds_per_block-1; i>=0; i--) { void *rxdblock_priv; xge_hal_ring_rxd_priv_t *rxd_priv; xge_hal_ring_rxd_1_t *rxdp; int reserve_index = index * ring->rxds_per_block + i; int memblock_item_idx; ring->reserved_rxds_arr[reserve_index] = (char *)item + (ring->rxds_per_block - 1 - i) * ring->rxd_size; /* Note: memblock_item_idx is index of the item within * the memblock. For instance, in case of three RxD-blocks * per memblock this value can be 0,1 or 2. */ rxdblock_priv = __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh, memblock_index, item, &memblock_item_idx); rxdp = (xge_hal_ring_rxd_1_t *) ring->reserved_rxds_arr[reserve_index]; rxd_priv = (xge_hal_ring_rxd_priv_t *) (void *) ((char*)rxdblock_priv + ring->rxd_priv_size * i); /* pre-format per-RxD Ring's private */ rxd_priv->dma_offset = (char*)rxdp - (char*)memblock; rxd_priv->dma_addr = dma_object->addr + rxd_priv->dma_offset; rxd_priv->dma_handle = dma_object->handle; #ifdef XGE_DEBUG_ASSERT rxd_priv->dma_object = dma_object; #endif /* pre-format Host_Control */ #if defined(XGE_HAL_USE_5B_MODE) if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) { xge_hal_ring_rxd_5_t *rxdp_5 = (xge_hal_ring_rxd_5_t *)rxdp; #if defined(XGE_OS_PLATFORM_64BIT) xge_assert(memblock_index <= 0xFFFF); xge_assert(i <= 0xFFFF); /* store memblock's index */ rxdp_5->host_control = (u32)memblock_index << 16; /* store index of memblock's private */ rxdp_5->host_control |= (u32)(memblock_item_idx * ring->rxds_per_block + i); #else /* 32-bit case */ rxdp_5->host_control = (u32)rxd_priv; #endif } else { /* 1b and 3b modes */ rxdp->host_control = (u64)(ulong_t)rxd_priv; } #else /* 1b and 3b modes */ rxdp->host_control = (u64)(ulong_t)rxd_priv; #endif } __hal_ring_block_memblock_idx_set((xge_hal_ring_block_t *) item, memblock_index); if (is_last) { /* link last one with first one */ __hal_ring_rxdblock_link(mempoolh, ring, 0, index); } if (index > 0 ) { /* link this RxD block with previous one */ __hal_ring_rxdblock_link(mempoolh, ring, index, index-1); } return XGE_HAL_OK; } xge_hal_status_e __hal_ring_initial_replenish(xge_hal_channel_t *channel, xge_hal_channel_reopen_e reopen) { xge_hal_dtr_h dtr = NULL; while (xge_hal_channel_dtr_count(channel) > 0) { xge_hal_status_e status; status = xge_hal_ring_dtr_reserve(channel, &dtr); xge_assert(status == XGE_HAL_OK); if (channel->dtr_init) { status = channel->dtr_init(channel, dtr, channel->reserve_length, channel->userdata, reopen); if (status != XGE_HAL_OK) { xge_hal_ring_dtr_free(channel, dtr); xge_hal_channel_abort(channel, XGE_HAL_CHANNEL_OC_NORMAL); return status; } } xge_hal_ring_dtr_post(channel, dtr); } return XGE_HAL_OK; } xge_hal_status_e __hal_ring_open(xge_hal_channel_h channelh, xge_hal_channel_attr_t *attr) { xge_hal_status_e status; xge_hal_device_t *hldev; xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh; xge_hal_ring_queue_t *queue; /* Note: at this point we have channel.devh and channel.pdev * pre-set only! */ hldev = (xge_hal_device_t *)ring->channel.devh; ring->config = &hldev->config.ring; queue = &ring->config->queue[attr->post_qid]; ring->indicate_max_pkts = queue->indicate_max_pkts; ring->buffer_mode = queue->buffer_mode; xge_assert(queue->configured); #if defined(XGE_HAL_RX_MULTI_RESERVE) xge_os_spin_lock_init(&ring->channel.reserve_lock, hldev->pdev); #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ) xge_os_spin_lock_init_irq(&ring->channel.reserve_lock, hldev->irqh); #endif #if defined(XGE_HAL_RX_MULTI_POST) xge_os_spin_lock_init(&ring->channel.post_lock, hldev->pdev); #elif defined(XGE_HAL_RX_MULTI_POST_IRQ) xge_os_spin_lock_init_irq(&ring->channel.post_lock, hldev->irqh); #endif ring->rxd_size = XGE_HAL_RING_RXD_SIZEOF(queue->buffer_mode); ring->rxd_priv_size = sizeof(xge_hal_ring_rxd_priv_t) + attr->per_dtr_space; /* how many RxDs can fit into one block. Depends on configured * buffer_mode. */ ring->rxds_per_block = XGE_HAL_RING_RXDS_PER_BLOCK(queue->buffer_mode); /* calculate actual RxD block private size */ ring->rxdblock_priv_size = ring->rxd_priv_size * ring->rxds_per_block; ring->reserved_rxds_arr = (void **) xge_os_malloc(ring->channel.pdev, sizeof(void*) * queue->max * ring->rxds_per_block); if (ring->reserved_rxds_arr == NULL) { __hal_ring_close(channelh); return XGE_HAL_ERR_OUT_OF_MEMORY; } ring->mempool = __hal_mempool_create( hldev->pdev, ring->config->memblock_size, XGE_HAL_RING_RXDBLOCK_SIZE, ring->rxdblock_priv_size, queue->initial, queue->max, __hal_ring_mempool_item_alloc, NULL, /* nothing to free */ ring); if (ring->mempool == NULL) { __hal_ring_close(channelh); return XGE_HAL_ERR_OUT_OF_MEMORY; } status = __hal_channel_initialize(channelh, attr, ring->reserved_rxds_arr, queue->initial * ring->rxds_per_block, queue->max * ring->rxds_per_block, 0 /* no threshold for ring! */); if (status != XGE_HAL_OK) { __hal_ring_close(channelh); return status; } /* sanity check that everything formatted ok */ xge_assert(ring->reserved_rxds_arr[0] == (char *)ring->mempool->items_arr[0] + (ring->rxds_per_block * ring->rxd_size - ring->rxd_size)); /* Note: * Specifying dtr_init callback means two things: * 1) dtrs need to be initialized by ULD at channel-open time; * 2) dtrs need to be posted at channel-open time * (that's what the initial_replenish() below does) * Currently we don't have a case when the 1) is done without the 2). */ if (ring->channel.dtr_init) { if ((status = __hal_ring_initial_replenish ( (xge_hal_channel_t *) channelh, XGE_HAL_CHANNEL_OC_NORMAL) ) != XGE_HAL_OK) { __hal_ring_close(channelh); return status; } } /* initial replenish will increment the counter in its post() routine, * we have to reset it */ ring->channel.usage_cnt = 0; return XGE_HAL_OK; } void __hal_ring_close(xge_hal_channel_h channelh) { xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh; xge_hal_ring_queue_t *queue; #if defined(XGE_HAL_RX_MULTI_RESERVE)||defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)||\ defined(XGE_HAL_RX_MULTI_POST) || defined(XGE_HAL_RX_MULTI_POST_IRQ) xge_hal_device_t *hldev = (xge_hal_device_t *)ring->channel.devh; #endif xge_assert(ring->channel.pdev); queue = &ring->config->queue[ring->channel.post_qid]; if (ring->mempool) { __hal_mempool_destroy(ring->mempool); } if (ring->reserved_rxds_arr) { xge_os_free(ring->channel.pdev, ring->reserved_rxds_arr, sizeof(void*) * queue->max * ring->rxds_per_block); } __hal_channel_terminate(channelh); #if defined(XGE_HAL_RX_MULTI_RESERVE) xge_os_spin_lock_destroy(&ring->channel.reserve_lock, hldev->pdev); #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ) xge_os_spin_lock_destroy_irq(&ring->channel.reserve_lock, hldev->pdev); #endif #if defined(XGE_HAL_RX_MULTI_POST) xge_os_spin_lock_destroy(&ring->channel.post_lock, hldev->pdev); #elif defined(XGE_HAL_RX_MULTI_POST_IRQ) xge_os_spin_lock_destroy_irq(&ring->channel.post_lock, hldev->pdev); #endif } void __hal_ring_prc_enable(xge_hal_channel_h channelh) { xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh; xge_hal_device_t *hldev = (xge_hal_device_t *)ring->channel.devh; xge_hal_pci_bar0_t *bar0; u64 val64; void *first_block; int block_num; xge_hal_ring_queue_t *queue; pci_dma_h dma_handle; xge_assert(ring); xge_assert(ring->channel.pdev); bar0 = (xge_hal_pci_bar0_t *) (void *) ((xge_hal_device_t *)ring->channel.devh)->bar0; queue = &ring->config->queue[ring->channel.post_qid]; xge_assert(queue->buffer_mode == 1 || queue->buffer_mode == 3 || queue->buffer_mode == 5); /* last block in fact becomes first. This is just the way it * is filled up and linked by item_alloc() */ block_num = queue->initial; first_block = __hal_mempool_item(ring->mempool, block_num - 1); val64 = __hal_ring_item_dma_addr(ring->mempool, first_block, &dma_handle); xge_os_pio_mem_write64(ring->channel.pdev, ring->channel.regh0, val64, &bar0->prc_rxd0_n[ring->channel.post_qid]); xge_debug_ring(XGE_TRACE, "ring%d PRC DMA addr 0x"XGE_OS_LLXFMT" initialized", ring->channel.post_qid, (unsigned long long)val64); val64 = xge_os_pio_mem_read64(ring->channel.pdev, ring->channel.regh0, &bar0->prc_ctrl_n[ring->channel.post_qid]); if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC && !queue->rth_en) { val64 |= XGE_HAL_PRC_CTRL_RTH_DISABLE; } val64 |= XGE_HAL_PRC_CTRL_RC_ENABLED; val64 |= vBIT((queue->buffer_mode >> 1),14,2);/* 1,3 or 5 => 0,1 or 2 */ val64 &= ~XGE_HAL_PRC_CTRL_RXD_BACKOFF_INTERVAL(0xFFFFFF); val64 |= XGE_HAL_PRC_CTRL_RXD_BACKOFF_INTERVAL( (hldev->config.pci_freq_mherz * queue->backoff_interval_us)); /* Beware: no snoop by the bridge if (no_snoop_bits) */ val64 |= XGE_HAL_PRC_CTRL_NO_SNOOP(queue->no_snoop_bits); /* Herc: always use group_reads */ if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) val64 |= XGE_HAL_PRC_CTRL_GROUP_READS; if (hldev->config.bimodal_interrupts) if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) val64 |= XGE_HAL_PRC_CTRL_BIMODAL_INTERRUPT; xge_os_pio_mem_write64(ring->channel.pdev, ring->channel.regh0, val64, &bar0->prc_ctrl_n[ring->channel.post_qid]); /* Configure Receive Protocol Assist */ val64 = xge_os_pio_mem_read64(ring->channel.pdev, ring->channel.regh0, &bar0->rx_pa_cfg); val64 |= XGE_HAL_RX_PA_CFG_SCATTER_MODE(ring->config->scatter_mode); val64 |= (XGE_HAL_RX_PA_CFG_IGNORE_SNAP_OUI | XGE_HAL_RX_PA_CFG_IGNORE_LLC_CTRL); /* Clean STRIP_VLAN_TAG bit and set as config from upper layer */ val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1); val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(ring->config->strip_vlan_tag); xge_os_pio_mem_write64(ring->channel.pdev, ring->channel.regh0, val64, &bar0->rx_pa_cfg); xge_debug_ring(XGE_TRACE, "ring%d enabled in buffer_mode %d", ring->channel.post_qid, queue->buffer_mode); } void __hal_ring_prc_disable(xge_hal_channel_h channelh) { xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh; xge_hal_pci_bar0_t *bar0; u64 val64; xge_assert(ring); xge_assert(ring->channel.pdev); bar0 = (xge_hal_pci_bar0_t *) (void *) ((xge_hal_device_t *)ring->channel.devh)->bar0; val64 = xge_os_pio_mem_read64(ring->channel.pdev, ring->channel.regh0, &bar0->prc_ctrl_n[ring->channel.post_qid]); val64 &= ~((u64) XGE_HAL_PRC_CTRL_RC_ENABLED); xge_os_pio_mem_write64(ring->channel.pdev, ring->channel.regh0, val64, &bar0->prc_ctrl_n[ring->channel.post_qid]); } void __hal_ring_hw_initialize(xge_hal_device_h devh) { xge_hal_device_t *hldev = (xge_hal_device_t *)devh; xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void *)hldev->bar0; u64 val64; int i, j; /* Rx DMA intialization. */ val64 = 0; for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { if (!hldev->config.ring.queue[i].configured) continue; val64 |= vBIT(hldev->config.ring.queue[i].priority, (5 + (i * 8)), 3); } xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->rx_queue_priority); xge_debug_ring(XGE_TRACE, "Rings priority configured to 0x"XGE_OS_LLXFMT, (unsigned long long)val64); /* Configuring ring queues according to per-ring configuration */ val64 = 0; for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { if (!hldev->config.ring.queue[i].configured) continue; val64 |= vBIT(hldev->config.ring.queue[i].dram_size_mb,(i*8),8); } xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->rx_queue_cfg); xge_debug_ring(XGE_TRACE, "DRAM configured to 0x"XGE_OS_LLXFMT, (unsigned long long)val64); if (!hldev->config.rts_qos_en && !hldev->config.rts_port_en && !hldev->config.rts_mac_en) { /* * Activate default (QoS-based) Rx steering */ val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, &bar0->rts_qos_steering); for (j = 0; j < 8 /* QoS max */; j++) { for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { if (!hldev->config.ring.queue[i].configured) continue; if (!hldev->config.ring.queue[i].rth_en) val64 |= (BIT(i) >> (j*8)); } } xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->rts_qos_steering); xge_debug_ring(XGE_TRACE, "QoS steering configured to 0x"XGE_OS_LLXFMT, (unsigned long long)val64); } /* Note: If a queue does not exist, it should be assigned a maximum * length of zero. Otherwise, packet loss could occur. * P. 4-4 User guide. * * All configured rings will be properly set at device open time * by utilizing device_mtu_set() API call. */ for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { if (hldev->config.ring.queue[i].configured) continue; xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0ULL, &bar0->rts_frm_len_n[i]); } #ifdef XGE_HAL_HERC_EMULATION val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, ((u8 *)bar0 + 0x2e60)); /* mc_rldram_mrs_herc */ val64 |= 0x0000000000010000; xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, ((u8 *)bar0 + 0x2e60)); val64 |= 0x003a000000000000; xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, ((u8 *)bar0 + 0x2e40)); /* mc_rldram_ref_herc */ xge_os_mdelay(2000); #endif /* now enabling MC-RLDRAM after setting MC_QUEUE sizes */ val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, &bar0->mc_rldram_mrs); val64 |= XGE_HAL_MC_RLDRAM_QUEUE_SIZE_ENABLE | XGE_HAL_MC_RLDRAM_MRS_ENABLE; __hal_pio_mem_write32_upper(hldev->pdev, hldev->regh0, (u32)(val64>>32), &bar0->mc_rldram_mrs); xge_os_wmb(); __hal_pio_mem_write32_lower(hldev->pdev, hldev->regh0, (u32)val64, &bar0->mc_rldram_mrs); /* RLDRAM initialization procedure require 500us to complete */ xge_os_mdelay(1); /* Temporary fixes for Herc RLDRAM */ if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) { val64 = XGE_HAL_MC_RLDRAM_SET_REF_PERIOD(0x0279); xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->mc_rldram_ref_per_herc); val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, &bar0->mc_rldram_mrs_herc); xge_debug_ring(XGE_TRACE, "default mc_rldram_mrs_herc 0x"XGE_OS_LLXFMT, (unsigned long long)val64); val64 = 0x0003570003010300ULL; xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, &bar0->mc_rldram_mrs_herc); xge_os_mdelay(1); } /* * Assign MSI-X vectors */ for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { xge_list_t *item; xge_hal_channel_t *channel = NULL; if (!hldev->config.ring.queue[i].configured || !hldev->config.ring.queue[i].intr_vector || - !hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX) + hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX) continue; /* find channel */ xge_list_for_each(item, &hldev->free_channels) { xge_hal_channel_t *tmp; tmp = xge_container_of(item, xge_hal_channel_t, item); if (tmp->type == XGE_HAL_CHANNEL_TYPE_RING && tmp->post_qid == i) { channel = tmp; break; } } if (channel) { xge_hal_channel_msix_set(channel, hldev->config.ring.queue[i].intr_vector); } } xge_debug_ring(XGE_TRACE, "%s", "ring channels initialized"); } void __hal_ring_mtu_set(xge_hal_device_h devh, int new_frmlen) { int i; xge_hal_device_t *hldev = (xge_hal_device_t *)devh; xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void *)hldev->bar0; for (i = 0; i < XGE_HAL_MAX_RING_NUM; i++) { if (!hldev->config.ring.queue[i].configured) continue; if (hldev->config.ring.queue[i].max_frm_len != XGE_HAL_RING_USE_MTU) { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, XGE_HAL_MAC_RTS_FRM_LEN_SET( hldev->config.ring.queue[i].max_frm_len), &bar0->rts_frm_len_n[i]); } else { xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, XGE_HAL_MAC_RTS_FRM_LEN_SET(new_frmlen), &bar0->rts_frm_len_n[i]); } } xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, XGE_HAL_RMAC_MAX_PYLD_LEN(new_frmlen), &bar0->rmac_max_pyld_len); }