Index: head/sys/mips/cavium/octe/ethernet-defines.h =================================================================== --- head/sys/mips/cavium/octe/ethernet-defines.h (revision 343968) +++ head/sys/mips/cavium/octe/ethernet-defines.h (revision 343969) @@ -1,53 +1,53 @@ /************************************************************************* SPDX-License-Identifier: BSD-3-Clause Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Cavium Networks nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. *************************************************************************/ /* $FreeBSD$ */ /* * A few defines are used to control the operation of this driver: * CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS * This kernel config option allows the user to control the number of * packet and work queue buffers allocated by the driver. If this is zero, * the driver uses the default from below. */ -#define INTERRUPT_LIMIT 10000 /* Max interrupts per second per core */ +#define INTERRUPT_LIMIT 1000 /* Max interrupts per second per core */ /*#define INTERRUPT_LIMIT 0 *//* Don't limit the number of interrupts */ #define USE_RED 1 /* Enable Random Early Dropping under load */ #define USE_10MBPS_PREAMBLE_WORKAROUND 1 /* Allow SW based preamble removal at 10Mbps to workaround PHYs giving us bad preambles */ #define DONT_WRITEBACK(x) (x) /* Use this to have all FPA frees also tell the L2 not to write data to memory */ /*#define DONT_WRITEBACK(x) 0 *//* Use this to not have FPA frees control L2 */ -#define MAX_RX_PACKETS 120 /* Maximum number of packets to process per interrupt. */ +#define MAX_RX_PACKETS 1024 /* Maximum number of packets to process per interrupt. */ #define MAX_OUT_QUEUE_DEPTH 1000 #define FAU_NUM_PACKET_BUFFERS_TO_FREE (CVMX_FAU_REG_END - sizeof(uint32_t)) #define TOTAL_NUMBER_OF_PORTS (CVMX_PIP_NUM_INPUT_PORTS+1) Index: head/sys/mips/cavium/octe/ethernet-rx.c =================================================================== --- head/sys/mips/cavium/octe/ethernet-rx.c (revision 343968) +++ head/sys/mips/cavium/octe/ethernet-rx.c (revision 343969) @@ -1,393 +1,388 @@ /************************************************************************* SPDX-License-Identifier: BSD-3-Clause Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Cavium Networks nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. *************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "wrapper-cvmx-includes.h" #include "ethernet-headers.h" extern int pow_receive_group; extern struct ifnet *cvm_oct_device[]; static struct task cvm_oct_task; static struct taskqueue *cvm_oct_taskq; -static int cvm_oct_rx_active; - /** * Interrupt handler. The interrupt occurs whenever the POW * transitions from 0->1 packets in our group. * * @param cpl * @param dev_id * @param regs * @return */ int cvm_oct_do_interrupt(void *dev_id) { /* Acknowledge the interrupt */ if (INTERRUPT_LIMIT) cvmx_write_csr(CVMX_POW_WQ_INT, 1<word2.snoip.err_code == 10) && (work->word1.s.len <= 64)) { /* Ignore length errors on min size packets. Some equipment incorrectly pads packets to 64+4FCS instead of 60+4FCS. Note these packets still get counted as frame errors. */ } else if (USE_10MBPS_PREAMBLE_WORKAROUND && ((work->word2.snoip.err_code == 5) || (work->word2.snoip.err_code == 7))) { /* We received a packet with either an alignment error or a FCS error. This may be signalling that we are running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK} off. If this is the case we need to parse the packet to determine if we can remove a non spec preamble and generate a correct packet */ int interface = cvmx_helper_get_interface_num(work->word1.cn38xx.ipprt); int index = cvmx_helper_get_interface_index_num(work->word1.cn38xx.ipprt); cvmx_gmxx_rxx_frm_ctl_t gmxx_rxx_frm_ctl; gmxx_rxx_frm_ctl.u64 = cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { uint8_t *ptr = cvmx_phys_to_ptr(work->packet_ptr.s.addr); int i = 0; while (i < work->word1.s.len-1) { if (*ptr != 0x55) break; ptr++; i++; } if (*ptr == 0xd5) { /* DEBUGPRINT("Port %d received 0xd5 preamble\n", work->word1.cn38xx.ipprt); */ work->packet_ptr.s.addr += i+1; work->word1.s.len -= i+5; } else if ((*ptr & 0xf) == 0xd) { /* DEBUGPRINT("Port %d received 0x?d preamble\n", work->word1.cn38xx.ipprt); */ work->packet_ptr.s.addr += i; work->word1.s.len -= i+4; for (i = 0; i < work->word1.s.len; i++) { *ptr = ((*ptr&0xf0)>>4) | ((*(ptr+1)&0xf)<<4); ptr++; } } else { DEBUGPRINT("Port %d unknown preamble, packet dropped\n", work->word1.cn38xx.ipprt); /* cvmx_helper_dump_packet(work); */ cvm_oct_free_work(work); return 1; } } } else { DEBUGPRINT("Port %d receive error code %d, packet dropped\n", work->word1.cn38xx.ipprt, work->word2.snoip.err_code); cvm_oct_free_work(work); return 1; } return 0; } /** * Tasklet function that is scheduled on a core when an interrupt occurs. * * @param unused */ void cvm_oct_tasklet_rx(void *context, int pending) { int coreid; uint64_t old_group_mask; int rx_count = 0; int number_to_free; int num_freed; int packet_not_copied; - sched_pin(); coreid = cvmx_get_core_num(); /* Prefetch cvm_oct_device since we know we need it soon */ CVMX_PREFETCH(cvm_oct_device, 0); /* Only allow work for our group (and preserve priorities) */ old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), (old_group_mask & ~0xFFFFull) | 1<word2.s.bufs == 1; if ((mbuf_in_hw)) { m = *(struct mbuf **)(cvm_oct_get_buffer_ptr(work->packet_ptr) - sizeof(void *)); CVMX_PREFETCH(m, offsetof(struct mbuf, m_data)); CVMX_PREFETCH(m, offsetof(struct mbuf, m_pkthdr)); } CVMX_PREFETCH(cvm_oct_device[work->word1.cn38xx.ipprt], 0); //CVMX_PREFETCH(m, 0); rx_count++; /* Immediately throw away all packets with receive errors */ if ((work->word2.snoip.rcv_error)) { if (cvm_oct_check_rcv_error(work)) continue; } /* We can only use the zero copy path if mbufs are in the FPA pool and the packet fits in a single buffer */ if ((mbuf_in_hw)) { CVMX_PREFETCH(m->m_data, 0); m->m_pkthdr.len = m->m_len = work->word1.s.len; packet_not_copied = 1; /* * Adjust the data pointer based on the offset * of the packet within the buffer. */ m->m_data += (work->packet_ptr.s.back << 7) + (work->packet_ptr.s.addr & 0x7f); } else { /* We have to copy the packet. First allocate an mbuf for it */ MGETHDR(m, M_NOWAIT, MT_DATA); if (m == NULL) { DEBUGPRINT("Port %d failed to allocate mbuf, packet dropped\n", work->word1.cn38xx.ipprt); cvm_oct_free_work(work); continue; } /* Check if we've received a packet that was entirely stored in the work entry. This is untested */ if ((work->word2.s.bufs == 0)) { uint8_t *ptr = work->packet_data; if (cvmx_likely(!work->word2.s.not_IP)) { /* The beginning of the packet moves for IP packets */ if (work->word2.s.is_v6) ptr += 2; else ptr += 6; } panic("%s: not yet implemented; copy in small packet.", __func__); /* No packet buffers to free */ } else { int segments = work->word2.s.bufs; cvmx_buf_ptr_t segment_ptr = work->packet_ptr; int len = work->word1.s.len; while (segments--) { cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8); /* Octeon Errata PKI-100: The segment size is wrong. Until it is fixed, calculate the segment size based on the packet pool buffer size. When it is fixed, the following line should be replaced with this one: int segment_size = segment_ptr.s.size; */ int segment_size = CVMX_FPA_PACKET_POOL_SIZE - (segment_ptr.s.addr - (((segment_ptr.s.addr >> 7) - segment_ptr.s.back) << 7)); /* Don't copy more than what is left in the packet */ if (segment_size > len) segment_size = len; /* Copy the data into the packet */ panic("%s: not yet implemented; copy in packet segments.", __func__); #if 0 memcpy(m_put(m, segment_size), cvmx_phys_to_ptr(segment_ptr.s.addr), segment_size); #endif /* Reduce the amount of bytes left to copy */ len -= segment_size; segment_ptr = next_ptr; } } packet_not_copied = 0; } if (((work->word1.cn38xx.ipprt < TOTAL_NUMBER_OF_PORTS) && cvm_oct_device[work->word1.cn38xx.ipprt])) { struct ifnet *ifp = cvm_oct_device[work->word1.cn38xx.ipprt]; /* Only accept packets for devices that are currently up */ if ((ifp->if_flags & IFF_UP)) { m->m_pkthdr.rcvif = ifp; if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { if ((work->word2.s.not_IP || work->word2.s.IP_exc || work->word2.s.L4_error)) m->m_pkthdr.csum_flags = 0; /* XXX */ else { m->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xffff; } } else { m->m_pkthdr.csum_flags = 0; /* XXX */ } if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); (*ifp->if_input)(ifp, m); } else { /* Drop any packet received for a device that isn't up */ /* DEBUGPRINT("%s: Device not up, packet dropped\n", if_name(ifp)); */ m_freem(m); } } else { /* Drop any packet received for a device that doesn't exist */ DEBUGPRINT("Port %d not controlled by FreeBSD, packet dropped\n", work->word1.cn38xx.ipprt); m_freem(m); } /* Check to see if the mbuf and work share the same packet buffer */ if ((packet_not_copied)) { /* This buffer needs to be replaced, increment the number of buffers we need to free by one */ cvmx_fau_atomic_add32( FAU_NUM_PACKET_BUFFERS_TO_FREE, 1); cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, DONT_WRITEBACK(1)); } else cvm_oct_free_work(work); } /* * If we hit our limit, schedule another task while we clean up. */ if (INTERRUPT_LIMIT != 0 && rx_count == MAX_RX_PACKETS) { taskqueue_enqueue(cvm_oct_taskq, &cvm_oct_task); - } else { - /* - * No more packets, all done. - */ - if (!atomic_cmpset_int(&cvm_oct_rx_active, 1, 0)) - panic("%s: inconsistent rx active state.", __func__); } /* Restore the original POW group mask */ cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask); /* Refill the packet buffer pool */ number_to_free = cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); if (number_to_free > 0) { cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, -number_to_free); num_freed = cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, number_to_free); if (num_freed != number_to_free) { cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, number_to_free - num_freed); } } - sched_unpin(); } void cvm_oct_rx_initialize(void) { + int cpu; TASK_INIT(&cvm_oct_task, 0, cvm_oct_tasklet_rx, NULL); cvm_oct_taskq = taskqueue_create_fast("oct_rx", M_NOWAIT, taskqueue_thread_enqueue, &cvm_oct_taskq); - taskqueue_start_threads(&cvm_oct_taskq, min(mp_ncpus, MAXCPU), - PI_NET, "octe taskq"); + + CPU_FOREACH(cpu) { + cpuset_t cpu_mask; + CPU_SETOF(cpu, &cpu_mask); + taskqueue_start_threads_cpuset(&cvm_oct_taskq, 1, PI_NET, + &cpu_mask, "octe taskq"); + } } void cvm_oct_rx_shutdown(void) { panic("%s: not yet implemented.", __func__); } Index: head/sys/mips/cavium/octe/ethernet.c =================================================================== --- head/sys/mips/cavium/octe/ethernet.c (revision 343968) +++ head/sys/mips/cavium/octe/ethernet.c (revision 343969) @@ -1,510 +1,510 @@ /************************************************************************* SPDX-License-Identifier: BSD-3-Clause Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Cavium Networks nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. This Software, including technical data, may be subject to U.S. export control laws, including the U.S. Export Administration Act and its associated regulations, and may be subject to export or import regulations in other countries. TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. *************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "wrapper-cvmx-includes.h" #include "ethernet-headers.h" #include "octebusvar.h" /* * XXX/juli * Convert 0444 to tunables, 0644 to sysctls. */ #if defined(CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS) && CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS int num_packet_buffers = CONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS; #else -int num_packet_buffers = 1024; +int num_packet_buffers = 2048; #endif TUNABLE_INT("hw.octe.num_packet_buffers", &num_packet_buffers); /* "\t\tNumber of packet buffers to allocate and store in the\n" "\t\tFPA. By default, 1024 packet buffers are used unless\n" "\t\tCONFIG_CAVIUM_OCTEON_NUM_PACKET_BUFFERS is defined." */ int pow_receive_group = 15; TUNABLE_INT("hw.octe.pow_receive_group", &pow_receive_group); /* "\t\tPOW group to receive packets from. All ethernet hardware\n" "\t\twill be configured to send incomming packets to this POW\n" "\t\tgroup. Also any other software can submit packets to this\n" "\t\tgroup for the kernel to process." */ /** * Periodic timer to check auto negotiation */ static struct callout cvm_oct_poll_timer; /** * Array of every ethernet device owned by this driver indexed by * the ipd input port number. */ struct ifnet *cvm_oct_device[TOTAL_NUMBER_OF_PORTS]; /** * Task to handle link status changes. */ static struct taskqueue *cvm_oct_link_taskq; /* * Number of buffers in output buffer pool. */ static int cvm_oct_num_output_buffers; /** * Function to update link status. */ static void cvm_oct_update_link(void *context, int pending) { cvm_oct_private_t *priv = (cvm_oct_private_t *)context; struct ifnet *ifp = priv->ifp; cvmx_helper_link_info_t link_info; link_info.u64 = priv->link_info; if (link_info.s.link_up) { if_link_state_change(ifp, LINK_STATE_UP); DEBUGPRINT("%s: %u Mbps %s duplex, port %2d, queue %2d\n", if_name(ifp), link_info.s.speed, (link_info.s.full_duplex) ? "Full" : "Half", priv->port, priv->queue); } else { if_link_state_change(ifp, LINK_STATE_DOWN); DEBUGPRINT("%s: Link down\n", if_name(ifp)); } priv->need_link_update = 0; } /** * Periodic timer tick for slow management operations * * @param arg Device to check */ static void cvm_do_timer(void *arg) { static int port; static int updated; if (port < CVMX_PIP_NUM_INPUT_PORTS) { if (cvm_oct_device[port]) { int queues_per_port; int qos; cvm_oct_private_t *priv = (cvm_oct_private_t *)cvm_oct_device[port]->if_softc; cvm_oct_common_poll(priv->ifp); if (priv->need_link_update) { updated++; taskqueue_enqueue(cvm_oct_link_taskq, &priv->link_task); } queues_per_port = cvmx_pko_get_num_queues(port); /* Drain any pending packets in the free list */ for (qos = 0; qos < queues_per_port; qos++) { if (_IF_QLEN(&priv->tx_free_queue[qos]) > 0) { IF_LOCK(&priv->tx_free_queue[qos]); while (_IF_QLEN(&priv->tx_free_queue[qos]) > cvmx_fau_fetch_and_add32(priv->fau+qos*4, 0)) { struct mbuf *m; _IF_DEQUEUE(&priv->tx_free_queue[qos], m); m_freem(m); } IF_UNLOCK(&priv->tx_free_queue[qos]); /* * XXX locking! */ priv->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; } } } port++; /* Poll the next port in a 50th of a second. This spreads the polling of ports out a little bit */ callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL); } else { port = 0; /* If any updates were made in this run, continue iterating at * 1/50th of a second, so that if a link has merely gone down * temporarily (e.g. because of interface reinitialization) it * will not be forced to stay down for an entire second. */ if (updated > 0) { updated = 0; callout_reset(&cvm_oct_poll_timer, hz / 50, cvm_do_timer, NULL); } else { /* All ports have been polled. Start the next iteration through the ports in one second */ callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL); } } } /** * Configure common hardware for all interfaces */ static void cvm_oct_configure_common_hw(device_t bus) { struct octebus_softc *sc; int pko_queues; int error; int rid; sc = device_get_softc(bus); /* Setup the FPA */ cvmx_fpa_enable(); cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers); cvm_oct_mem_fill_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers); if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL) { /* * If the FPA uses different pools for output buffers and * packets, size the output buffer pool based on the number * of PKO queues. */ if (OCTEON_IS_MODEL(OCTEON_CN38XX)) pko_queues = 128; else if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) pko_queues = 32; else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) pko_queues = 32; else pko_queues = 256; cvm_oct_num_output_buffers = 4 * pko_queues; cvm_oct_mem_fill_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, cvm_oct_num_output_buffers); } if (USE_RED) cvmx_helper_setup_red(num_packet_buffers/4, num_packet_buffers/8); /* Enable the MII interface */ if (cvmx_sysinfo_get()->board_type != CVMX_BOARD_TYPE_SIM) cvmx_write_csr(CVMX_SMI_EN, 1); /* Register an IRQ hander for to receive POW interrupts */ rid = 0; sc->sc_rx_irq = bus_alloc_resource(bus, SYS_RES_IRQ, &rid, OCTEON_IRQ_WORKQ0 + pow_receive_group, OCTEON_IRQ_WORKQ0 + pow_receive_group, 1, RF_ACTIVE); if (sc->sc_rx_irq == NULL) { device_printf(bus, "could not allocate workq irq"); return; } error = bus_setup_intr(bus, sc->sc_rx_irq, INTR_TYPE_NET | INTR_MPSAFE, cvm_oct_do_interrupt, NULL, cvm_oct_device, &sc->sc_rx_intr_cookie); if (error != 0) { device_printf(bus, "could not setup workq irq"); return; } #ifdef SMP { cvmx_ciu_intx0_t en; int core; CPU_FOREACH(core) { if (core == PCPU_GET(cpuid)) continue; en.u64 = cvmx_read_csr(CVMX_CIU_INTX_EN0(core*2)); en.s.workq |= (1<word2.s.bufs; cvmx_buf_ptr_t segment_ptr = work->packet_ptr; while (segments--) { cvmx_buf_ptr_t next_ptr = *(cvmx_buf_ptr_t *)cvmx_phys_to_ptr(segment_ptr.s.addr-8); if (__predict_false(!segment_ptr.s.i)) cvmx_fpa_free(cvm_oct_get_buffer_ptr(segment_ptr), segment_ptr.s.pool, DONT_WRITEBACK(CVMX_FPA_PACKET_POOL_SIZE/128)); segment_ptr = next_ptr; } cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, DONT_WRITEBACK(1)); return 0; } /** * Module/ driver initialization. Creates the linux network * devices. * * @return Zero on success */ int cvm_oct_init_module(device_t bus) { device_t dev; int ifnum; int num_interfaces; int interface; int fau = FAU_NUM_PACKET_BUFFERS_TO_FREE; int qos; cvm_oct_rx_initialize(); cvm_oct_configure_common_hw(bus); cvmx_helper_initialize_packet_io_global(); /* Change the input group for all ports before input is enabled */ num_interfaces = cvmx_helper_get_number_of_interfaces(); for (interface = 0; interface < num_interfaces; interface++) { int num_ports = cvmx_helper_ports_on_interface(interface); int port; for (port = 0; port < num_ports; port++) { cvmx_pip_prt_tagx_t pip_prt_tagx; int pkind = cvmx_helper_get_ipd_port(interface, port); pip_prt_tagx.u64 = cvmx_read_csr(CVMX_PIP_PRT_TAGX(pkind)); pip_prt_tagx.s.grp = pow_receive_group; cvmx_write_csr(CVMX_PIP_PRT_TAGX(pkind), pip_prt_tagx.u64); } } cvmx_helper_ipd_and_packet_input_enable(); memset(cvm_oct_device, 0, sizeof(cvm_oct_device)); cvm_oct_link_taskq = taskqueue_create("octe link", M_NOWAIT, taskqueue_thread_enqueue, &cvm_oct_link_taskq); taskqueue_start_threads(&cvm_oct_link_taskq, 1, PI_NET, "octe link taskq"); /* Initialize the FAU used for counting packet buffers that need to be freed */ cvmx_fau_atomic_write32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); ifnum = 0; num_interfaces = cvmx_helper_get_number_of_interfaces(); for (interface = 0; interface < num_interfaces; interface++) { cvmx_helper_interface_mode_t imode = cvmx_helper_interface_get_mode(interface); int num_ports = cvmx_helper_ports_on_interface(interface); int port; for (port = cvmx_helper_get_ipd_port(interface, 0); port < cvmx_helper_get_ipd_port(interface, num_ports); ifnum++, port++) { cvm_oct_private_t *priv; struct ifnet *ifp; dev = BUS_ADD_CHILD(bus, 0, "octe", ifnum); if (dev != NULL) ifp = if_alloc(IFT_ETHER); if (dev == NULL || ifp == NULL) { printf("Failed to allocate ethernet device for interface %d port %d\n", interface, port); continue; } /* Initialize the device private structure. */ device_probe(dev); priv = device_get_softc(dev); priv->dev = dev; priv->ifp = ifp; priv->imode = imode; priv->port = port; priv->queue = cvmx_pko_get_base_queue(priv->port); priv->fau = fau - cvmx_pko_get_num_queues(port) * 4; for (qos = 0; qos < cvmx_pko_get_num_queues(port); qos++) cvmx_fau_atomic_write32(priv->fau+qos*4, 0); TASK_INIT(&priv->link_task, 0, cvm_oct_update_link, priv); switch (priv->imode) { /* These types don't support ports to IPD/PKO */ case CVMX_HELPER_INTERFACE_MODE_DISABLED: case CVMX_HELPER_INTERFACE_MODE_PCIE: case CVMX_HELPER_INTERFACE_MODE_PICMG: break; case CVMX_HELPER_INTERFACE_MODE_NPI: priv->init = cvm_oct_common_init; priv->uninit = cvm_oct_common_uninit; device_set_desc(dev, "Cavium Octeon NPI Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_XAUI: priv->init = cvm_oct_xaui_init; priv->uninit = cvm_oct_common_uninit; device_set_desc(dev, "Cavium Octeon XAUI Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_LOOP: priv->init = cvm_oct_common_init; priv->uninit = cvm_oct_common_uninit; device_set_desc(dev, "Cavium Octeon LOOP Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_SGMII: priv->init = cvm_oct_sgmii_init; priv->uninit = cvm_oct_common_uninit; device_set_desc(dev, "Cavium Octeon SGMII Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_SPI: priv->init = cvm_oct_spi_init; priv->uninit = cvm_oct_spi_uninit; device_set_desc(dev, "Cavium Octeon SPI Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_RGMII: priv->init = cvm_oct_rgmii_init; priv->uninit = cvm_oct_rgmii_uninit; device_set_desc(dev, "Cavium Octeon RGMII Ethernet"); break; case CVMX_HELPER_INTERFACE_MODE_GMII: priv->init = cvm_oct_rgmii_init; priv->uninit = cvm_oct_rgmii_uninit; device_set_desc(dev, "Cavium Octeon GMII Ethernet"); break; } ifp->if_softc = priv; if (!priv->init) { printf("octe%d: unsupported device type interface %d, port %d\n", ifnum, interface, priv->port); if_free(ifp); } else if (priv->init(ifp) != 0) { printf("octe%d: failed to register device for interface %d, port %d\n", ifnum, interface, priv->port); if_free(ifp); } else { cvm_oct_device[priv->port] = ifp; fau -= cvmx_pko_get_num_queues(priv->port) * sizeof(uint32_t); } } } if (INTERRUPT_LIMIT) { /* Set the POW timer rate to give an interrupt at most INTERRUPT_LIMIT times per second */ - cvmx_write_csr(CVMX_POW_WQ_INT_PC, cvmx_clock_get_rate(CVMX_CLOCK_CORE)/(INTERRUPT_LIMIT*16*256)<<8); + cvmx_write_csr(CVMX_POW_WQ_INT_PC, cvmx_clock_get_rate(CVMX_CLOCK_CORE)/((INTERRUPT_LIMIT+1)*16*256)<<8); /* Enable POW timer interrupt. It will count when there are packets available */ cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1ful<<24); } else { /* Enable POW interrupt when our port has at least one packet */ cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0x1001); } callout_init(&cvm_oct_poll_timer, 1); callout_reset(&cvm_oct_poll_timer, hz, cvm_do_timer, NULL); return 0; } /** * Module / driver shutdown * * @return Zero on success */ void cvm_oct_cleanup_module(device_t bus) { int port; struct octebus_softc *sc = device_get_softc(bus); /* Disable POW interrupt */ cvmx_write_csr(CVMX_POW_WQ_INT_THRX(pow_receive_group), 0); /* Free the interrupt handler */ bus_teardown_intr(bus, sc->sc_rx_irq, sc->sc_rx_intr_cookie); callout_stop(&cvm_oct_poll_timer); cvm_oct_rx_shutdown(); cvmx_helper_shutdown_packet_io_global(); /* Free the ethernet devices */ for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) { if (cvm_oct_device[port]) { cvm_oct_tx_shutdown(cvm_oct_device[port]); #if 0 unregister_netdev(cvm_oct_device[port]); kfree(cvm_oct_device[port]); #else panic("%s: need to detach and free interface.", __func__); #endif cvm_oct_device[port] = NULL; } } /* Free the HW pools */ cvm_oct_mem_empty_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers); cvm_oct_mem_empty_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers); if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL) cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, cvm_oct_num_output_buffers); /* Disable FPA, all buffers are free, not done by helper shutdown. */ cvmx_fpa_disable(); }