diff --git a/sys/dev/liquidio/base/lio_console.c b/sys/dev/liquidio/base/lio_console.c index c36fb5bf2247..397c2f963800 100644 --- a/sys/dev/liquidio/base/lio_console.c +++ b/sys/dev/liquidio/base/lio_console.c @@ -1,941 +1,940 @@ /* * BSD LICENSE * * Copyright(c) 2017 Cavium, Inc.. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Cavium, Inc. nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD$*/ /* * @file lio_console.c */ #include "lio_bsd.h" #include "lio_common.h" #include "lio_droq.h" #include "lio_iq.h" #include "lio_response_manager.h" #include "lio_device.h" #include "lio_image.h" #include "lio_mem_ops.h" #include "lio_main.h" static void lio_get_uboot_version(struct octeon_device *oct); static void lio_remote_lock(void); static void lio_remote_unlock(void); static uint64_t cvmx_bootmem_phy_named_block_find(struct octeon_device *oct, const char *name, uint32_t flags); static int lio_console_read(struct octeon_device *oct, uint32_t console_num, char *buffer, uint32_t buf_size); #define CAST_ULL(v) ((unsigned long long)(v)) #define LIO_BOOTLOADER_PCI_READ_BUFFER_DATA_ADDR 0x0006c008 #define LIO_BOOTLOADER_PCI_READ_BUFFER_LEN_ADDR 0x0006c004 #define LIO_BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR 0x0006c000 #define LIO_BOOTLOADER_PCI_READ_DESC_ADDR 0x0006c100 #define LIO_BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN 248 #define LIO_PCI_IO_BUF_OWNER_OCTEON 0x00000001 #define LIO_PCI_IO_BUF_OWNER_HOST 0x00000002 #define LIO_PCI_CONSOLE_BLOCK_NAME "__pci_console" #define LIO_CONSOLE_POLL_INTERVAL_MS 100 /* 10 times per second */ /* * First three members of cvmx_bootmem_desc are left in original positions * for backwards compatibility. Assumes big endian target */ struct cvmx_bootmem_desc { /* lock to control access to list */ uint32_t lock; /* flags for indicating various conditions */ uint32_t flags; uint64_t head_addr; /* incremented changed when incompatible changes made */ uint32_t major_version; /* * incremented changed when compatible changes made, reset to zero * when major incremented */ uint32_t minor_version; uint64_t app_data_addr; uint64_t app_data_size; /* number of elements in named blocks array */ uint32_t nb_num_blocks; /* length of name array in bootmem blocks */ uint32_t named_block_name_len; /* address of named memory block descriptors */ uint64_t named_block_array_addr; }; /* * Structure that defines a single console. * * Note: when read_index == write_index, the buffer is empty. The actual usable * size of each console is console_buf_size -1; */ struct lio_pci_console { uint64_t input_base_addr; uint32_t input_read_index; uint32_t input_write_index; uint64_t output_base_addr; uint32_t output_read_index; uint32_t output_write_index; uint32_t lock; uint32_t buf_size; }; /* * This is the main container structure that contains all the information * about all PCI consoles. The address of this structure is passed to * various routines that operation on PCI consoles. */ struct lio_pci_console_desc { uint32_t major_version; uint32_t minor_version; uint32_t lock; uint32_t flags; uint32_t num_consoles; uint32_t pad; /* must be 64 bit aligned here... */ /* Array of addresses of octeon_pci_console structures */ uint64_t console_addr_array[1]; /* Implicit storage for console_addr_array */ }; /* * This macro returns the size of a member of a structure. Logically it is * the same as "sizeof(s::field)" in C++, but C lacks the "::" operator. */ #define SIZEOF_FIELD(s, field) sizeof(((s *)NULL)->field) /* * This function is the implementation of the get macros defined * for individual structure members. The argument are generated * by the macros inorder to read only the needed memory. * * @param oct Pointer to current octeon device * @param base 64bit physical address of the complete structure * @param offset Offset from the beginning of the structure to the member being * accessed. * @param size Size of the structure member. * * @return Value of the structure member promoted into a uint64_t. */ static inline uint64_t __cvmx_bootmem_desc_get(struct octeon_device *oct, uint64_t base, uint32_t offset, uint32_t size) { base = (1ull << 63) | (base + offset); switch (size) { case 4: return (lio_read_device_mem32(oct, base)); case 8: return (lio_read_device_mem64(oct, base)); default: return (0); } } /* * This function retrieves the string name of a named block. It is * more complicated than a simple memcpy() since the named block * descriptor may not be directly accessible. * * @param oct Pointer to current octeon device * @param addr Physical address of the named block descriptor * @param str String to receive the named block string name * @param len Length of the string buffer, which must match the length * stored in the bootmem descriptor. */ static void lio_bootmem_named_get_name(struct octeon_device *oct, uint64_t addr, char *str, uint32_t len) { addr += offsetof(struct cvmx_bootmem_named_block_desc, name); lio_pci_read_core_mem(oct, addr, (uint8_t *) str, len); str[len] = 0; } /* See header file for descriptions of functions */ /* * Check the version information on the bootmem descriptor * * @param oct Pointer to current octeon device * @param exact_match * Exact major version to check against. A zero means * check that the version supports named blocks. * * @return Zero if the version is correct. Negative if the version is * incorrect. Failures also cause a message to be displayed. */ static int __cvmx_bootmem_check_version(struct octeon_device *oct, uint32_t exact_match) { uint32_t major_version; uint32_t minor_version; if (!oct->bootmem_desc_addr) oct->bootmem_desc_addr = lio_read_device_mem64(oct, LIO_BOOTLOADER_PCI_READ_DESC_ADDR); major_version = (uint32_t) __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, offsetof(struct cvmx_bootmem_desc, major_version), SIZEOF_FIELD(struct cvmx_bootmem_desc, major_version)); minor_version = (uint32_t) __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, offsetof(struct cvmx_bootmem_desc, minor_version), SIZEOF_FIELD(struct cvmx_bootmem_desc, minor_version)); lio_dev_dbg(oct, "%s: major_version=%d\n", __func__, major_version); if ((major_version > 3) || (exact_match && major_version != exact_match)) { lio_dev_err(oct, "bootmem ver mismatch %d.%d addr:0x%llx\n", major_version, minor_version, CAST_ULL(oct->bootmem_desc_addr)); return (-1); } else { return (0); } } static const struct cvmx_bootmem_named_block_desc * __cvmx_bootmem_find_named_block_flags(struct octeon_device *oct, const char *name, uint32_t flags) { struct cvmx_bootmem_named_block_desc *desc = &oct->bootmem_named_block_desc; uint64_t named_addr; named_addr = cvmx_bootmem_phy_named_block_find(oct, name, flags); if (named_addr) { desc->base_addr = __cvmx_bootmem_desc_get(oct, named_addr, offsetof(struct cvmx_bootmem_named_block_desc, base_addr), SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, base_addr)); desc->size = __cvmx_bootmem_desc_get(oct, named_addr, offsetof(struct cvmx_bootmem_named_block_desc, size), SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, size)); strncpy(desc->name, name, sizeof(desc->name)); desc->name[sizeof(desc->name) - 1] = 0; return (&oct->bootmem_named_block_desc); } else { return (NULL); } } static uint64_t cvmx_bootmem_phy_named_block_find(struct octeon_device *oct, const char *name, uint32_t flags) { uint64_t result = 0; if (!__cvmx_bootmem_check_version(oct, 3)) { uint32_t i; uint64_t named_block_array_addr = __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, offsetof(struct cvmx_bootmem_desc, named_block_array_addr), SIZEOF_FIELD(struct cvmx_bootmem_desc, named_block_array_addr)); uint32_t num_blocks = (uint32_t) __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, offsetof(struct cvmx_bootmem_desc, nb_num_blocks), SIZEOF_FIELD(struct cvmx_bootmem_desc, nb_num_blocks)); uint32_t name_length = (uint32_t) __cvmx_bootmem_desc_get(oct, oct->bootmem_desc_addr, offsetof(struct cvmx_bootmem_desc, named_block_name_len), SIZEOF_FIELD(struct cvmx_bootmem_desc, named_block_name_len)); uint64_t named_addr = named_block_array_addr; for (i = 0; i < num_blocks; i++) { uint64_t named_size = __cvmx_bootmem_desc_get(oct, named_addr, offsetof(struct cvmx_bootmem_named_block_desc, size), SIZEOF_FIELD(struct cvmx_bootmem_named_block_desc, size)); if (name && named_size) { char *name_tmp = malloc(name_length + 1, M_DEVBUF, M_NOWAIT | M_ZERO); if (!name_tmp) break; lio_bootmem_named_get_name(oct, named_addr, name_tmp, name_length); if (!strncmp(name, name_tmp, name_length)) { result = named_addr; free(name_tmp, M_DEVBUF); break; } free(name_tmp, M_DEVBUF); } else if (!name && !named_size) { result = named_addr; break; } named_addr += sizeof(struct cvmx_bootmem_named_block_desc); } } return (result); } /* * Find a named block on the remote Octeon * * @param oct Pointer to current octeon device * @param name Name of block to find * @param base_addr Address the block is at (OUTPUT) * @param size The size of the block (OUTPUT) * * @return Zero on success, One on failure. */ static int lio_named_block_find(struct octeon_device *oct, const char *name, uint64_t * base_addr, uint64_t * size) { const struct cvmx_bootmem_named_block_desc *named_block; lio_remote_lock(); named_block = __cvmx_bootmem_find_named_block_flags(oct, name, 0); lio_remote_unlock(); if (named_block != NULL) { *base_addr = named_block->base_addr; *size = named_block->size; return (0); } return (1); } static void lio_remote_lock(void) { /* fill this in if any sharing is needed */ } static void lio_remote_unlock(void) { /* fill this in if any sharing is needed */ } int lio_console_send_cmd(struct octeon_device *oct, char *cmd_str, uint32_t wait_hundredths) { uint32_t len = (uint32_t) strlen(cmd_str); lio_dev_dbg(oct, "sending \"%s\" to bootloader\n", cmd_str); if (len > LIO_BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN - 1) { lio_dev_err(oct, "Command string too long, max length is: %d\n", LIO_BOOTLOADER_PCI_WRITE_BUFFER_STR_LEN - 1); return (-1); } if (lio_wait_for_bootloader(oct, wait_hundredths)) { lio_dev_err(oct, "Bootloader not ready for command.\n"); return (-1); } /* Write command to bootloader */ lio_remote_lock(); lio_pci_write_core_mem(oct, LIO_BOOTLOADER_PCI_READ_BUFFER_DATA_ADDR, (uint8_t *) cmd_str, len); lio_write_device_mem32(oct, LIO_BOOTLOADER_PCI_READ_BUFFER_LEN_ADDR, len); lio_write_device_mem32(oct, LIO_BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR, LIO_PCI_IO_BUF_OWNER_OCTEON); /* * Bootloader should accept command very quickly if it really was * ready */ if (lio_wait_for_bootloader(oct, 200)) { lio_remote_unlock(); lio_dev_err(oct, "Bootloader did not accept command.\n"); return (-1); } lio_remote_unlock(); return (0); } int lio_wait_for_bootloader(struct octeon_device *oct, uint32_t wait_time_hundredths) { lio_dev_dbg(oct, "waiting %d0 ms for bootloader\n", wait_time_hundredths); if (lio_mem_access_ok(oct)) return (-1); while (wait_time_hundredths > 0 && lio_read_device_mem32(oct, LIO_BOOTLOADER_PCI_READ_BUFFER_OWNER_ADDR) != LIO_PCI_IO_BUF_OWNER_HOST) { if (--wait_time_hundredths <= 0) return (-1); lio_sleep_timeout(10); } return (0); } static void lio_console_handle_result(struct octeon_device *oct, size_t console_num) { struct lio_console *console; console = &oct->console[console_num]; console->waiting = 0; } static char console_buffer[LIO_MAX_CONSOLE_READ_BYTES]; static void lio_output_console_line(struct octeon_device *oct, struct lio_console *console, size_t console_num, char *console_buffer, int32_t bytes_read) { size_t len; int32_t i; char *line; line = console_buffer; for (i = 0; i < bytes_read; i++) { /* Output a line at a time, prefixed */ if (console_buffer[i] == '\n') { console_buffer[i] = '\0'; /* We need to output 'line', prefaced by 'leftover'. * However, it is possible we're being called to * output 'leftover' by itself (in the case of nothing * having been read from the console). * * To avoid duplication, check for this condition. */ if (console->leftover[0] && (line != console->leftover)) { if (console->print) (*console->print)(oct, (uint32_t)console_num, console->leftover,line); console->leftover[0] = '\0'; } else { if (console->print) (*console->print)(oct, (uint32_t)console_num, line, NULL); } line = &console_buffer[i + 1]; } } /* Save off any leftovers */ if (line != &console_buffer[bytes_read]) { console_buffer[bytes_read] = '\0'; len = strlen(console->leftover); strncpy(&console->leftover[len], line, sizeof(console->leftover) - len); } } static void lio_check_console(void *arg) { struct lio_console *console; struct lio_callout *console_callout = arg; struct octeon_device *oct = (struct octeon_device *)console_callout->ctxptr; size_t len; uint32_t console_num = (uint32_t) console_callout->ctxul; int32_t bytes_read, total_read, tries; console = &oct->console[console_num]; tries = 0; total_read = 0; if (callout_pending(&console_callout->timer) || (callout_active(&console_callout->timer) == 0)) return; do { /* * Take console output regardless of whether it will be * logged */ bytes_read = lio_console_read(oct, console_num, console_buffer, sizeof(console_buffer) - 1); if (bytes_read > 0) { total_read += bytes_read; if (console->waiting) lio_console_handle_result(oct, console_num); if (console->print) { lio_output_console_line(oct, console, console_num, console_buffer, bytes_read); } } else if (bytes_read < 0) { lio_dev_err(oct, "Error reading console %u, ret=%d\n", console_num, bytes_read); } tries++; } while ((bytes_read > 0) && (tries < 16)); /* * If nothing is read after polling the console, output any leftovers * if any */ if (console->print && (total_read == 0) && (console->leftover[0])) { /* append '\n' as terminator for 'output_console_line' */ len = strlen(console->leftover); console->leftover[len] = '\n'; lio_output_console_line(oct, console, console_num, console->leftover, (int32_t)(len + 1)); console->leftover[0] = '\0'; } callout_schedule(&oct->console_timer[console_num].timer, lio_ms_to_ticks(LIO_CONSOLE_POLL_INTERVAL_MS)); } int lio_init_consoles(struct octeon_device *oct) { uint64_t addr, size; int ret = 0; ret = lio_mem_access_ok(oct); if (ret) { lio_dev_err(oct, "Memory access not okay'\n"); return (ret); } ret = lio_named_block_find(oct, LIO_PCI_CONSOLE_BLOCK_NAME, &addr, &size); if (ret) { lio_dev_err(oct, "Could not find console '%s'\n", LIO_PCI_CONSOLE_BLOCK_NAME); return (ret); } /* * Use BAR1_INDEX15 to create a static mapping to a region of * Octeon's DRAM that contains the PCI console named block. */ oct->console_nb_info.bar1_index = 15; oct->fn_list.bar1_idx_setup(oct, addr, oct->console_nb_info.bar1_index, 1); oct->console_nb_info.dram_region_base = addr & 0xFFFFFFFFFFC00000ULL; /* * num_consoles > 0, is an indication that the consoles are * accessible */ oct->num_consoles = lio_read_device_mem32(oct, addr + offsetof(struct lio_pci_console_desc, num_consoles)); oct->console_desc_addr = addr; lio_dev_dbg(oct, "Initialized consoles. %d available\n", oct->num_consoles); return (ret); } int lio_add_console(struct octeon_device *oct, uint32_t console_num, char *dbg_enb) { struct callout *timer; struct lio_console *console; uint64_t coreaddr; int ret = 0; if (console_num >= oct->num_consoles) { lio_dev_err(oct, "trying to read from console number %d when only 0 to %d exist\n", console_num, oct->num_consoles); } else { console = &oct->console[console_num]; console->waiting = 0; coreaddr = oct->console_desc_addr + console_num * 8 + offsetof(struct lio_pci_console_desc, console_addr_array); console->addr = lio_read_device_mem64(oct, coreaddr); coreaddr = console->addr + offsetof(struct lio_pci_console, buf_size); console->buffer_size = lio_read_device_mem32(oct, coreaddr); coreaddr = console->addr + offsetof(struct lio_pci_console, input_base_addr); console->input_base_addr = lio_read_device_mem64(oct, coreaddr); coreaddr = console->addr + offsetof(struct lio_pci_console, output_base_addr); console->output_base_addr = lio_read_device_mem64(oct, coreaddr); console->leftover[0] = '\0'; timer = &oct->console_timer[console_num].timer; if (oct->uboot_len == 0) lio_get_uboot_version(oct); callout_init(timer, 0); oct->console_timer[console_num].ctxptr = (void *)oct; oct->console_timer[console_num].ctxul = console_num; callout_reset(timer, lio_ms_to_ticks(LIO_CONSOLE_POLL_INTERVAL_MS), lio_check_console, timer); /* an empty string means use default debug console enablement */ if (dbg_enb && !dbg_enb[0]) dbg_enb = "setenv pci_console_active 1"; if (dbg_enb) ret = lio_console_send_cmd(oct, dbg_enb, 2000); console->active = 1; } return (ret); } /* * Removes all consoles * * @param oct octeon device */ void lio_remove_consoles(struct octeon_device *oct) { struct lio_console *console; uint32_t i; for (i = 0; i < oct->num_consoles; i++) { console = &oct->console[i]; if (!console->active) continue; callout_stop(&oct->console_timer[i].timer); console->addr = 0; console->buffer_size = 0; console->input_base_addr = 0; console->output_base_addr = 0; } oct->num_consoles = 0; } static inline int lio_console_free_bytes(uint32_t buffer_size, uint32_t wr_idx, uint32_t rd_idx) { if (rd_idx >= buffer_size || wr_idx >= buffer_size) return (-1); return (((buffer_size - 1) - (wr_idx - rd_idx)) % buffer_size); } static inline int lio_console_avail_bytes(uint32_t buffer_size, uint32_t wr_idx, uint32_t rd_idx) { if (rd_idx >= buffer_size || wr_idx >= buffer_size) return (-1); return (buffer_size - 1 - lio_console_free_bytes(buffer_size, wr_idx, rd_idx)); } static int lio_console_read(struct octeon_device *oct, uint32_t console_num, char *buffer, uint32_t buf_size) { struct lio_console *console; int bytes_to_read; uint32_t rd_idx, wr_idx; if (console_num >= oct->num_consoles) { lio_dev_err(oct, "Attempted to read from disabled console %d\n", console_num); return (0); } console = &oct->console[console_num]; /* * Check to see if any data is available. Maybe optimize this with * 64-bit read. */ rd_idx = lio_read_device_mem32(oct, console->addr + offsetof(struct lio_pci_console, output_read_index)); wr_idx = lio_read_device_mem32(oct, console->addr + offsetof(struct lio_pci_console, output_write_index)); bytes_to_read = lio_console_avail_bytes(console->buffer_size, wr_idx, rd_idx); if (bytes_to_read <= 0) return (bytes_to_read); bytes_to_read = min(bytes_to_read, buf_size); /* * Check to see if what we want to read is not contiguous, and limit * ourselves to the contiguous block */ if (rd_idx + bytes_to_read >= console->buffer_size) bytes_to_read = console->buffer_size - rd_idx; lio_pci_read_core_mem(oct, console->output_base_addr + rd_idx, (uint8_t *) buffer, bytes_to_read); lio_write_device_mem32(oct, console->addr + offsetof(struct lio_pci_console, output_read_index), (rd_idx + bytes_to_read) % console->buffer_size); return (bytes_to_read); } static void lio_get_uboot_version(struct octeon_device *oct) { struct lio_console *console; int32_t bytes_read, total_read, tries; uint32_t console_num = 0; - int i, ret = 0; + int i, ret __unused = 0; ret = lio_console_send_cmd(oct, "setenv stdout pci", 50); console = &oct->console[console_num]; tries = 0; total_read = 0; ret = lio_console_send_cmd(oct, "version", 1); do { /* * Take console output regardless of whether it will be * logged */ bytes_read = lio_console_read(oct, console_num, oct->uboot_version + total_read, OCTEON_UBOOT_BUFFER_SIZE - 1 - total_read); if (bytes_read > 0) { oct->uboot_version[bytes_read] = 0x0; total_read += bytes_read; if (console->waiting) lio_console_handle_result(oct, console_num); } else if (bytes_read < 0) { lio_dev_err(oct, "Error reading console %u, ret=%d\n", console_num, bytes_read); } tries++; } while ((bytes_read > 0) && (tries < 16)); /* * If nothing is read after polling the console, output any leftovers * if any */ if ((total_read == 0) && (console->leftover[0])) { lio_dev_dbg(oct, "%u: %s\n", console_num, console->leftover); console->leftover[0] = '\0'; } ret = lio_console_send_cmd(oct, "setenv stdout serial", 50); /* U-Boot */ for (i = 0; i < (OCTEON_UBOOT_BUFFER_SIZE - 9); i++) { if (oct->uboot_version[i] == 'U' && oct->uboot_version[i + 2] == 'B' && oct->uboot_version[i + 3] == 'o' && oct->uboot_version[i + 4] == 'o' && oct->uboot_version[i + 5] == 't') { oct->uboot_sidx = i; i++; for (; oct->uboot_version[i] != 0x0; i++) { if (oct->uboot_version[i] == 'm' && oct->uboot_version[i + 1] == 'i' && oct->uboot_version[i + 2] == 'p' && oct->uboot_version[i + 3] == 's') { oct->uboot_eidx = i - 1; oct->uboot_version[i - 1] = 0x0; oct->uboot_len = oct->uboot_eidx - oct->uboot_sidx + 1; lio_dev_info(oct, "%s\n", &oct->uboot_version [oct->uboot_sidx]); return; } } } } } #define FBUF_SIZE (4 * 1024 * 1024) int lio_download_firmware(struct octeon_device *oct, const uint8_t * data, size_t size) { struct lio_firmware_file_header *h; uint64_t load_addr; uint32_t crc32_result, i, image_len, rem; - int ret = 0; if (size < sizeof(struct lio_firmware_file_header)) { lio_dev_err(oct, "Firmware file too small (%d < %d).\n", (uint32_t) size, (uint32_t) sizeof(struct lio_firmware_file_header)); return (-EINVAL); } h = __DECONST(struct lio_firmware_file_header *, data); if (be32toh(h->magic) != LIO_NIC_MAGIC) { lio_dev_err(oct, "Unrecognized firmware file.\n"); return (-EINVAL); } crc32_result = crc32(data, sizeof(struct lio_firmware_file_header) - sizeof(uint32_t)); if (crc32_result != be32toh(h->crc32)) { lio_dev_err(oct, "Firmware CRC mismatch (0x%08x != 0x%08x).\n", crc32_result, be32toh(h->crc32)); return (-EINVAL); } if (memcmp(LIO_BASE_VERSION, h->version, strlen(LIO_BASE_VERSION))) { lio_dev_err(oct, "Unmatched firmware version. Expected %s.x, got %s.\n", LIO_BASE_VERSION, h->version); return (-EINVAL); } if (be32toh(h->num_images) > LIO_MAX_IMAGES) { lio_dev_err(oct, "Too many images in firmware file (%d).\n", be32toh(h->num_images)); return (-EINVAL); } lio_dev_info(oct, "Firmware version: %s\n", h->version); snprintf(oct->fw_info.lio_firmware_version, 32, "LIQUIDIO: %s", h->version); data += sizeof(struct lio_firmware_file_header); lio_dev_info(oct, "Loading %d image(s)\n", be32toh(h->num_images)); /* load all images */ for (i = 0; i < be32toh(h->num_images); i++) { load_addr = be64toh(h->desc[i].addr); image_len = be32toh(h->desc[i].len); lio_dev_info(oct, "Loading firmware %d at %llx\n", image_len, (unsigned long long)load_addr); /* Write in 4MB chunks */ rem = image_len; while (rem) { if (rem < FBUF_SIZE) size = rem; else size = FBUF_SIZE; /* download the image */ lio_pci_write_core_mem(oct, load_addr, __DECONST(uint8_t *, data), (uint32_t) size); data += size; rem -= (uint32_t) size; load_addr += size; } } lio_dev_info(oct, "Writing boot command: %s\n", h->bootcmd); /* Invoke the bootcmd */ - ret = lio_console_send_cmd(oct, h->bootcmd, 50); + lio_console_send_cmd(oct, h->bootcmd, 50); return (0); } diff --git a/sys/dev/liquidio/base/lio_device.c b/sys/dev/liquidio/base/lio_device.c index 8bcf4f329848..a673a9bf414a 100644 --- a/sys/dev/liquidio/base/lio_device.c +++ b/sys/dev/liquidio/base/lio_device.c @@ -1,1062 +1,1061 @@ /* * BSD LICENSE * * Copyright(c) 2017 Cavium, Inc.. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Cavium, Inc. nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD$*/ #include "lio_bsd.h" #include "lio_common.h" #include "lio_droq.h" #include "lio_iq.h" #include "lio_response_manager.h" #include "lio_device.h" #include "lio_main.h" #include "lio_network.h" #include "cn23xx_pf_device.h" #include "lio_image.h" #include "lio_mem_ops.h" static struct lio_config default_cn23xx_conf = { .card_type = LIO_23XX, .card_name = LIO_23XX_NAME, /* IQ attributes */ .iq = { .max_iqs = LIO_CN23XX_CFG_IO_QUEUES, .pending_list_size = (LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS * LIO_CN23XX_CFG_IO_QUEUES), .instr_type = LIO_64BYTE_INSTR, .db_min = LIO_CN23XX_DB_MIN, .db_timeout = LIO_CN23XX_DB_TIMEOUT, .iq_intr_pkt = LIO_CN23XX_DEF_IQ_INTR_THRESHOLD, }, /* OQ attributes */ .oq = { .max_oqs = LIO_CN23XX_CFG_IO_QUEUES, .pkts_per_intr = LIO_CN23XX_OQ_PKTS_PER_INTR, .refill_threshold = LIO_CN23XX_OQ_REFIL_THRESHOLD, .oq_intr_pkt = LIO_CN23XX_OQ_INTR_PKT, .oq_intr_time = LIO_CN23XX_OQ_INTR_TIME, }, .num_nic_ports = LIO_CN23XX_DEFAULT_NUM_PORTS, .num_def_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, .num_def_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, .def_rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, /* For ethernet interface 0: Port cfg Attributes */ .nic_if_cfg[0] = { /* Max Txqs: Half for each of the two ports :max_iq/2 */ .max_txqs = LIO_MAX_TXQS_PER_INTF, /* Actual configured value. Range could be: 1...max_txqs */ .num_txqs = LIO_DEF_TXQS_PER_INTF, /* Max Rxqs: Half for each of the two ports :max_oq/2 */ .max_rxqs = LIO_MAX_RXQS_PER_INTF, /* Actual configured value. Range could be: 1...max_rxqs */ .num_rxqs = LIO_DEF_RXQS_PER_INTF, /* Num of desc for rx rings */ .num_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, /* Num of desc for tx rings */ .num_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, /* * Mbuf size, We need not change buf size even for Jumbo frames. * Octeon can send jumbo frames in 4 consecutive descriptors, */ .rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, .base_queue = LIO_BASE_QUEUE_NOT_REQUESTED, .gmx_port_id = 0, }, .nic_if_cfg[1] = { /* Max Txqs: Half for each of the two ports :max_iq/2 */ .max_txqs = LIO_MAX_TXQS_PER_INTF, /* Actual configured value. Range could be: 1...max_txqs */ .num_txqs = LIO_DEF_TXQS_PER_INTF, /* Max Rxqs: Half for each of the two ports :max_oq/2 */ .max_rxqs = LIO_MAX_RXQS_PER_INTF, /* Actual configured value. Range could be: 1...max_rxqs */ .num_rxqs = LIO_DEF_RXQS_PER_INTF, /* Num of desc for rx rings */ .num_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, /* Num of desc for tx rings */ .num_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, /* * Mbuf size, We need not change buf size even for Jumbo frames. * Octeon can send jumbo frames in 4 consecutive descriptors, */ .rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, .base_queue = LIO_BASE_QUEUE_NOT_REQUESTED, .gmx_port_id = 1, }, .misc = { /* Host driver link query interval */ .oct_link_query_interval = 100, /* Octeon link query interval */ .host_link_query_interval = 500, .enable_sli_oq_bp = 0, /* Control queue group */ .ctrlq_grp = 1, } }; static struct lio_config_ptr { uint32_t conf_type; } oct_conf_info[LIO_MAX_DEVICES] = { { LIO_CFG_TYPE_DEFAULT, }, { LIO_CFG_TYPE_DEFAULT, }, { LIO_CFG_TYPE_DEFAULT, }, { LIO_CFG_TYPE_DEFAULT, }, }; static char lio_state_str[LIO_DEV_STATES + 1][32] = { "BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE", "IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE", "DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE", "INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE", "HOST-READY", "CORE-READY", "RUNNING", "IN-RESET", "INVALID" }; static char lio_app_str[LIO_DRV_APP_COUNT + 1][32] = {"BASE", "NIC", "UNKNOWN"}; static struct octeon_device *octeon_device[LIO_MAX_DEVICES]; static volatile int lio_adapter_refcounts[LIO_MAX_DEVICES]; static uint32_t octeon_device_count; /* locks device array (i.e. octeon_device[]) */ struct mtx octeon_devices_lock; static struct lio_core_setup core_setup[LIO_MAX_DEVICES]; static void oct_set_config_info(int oct_id, int conf_type) { if (conf_type < 0 || conf_type > (LIO_NUM_CFGS - 1)) conf_type = LIO_CFG_TYPE_DEFAULT; oct_conf_info[oct_id].conf_type = conf_type; } void lio_init_device_list(int conf_type) { int i; bzero(octeon_device, (sizeof(void *) * LIO_MAX_DEVICES)); for (i = 0; i < LIO_MAX_DEVICES; i++) oct_set_config_info(i, conf_type); mtx_init(&octeon_devices_lock, "octeon_devices_lock", NULL, MTX_DEF); } static void * __lio_retrieve_config_info(struct octeon_device *oct, uint16_t card_type) { void *ret = NULL; uint32_t oct_id = oct->octeon_id; switch (oct_conf_info[oct_id].conf_type) { case LIO_CFG_TYPE_DEFAULT: if (oct->chip_id == LIO_CN23XX_PF_VID) { ret = &default_cn23xx_conf; } break; default: break; } return (ret); } void * lio_get_config_info(struct octeon_device *oct, uint16_t card_type) { void *conf = NULL; conf = __lio_retrieve_config_info(oct, card_type); if (conf == NULL) return (NULL); return (conf); } char * lio_get_state_string(volatile int *state_ptr) { int32_t istate = (int32_t)atomic_load_acq_int(state_ptr); if (istate > LIO_DEV_STATES || istate < 0) return (lio_state_str[LIO_DEV_STATE_INVALID]); return (lio_state_str[istate]); } static char * lio_get_app_string(uint32_t app_mode) { if (app_mode <= LIO_DRV_APP_END) return (lio_app_str[app_mode - LIO_DRV_APP_START]); return (lio_app_str[LIO_DRV_INVALID_APP - LIO_DRV_APP_START]); } void lio_free_device_mem(struct octeon_device *oct) { int i; for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i])) free(oct->droq[i], M_DEVBUF); } for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i])) free(oct->instr_queue[i], M_DEVBUF); } i = oct->octeon_id; free(oct->chip, M_DEVBUF); octeon_device[i] = NULL; octeon_device_count--; } static struct octeon_device * lio_allocate_device_mem(device_t device) { struct octeon_device *oct; uint32_t configsize = 0, pci_id = 0, size; uint8_t *buf = NULL; pci_id = pci_get_device(device); switch (pci_id) { case LIO_CN23XX_PF_VID: configsize = sizeof(struct lio_cn23xx_pf); break; default: device_printf(device, "Error: Unknown PCI Device: 0x%x\n", pci_id); return (NULL); } if (configsize & 0x7) configsize += (8 - (configsize & 0x7)); size = configsize + (sizeof(struct lio_dispatch) * LIO_DISPATCH_LIST_SIZE); buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); if (buf == NULL) return (NULL); oct = (struct octeon_device *)device_get_softc(device); oct->chip = (void *)(buf); oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize); return (oct); } struct octeon_device * lio_allocate_device(device_t device) { struct octeon_device *oct = NULL; uint32_t oct_idx = 0; mtx_lock(&octeon_devices_lock); for (oct_idx = 0; oct_idx < LIO_MAX_DEVICES; oct_idx++) if (!octeon_device[oct_idx]) break; if (oct_idx < LIO_MAX_DEVICES) { oct = lio_allocate_device_mem(device); if (oct != NULL) { octeon_device_count++; octeon_device[oct_idx] = oct; } } mtx_unlock(&octeon_devices_lock); if (oct == NULL) return (NULL); mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF); mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF); oct->octeon_id = oct_idx; snprintf(oct->device_name, sizeof(oct->device_name), "%s%d", LIO_DRV_NAME, oct->octeon_id); return (oct); } /* * Register a device's bus location at initialization time. * @param oct - pointer to the octeon device structure. * @param bus - PCIe bus # * @param dev - PCIe device # * @param func - PCIe function # * @param is_pf - TRUE for PF, FALSE for VF * @return reference count of device's adapter */ int lio_register_device(struct octeon_device *oct, int bus, int dev, int func, int is_pf) { int idx, refcount; oct->loc.bus = bus; oct->loc.dev = dev; oct->loc.func = func; oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id]; atomic_store_rel_int(oct->adapter_refcount, 0); mtx_lock(&octeon_devices_lock); for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) { if (octeon_device[idx] == NULL) { lio_dev_err(oct, "%s: Internal driver error, missing dev\n", __func__); mtx_unlock(&octeon_devices_lock); atomic_add_int(oct->adapter_refcount, 1); return (1); /* here, refcount is guaranteed to be 1 */ } /* if another device is at same bus/dev, use its refcounter */ if ((octeon_device[idx]->loc.bus == bus) && (octeon_device[idx]->loc.dev == dev)) { oct->adapter_refcount = octeon_device[idx]->adapter_refcount; break; } } mtx_unlock(&octeon_devices_lock); atomic_add_int(oct->adapter_refcount, 1); refcount = atomic_load_acq_int(oct->adapter_refcount); lio_dev_dbg(oct, "%s: %02x:%02x:%d refcount %u\n", __func__, oct->loc.bus, oct->loc.dev, oct->loc.func, refcount); return (refcount); } /* * Deregister a device at de-initialization time. * @param oct - pointer to the octeon device structure. * @return reference count of device's adapter */ int lio_deregister_device(struct octeon_device *oct) { int refcount; atomic_subtract_int(oct->adapter_refcount, 1); refcount = atomic_load_acq_int(oct->adapter_refcount); lio_dev_dbg(oct, "%s: %04d:%02d:%d refcount %u\n", __func__, oct->loc.bus, oct->loc.dev, oct->loc.func, refcount); return (refcount); } int lio_allocate_ioq_vector(struct octeon_device *oct) { struct lio_ioq_vector *ioq_vector; int i, cpu_num, num_ioqs = 0, size; if (LIO_CN23XX_PF(oct)) num_ioqs = oct->sriov_info.num_pf_rings; size = sizeof(struct lio_ioq_vector) * num_ioqs; oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); if (oct->ioq_vector == NULL) return (1); for (i = 0; i < num_ioqs; i++) { ioq_vector = &oct->ioq_vector[i]; ioq_vector->oct_dev = oct; ioq_vector->droq_index = i; cpu_num = i % mp_ncpus; CPU_SETOF(cpu_num, &ioq_vector->affinity_mask); if (oct->chip_id == LIO_CN23XX_PF_VID) ioq_vector->ioq_num = i + oct->sriov_info.pf_srn; else ioq_vector->ioq_num = i; } return (0); } void lio_free_ioq_vector(struct octeon_device *oct) { free(oct->ioq_vector, M_DEVBUF); oct->ioq_vector = NULL; } /* this function is only for setting up the first queue */ int lio_setup_instr_queue0(struct octeon_device *oct) { union octeon_txpciq txpciq; uint32_t iq_no = 0; uint32_t num_descs = 0; if (LIO_CN23XX_PF(oct)) num_descs = LIO_GET_NUM_DEF_TX_DESCS_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)); oct->num_iqs = 0; oct->instr_queue[0]->q_index = 0; oct->instr_queue[0]->app_ctx = (void *)(size_t)0; oct->instr_queue[0]->ifidx = 0; txpciq.txpciq64 = 0; txpciq.s.q_no = iq_no; txpciq.s.pkind = oct->pfvf_hsword.pkind; txpciq.s.use_qpg = 0; txpciq.s.qpg = 0; if (lio_init_instr_queue(oct, txpciq, num_descs)) { /* prevent memory leak */ lio_delete_instr_queue(oct, 0); return (1); } oct->num_iqs++; return (0); } int lio_setup_output_queue0(struct octeon_device *oct) { uint32_t desc_size = 0, num_descs = 0, oq_no = 0; if (LIO_CN23XX_PF(oct)) { num_descs = LIO_GET_NUM_DEF_RX_DESCS_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)); desc_size = LIO_GET_DEF_RX_BUF_SIZE_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)); } oct->num_oqs = 0; if (lio_init_droq(oct, oq_no, num_descs, desc_size, NULL)) { return (1); } oct->num_oqs++; return (0); } int lio_init_dispatch_list(struct octeon_device *oct) { uint32_t i; oct->dispatch.count = 0; for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) { oct->dispatch.dlist[i].opcode = 0; STAILQ_INIT(&oct->dispatch.dlist[i].head); } mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF); return (0); } void lio_delete_dispatch_list(struct octeon_device *oct) { struct lio_stailq_head freelist; struct lio_stailq_node *temp, *tmp2; uint32_t i; STAILQ_INIT(&freelist); mtx_lock(&oct->dispatch.lock); for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) { struct lio_stailq_head *dispatch; dispatch = &oct->dispatch.dlist[i].head; while (!STAILQ_EMPTY(dispatch)) { temp = STAILQ_FIRST(dispatch); STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head, entries); STAILQ_INSERT_TAIL(&freelist, temp, entries); } oct->dispatch.dlist[i].opcode = 0; } oct->dispatch.count = 0; mtx_unlock(&oct->dispatch.lock); STAILQ_FOREACH_SAFE(temp, &freelist, entries, tmp2) { STAILQ_REMOVE_HEAD(&freelist, entries); free(temp, M_DEVBUF); } } lio_dispatch_fn_t lio_get_dispatch(struct octeon_device *octeon_dev, uint16_t opcode, uint16_t subcode) { struct lio_stailq_node *dispatch; lio_dispatch_fn_t fn = NULL; uint32_t idx; uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); idx = combined_opcode & LIO_OPCODE_MASK; mtx_lock(&octeon_dev->dispatch.lock); if (octeon_dev->dispatch.count == 0) { mtx_unlock(&octeon_dev->dispatch.lock); return (NULL); } if (!(octeon_dev->dispatch.dlist[idx].opcode)) { mtx_unlock(&octeon_dev->dispatch.lock); return (NULL); } if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { fn = octeon_dev->dispatch.dlist[idx].dispatch_fn; } else { STAILQ_FOREACH(dispatch, &octeon_dev->dispatch.dlist[idx].head, entries) { if (((struct lio_dispatch *)dispatch)->opcode == combined_opcode) { fn = ((struct lio_dispatch *) dispatch)->dispatch_fn; break; } } } mtx_unlock(&octeon_dev->dispatch.lock); return (fn); } /* * lio_register_dispatch_fn * Parameters: * octeon_id - id of the octeon device. * opcode - opcode for which driver should call the registered function * subcode - subcode for which driver should call the registered function * fn - The function to call when a packet with "opcode" arrives in * octeon output queues. * fn_arg - The argument to be passed when calling function "fn". * Description: * Registers a function and its argument to be called when a packet * arrives in Octeon output queues with "opcode". * Returns: * Success: 0 * Failure: 1 * Locks: * No locks are held. */ int lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode, uint16_t subcode, lio_dispatch_fn_t fn, void *fn_arg) { lio_dispatch_fn_t pfn; uint32_t idx; uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); idx = combined_opcode & LIO_OPCODE_MASK; mtx_lock(&oct->dispatch.lock); /* Add dispatch function to first level of lookup table */ if (oct->dispatch.dlist[idx].opcode == 0) { oct->dispatch.dlist[idx].opcode = combined_opcode; oct->dispatch.dlist[idx].dispatch_fn = fn; oct->dispatch.dlist[idx].arg = fn_arg; oct->dispatch.count++; mtx_unlock(&oct->dispatch.lock); return (0); } mtx_unlock(&oct->dispatch.lock); /* * Check if there was a function already registered for this * opcode/subcode. */ pfn = lio_get_dispatch(oct, opcode, subcode); if (!pfn) { struct lio_dispatch *dispatch; lio_dev_dbg(oct, "Adding opcode to dispatch list linked list\n"); dispatch = (struct lio_dispatch *) malloc(sizeof(struct lio_dispatch), M_DEVBUF, M_NOWAIT | M_ZERO); if (dispatch == NULL) { lio_dev_err(oct, "No memory to add dispatch function\n"); return (1); } dispatch->opcode = combined_opcode; dispatch->dispatch_fn = fn; dispatch->arg = fn_arg; /* * Add dispatch function to linked list of fn ptrs * at the hashed index. */ mtx_lock(&oct->dispatch.lock); STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head, &dispatch->node, entries); oct->dispatch.count++; mtx_unlock(&oct->dispatch.lock); } else { lio_dev_err(oct, "Found previously registered dispatch fn for opcode/subcode: %x/%x\n", opcode, subcode); return (1); } return (0); } /* * lio_unregister_dispatch_fn * Parameters: * oct - octeon device * opcode - driver should unregister the function for this opcode * subcode - driver should unregister the function for this subcode * Description: * Unregister the function set for this opcode+subcode. * Returns: * Success: 0 * Failure: 1 * Locks: * No locks are held. */ int lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode, uint16_t subcode) { struct lio_stailq_head *dispatch_head; struct lio_stailq_node *dispatch, *dfree = NULL, *tmp2; int retval = 0; uint32_t idx; uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); idx = combined_opcode & LIO_OPCODE_MASK; mtx_lock(&oct->dispatch.lock); if (oct->dispatch.count == 0) { mtx_unlock(&oct->dispatch.lock); lio_dev_err(oct, "No dispatch functions registered for this device\n"); return (1); } if (oct->dispatch.dlist[idx].opcode == combined_opcode) { dispatch_head = &oct->dispatch.dlist[idx].head; if (!STAILQ_EMPTY(dispatch_head)) { dispatch = STAILQ_FIRST(dispatch_head); oct->dispatch.dlist[idx].opcode = ((struct lio_dispatch *)dispatch)->opcode; oct->dispatch.dlist[idx].dispatch_fn = ((struct lio_dispatch *)dispatch)->dispatch_fn; oct->dispatch.dlist[idx].arg = ((struct lio_dispatch *)dispatch)->arg; STAILQ_REMOVE_HEAD(dispatch_head, entries); dfree = dispatch; } else { oct->dispatch.dlist[idx].opcode = 0; oct->dispatch.dlist[idx].dispatch_fn = NULL; oct->dispatch.dlist[idx].arg = NULL; } } else { retval = 1; STAILQ_FOREACH_SAFE(dispatch, &oct->dispatch.dlist[idx].head, entries, tmp2) { if (((struct lio_dispatch *)dispatch)->opcode == combined_opcode) { STAILQ_REMOVE(&oct->dispatch.dlist[idx].head, dispatch, lio_stailq_node, entries); dfree = dispatch; retval = 0; } } } if (!retval) oct->dispatch.count--; mtx_unlock(&oct->dispatch.lock); free(dfree, M_DEVBUF); return (retval); } int lio_core_drv_init(struct lio_recv_info *recv_info, void *buf) { struct octeon_device *oct = (struct octeon_device *)buf; struct lio_recv_pkt *recv_pkt = recv_info->recv_pkt; struct lio_core_setup *cs = NULL; uint32_t i; uint32_t num_nic_ports = 0; char app_name[16]; if (LIO_CN23XX_PF(oct)) num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG( LIO_CHIP_CONF(oct, cn23xx_pf)); if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) { lio_dev_err(oct, "Received CORE OK when device state is 0x%x\n", atomic_load_acq_int(&oct->status)); goto core_drv_init_err; } strncpy(app_name, lio_get_app_string((uint32_t) recv_pkt->rh.r_core_drv_init.app_mode), sizeof(app_name) - 1); oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; if (recv_pkt->rh.r_core_drv_init.app_mode == LIO_DRV_NIC_APP) { oct->fw_info.max_nic_ports = (uint32_t)recv_pkt->rh.r_core_drv_init.max_nic_ports; oct->fw_info.num_gmx_ports = (uint32_t)recv_pkt->rh.r_core_drv_init.num_gmx_ports; } if (oct->fw_info.max_nic_ports < num_nic_ports) { lio_dev_err(oct, "Config has more ports than firmware allows (%d > %d).\n", num_nic_ports, oct->fw_info.max_nic_ports); goto core_drv_init_err; } oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags; oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; oct->pfvf_hsword.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind; for (i = 0; i < oct->num_iqs; i++) oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind; atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK); cs = &core_setup[oct->octeon_id]; if (recv_pkt->buffer_size[0] != (sizeof(*cs) + LIO_DROQ_INFO_SIZE)) { lio_dev_dbg(oct, "Core setup bytes expected %llu found %d\n", LIO_CAST64(sizeof(*cs) + LIO_DROQ_INFO_SIZE), recv_pkt->buffer_size[0]); } memcpy(cs, recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE, sizeof(*cs)); strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME); strncpy(oct->boardinfo.serial_number, cs->board_serial_number, LIO_SERIAL_NUM_LEN); lio_swap_8B_data((uint64_t *)cs, (sizeof(*cs) >> 3)); oct->boardinfo.major = cs->board_rev_major; oct->boardinfo.minor = cs->board_rev_minor; lio_dev_info(oct, "Running %s (%llu Hz)\n", app_name, LIO_CAST64(cs->corefreq)); core_drv_init_err: for (i = 0; i < recv_pkt->buffer_count; i++) lio_recv_buffer_free(recv_pkt->buffer_ptr[i]); lio_free_recv_info(recv_info); return (0); } int lio_get_tx_qsize(struct octeon_device *oct, uint32_t q_no) { if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_INSTR_QUEUES(oct)) && (oct->io_qmask.iq & BIT_ULL(q_no))) return (oct->instr_queue[q_no]->max_count); return (-1); } int lio_get_rx_qsize(struct octeon_device *oct, uint32_t q_no) { if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_OUTPUT_QUEUES(oct)) && (oct->io_qmask.oq & BIT_ULL(q_no))) return (oct->droq[q_no]->max_count); return (-1); } /* Returns the host firmware handshake OCTEON specific configuration */ struct lio_config * lio_get_conf(struct octeon_device *oct) { struct lio_config *default_oct_conf = NULL; /* * check the OCTEON Device model & return the corresponding octeon * configuration */ if (LIO_CN23XX_PF(oct)) { default_oct_conf = (struct lio_config *)( LIO_CHIP_CONF(oct, cn23xx_pf)); } return (default_oct_conf); } /* * Get the octeon device pointer. * @param octeon_id - The id for which the octeon device pointer is required. * @return Success: Octeon device pointer. * @return Failure: NULL. */ struct octeon_device * lio_get_device(uint32_t octeon_id) { if (octeon_id >= LIO_MAX_DEVICES) return (NULL); else return (octeon_device[octeon_id]); } uint64_t lio_pci_readq(struct octeon_device *oct, uint64_t addr) { uint64_t val64; - volatile uint32_t val32, addrhi; + volatile uint32_t addrhi; mtx_lock(&oct->pci_win_lock); /* * The windowed read happens when the LSB of the addr is written. * So write MSB first */ addrhi = (addr >> 32); if (oct->chip_id == LIO_CN23XX_PF_VID) addrhi |= 0x00060000; lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi); /* Read back to preserve ordering of writes */ - val32 = lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi); + (void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi); lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo, addr & 0xffffffff); - val32 = lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo); + (void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo); val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data); mtx_unlock(&oct->pci_win_lock); return (val64); } void lio_pci_writeq(struct octeon_device *oct, uint64_t val, uint64_t addr) { - volatile uint32_t val32; mtx_lock(&oct->pci_win_lock); lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr); /* The write happens when the LSB is written. So write MSB first. */ lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32); /* Read the MSB to ensure ordering of writes. */ - val32 = lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi); + (void)lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi); lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo, val & 0xffffffff); mtx_unlock(&oct->pci_win_lock); } int lio_mem_access_ok(struct octeon_device *oct) { uint64_t access_okay = 0; uint64_t lmc0_reset_ctl; /* Check to make sure a DDR interface is enabled */ if (LIO_CN23XX_PF(oct)) { lmc0_reset_ctl = lio_pci_readq(oct, LIO_CN23XX_LMC0_RESET_CTL); access_okay = (lmc0_reset_ctl & LIO_CN23XX_LMC0_RESET_CTL_DDR3RST_MASK); } return (access_okay ? 0 : 1); } int lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout) { int ret = 1; uint32_t ms; if (timeout == NULL) return (ret); for (ms = 0; ret && ((*timeout == 0) || (ms <= *timeout)); ms += 100) { ret = lio_mem_access_ok(oct); /* wait 100 ms */ if (ret) lio_sleep_timeout(100); } return (ret); } /* * Get the octeon id assigned to the octeon device passed as argument. * This function is exported to other modules. * @param dev - octeon device pointer passed as a void *. * @return octeon device id */ int lio_get_device_id(void *dev) { struct octeon_device *octeon_dev = (struct octeon_device *)dev; uint32_t i; for (i = 0; i < LIO_MAX_DEVICES; i++) if (octeon_device[i] == octeon_dev) return (octeon_dev->octeon_id); return (-1); } void lio_enable_irq(struct lio_droq *droq, struct lio_instr_queue *iq) { struct octeon_device *oct = NULL; uint64_t instr_cnt; uint32_t pkts_pend; /* the whole thing needs to be atomic, ideally */ if (droq != NULL) { oct = droq->oct_dev; pkts_pend = atomic_load_acq_int(&droq->pkts_pending); mtx_lock(&droq->lock); lio_write_csr32(oct, droq->pkts_sent_reg, droq->pkt_count - pkts_pend); droq->pkt_count = pkts_pend; /* this write needs to be flushed before we release the lock */ __compiler_membar(); mtx_unlock(&droq->lock); } if (iq != NULL) { oct = iq->oct_dev; mtx_lock(&iq->lock); lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done); iq->pkt_in_done = 0; /* this write needs to be flushed before we release the lock */ __compiler_membar(); mtx_unlock(&iq->lock); } /* * Implementation note: * * SLI_PKT(x)_CNTS[RESEND] is written separately so that if an interrupt * DOES occur as a result of RESEND, the DROQ lock will NOT be held. * * Write resend. Writing RESEND in SLI_PKTX_CNTS should be enough * to trigger tx interrupts as well, if they are pending. */ if ((oct != NULL) && (LIO_CN23XX_PF(oct))) { if (droq != NULL) lio_write_csr64(oct, droq->pkts_sent_reg, LIO_CN23XX_INTR_RESEND); /* we race with firmrware here. */ /* read and write the IN_DONE_CNTS */ else if (iq != NULL) { instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg); lio_write_csr64(oct, iq->inst_cnt_reg, ((instr_cnt & 0xFFFFFFFF00000000ULL) | LIO_CN23XX_INTR_RESEND)); } } } diff --git a/sys/dev/liquidio/base/lio_request_manager.c b/sys/dev/liquidio/base/lio_request_manager.c index db7548e7ddff..e07d65a6c33b 100644 --- a/sys/dev/liquidio/base/lio_request_manager.c +++ b/sys/dev/liquidio/base/lio_request_manager.c @@ -1,858 +1,853 @@ /* * BSD LICENSE * * Copyright(c) 2017 Cavium, Inc.. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Cavium, Inc. nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD$*/ #include "lio_bsd.h" #include "lio_common.h" #include "lio_droq.h" #include "lio_iq.h" #include "lio_response_manager.h" #include "lio_device.h" #include "lio_main.h" #include "lio_network.h" #include "cn23xx_pf_device.h" #include "lio_rxtx.h" struct lio_iq_post_status { int status; int index; }; static void lio_check_db_timeout(void *arg, int pending); static void __lio_check_db_timeout(struct octeon_device *oct, uint64_t iq_no); /* Return 0 on success, 1 on failure */ int lio_init_instr_queue(struct octeon_device *oct, union octeon_txpciq txpciq, uint32_t num_descs) { struct lio_instr_queue *iq; struct lio_iq_config *conf = NULL; struct lio_tq *db_tq; struct lio_request_list *request_buf; bus_size_t max_size; uint32_t iq_no = (uint32_t)txpciq.s.q_no; uint32_t q_size; int error, i; if (LIO_CN23XX_PF(oct)) conf = &(LIO_GET_IQ_CFG(LIO_CHIP_CONF(oct, cn23xx_pf))); if (conf == NULL) { lio_dev_err(oct, "Unsupported Chip %x\n", oct->chip_id); return (1); } q_size = (uint32_t)conf->instr_type * num_descs; iq = oct->instr_queue[iq_no]; iq->oct_dev = oct; max_size = LIO_CN23XX_PKI_MAX_FRAME_SIZE * num_descs; error = bus_dma_tag_create(bus_get_dma_tag(oct->device), /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ max_size, /* maxsize */ LIO_MAX_SG, /* nsegments */ PAGE_SIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &iq->txtag); if (error) { lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n", iq_no); return (1); } iq->base_addr = lio_dma_alloc(q_size, (vm_paddr_t *)&iq->base_addr_dma); if (!iq->base_addr) { lio_dev_err(oct, "Cannot allocate memory for instr queue %d\n", iq_no); return (1); } iq->max_count = num_descs; /* * Initialize a list to holds requests that have been posted to * Octeon but has yet to be fetched by octeon */ iq->request_list = malloc(sizeof(*iq->request_list) * num_descs, M_DEVBUF, M_NOWAIT | M_ZERO); if (iq->request_list == NULL) { lio_dev_err(oct, "Alloc failed for IQ[%d] nr free list\n", iq_no); return (1); } lio_dev_dbg(oct, "IQ[%d]: base: %p basedma: %llx count: %d\n", iq_no, iq->base_addr, LIO_CAST64(iq->base_addr_dma), iq->max_count); /* Create the descriptor buffer dma maps */ request_buf = iq->request_list; for (i = 0; i < num_descs; i++, request_buf++) { error = bus_dmamap_create(iq->txtag, 0, &request_buf->map); if (error) { lio_dev_err(oct, "Unable to create TX DMA map\n"); return (1); } } iq->txpciq.txpciq64 = txpciq.txpciq64; iq->fill_cnt = 0; iq->host_write_index = 0; iq->octeon_read_index = 0; iq->flush_index = 0; iq->last_db_time = 0; iq->db_timeout = (uint32_t)conf->db_timeout; atomic_store_rel_int(&iq->instr_pending, 0); /* Initialize the lock for this instruction queue */ mtx_init(&iq->lock, "Tx_lock", NULL, MTX_DEF); mtx_init(&iq->post_lock, "iq_post_lock", NULL, MTX_DEF); mtx_init(&iq->enq_lock, "enq_lock", NULL, MTX_DEF); mtx_init(&iq->iq_flush_running_lock, "iq_flush_running_lock", NULL, MTX_DEF); oct->io_qmask.iq |= BIT_ULL(iq_no); /* Set the 32B/64B mode for each input queue */ oct->io_qmask.iq64B |= ((conf->instr_type == 64) << iq_no); iq->iqcmd_64B = (conf->instr_type == 64); oct->fn_list.setup_iq_regs(oct, iq_no); db_tq = &oct->check_db_tq[iq_no]; db_tq->tq = taskqueue_create("lio_check_db_timeout", M_WAITOK, taskqueue_thread_enqueue, &db_tq->tq); if (db_tq->tq == NULL) { lio_dev_err(oct, "check db wq create failed for iq %d\n", iq_no); return (1); } TIMEOUT_TASK_INIT(db_tq->tq, &db_tq->work, 0, lio_check_db_timeout, (void *)db_tq); db_tq->ctxul = iq_no; db_tq->ctxptr = oct; taskqueue_start_threads(&db_tq->tq, 1, PI_NET, "lio%d_check_db_timeout:%d", oct->octeon_id, iq_no); taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, 1); /* Allocate a buf ring */ oct->instr_queue[iq_no]->br = buf_ring_alloc(LIO_BR_SIZE, M_DEVBUF, M_WAITOK, &oct->instr_queue[iq_no]->enq_lock); if (oct->instr_queue[iq_no]->br == NULL) { lio_dev_err(oct, "Critical Failure setting up buf ring\n"); return (1); } return (0); } int lio_delete_instr_queue(struct octeon_device *oct, uint32_t iq_no) { struct lio_instr_queue *iq = oct->instr_queue[iq_no]; struct lio_request_list *request_buf; struct lio_mbuf_free_info *finfo; uint64_t desc_size = 0, q_size; int i; lio_dev_dbg(oct, "%s[%d]\n", __func__, iq_no); if (oct->check_db_tq[iq_no].tq != NULL) { while (taskqueue_cancel_timeout(oct->check_db_tq[iq_no].tq, &oct->check_db_tq[iq_no].work, NULL)) taskqueue_drain_timeout(oct->check_db_tq[iq_no].tq, &oct->check_db_tq[iq_no].work); taskqueue_free(oct->check_db_tq[iq_no].tq); oct->check_db_tq[iq_no].tq = NULL; } if (LIO_CN23XX_PF(oct)) desc_size = LIO_GET_IQ_INSTR_TYPE_CFG(LIO_CHIP_CONF(oct, cn23xx_pf)); request_buf = iq->request_list; for (i = 0; i < iq->max_count; i++, request_buf++) { if ((request_buf->reqtype == LIO_REQTYPE_NORESP_NET) || (request_buf->reqtype == LIO_REQTYPE_NORESP_NET_SG)) { if (request_buf->buf != NULL) { finfo = request_buf->buf; bus_dmamap_sync(iq->txtag, request_buf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(iq->txtag, request_buf->map); m_freem(finfo->mb); request_buf->buf = NULL; if (request_buf->map != NULL) { bus_dmamap_destroy(iq->txtag, request_buf->map); request_buf->map = NULL; } } else if (request_buf->map != NULL) { bus_dmamap_unload(iq->txtag, request_buf->map); bus_dmamap_destroy(iq->txtag, request_buf->map); request_buf->map = NULL; } } } if (iq->br != NULL) { buf_ring_free(iq->br, M_DEVBUF); iq->br = NULL; } if (iq->request_list != NULL) { free(iq->request_list, M_DEVBUF); iq->request_list = NULL; } if (iq->txtag != NULL) { bus_dma_tag_destroy(iq->txtag); iq->txtag = NULL; } if (iq->base_addr) { q_size = iq->max_count * desc_size; lio_dma_free((uint32_t)q_size, iq->base_addr); oct->io_qmask.iq &= ~(1ULL << iq_no); bzero(oct->instr_queue[iq_no], sizeof(struct lio_instr_queue)); oct->num_iqs--; return (0); } return (1); } /* Return 0 on success, 1 on failure */ int lio_setup_iq(struct octeon_device *oct, int ifidx, int q_index, union octeon_txpciq txpciq, uint32_t num_descs) { uint32_t iq_no = (uint32_t)txpciq.s.q_no; if (oct->instr_queue[iq_no]->oct_dev != NULL) { lio_dev_dbg(oct, "IQ is in use. Cannot create the IQ: %d again\n", iq_no); oct->instr_queue[iq_no]->txpciq.txpciq64 = txpciq.txpciq64; return (0); } oct->instr_queue[iq_no]->q_index = q_index; oct->instr_queue[iq_no]->ifidx = ifidx; if (lio_init_instr_queue(oct, txpciq, num_descs)) { lio_delete_instr_queue(oct, iq_no); return (1); } oct->num_iqs++; if (oct->fn_list.enable_io_queues(oct)) return (1); return (0); } int lio_wait_for_instr_fetch(struct octeon_device *oct) { int i, retry = 1000, pending, instr_cnt = 0; do { instr_cnt = 0; for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { if (!(oct->io_qmask.iq & BIT_ULL(i))) continue; pending = atomic_load_acq_int( &oct->instr_queue[i]->instr_pending); if (pending) __lio_check_db_timeout(oct, i); instr_cnt += pending; } if (instr_cnt == 0) break; lio_sleep_timeout(1); } while (retry-- && instr_cnt); return (instr_cnt); } static inline void lio_ring_doorbell(struct octeon_device *oct, struct lio_instr_queue *iq) { if (atomic_load_acq_int(&oct->status) == LIO_DEV_RUNNING) { lio_write_csr32(oct, iq->doorbell_reg, iq->fill_cnt); /* make sure doorbell write goes through */ __compiler_membar(); iq->fill_cnt = 0; iq->last_db_time = ticks; return; } } static inline void __lio_copy_cmd_into_iq(struct lio_instr_queue *iq, uint8_t *cmd) { uint8_t *iqptr, cmdsize; cmdsize = ((iq->iqcmd_64B) ? 64 : 32); iqptr = iq->base_addr + (cmdsize * iq->host_write_index); memcpy(iqptr, cmd, cmdsize); } static inline struct lio_iq_post_status __lio_post_command2(struct lio_instr_queue *iq, uint8_t *cmd) { struct lio_iq_post_status st; st.status = LIO_IQ_SEND_OK; /* * This ensures that the read index does not wrap around to the same * position if queue gets full before Octeon could fetch any instr. */ if (atomic_load_acq_int(&iq->instr_pending) >= (int32_t)(iq->max_count - 1)) { st.status = LIO_IQ_SEND_FAILED; st.index = -1; return (st); } if (atomic_load_acq_int(&iq->instr_pending) >= (int32_t)(iq->max_count - 2)) st.status = LIO_IQ_SEND_STOP; __lio_copy_cmd_into_iq(iq, cmd); /* "index" is returned, host_write_index is modified. */ st.index = iq->host_write_index; iq->host_write_index = lio_incr_index(iq->host_write_index, 1, iq->max_count); iq->fill_cnt++; /* * Flush the command into memory. We need to be sure the data is in * memory before indicating that the instruction is pending. */ wmb(); atomic_add_int(&iq->instr_pending, 1); return (st); } static inline void __lio_add_to_request_list(struct lio_instr_queue *iq, int idx, void *buf, int reqtype) { iq->request_list[idx].buf = buf; iq->request_list[idx].reqtype = reqtype; } /* Can only run in process context */ int lio_process_iq_request_list(struct octeon_device *oct, struct lio_instr_queue *iq, uint32_t budget) { struct lio_soft_command *sc; struct octeon_instr_irh *irh = NULL; - struct lio_mbuf_free_info *finfo; void *buf; uint32_t inst_count = 0; uint32_t old = iq->flush_index; int reqtype; while (old != iq->octeon_read_index) { reqtype = iq->request_list[old].reqtype; buf = iq->request_list[old].buf; - finfo = buf; if (reqtype == LIO_REQTYPE_NONE) goto skip_this; switch (reqtype) { case LIO_REQTYPE_NORESP_NET: lio_free_mbuf(iq, buf); break; case LIO_REQTYPE_NORESP_NET_SG: lio_free_sgmbuf(iq, buf); break; case LIO_REQTYPE_RESP_NET: case LIO_REQTYPE_SOFT_COMMAND: sc = buf; if (LIO_CN23XX_PF(oct)) irh = (struct octeon_instr_irh *) &sc->cmd.cmd3.irh; if (irh->rflag) { /* * We're expecting a response from Octeon. * It's up to lio_process_ordered_list() to * process sc. Add sc to the ordered soft * command response list because we expect * a response from Octeon. */ mtx_lock(&oct->response_list [LIO_ORDERED_SC_LIST].lock); atomic_add_int(&oct->response_list [LIO_ORDERED_SC_LIST]. pending_req_count, 1); STAILQ_INSERT_TAIL(&oct->response_list [LIO_ORDERED_SC_LIST]. head, &sc->node, entries); mtx_unlock(&oct->response_list [LIO_ORDERED_SC_LIST].lock); } else { if (sc->callback != NULL) { /* This callback must not sleep */ sc->callback(oct, LIO_REQUEST_DONE, sc->callback_arg); } } break; default: lio_dev_err(oct, "%s Unknown reqtype: %d buf: %p at idx %d\n", __func__, reqtype, buf, old); } iq->request_list[old].buf = NULL; iq->request_list[old].reqtype = 0; skip_this: inst_count++; old = lio_incr_index(old, 1, iq->max_count); if ((budget) && (inst_count >= budget)) break; } iq->flush_index = old; return (inst_count); } /* Can only be called from process context */ int lio_flush_iq(struct octeon_device *oct, struct lio_instr_queue *iq, uint32_t budget) { uint32_t inst_processed = 0; uint32_t tot_inst_processed = 0; int tx_done = 1; if (!mtx_trylock(&iq->iq_flush_running_lock)) return (tx_done); mtx_lock(&iq->lock); iq->octeon_read_index = oct->fn_list.update_iq_read_idx(iq); do { /* Process any outstanding IQ packets. */ if (iq->flush_index == iq->octeon_read_index) break; if (budget) inst_processed = lio_process_iq_request_list(oct, iq, budget - tot_inst_processed); else inst_processed = lio_process_iq_request_list(oct, iq, 0); if (inst_processed) { atomic_subtract_int(&iq->instr_pending, inst_processed); iq->stats.instr_processed += inst_processed; } tot_inst_processed += inst_processed; inst_processed = 0; } while (tot_inst_processed < budget); if (budget && (tot_inst_processed >= budget)) tx_done = 0; iq->last_db_time = ticks; mtx_unlock(&iq->lock); mtx_unlock(&iq->iq_flush_running_lock); return (tx_done); } /* * Process instruction queue after timeout. * This routine gets called from a taskqueue or when removing the module. */ static void __lio_check_db_timeout(struct octeon_device *oct, uint64_t iq_no) { struct lio_instr_queue *iq; uint64_t next_time; if (oct == NULL) return; iq = oct->instr_queue[iq_no]; if (iq == NULL) return; if (atomic_load_acq_int(&iq->instr_pending)) { /* If ticks - last_db_time < db_timeout do nothing */ next_time = iq->last_db_time + lio_ms_to_ticks(iq->db_timeout); if (!lio_check_timeout(ticks, next_time)) return; iq->last_db_time = ticks; /* Flush the instruction queue */ lio_flush_iq(oct, iq, 0); lio_enable_irq(NULL, iq); } if (oct->props.ifp != NULL && iq->br != NULL) { if (mtx_trylock(&iq->enq_lock)) { if (!drbr_empty(oct->props.ifp, iq->br)) lio_mq_start_locked(oct->props.ifp, iq); mtx_unlock(&iq->enq_lock); } } } /* * Called by the Poll thread at regular intervals to check the instruction * queue for commands to be posted and for commands that were fetched by Octeon. */ static void lio_check_db_timeout(void *arg, int pending) { struct lio_tq *db_tq = (struct lio_tq *)arg; struct octeon_device *oct = db_tq->ctxptr; uint64_t iq_no = db_tq->ctxul; uint32_t delay = 10; __lio_check_db_timeout(oct, iq_no); taskqueue_enqueue_timeout(db_tq->tq, &db_tq->work, lio_ms_to_ticks(delay)); } int lio_send_command(struct octeon_device *oct, uint32_t iq_no, uint32_t force_db, void *cmd, void *buf, uint32_t datasize, uint32_t reqtype) { struct lio_iq_post_status st; struct lio_instr_queue *iq = oct->instr_queue[iq_no]; /* * Get the lock and prevent other tasks and tx interrupt handler * from running. */ mtx_lock(&iq->post_lock); st = __lio_post_command2(iq, cmd); if (st.status != LIO_IQ_SEND_FAILED) { __lio_add_to_request_list(iq, st.index, buf, reqtype); LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, bytes_sent, datasize); LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_posted, 1); if (force_db || (st.status == LIO_IQ_SEND_STOP)) lio_ring_doorbell(oct, iq); } else { LIO_INCR_INSTRQUEUE_PKT_COUNT(oct, iq_no, instr_dropped, 1); } mtx_unlock(&iq->post_lock); /* * This is only done here to expedite packets being flushed for * cases where there are no IQ completion interrupts. */ return (st.status); } void lio_prepare_soft_command(struct octeon_device *oct, struct lio_soft_command *sc, uint8_t opcode, uint8_t subcode, uint32_t irh_ossp, uint64_t ossp0, uint64_t ossp1) { - struct lio_config *lio_cfg; struct octeon_instr_ih3 *ih3; struct octeon_instr_pki_ih3 *pki_ih3; struct octeon_instr_irh *irh; struct octeon_instr_rdp *rdp; KASSERT(opcode <= 15, ("%s, %d, opcode > 15", __func__, __LINE__)); KASSERT(subcode <= 127, ("%s, %d, opcode > 127", __func__, __LINE__)); - lio_cfg = lio_get_conf(oct); - if (LIO_CN23XX_PF(oct)) { ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; ih3->pkind = oct->instr_queue[sc->iq_no]->txpciq.s.pkind; pki_ih3 = (struct octeon_instr_pki_ih3 *)&sc->cmd.cmd3.pki_ih3; pki_ih3->w = 1; pki_ih3->raw = 1; pki_ih3->utag = 1; pki_ih3->uqpg = oct->instr_queue[sc->iq_no]->txpciq.s.use_qpg; pki_ih3->utt = 1; pki_ih3->tag = LIO_CONTROL; pki_ih3->tagtype = LIO_ATOMIC_TAG; pki_ih3->qpg = oct->instr_queue[sc->iq_no]->txpciq.s.qpg; pki_ih3->pm = 0x7; pki_ih3->sl = 8; if (sc->datasize) ih3->dlengsz = sc->datasize; irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; irh->opcode = opcode; irh->subcode = subcode; /* opcode/subcode specific parameters (ossp) */ irh->ossp = irh_ossp; sc->cmd.cmd3.ossp[0] = ossp0; sc->cmd.cmd3.ossp[1] = ossp1; if (sc->rdatasize) { rdp = (struct octeon_instr_rdp *)&sc->cmd.cmd3.rdp; rdp->pcie_port = oct->pcie_port; rdp->rlen = sc->rdatasize; irh->rflag = 1; /* PKI IH3 */ /* pki_ih3 irh+ossp[0]+ossp[1]+rdp+rptr = 48 bytes */ ih3->fsz = LIO_SOFTCMDRESP_IH3; } else { irh->rflag = 0; /* PKI IH3 */ /* pki_h3 + irh + ossp[0] + ossp[1] = 32 bytes */ ih3->fsz = LIO_PCICMD_O3; } } } int lio_send_soft_command(struct octeon_device *oct, struct lio_soft_command *sc) { struct octeon_instr_ih3 *ih3; struct octeon_instr_irh *irh; uint32_t len = 0; if (LIO_CN23XX_PF(oct)) { ih3 = (struct octeon_instr_ih3 *)&sc->cmd.cmd3.ih3; if (ih3->dlengsz) { KASSERT(sc->dmadptr, ("%s, %d, sc->dmadptr is NULL", __func__, __LINE__)); sc->cmd.cmd3.dptr = sc->dmadptr; } irh = (struct octeon_instr_irh *)&sc->cmd.cmd3.irh; if (irh->rflag) { KASSERT(sc->dmarptr, ("%s, %d, sc->dmarptr is NULL", __func__, __LINE__)); KASSERT(sc->status_word, ("%s, %d, sc->status_word is NULL", __func__, __LINE__)); *sc->status_word = COMPLETION_WORD_INIT; sc->cmd.cmd3.rptr = sc->dmarptr; } len = (uint32_t)ih3->dlengsz; } if (sc->wait_time) sc->timeout = ticks + lio_ms_to_ticks(sc->wait_time); return (lio_send_command(oct, sc->iq_no, 1, &sc->cmd, sc, len, LIO_REQTYPE_SOFT_COMMAND)); } int lio_setup_sc_buffer_pool(struct octeon_device *oct) { struct lio_soft_command *sc; uint64_t dma_addr; int i; STAILQ_INIT(&oct->sc_buf_pool.head); mtx_init(&oct->sc_buf_pool.lock, "sc_pool_lock", NULL, MTX_DEF); atomic_store_rel_int(&oct->sc_buf_pool.alloc_buf_count, 0); for (i = 0; i < LIO_MAX_SOFT_COMMAND_BUFFERS; i++) { sc = (struct lio_soft_command *) lio_dma_alloc(LIO_SOFT_COMMAND_BUFFER_SIZE, (vm_paddr_t *)&dma_addr); if (sc == NULL) { lio_free_sc_buffer_pool(oct); return (1); } sc->dma_addr = dma_addr; sc->size = LIO_SOFT_COMMAND_BUFFER_SIZE; STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries); } return (0); } int lio_free_sc_buffer_pool(struct octeon_device *oct) { struct lio_stailq_node *tmp, *tmp2; struct lio_soft_command *sc; mtx_lock(&oct->sc_buf_pool.lock); STAILQ_FOREACH_SAFE(tmp, &oct->sc_buf_pool.head, entries, tmp2) { sc = LIO_STAILQ_FIRST_ENTRY(&oct->sc_buf_pool.head, struct lio_soft_command, node); STAILQ_REMOVE_HEAD(&oct->sc_buf_pool.head, entries); lio_dma_free(sc->size, sc); } STAILQ_INIT(&oct->sc_buf_pool.head); mtx_unlock(&oct->sc_buf_pool.lock); return (0); } struct lio_soft_command * lio_alloc_soft_command(struct octeon_device *oct, uint32_t datasize, uint32_t rdatasize, uint32_t ctxsize) { struct lio_soft_command *sc = NULL; struct lio_stailq_node *tmp; uint64_t dma_addr; uint32_t size; uint32_t offset = sizeof(struct lio_soft_command); KASSERT((offset + datasize + rdatasize + ctxsize) <= LIO_SOFT_COMMAND_BUFFER_SIZE, ("%s, %d, offset + datasize + rdatasize + ctxsize > LIO_SOFT_COMMAND_BUFFER_SIZE", __func__, __LINE__)); mtx_lock(&oct->sc_buf_pool.lock); if (STAILQ_EMPTY(&oct->sc_buf_pool.head)) { mtx_unlock(&oct->sc_buf_pool.lock); return (NULL); } tmp = STAILQ_LAST(&oct->sc_buf_pool.head, lio_stailq_node, entries); STAILQ_REMOVE(&oct->sc_buf_pool.head, tmp, lio_stailq_node, entries); atomic_add_int(&oct->sc_buf_pool.alloc_buf_count, 1); mtx_unlock(&oct->sc_buf_pool.lock); sc = (struct lio_soft_command *)tmp; dma_addr = sc->dma_addr; size = sc->size; bzero(sc, sc->size); sc->dma_addr = dma_addr; sc->size = size; if (ctxsize) { sc->ctxptr = (uint8_t *)sc + offset; sc->ctxsize = ctxsize; } /* Start data at 128 byte boundary */ offset = (offset + ctxsize + 127) & 0xffffff80; if (datasize) { sc->virtdptr = (uint8_t *)sc + offset; sc->dmadptr = dma_addr + offset; sc->datasize = datasize; } /* Start rdata at 128 byte boundary */ offset = (offset + datasize + 127) & 0xffffff80; if (rdatasize) { KASSERT(rdatasize >= 16, ("%s, %d, rdatasize < 16", __func__, __LINE__)); sc->virtrptr = (uint8_t *)sc + offset; sc->dmarptr = dma_addr + offset; sc->rdatasize = rdatasize; sc->status_word = (uint64_t *)((uint8_t *)(sc->virtrptr) + rdatasize - 8); } return (sc); } void lio_free_soft_command(struct octeon_device *oct, struct lio_soft_command *sc) { mtx_lock(&oct->sc_buf_pool.lock); STAILQ_INSERT_TAIL(&oct->sc_buf_pool.head, &sc->node, entries); atomic_subtract_int(&oct->sc_buf_pool.alloc_buf_count, 1); mtx_unlock(&oct->sc_buf_pool.lock); } diff --git a/sys/dev/liquidio/lio_main.c b/sys/dev/liquidio/lio_main.c index a7c1308c10d9..79cf7105bed3 100644 --- a/sys/dev/liquidio/lio_main.c +++ b/sys/dev/liquidio/lio_main.c @@ -1,2310 +1,2309 @@ /* * BSD LICENSE * * Copyright(c) 2017 Cavium, Inc.. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Cavium, Inc. nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*$FreeBSD$*/ #include "lio_bsd.h" #include "lio_common.h" #include "lio_droq.h" #include "lio_iq.h" #include "lio_response_manager.h" #include "lio_device.h" #include "lio_ctrl.h" #include "lio_main.h" #include "lio_network.h" #include "cn23xx_pf_device.h" #include "lio_image.h" #include "lio_ioctl.h" #include "lio_rxtx.h" #include "lio_rss.h" /* Number of milliseconds to wait for DDR initialization */ #define LIO_DDR_TIMEOUT 10000 #define LIO_MAX_FW_TYPE_LEN 8 static char fw_type[LIO_MAX_FW_TYPE_LEN]; TUNABLE_STR("hw.lio.fw_type", fw_type, sizeof(fw_type)); /* * Integers that specify number of queues per PF. * Valid range is 0 to 64. * Use 0 to derive from CPU count. */ static int num_queues_per_pf0; static int num_queues_per_pf1; TUNABLE_INT("hw.lio.num_queues_per_pf0", &num_queues_per_pf0); TUNABLE_INT("hw.lio.num_queues_per_pf1", &num_queues_per_pf1); #ifdef RSS static int lio_rss = 1; TUNABLE_INT("hw.lio.rss", &lio_rss); #endif /* RSS */ /* Hardware LRO */ unsigned int lio_hwlro = 0; TUNABLE_INT("hw.lio.hwlro", &lio_hwlro); /* * Bitmask indicating which consoles have debug * output redirected to syslog. */ static unsigned long console_bitmask; TUNABLE_ULONG("hw.lio.console_bitmask", &console_bitmask); /* * \brief determines if a given console has debug enabled. * @param console console to check * @returns 1 = enabled. 0 otherwise */ int lio_console_debug_enabled(uint32_t console) { return (console_bitmask >> (console)) & 0x1; } static int lio_detach(device_t dev); static int lio_device_init(struct octeon_device *octeon_dev); static int lio_chip_specific_setup(struct octeon_device *oct); static void lio_watchdog(void *param); static int lio_load_firmware(struct octeon_device *oct); static int lio_nic_starter(struct octeon_device *oct); static int lio_init_nic_module(struct octeon_device *oct); static int lio_setup_nic_devices(struct octeon_device *octeon_dev); static int lio_link_info(struct lio_recv_info *recv_info, void *ptr); static void lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf); static int lio_set_rxcsum_command(struct ifnet *ifp, int command, uint8_t rx_cmd); static int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs); static void lio_destroy_nic_device(struct octeon_device *oct, int ifidx); static inline void lio_update_link_status(struct ifnet *ifp, union octeon_link_status *ls); static void lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop); static int lio_stop_nic_module(struct octeon_device *oct); static void lio_destroy_resources(struct octeon_device *oct); static int lio_setup_rx_oom_poll_fn(struct ifnet *ifp); static void lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid); static void lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, uint16_t vid); static struct octeon_device * lio_get_other_octeon_device(struct octeon_device *oct); static int lio_wait_for_oq_pkts(struct octeon_device *oct); int lio_send_rss_param(struct lio *lio); static int lio_dbg_console_print(struct octeon_device *oct, uint32_t console_num, char *prefix, char *suffix); /* Polling interval for determining when NIC application is alive */ #define LIO_STARTER_POLL_INTERVAL_MS 100 /* * vendor_info_array. * This array contains the list of IDs on which the driver should load. */ struct lio_vendor_info { uint16_t vendor_id; uint16_t device_id; uint16_t subdevice_id; uint8_t revision_id; uint8_t index; }; static struct lio_vendor_info lio_pci_tbl[] = { /* CN2350 10G */ {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE, 0x02, 0}, /* CN2350 10G */ {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_10G_SUBDEVICE1, 0x02, 0}, /* CN2360 10G */ {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_10G_SUBDEVICE, 0x02, 1}, /* CN2350 25G */ {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2350_25G_SUBDEVICE, 0x02, 2}, /* CN2360 25G */ {PCI_VENDOR_ID_CAVIUM, LIO_CN23XX_PF_VID, LIO_CN2360_25G_SUBDEVICE, 0x02, 3}, {0, 0, 0, 0, 0} }; static char *lio_strings[] = { "LiquidIO 2350 10GbE Server Adapter", "LiquidIO 2360 10GbE Server Adapter", "LiquidIO 2350 25GbE Server Adapter", "LiquidIO 2360 25GbE Server Adapter", }; struct lio_if_cfg_resp { uint64_t rh; struct octeon_if_cfg_info cfg_info; uint64_t status; }; struct lio_if_cfg_context { int octeon_id; volatile int cond; }; struct lio_rx_ctl_context { int octeon_id; volatile int cond; }; static int lio_probe(device_t dev) { struct lio_vendor_info *tbl; uint16_t vendor_id; uint16_t device_id; uint16_t subdevice_id; uint8_t revision_id; char device_ver[256]; vendor_id = pci_get_vendor(dev); if (vendor_id != PCI_VENDOR_ID_CAVIUM) return (ENXIO); device_id = pci_get_device(dev); subdevice_id = pci_get_subdevice(dev); revision_id = pci_get_revid(dev); tbl = lio_pci_tbl; while (tbl->vendor_id) { if ((vendor_id == tbl->vendor_id) && (device_id == tbl->device_id) && (subdevice_id == tbl->subdevice_id) && (revision_id == tbl->revision_id)) { sprintf(device_ver, "%s, Version - %s", lio_strings[tbl->index], LIO_VERSION); device_set_desc_copy(dev, device_ver); return (BUS_PROBE_DEFAULT); } tbl++; } return (ENXIO); } static int lio_attach(device_t device) { struct octeon_device *oct_dev = NULL; uint64_t scratch1; uint32_t error; int timeout, ret = 1; uint8_t bus, dev, function; oct_dev = lio_allocate_device(device); if (oct_dev == NULL) { device_printf(device, "Error: Unable to allocate device\n"); return (-ENOMEM); } oct_dev->tx_budget = LIO_DEFAULT_TX_PKTS_PROCESS_BUDGET; oct_dev->rx_budget = LIO_DEFAULT_RX_PKTS_PROCESS_BUDGET; oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; oct_dev->device = device; bus = pci_get_bus(device); dev = pci_get_slot(device); function = pci_get_function(device); lio_dev_info(oct_dev, "Initializing device %x:%x %02x:%02x.%01x\n", pci_get_vendor(device), pci_get_device(device), bus, dev, function); if (lio_device_init(oct_dev)) { lio_dev_err(oct_dev, "Failed to init device\n"); lio_detach(device); return (-ENOMEM); } scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1); if (!(scratch1 & 4ULL)) { /* * Bit 2 of SLI_SCRATCH_1 is a flag that indicates that * the lio watchdog kernel thread is running for this * NIC. Each NIC gets one watchdog kernel thread. */ scratch1 |= 4ULL; lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1); error = kproc_create(lio_watchdog, oct_dev, &oct_dev->watchdog_task, 0, 0, "liowd/%02hhx:%02hhx.%hhx", bus, dev, function); if (!error) { kproc_resume(oct_dev->watchdog_task); } else { oct_dev->watchdog_task = NULL; lio_dev_err(oct_dev, "failed to create kernel_thread\n"); lio_detach(device); return (-1); } } oct_dev->rx_pause = 1; oct_dev->tx_pause = 1; timeout = 0; while (timeout < LIO_NIC_STARTER_TIMEOUT) { lio_mdelay(LIO_STARTER_POLL_INTERVAL_MS); timeout += LIO_STARTER_POLL_INTERVAL_MS; /* * During the boot process interrupts are not available. * So polling for first control message from FW. */ if (cold) lio_droq_bh(oct_dev->droq[0], 0); if (atomic_load_acq_int(&oct_dev->status) == LIO_DEV_CORE_OK) { ret = lio_nic_starter(oct_dev); break; } } if (ret) { lio_dev_err(oct_dev, "Firmware failed to start\n"); lio_detach(device); return (-EIO); } lio_dev_dbg(oct_dev, "Device is ready\n"); return (0); } static int lio_detach(device_t dev) { struct octeon_device *oct_dev = device_get_softc(dev); lio_dev_dbg(oct_dev, "Stopping device\n"); if (oct_dev->watchdog_task) { uint64_t scratch1; kproc_suspend(oct_dev->watchdog_task, 0); scratch1 = lio_read_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1); scratch1 &= ~4ULL; lio_write_csr64(oct_dev, LIO_CN23XX_SLI_SCRATCH1, scratch1); } if (oct_dev->app_mode && (oct_dev->app_mode == LIO_DRV_NIC_APP)) lio_stop_nic_module(oct_dev); /* * Reset the octeon device and cleanup all memory allocated for * the octeon device by driver. */ lio_destroy_resources(oct_dev); lio_dev_info(oct_dev, "Device removed\n"); /* * This octeon device has been removed. Update the global * data structure to reflect this. Free the device structure. */ lio_free_device_mem(oct_dev); return (0); } static int lio_shutdown(device_t dev) { struct octeon_device *oct_dev = device_get_softc(dev); struct lio *lio = if_getsoftc(oct_dev->props.ifp); lio_send_rx_ctrl_cmd(lio, 0); return (0); } static int lio_suspend(device_t dev) { return (ENXIO); } static int lio_resume(device_t dev) { return (ENXIO); } static int lio_event(struct module *mod, int event, void *junk) { switch (event) { case MOD_LOAD: lio_init_device_list(LIO_CFG_TYPE_DEFAULT); break; default: break; } return (0); } /********************************************************************* * FreeBSD Device Interface Entry Points * *******************************************************************/ static device_method_t lio_methods[] = { /* Device interface */ DEVMETHOD(device_probe, lio_probe), DEVMETHOD(device_attach, lio_attach), DEVMETHOD(device_detach, lio_detach), DEVMETHOD(device_shutdown, lio_shutdown), DEVMETHOD(device_suspend, lio_suspend), DEVMETHOD(device_resume, lio_resume), DEVMETHOD_END }; static driver_t lio_driver = { LIO_DRV_NAME, lio_methods, sizeof(struct octeon_device), }; devclass_t lio_devclass; DRIVER_MODULE(lio, pci, lio_driver, lio_devclass, lio_event, 0); MODULE_DEPEND(lio, pci, 1, 1, 1); MODULE_DEPEND(lio, ether, 1, 1, 1); MODULE_DEPEND(lio, firmware, 1, 1, 1); static bool fw_type_is_none(void) { return strncmp(fw_type, LIO_FW_NAME_TYPE_NONE, sizeof(LIO_FW_NAME_TYPE_NONE)) == 0; } /* * \brief Device initialization for each Octeon device that is probed * @param octeon_dev octeon device */ static int lio_device_init(struct octeon_device *octeon_dev) { unsigned long ddr_timeout = LIO_DDR_TIMEOUT; char *dbg_enb = NULL; int fw_loaded = 0; int i, j, ret; uint8_t bus, dev, function; char bootcmd[] = "\n"; bus = pci_get_bus(octeon_dev->device); dev = pci_get_slot(octeon_dev->device); function = pci_get_function(octeon_dev->device); atomic_store_rel_int(&octeon_dev->status, LIO_DEV_BEGIN_STATE); /* Enable access to the octeon device */ if (pci_enable_busmaster(octeon_dev->device)) { lio_dev_err(octeon_dev, "pci_enable_device failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_ENABLE_DONE); /* Identify the Octeon type and map the BAR address space. */ if (lio_chip_specific_setup(octeon_dev)) { lio_dev_err(octeon_dev, "Chip specific setup failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_PCI_MAP_DONE); /* * Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE', * since that is what is required for the reference to be removed * during de-initialization (see 'octeon_destroy_resources'). */ lio_register_device(octeon_dev, bus, dev, function, true); octeon_dev->app_mode = LIO_DRV_INVALID_APP; if (!lio_cn23xx_pf_fw_loaded(octeon_dev) && !fw_type_is_none()) { fw_loaded = 0; /* Do a soft reset of the Octeon device. */ if (octeon_dev->fn_list.soft_reset(octeon_dev)) return (1); /* things might have changed */ if (!lio_cn23xx_pf_fw_loaded(octeon_dev)) fw_loaded = 0; else fw_loaded = 1; } else { fw_loaded = 1; } /* * Initialize the dispatch mechanism used to push packets arriving on * Octeon Output queues. */ if (lio_init_dispatch_list(octeon_dev)) return (1); lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CORE_DRV_ACTIVE, lio_core_drv_init, octeon_dev); atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DISPATCH_INIT_DONE); ret = octeon_dev->fn_list.setup_device_regs(octeon_dev); if (ret) { lio_dev_err(octeon_dev, "Failed to configure device registers\n"); return (ret); } /* Initialize soft command buffer pool */ if (lio_setup_sc_buffer_pool(octeon_dev)) { lio_dev_err(octeon_dev, "sc buffer pool allocation failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_SC_BUFF_POOL_INIT_DONE); if (lio_allocate_ioq_vector(octeon_dev)) { lio_dev_err(octeon_dev, "IOQ vector allocation failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_MSIX_ALLOC_VECTOR_DONE); for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) { octeon_dev->instr_queue[i] = malloc(sizeof(struct lio_instr_queue), M_DEVBUF, M_NOWAIT | M_ZERO); if (octeon_dev->instr_queue[i] == NULL) return (1); } /* Setup the data structures that manage this Octeon's Input queues. */ if (lio_setup_instr_queue0(octeon_dev)) { lio_dev_err(octeon_dev, "Instruction queue initialization failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_INSTR_QUEUE_INIT_DONE); /* * Initialize lists to manage the requests of different types that * arrive from user & kernel applications for this octeon device. */ if (lio_setup_response_list(octeon_dev)) { lio_dev_err(octeon_dev, "Response list allocation failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_RESP_LIST_INIT_DONE); for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) { octeon_dev->droq[i] = malloc(sizeof(*octeon_dev->droq[i]), M_DEVBUF, M_NOWAIT | M_ZERO); if (octeon_dev->droq[i] == NULL) return (1); } if (lio_setup_output_queue0(octeon_dev)) { lio_dev_err(octeon_dev, "Output queue initialization failed\n"); return (1); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_DROQ_INIT_DONE); /* * Setup the interrupt handler and record the INT SUM register address */ if (lio_setup_interrupt(octeon_dev, octeon_dev->sriov_info.num_pf_rings)) return (1); /* Enable Octeon device interrupts */ octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR); atomic_store_rel_int(&octeon_dev->status, LIO_DEV_INTR_SET_DONE); /* * Send Credit for Octeon Output queues. Credits are always sent BEFORE * the output queue is enabled. * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0. * Otherwise, it is possible that the DRV_ACTIVE message will be sent * before any credits have been issued, causing the ring to be reset * (and the f/w appear to never have started). */ for (j = 0; j < octeon_dev->num_oqs; j++) lio_write_csr32(octeon_dev, octeon_dev->droq[j]->pkts_credit_reg, octeon_dev->droq[j]->max_count); /* Enable the input and output queues for this Octeon device */ ret = octeon_dev->fn_list.enable_io_queues(octeon_dev); if (ret) { lio_dev_err(octeon_dev, "Failed to enable input/output queues"); return (ret); } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_IO_QUEUES_DONE); if (!fw_loaded) { lio_dev_dbg(octeon_dev, "Waiting for DDR initialization...\n"); if (!ddr_timeout) { lio_dev_info(octeon_dev, "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n"); } lio_sleep_timeout(LIO_RESET_MSECS); /* * Wait for the octeon to initialize DDR after the * soft-reset. */ while (!ddr_timeout) { if (pause("-", lio_ms_to_ticks(100))) { /* user probably pressed Control-C */ return (1); } } ret = lio_wait_for_ddr_init(octeon_dev, &ddr_timeout); if (ret) { lio_dev_err(octeon_dev, "DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n", ret); return (1); } if (lio_wait_for_bootloader(octeon_dev, 1100)) { lio_dev_err(octeon_dev, "Board not responding\n"); return (1); } /* Divert uboot to take commands from host instead. */ ret = lio_console_send_cmd(octeon_dev, bootcmd, 50); lio_dev_dbg(octeon_dev, "Initializing consoles\n"); ret = lio_init_consoles(octeon_dev); if (ret) { lio_dev_err(octeon_dev, "Could not access board consoles\n"); return (1); } /* * If console debug enabled, specify empty string to * use default enablement ELSE specify NULL string for * 'disabled'. */ dbg_enb = lio_console_debug_enabled(0) ? "" : NULL; ret = lio_add_console(octeon_dev, 0, dbg_enb); if (ret) { lio_dev_err(octeon_dev, "Could not access board console\n"); return (1); } else if (lio_console_debug_enabled(0)) { /* * If console was added AND we're logging console output * then set our console print function. */ octeon_dev->console[0].print = lio_dbg_console_print; } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_CONSOLE_INIT_DONE); lio_dev_dbg(octeon_dev, "Loading firmware\n"); ret = lio_load_firmware(octeon_dev); if (ret) { lio_dev_err(octeon_dev, "Could not load firmware to board\n"); return (1); } } atomic_store_rel_int(&octeon_dev->status, LIO_DEV_HOST_OK); return (0); } /* * \brief PCI FLR for each Octeon device. * @param oct octeon device */ static void lio_pci_flr(struct octeon_device *oct) { uint32_t exppos, status; pci_find_cap(oct->device, PCIY_EXPRESS, &exppos); pci_save_state(oct->device); /* Quiesce the device completely */ pci_write_config(oct->device, PCIR_COMMAND, PCIM_CMD_INTxDIS, 2); /* Wait for Transaction Pending bit clean */ lio_mdelay(100); status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2); if (status & PCIEM_STA_TRANSACTION_PND) { lio_dev_info(oct, "Function reset incomplete after 100ms, sleeping for 5 seconds\n"); lio_mdelay(5); status = pci_read_config(oct->device, exppos + PCIER_DEVICE_STA, 2); if (status & PCIEM_STA_TRANSACTION_PND) lio_dev_info(oct, "Function reset still incomplete after 5s, reset anyway\n"); } pci_write_config(oct->device, exppos + PCIER_DEVICE_CTL, PCIEM_CTL_INITIATE_FLR, 2); lio_mdelay(100); pci_restore_state(oct->device); } /* * \brief Debug console print function * @param octeon_dev octeon device * @param console_num console number * @param prefix first portion of line to display * @param suffix second portion of line to display * * The OCTEON debug console outputs entire lines (excluding '\n'). * Normally, the line will be passed in the 'prefix' parameter. * However, due to buffering, it is possible for a line to be split into two * parts, in which case they will be passed as the 'prefix' parameter and * 'suffix' parameter. */ static int lio_dbg_console_print(struct octeon_device *oct, uint32_t console_num, char *prefix, char *suffix) { if (prefix != NULL && suffix != NULL) lio_dev_info(oct, "%u: %s%s\n", console_num, prefix, suffix); else if (prefix != NULL) lio_dev_info(oct, "%u: %s\n", console_num, prefix); else if (suffix != NULL) lio_dev_info(oct, "%u: %s\n", console_num, suffix); return (0); } static void lio_watchdog(void *param) { int core_num; uint16_t mask_of_crashed_or_stuck_cores = 0; struct octeon_device *oct = param; bool err_msg_was_printed[12]; bzero(err_msg_was_printed, sizeof(err_msg_was_printed)); while (1) { kproc_suspend_check(oct->watchdog_task); mask_of_crashed_or_stuck_cores = (uint16_t)lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2); if (mask_of_crashed_or_stuck_cores) { struct octeon_device *other_oct; oct->cores_crashed = true; other_oct = lio_get_other_octeon_device(oct); if (other_oct != NULL) other_oct->cores_crashed = true; for (core_num = 0; core_num < LIO_MAX_CORES; core_num++) { bool core_crashed_or_got_stuck; core_crashed_or_got_stuck = (mask_of_crashed_or_stuck_cores >> core_num) & 1; if (core_crashed_or_got_stuck && !err_msg_was_printed[core_num]) { lio_dev_err(oct, "ERROR: Octeon core %d crashed or got stuck! See oct-fwdump for details.\n", core_num); err_msg_was_printed[core_num] = true; } } } /* sleep for two seconds */ pause("-", lio_ms_to_ticks(2000)); } } static int lio_chip_specific_setup(struct octeon_device *oct) { char *s; - uint32_t dev_id, rev_id; + uint32_t dev_id; int ret = 1; dev_id = lio_read_pci_cfg(oct, 0); - rev_id = pci_get_revid(oct->device); oct->subdevice_id = pci_get_subdevice(oct->device); switch (dev_id) { case LIO_CN23XX_PF_PCIID: oct->chip_id = LIO_CN23XX_PF_VID; if (pci_get_function(oct->device) == 0) { if (num_queues_per_pf0 < 0) { lio_dev_info(oct, "Invalid num_queues_per_pf0: %d, Setting it to default\n", num_queues_per_pf0); num_queues_per_pf0 = 0; } oct->sriov_info.num_pf_rings = num_queues_per_pf0; } else { if (num_queues_per_pf1 < 0) { lio_dev_info(oct, "Invalid num_queues_per_pf1: %d, Setting it to default\n", num_queues_per_pf1); num_queues_per_pf1 = 0; } oct->sriov_info.num_pf_rings = num_queues_per_pf1; } ret = lio_cn23xx_pf_setup_device(oct); s = "CN23XX"; break; default: s = "?"; lio_dev_err(oct, "Unknown device found (dev_id: %x)\n", dev_id); } if (!ret) lio_dev_info(oct, "%s PASS%d.%d %s Version: %s\n", s, OCTEON_MAJOR_REV(oct), OCTEON_MINOR_REV(oct), lio_get_conf(oct)->card_name, LIO_VERSION); return (ret); } static struct octeon_device * lio_get_other_octeon_device(struct octeon_device *oct) { struct octeon_device *other_oct; other_oct = lio_get_device(oct->octeon_id + 1); if ((other_oct != NULL) && other_oct->device) { int oct_busnum, other_oct_busnum; oct_busnum = pci_get_bus(oct->device); other_oct_busnum = pci_get_bus(other_oct->device); if (oct_busnum == other_oct_busnum) { int oct_slot, other_oct_slot; oct_slot = pci_get_slot(oct->device); other_oct_slot = pci_get_slot(other_oct->device); if (oct_slot == other_oct_slot) return (other_oct); } } return (NULL); } /* * \brief Load firmware to device * @param oct octeon device * * Maps device to firmware filename, requests firmware, and downloads it */ static int lio_load_firmware(struct octeon_device *oct) { const struct firmware *fw; char *tmp_fw_type = NULL; int ret = 0; char fw_name[LIO_MAX_FW_FILENAME_LEN]; if (fw_type[0] == '\0') tmp_fw_type = LIO_FW_NAME_TYPE_NIC; else tmp_fw_type = fw_type; sprintf(fw_name, "%s%s_%s%s", LIO_FW_BASE_NAME, lio_get_conf(oct)->card_name, tmp_fw_type, LIO_FW_NAME_SUFFIX); fw = firmware_get(fw_name); if (fw == NULL) { lio_dev_err(oct, "Request firmware failed. Could not find file %s.\n", fw_name); return (EINVAL); } ret = lio_download_firmware(oct, fw->data, fw->datasize); firmware_put(fw, FIRMWARE_UNLOAD); return (ret); } static int lio_nic_starter(struct octeon_device *oct) { int ret = 0; atomic_store_rel_int(&oct->status, LIO_DEV_RUNNING); if (oct->app_mode && oct->app_mode == LIO_DRV_NIC_APP) { if (lio_init_nic_module(oct)) { lio_dev_err(oct, "NIC initialization failed\n"); ret = -1; #ifdef CAVIUM_ONiLY_23XX_VF } else { if (octeon_enable_sriov(oct) < 0) ret = -1; #endif } } else { lio_dev_err(oct, "Unexpected application running on NIC (%d). Check firmware.\n", oct->app_mode); ret = -1; } return (ret); } static int lio_init_nic_module(struct octeon_device *oct) { int num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(lio_get_conf(oct)); int retval = 0; lio_dev_dbg(oct, "Initializing network interfaces\n"); /* * only default iq and oq were initialized * initialize the rest as well */ /* run port_config command for each port */ oct->ifcount = num_nic_ports; bzero(&oct->props, sizeof(struct lio_if_props)); oct->props.gmxport = -1; retval = lio_setup_nic_devices(oct); if (retval) { lio_dev_err(oct, "Setup NIC devices failed\n"); goto lio_init_failure; } lio_dev_dbg(oct, "Network interfaces ready\n"); return (retval); lio_init_failure: oct->ifcount = 0; return (retval); } static int lio_ifmedia_update(struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); struct ifmedia *ifm; ifm = &lio->ifmedia; /* We only support Ethernet media type. */ if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: break; case IFM_10G_CX4: case IFM_10G_SR: case IFM_10G_T: case IFM_10G_TWINAX: default: /* We don't support changing the media type. */ lio_dev_err(lio->oct_dev, "Invalid media type (%d)\n", IFM_SUBTYPE(ifm->ifm_media)); return (EINVAL); } return (0); } static int lio_get_media_subtype(struct octeon_device *oct) { switch(oct->subdevice_id) { case LIO_CN2350_10G_SUBDEVICE: case LIO_CN2350_10G_SUBDEVICE1: case LIO_CN2360_10G_SUBDEVICE: return (IFM_10G_SR); case LIO_CN2350_25G_SUBDEVICE: case LIO_CN2360_25G_SUBDEVICE: return (IFM_25G_SR); } return (IFM_10G_SR); } static uint64_t lio_get_baudrate(struct octeon_device *oct) { switch(oct->subdevice_id) { case LIO_CN2350_10G_SUBDEVICE: case LIO_CN2350_10G_SUBDEVICE1: case LIO_CN2360_10G_SUBDEVICE: return (IF_Gbps(10)); case LIO_CN2350_25G_SUBDEVICE: case LIO_CN2360_25G_SUBDEVICE: return (IF_Gbps(25)); } return (IF_Gbps(10)); } static void lio_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct lio *lio = if_getsoftc(ifp); /* Report link down if the driver isn't running. */ if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) { ifmr->ifm_active |= IFM_NONE; return; } /* Setup the default interface info. */ ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (lio->linfo.link.s.link_up) { ifmr->ifm_status |= IFM_ACTIVE; } else { ifmr->ifm_active |= IFM_NONE; return; } ifmr->ifm_active |= lio_get_media_subtype(lio->oct_dev); if (lio->linfo.link.s.duplex) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; } static uint64_t lio_get_counter(if_t ifp, ift_counter cnt) { struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; uint64_t counter = 0; int i, q_no; switch (cnt) { case IFCOUNTER_IPACKETS: for (i = 0; i < oct->num_oqs; i++) { q_no = lio->linfo.rxpciq[i].s.q_no; counter += oct->droq[q_no]->stats.rx_pkts_received; } break; case IFCOUNTER_OPACKETS: for (i = 0; i < oct->num_iqs; i++) { q_no = lio->linfo.txpciq[i].s.q_no; counter += oct->instr_queue[q_no]->stats.tx_done; } break; case IFCOUNTER_IBYTES: for (i = 0; i < oct->num_oqs; i++) { q_no = lio->linfo.rxpciq[i].s.q_no; counter += oct->droq[q_no]->stats.rx_bytes_received; } break; case IFCOUNTER_OBYTES: for (i = 0; i < oct->num_iqs; i++) { q_no = lio->linfo.txpciq[i].s.q_no; counter += oct->instr_queue[q_no]->stats.tx_tot_bytes; } break; case IFCOUNTER_IQDROPS: for (i = 0; i < oct->num_oqs; i++) { q_no = lio->linfo.rxpciq[i].s.q_no; counter += oct->droq[q_no]->stats.rx_dropped; } break; case IFCOUNTER_OQDROPS: for (i = 0; i < oct->num_iqs; i++) { q_no = lio->linfo.txpciq[i].s.q_no; counter += oct->instr_queue[q_no]->stats.tx_dropped; } break; case IFCOUNTER_IMCASTS: counter = oct->link_stats.fromwire.total_mcst; break; case IFCOUNTER_OMCASTS: counter = oct->link_stats.fromhost.mcast_pkts_sent; break; case IFCOUNTER_COLLISIONS: counter = oct->link_stats.fromhost.total_collisions; break; case IFCOUNTER_IERRORS: counter = oct->link_stats.fromwire.fcs_err + oct->link_stats.fromwire.l2_err + oct->link_stats.fromwire.frame_err; break; default: return (if_get_counter_default(ifp, cnt)); } return (counter); } static int lio_init_ifnet(struct lio *lio) { struct octeon_device *oct = lio->oct_dev; if_t ifp = lio->ifp; /* ifconfig entrypoint for media type/status reporting */ ifmedia_init(&lio->ifmedia, IFM_IMASK, lio_ifmedia_update, lio_ifmedia_status); /* set the default interface values */ ifmedia_add(&lio->ifmedia, (IFM_ETHER | IFM_FDX | lio_get_media_subtype(oct)), 0, NULL); ifmedia_add(&lio->ifmedia, (IFM_ETHER | IFM_AUTO), 0, NULL); ifmedia_set(&lio->ifmedia, (IFM_ETHER | IFM_AUTO)); lio->ifmedia.ifm_media = lio->ifmedia.ifm_cur->ifm_media; lio_dev_dbg(oct, "IFMEDIA flags : %x\n", lio->ifmedia.ifm_media); if_initname(ifp, device_get_name(oct->device), device_get_unit(oct->device)); if_setflags(ifp, (IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST)); if_setioctlfn(ifp, lio_ioctl); if_setgetcounterfn(ifp, lio_get_counter); if_settransmitfn(ifp, lio_mq_start); if_setqflushfn(ifp, lio_qflush); if_setinitfn(ifp, lio_open); if_setmtu(ifp, lio->linfo.link.s.mtu); lio->mtu = lio->linfo.link.s.mtu; if_sethwassist(ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)); if_setcapabilitiesbit(ifp, (IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6 | IFCAP_TSO | IFCAP_LRO | IFCAP_JUMBO_MTU | IFCAP_HWSTATS | IFCAP_LINKSTATE | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU), 0); if_setcapenable(ifp, if_getcapabilities(ifp)); if_setbaudrate(ifp, lio_get_baudrate(oct)); return (0); } static void lio_tcp_lro_free(struct octeon_device *octeon_dev, struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); struct lio_droq *droq; int q_no; int i; for (i = 0; i < octeon_dev->num_oqs; i++) { q_no = lio->linfo.rxpciq[i].s.q_no; droq = octeon_dev->droq[q_no]; if (droq->lro.ifp) { tcp_lro_free(&droq->lro); droq->lro.ifp = NULL; } } } static int lio_tcp_lro_init(struct octeon_device *octeon_dev, struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); struct lio_droq *droq; struct lro_ctrl *lro; int i, q_no, ret = 0; for (i = 0; i < octeon_dev->num_oqs; i++) { q_no = lio->linfo.rxpciq[i].s.q_no; droq = octeon_dev->droq[q_no]; lro = &droq->lro; ret = tcp_lro_init(lro); if (ret) { lio_dev_err(octeon_dev, "LRO Initialization failed ret %d\n", ret); goto lro_init_failed; } lro->ifp = ifp; } return (ret); lro_init_failed: lio_tcp_lro_free(octeon_dev, ifp); return (ret); } static int lio_setup_nic_devices(struct octeon_device *octeon_dev) { union octeon_if_cfg if_cfg; struct lio *lio = NULL; struct ifnet *ifp = NULL; struct lio_version *vdata; struct lio_soft_command *sc; struct lio_if_cfg_context *ctx; struct lio_if_cfg_resp *resp; struct lio_if_props *props; int num_iqueues, num_oqueues, retval; unsigned int base_queue; unsigned int gmx_port_id; uint32_t ctx_size, data_size; uint32_t ifidx_or_pfnum, resp_size; uint8_t mac[ETHER_HDR_LEN], i, j; /* This is to handle link status changes */ lio_register_dispatch_fn(octeon_dev, LIO_OPCODE_NIC, LIO_OPCODE_NIC_INFO, lio_link_info, octeon_dev); for (i = 0; i < octeon_dev->ifcount; i++) { resp_size = sizeof(struct lio_if_cfg_resp); ctx_size = sizeof(struct lio_if_cfg_context); data_size = sizeof(struct lio_version); sc = lio_alloc_soft_command(octeon_dev, data_size, resp_size, ctx_size); if (sc == NULL) return (ENOMEM); resp = (struct lio_if_cfg_resp *)sc->virtrptr; ctx = (struct lio_if_cfg_context *)sc->ctxptr; vdata = (struct lio_version *)sc->virtdptr; *((uint64_t *)vdata) = 0; vdata->major = htobe16(LIO_BASE_MAJOR_VERSION); vdata->minor = htobe16(LIO_BASE_MINOR_VERSION); vdata->micro = htobe16(LIO_BASE_MICRO_VERSION); num_iqueues = octeon_dev->sriov_info.num_pf_rings; num_oqueues = octeon_dev->sriov_info.num_pf_rings; base_queue = octeon_dev->sriov_info.pf_srn; gmx_port_id = octeon_dev->pf_num; ifidx_or_pfnum = octeon_dev->pf_num; lio_dev_dbg(octeon_dev, "requesting config for interface %d, iqs %d, oqs %d\n", ifidx_or_pfnum, num_iqueues, num_oqueues); ctx->cond = 0; ctx->octeon_id = lio_get_device_id(octeon_dev); if_cfg.if_cfg64 = 0; if_cfg.s.num_iqueues = num_iqueues; if_cfg.s.num_oqueues = num_oqueues; if_cfg.s.base_queue = base_queue; if_cfg.s.gmx_port_id = gmx_port_id; sc->iq_no = 0; lio_prepare_soft_command(octeon_dev, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_IF_CFG, 0, if_cfg.if_cfg64, 0); sc->callback = lio_if_cfg_callback; sc->callback_arg = sc; sc->wait_time = 3000; retval = lio_send_soft_command(octeon_dev, sc); if (retval == LIO_IQ_SEND_FAILED) { lio_dev_err(octeon_dev, "iq/oq config failed status: %x\n", retval); /* Soft instr is freed by driver in case of failure. */ goto setup_nic_dev_fail; } /* * Sleep on a wait queue till the cond flag indicates that the * response arrived or timed-out. */ lio_sleep_cond(octeon_dev, &ctx->cond); retval = resp->status; if (retval) { lio_dev_err(octeon_dev, "iq/oq config failed\n"); goto setup_nic_dev_fail; } lio_swap_8B_data((uint64_t *)(&resp->cfg_info), (sizeof(struct octeon_if_cfg_info)) >> 3); num_iqueues = bitcount64(resp->cfg_info.iqmask); num_oqueues = bitcount64(resp->cfg_info.oqmask); if (!(num_iqueues) || !(num_oqueues)) { lio_dev_err(octeon_dev, "Got bad iqueues (%016llX) or oqueues (%016llX) from firmware.\n", LIO_CAST64(resp->cfg_info.iqmask), LIO_CAST64(resp->cfg_info.oqmask)); goto setup_nic_dev_fail; } lio_dev_dbg(octeon_dev, "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", i, LIO_CAST64(resp->cfg_info.iqmask), LIO_CAST64(resp->cfg_info.oqmask), num_iqueues, num_oqueues); ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { lio_dev_err(octeon_dev, "Device allocation failed\n"); goto setup_nic_dev_fail; } lio = malloc(sizeof(struct lio), M_DEVBUF, M_NOWAIT | M_ZERO); if (lio == NULL) { lio_dev_err(octeon_dev, "Lio allocation failed\n"); goto setup_nic_dev_fail; } if_setsoftc(ifp, lio); ifp->if_hw_tsomax = LIO_MAX_FRAME_SIZE; ifp->if_hw_tsomaxsegcount = LIO_MAX_SG; ifp->if_hw_tsomaxsegsize = PAGE_SIZE; lio->ifidx = ifidx_or_pfnum; props = &octeon_dev->props; props->gmxport = resp->cfg_info.linfo.gmxport; props->ifp = ifp; lio->linfo.num_rxpciq = num_oqueues; lio->linfo.num_txpciq = num_iqueues; for (j = 0; j < num_oqueues; j++) { lio->linfo.rxpciq[j].rxpciq64 = resp->cfg_info.linfo.rxpciq[j].rxpciq64; } for (j = 0; j < num_iqueues; j++) { lio->linfo.txpciq[j].txpciq64 = resp->cfg_info.linfo.txpciq[j].txpciq64; } lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; lio->linfo.link.link_status64 = resp->cfg_info.linfo.link.link_status64; /* * Point to the properties for octeon device to which this * interface belongs. */ lio->oct_dev = octeon_dev; lio->ifp = ifp; lio_dev_dbg(octeon_dev, "if%d gmx: %d hw_addr: 0x%llx\n", i, lio->linfo.gmxport, LIO_CAST64(lio->linfo.hw_addr)); lio_init_ifnet(lio); /* 64-bit swap required on LE machines */ lio_swap_8B_data(&lio->linfo.hw_addr, 1); for (j = 0; j < 6; j++) mac[j] = *((uint8_t *)( ((uint8_t *)&lio->linfo.hw_addr) + 2 + j)); ether_ifattach(ifp, mac); /* * By default all interfaces on a single Octeon uses the same * tx and rx queues */ lio->txq = lio->linfo.txpciq[0].s.q_no; lio->rxq = lio->linfo.rxpciq[0].s.q_no; if (lio_setup_io_queues(octeon_dev, i, lio->linfo.num_txpciq, lio->linfo.num_rxpciq)) { lio_dev_err(octeon_dev, "I/O queues creation failed\n"); goto setup_nic_dev_fail; } lio_ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); lio->tx_qsize = lio_get_tx_qsize(octeon_dev, lio->txq); lio->rx_qsize = lio_get_rx_qsize(octeon_dev, lio->rxq); if (lio_setup_glists(octeon_dev, lio, num_iqueues)) { lio_dev_err(octeon_dev, "Gather list allocation failed\n"); goto setup_nic_dev_fail; } if ((lio_hwlro == 0) && lio_tcp_lro_init(octeon_dev, ifp)) goto setup_nic_dev_fail; if (lio_hwlro && (if_getcapenable(ifp) & IFCAP_LRO) && (if_getcapenable(ifp) & IFCAP_RXCSUM) && (if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6)) lio_set_feature(ifp, LIO_CMD_LRO_ENABLE, LIO_LROIPV4 | LIO_LROIPV6); if ((if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)) lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 1); else lio_set_feature(ifp, LIO_CMD_VLAN_FILTER_CTL, 0); if (lio_setup_rx_oom_poll_fn(ifp)) goto setup_nic_dev_fail; lio_dev_dbg(octeon_dev, "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); lio->link_changes++; lio_ifstate_set(lio, LIO_IFSTATE_REGISTERED); /* * Sending command to firmware to enable Rx checksum offload * by default at the time of setup of Liquidio driver for * this device */ lio_set_rxcsum_command(ifp, LIO_CMD_TNL_RX_CSUM_CTL, LIO_CMD_RXCSUM_ENABLE); lio_set_feature(ifp, LIO_CMD_TNL_TX_CSUM_CTL, LIO_CMD_TXCSUM_ENABLE); #ifdef RSS if (lio_rss) { if (lio_send_rss_param(lio)) goto setup_nic_dev_fail; } else #endif /* RSS */ lio_set_feature(ifp, LIO_CMD_SET_FNV, LIO_CMD_FNV_ENABLE); lio_dev_dbg(octeon_dev, "NIC ifidx:%d Setup successful\n", i); lio_free_soft_command(octeon_dev, sc); lio->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, lio_vlan_rx_add_vid, lio, EVENTHANDLER_PRI_FIRST); lio->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, lio_vlan_rx_kill_vid, lio, EVENTHANDLER_PRI_FIRST); /* Update stats periodically */ callout_init(&lio->stats_timer, 0); lio->stats_interval = LIO_DEFAULT_STATS_INTERVAL; lio_add_hw_stats(lio); } return (0); setup_nic_dev_fail: lio_free_soft_command(octeon_dev, sc); while (i--) { lio_dev_err(octeon_dev, "NIC ifidx:%d Setup failed\n", i); lio_destroy_nic_device(octeon_dev, i); } return (ENODEV); } static int lio_link_info(struct lio_recv_info *recv_info, void *ptr) { struct octeon_device *oct = (struct octeon_device *)ptr; struct lio_recv_pkt *recv_pkt = recv_info->recv_pkt; union octeon_link_status *ls; int gmxport = 0, i; lio_dev_dbg(oct, "%s Called\n", __func__); if (recv_pkt->buffer_size[0] != (sizeof(*ls) + LIO_DROQ_INFO_SIZE)) { lio_dev_err(oct, "Malformed NIC_INFO, len=%d, ifidx=%d\n", recv_pkt->buffer_size[0], recv_pkt->rh.r_nic_info.gmxport); goto nic_info_err; } gmxport = recv_pkt->rh.r_nic_info.gmxport; ls = (union octeon_link_status *)(recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE); lio_swap_8B_data((uint64_t *)ls, (sizeof(union octeon_link_status)) >> 3); if (oct->props.gmxport == gmxport) lio_update_link_status(oct->props.ifp, ls); nic_info_err: for (i = 0; i < recv_pkt->buffer_count; i++) lio_recv_buffer_free(recv_pkt->buffer_ptr[i]); lio_free_recv_info(recv_info); return (0); } void lio_free_mbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo) { bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(iq->txtag, finfo->map); m_freem(finfo->mb); } void lio_free_sgmbuf(struct lio_instr_queue *iq, struct lio_mbuf_free_info *finfo) { struct lio_gather *g; struct octeon_device *oct; struct lio *lio; int iq_no; g = finfo->g; iq_no = iq->txpciq.s.q_no; oct = iq->oct_dev; lio = if_getsoftc(oct->props.ifp); mtx_lock(&lio->glist_lock[iq_no]); STAILQ_INSERT_TAIL(&lio->ghead[iq_no], &g->node, entries); mtx_unlock(&lio->glist_lock[iq_no]); bus_dmamap_sync(iq->txtag, finfo->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(iq->txtag, finfo->map); m_freem(finfo->mb); } static void lio_if_cfg_callback(struct octeon_device *oct, uint32_t status, void *buf) { struct lio_soft_command *sc = (struct lio_soft_command *)buf; struct lio_if_cfg_resp *resp; struct lio_if_cfg_context *ctx; resp = (struct lio_if_cfg_resp *)sc->virtrptr; ctx = (struct lio_if_cfg_context *)sc->ctxptr; oct = lio_get_device(ctx->octeon_id); if (resp->status) lio_dev_err(oct, "nic if cfg instruction failed. Status: %llx (0x%08x)\n", LIO_CAST64(resp->status), status); ctx->cond = 1; snprintf(oct->fw_info.lio_firmware_version, 32, "%s", resp->cfg_info.lio_firmware_version); /* * This barrier is required to be sure that the response has been * written fully before waking up the handler */ wmb(); } static int lio_is_mac_changed(uint8_t *new, uint8_t *old) { return ((new[0] != old[0]) || (new[1] != old[1]) || (new[2] != old[2]) || (new[3] != old[3]) || (new[4] != old[4]) || (new[5] != old[5])); } void lio_open(void *arg) { struct lio *lio = arg; struct ifnet *ifp = lio->ifp; struct octeon_device *oct = lio->oct_dev; uint8_t *mac_new, mac_old[ETHER_HDR_LEN]; int ret = 0; lio_ifstate_set(lio, LIO_IFSTATE_RUNNING); /* Ready for link status updates */ lio->intf_open = 1; lio_dev_info(oct, "Interface Open, ready for traffic\n"); /* tell Octeon to start forwarding packets to host */ lio_send_rx_ctrl_cmd(lio, 1); mac_new = IF_LLADDR(ifp); memcpy(mac_old, ((uint8_t *)&lio->linfo.hw_addr) + 2, ETHER_HDR_LEN); if (lio_is_mac_changed(mac_new, mac_old)) { ret = lio_set_mac(ifp, mac_new); if (ret) lio_dev_err(oct, "MAC change failed, error: %d\n", ret); } /* Now inform the stack we're ready */ if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); lio_dev_info(oct, "Interface is opened\n"); } static int lio_set_rxcsum_command(struct ifnet *ifp, int command, uint8_t rx_cmd) { struct lio_ctrl_pkt nctrl; struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; int ret = 0; nctrl.ncmd.cmd64 = 0; nctrl.ncmd.s.cmd = command; nctrl.ncmd.s.param1 = rx_cmd; nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; nctrl.wait_time = 100; nctrl.lio = lio; nctrl.cb_fn = lio_ctrl_cmd_completion; ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); if (ret < 0) { lio_dev_err(oct, "DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n", ret); } return (ret); } static int lio_stop_nic_module(struct octeon_device *oct) { int i, j; struct lio *lio; lio_dev_dbg(oct, "Stopping network interfaces\n"); if (!oct->ifcount) { lio_dev_err(oct, "Init for Octeon was not completed\n"); return (1); } mtx_lock(&oct->cmd_resp_wqlock); oct->cmd_resp_state = LIO_DRV_OFFLINE; mtx_unlock(&oct->cmd_resp_wqlock); for (i = 0; i < oct->ifcount; i++) { lio = if_getsoftc(oct->props.ifp); for (j = 0; j < oct->num_oqs; j++) lio_unregister_droq_ops(oct, lio->linfo.rxpciq[j].s.q_no); } callout_drain(&lio->stats_timer); for (i = 0; i < oct->ifcount; i++) lio_destroy_nic_device(oct, i); lio_dev_dbg(oct, "Network interface stopped\n"); return (0); } static void lio_delete_glists(struct octeon_device *oct, struct lio *lio) { struct lio_gather *g; int i; if (lio->glist_lock != NULL) { free((void *)lio->glist_lock, M_DEVBUF); lio->glist_lock = NULL; } if (lio->ghead == NULL) return; for (i = 0; i < lio->linfo.num_txpciq; i++) { do { g = (struct lio_gather *) lio_delete_first_node(&lio->ghead[i]); free(g, M_DEVBUF); } while (g); if ((lio->glists_virt_base != NULL) && (lio->glists_virt_base[i] != NULL)) { lio_dma_free(lio->glist_entry_size * lio->tx_qsize, lio->glists_virt_base[i]); } } free(lio->glists_virt_base, M_DEVBUF); lio->glists_virt_base = NULL; free(lio->glists_dma_base, M_DEVBUF); lio->glists_dma_base = NULL; free(lio->ghead, M_DEVBUF); lio->ghead = NULL; } static int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_iqs) { struct lio_gather *g; int i, j; lio->glist_lock = malloc(num_iqs * sizeof(*lio->glist_lock), M_DEVBUF, M_NOWAIT | M_ZERO); if (lio->glist_lock == NULL) return (1); lio->ghead = malloc(num_iqs * sizeof(*lio->ghead), M_DEVBUF, M_NOWAIT | M_ZERO); if (lio->ghead == NULL) { free((void *)lio->glist_lock, M_DEVBUF); lio->glist_lock = NULL; return (1); } lio->glist_entry_size = ROUNDUP8((ROUNDUP4(LIO_MAX_SG) >> 2) * LIO_SG_ENTRY_SIZE); /* * allocate memory to store virtual and dma base address of * per glist consistent memory */ lio->glists_virt_base = malloc(num_iqs * sizeof(void *), M_DEVBUF, M_NOWAIT | M_ZERO); lio->glists_dma_base = malloc(num_iqs * sizeof(vm_paddr_t), M_DEVBUF, M_NOWAIT | M_ZERO); if ((lio->glists_virt_base == NULL) || (lio->glists_dma_base == NULL)) { lio_delete_glists(oct, lio); return (1); } for (i = 0; i < num_iqs; i++) { mtx_init(&lio->glist_lock[i], "glist_lock", NULL, MTX_DEF); STAILQ_INIT(&lio->ghead[i]); lio->glists_virt_base[i] = lio_dma_alloc(lio->glist_entry_size * lio->tx_qsize, (vm_paddr_t *)&lio->glists_dma_base[i]); if (lio->glists_virt_base[i] == NULL) { lio_delete_glists(oct, lio); return (1); } for (j = 0; j < lio->tx_qsize; j++) { g = malloc(sizeof(*g), M_DEVBUF, M_NOWAIT | M_ZERO); if (g == NULL) break; g->sg = (struct lio_sg_entry *)(uintptr_t) ((uint64_t)(uintptr_t)lio->glists_virt_base[i] + (j * lio->glist_entry_size)); g->sg_dma_ptr = (uint64_t)lio->glists_dma_base[i] + (j * lio->glist_entry_size); STAILQ_INSERT_TAIL(&lio->ghead[i], &g->node, entries); } if (j != lio->tx_qsize) { lio_delete_glists(oct, lio); return (1); } } return (0); } void lio_stop(struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; lio_ifstate_reset(lio, LIO_IFSTATE_RUNNING); if_link_state_change(ifp, LINK_STATE_DOWN); lio->intf_open = 0; lio->linfo.link.s.link_up = 0; lio->link_changes++; lio_send_rx_ctrl_cmd(lio, 0); /* Tell the stack that the interface is no longer active */ if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); lio_dev_info(oct, "Interface is stopped\n"); } static void lio_check_rx_oom_status(struct lio *lio) { struct lio_droq *droq; struct octeon_device *oct = lio->oct_dev; int desc_refilled; int q, q_no = 0; for (q = 0; q < oct->num_oqs; q++) { q_no = lio->linfo.rxpciq[q].s.q_no; droq = oct->droq[q_no]; if (droq == NULL) continue; if (lio_read_csr32(oct, droq->pkts_credit_reg) <= 0x40) { mtx_lock(&droq->lock); desc_refilled = lio_droq_refill(oct, droq); /* * Flush the droq descriptor data to memory to be sure * that when we update the credits the data in memory * is accurate. */ wmb(); lio_write_csr32(oct, droq->pkts_credit_reg, desc_refilled); /* make sure mmio write completes */ __compiler_membar(); mtx_unlock(&droq->lock); } } } static void lio_poll_check_rx_oom_status(void *arg, int pending __unused) { struct lio_tq *rx_status_tq = arg; struct lio *lio = rx_status_tq->ctxptr; if (lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) lio_check_rx_oom_status(lio); taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work, lio_ms_to_ticks(50)); } static int lio_setup_rx_oom_poll_fn(struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; struct lio_tq *rx_status_tq; rx_status_tq = &lio->rx_status_tq; rx_status_tq->tq = taskqueue_create("lio_rx_oom_status", M_WAITOK, taskqueue_thread_enqueue, &rx_status_tq->tq); if (rx_status_tq->tq == NULL) { lio_dev_err(oct, "unable to create lio rx oom status tq\n"); return (-1); } TIMEOUT_TASK_INIT(rx_status_tq->tq, &rx_status_tq->work, 0, lio_poll_check_rx_oom_status, (void *)rx_status_tq); rx_status_tq->ctxptr = lio; taskqueue_start_threads(&rx_status_tq->tq, 1, PI_NET, "lio%d_rx_oom_status", oct->octeon_id); taskqueue_enqueue_timeout(rx_status_tq->tq, &rx_status_tq->work, lio_ms_to_ticks(50)); return (0); } static void lio_cleanup_rx_oom_poll_fn(struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); if (lio->rx_status_tq.tq != NULL) { while (taskqueue_cancel_timeout(lio->rx_status_tq.tq, &lio->rx_status_tq.work, NULL)) taskqueue_drain_timeout(lio->rx_status_tq.tq, &lio->rx_status_tq.work); taskqueue_free(lio->rx_status_tq.tq); lio->rx_status_tq.tq = NULL; } } static void lio_destroy_nic_device(struct octeon_device *oct, int ifidx) { struct ifnet *ifp = oct->props.ifp; struct lio *lio; if (ifp == NULL) { lio_dev_err(oct, "%s No ifp ptr for index %d\n", __func__, ifidx); return; } lio = if_getsoftc(ifp); lio_ifstate_set(lio, LIO_IFSTATE_DETACH); lio_dev_dbg(oct, "NIC device cleanup\n"); if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_RUNNING) lio_stop(ifp); if (lio_wait_for_pending_requests(oct)) lio_dev_err(oct, "There were pending requests\n"); if (lio_wait_for_instr_fetch(oct)) lio_dev_err(oct, "IQ had pending instructions\n"); if (lio_wait_for_oq_pkts(oct)) lio_dev_err(oct, "OQ had pending packets\n"); if (atomic_load_acq_int(&lio->ifstate) & LIO_IFSTATE_REGISTERED) ether_ifdetach(ifp); lio_tcp_lro_free(oct, ifp); lio_cleanup_rx_oom_poll_fn(ifp); lio_delete_glists(oct, lio); EVENTHANDLER_DEREGISTER(vlan_config, lio->vlan_attach); EVENTHANDLER_DEREGISTER(vlan_unconfig, lio->vlan_detach); free(lio, M_DEVBUF); if_free(ifp); oct->props.gmxport = -1; oct->props.ifp = NULL; } static void print_link_info(struct ifnet *ifp) { struct lio *lio = if_getsoftc(ifp); if (!lio_ifstate_check(lio, LIO_IFSTATE_RESETTING) && lio_ifstate_check(lio, LIO_IFSTATE_REGISTERED)) { struct octeon_link_info *linfo = &lio->linfo; if (linfo->link.s.link_up) { lio_dev_info(lio->oct_dev, "%d Mbps %s Duplex UP\n", linfo->link.s.speed, (linfo->link.s.duplex) ? "Full" : "Half"); } else { lio_dev_info(lio->oct_dev, "Link Down\n"); } } } static inline void lio_update_link_status(struct ifnet *ifp, union octeon_link_status *ls) { struct lio *lio = if_getsoftc(ifp); int changed = (lio->linfo.link.link_status64 != ls->link_status64); lio->linfo.link.link_status64 = ls->link_status64; if ((lio->intf_open) && (changed)) { print_link_info(ifp); lio->link_changes++; if (lio->linfo.link.s.link_up) if_link_state_change(ifp, LINK_STATE_UP); else if_link_state_change(ifp, LINK_STATE_DOWN); } } /* * \brief Callback for rx ctrl * @param status status of request * @param buf pointer to resp structure */ static void lio_rx_ctl_callback(struct octeon_device *oct, uint32_t status, void *buf) { struct lio_soft_command *sc = (struct lio_soft_command *)buf; struct lio_rx_ctl_context *ctx; ctx = (struct lio_rx_ctl_context *)sc->ctxptr; oct = lio_get_device(ctx->octeon_id); if (status) lio_dev_err(oct, "rx ctl instruction failed. Status: %llx\n", LIO_CAST64(status)); ctx->cond = 1; /* * This barrier is required to be sure that the response has been * written fully before waking up the handler */ wmb(); } static void lio_send_rx_ctrl_cmd(struct lio *lio, int start_stop) { struct lio_soft_command *sc; struct lio_rx_ctl_context *ctx; union octeon_cmd *ncmd; struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; int ctx_size = sizeof(struct lio_rx_ctl_context); int retval; if (oct->props.rx_on == start_stop) return; sc = lio_alloc_soft_command(oct, OCTEON_CMD_SIZE, 16, ctx_size); if (sc == NULL) return; ncmd = (union octeon_cmd *)sc->virtdptr; ctx = (struct lio_rx_ctl_context *)sc->ctxptr; ctx->cond = 0; ctx->octeon_id = lio_get_device_id(oct); ncmd->cmd64 = 0; ncmd->s.cmd = LIO_CMD_RX_CTL; ncmd->s.param1 = start_stop; lio_swap_8B_data((uint64_t *)ncmd, (OCTEON_CMD_SIZE >> 3)); sc->iq_no = lio->linfo.txpciq[0].s.q_no; lio_prepare_soft_command(oct, sc, LIO_OPCODE_NIC, LIO_OPCODE_NIC_CMD, 0, 0, 0); sc->callback = lio_rx_ctl_callback; sc->callback_arg = sc; sc->wait_time = 5000; retval = lio_send_soft_command(oct, sc); if (retval == LIO_IQ_SEND_FAILED) { lio_dev_err(oct, "Failed to send RX Control message\n"); } else { /* * Sleep on a wait queue till the cond flag indicates that the * response arrived or timed-out. */ lio_sleep_cond(oct, &ctx->cond); oct->props.rx_on = start_stop; } lio_free_soft_command(oct, sc); } static void lio_vlan_rx_add_vid(void *arg, struct ifnet *ifp, uint16_t vid) { struct lio_ctrl_pkt nctrl; struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; int ret = 0; if (if_getsoftc(ifp) != arg) /* Not our event */ return; if ((vid == 0) || (vid > 4095)) /* Invalid */ return; bzero(&nctrl, sizeof(struct lio_ctrl_pkt)); nctrl.ncmd.cmd64 = 0; nctrl.ncmd.s.cmd = LIO_CMD_ADD_VLAN_FILTER; nctrl.ncmd.s.param1 = vid; nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; nctrl.wait_time = 100; nctrl.lio = lio; nctrl.cb_fn = lio_ctrl_cmd_completion; ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); if (ret < 0) { lio_dev_err(oct, "Add VLAN filter failed in core (ret: 0x%x)\n", ret); } } static void lio_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, uint16_t vid) { struct lio_ctrl_pkt nctrl; struct lio *lio = if_getsoftc(ifp); struct octeon_device *oct = lio->oct_dev; int ret = 0; if (if_getsoftc(ifp) != arg) /* Not our event */ return; if ((vid == 0) || (vid > 4095)) /* Invalid */ return; bzero(&nctrl, sizeof(struct lio_ctrl_pkt)); nctrl.ncmd.cmd64 = 0; nctrl.ncmd.s.cmd = LIO_CMD_DEL_VLAN_FILTER; nctrl.ncmd.s.param1 = vid; nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; nctrl.wait_time = 100; nctrl.lio = lio; nctrl.cb_fn = lio_ctrl_cmd_completion; ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl); if (ret < 0) { lio_dev_err(oct, "Kill VLAN filter failed in core (ret: 0x%x)\n", ret); } } static int lio_wait_for_oq_pkts(struct octeon_device *oct) { int i, pending_pkts, pkt_cnt = 0, retry = 100; do { pending_pkts = 0; for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { if (!(oct->io_qmask.oq & BIT_ULL(i))) continue; pkt_cnt = lio_droq_check_hw_for_pkts(oct->droq[i]); if (pkt_cnt > 0) { pending_pkts += pkt_cnt; taskqueue_enqueue(oct->droq[i]->droq_taskqueue, &oct->droq[i]->droq_task); } } pkt_cnt = 0; lio_sleep_timeout(1); } while (retry-- && pending_pkts); return (pkt_cnt); } static void lio_destroy_resources(struct octeon_device *oct) { int i, refcount; switch (atomic_load_acq_int(&oct->status)) { case LIO_DEV_RUNNING: case LIO_DEV_CORE_OK: /* No more instructions will be forwarded. */ atomic_store_rel_int(&oct->status, LIO_DEV_IN_RESET); oct->app_mode = LIO_DRV_INVALID_APP; lio_dev_dbg(oct, "Device state is now %s\n", lio_get_state_string(&oct->status)); lio_sleep_timeout(100); /* fallthrough */ case LIO_DEV_HOST_OK: /* fallthrough */ case LIO_DEV_CONSOLE_INIT_DONE: /* Remove any consoles */ lio_remove_consoles(oct); /* fallthrough */ case LIO_DEV_IO_QUEUES_DONE: if (lio_wait_for_pending_requests(oct)) lio_dev_err(oct, "There were pending requests\n"); if (lio_wait_for_instr_fetch(oct)) lio_dev_err(oct, "IQ had pending instructions\n"); /* * Disable the input and output queues now. No more packets will * arrive from Octeon, but we should wait for all packet * processing to finish. */ oct->fn_list.disable_io_queues(oct); if (lio_wait_for_oq_pkts(oct)) lio_dev_err(oct, "OQ had pending packets\n"); /* fallthrough */ case LIO_DEV_INTR_SET_DONE: /* Disable interrupts */ oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); if (oct->msix_on) { for (i = 0; i < oct->num_msix_irqs - 1; i++) { if (oct->ioq_vector[i].tag != NULL) { bus_teardown_intr(oct->device, oct->ioq_vector[i].msix_res, oct->ioq_vector[i].tag); oct->ioq_vector[i].tag = NULL; } if (oct->ioq_vector[i].msix_res != NULL) { bus_release_resource(oct->device, SYS_RES_IRQ, oct->ioq_vector[i].vector, oct->ioq_vector[i].msix_res); oct->ioq_vector[i].msix_res = NULL; } } /* non-iov vector's argument is oct struct */ if (oct->tag != NULL) { bus_teardown_intr(oct->device, oct->msix_res, oct->tag); oct->tag = NULL; } if (oct->msix_res != NULL) { bus_release_resource(oct->device, SYS_RES_IRQ, oct->aux_vector, oct->msix_res); oct->msix_res = NULL; } pci_release_msi(oct->device); } /* fallthrough */ case LIO_DEV_IN_RESET: case LIO_DEV_DROQ_INIT_DONE: /* Wait for any pending operations */ lio_mdelay(100); for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { if (!(oct->io_qmask.oq & BIT_ULL(i))) continue; lio_delete_droq(oct, i); } /* fallthrough */ case LIO_DEV_RESP_LIST_INIT_DONE: for (i = 0; i < LIO_MAX_POSSIBLE_OUTPUT_QUEUES; i++) { if (oct->droq[i] != NULL) { free(oct->droq[i], M_DEVBUF); oct->droq[i] = NULL; } } lio_delete_response_list(oct); /* fallthrough */ case LIO_DEV_INSTR_QUEUE_INIT_DONE: for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { if (!(oct->io_qmask.iq & BIT_ULL(i))) continue; lio_delete_instr_queue(oct, i); } /* fallthrough */ case LIO_DEV_MSIX_ALLOC_VECTOR_DONE: for (i = 0; i < LIO_MAX_POSSIBLE_INSTR_QUEUES; i++) { if (oct->instr_queue[i] != NULL) { free(oct->instr_queue[i], M_DEVBUF); oct->instr_queue[i] = NULL; } } lio_free_ioq_vector(oct); /* fallthrough */ case LIO_DEV_SC_BUFF_POOL_INIT_DONE: lio_free_sc_buffer_pool(oct); /* fallthrough */ case LIO_DEV_DISPATCH_INIT_DONE: lio_delete_dispatch_list(oct); /* fallthrough */ case LIO_DEV_PCI_MAP_DONE: refcount = lio_deregister_device(oct); if (fw_type_is_none()) lio_pci_flr(oct); if (!refcount) oct->fn_list.soft_reset(oct); lio_unmap_pci_barx(oct, 0); lio_unmap_pci_barx(oct, 1); /* fallthrough */ case LIO_DEV_PCI_ENABLE_DONE: /* Disable the device, releasing the PCI INT */ pci_disable_busmaster(oct->device); /* fallthrough */ case LIO_DEV_BEGIN_STATE: break; } /* end switch (oct->status) */ }