Index: user/imp/tbemd/gnu/usr.bin/gdb/kgdb/kthr.c =================================================================== --- user/imp/tbemd/gnu/usr.bin/gdb/kgdb/kthr.c (revision 203849) +++ user/imp/tbemd/gnu/usr.bin/gdb/kgdb/kthr.c (revision 203850) @@ -1,241 +1,242 @@ /* * Copyright (c) 2004 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "kgdb.h" #include static uintptr_t dumppcb; static int dumptid; static uintptr_t stoppcbs; static __cpumask_t stopped_cpus; static struct kthr *first; struct kthr *curkthr; uintptr_t kgdb_lookup(const char *sym) { struct nlist nl[2]; + nl[0].n_type = N_UNDF; nl[0].n_name = (char *)(uintptr_t)sym; nl[1].n_name = NULL; if (kvm_nlist(kvm, nl) != 0) return (0); return (nl[0].n_value); } struct kthr * kgdb_thr_first(void) { return (first); } struct kthr * kgdb_thr_init(void) { struct proc p; struct thread td; struct kthr *kt; uintptr_t addr, paddr; while (first != NULL) { kt = first; first = kt->next; free(kt); } addr = kgdb_lookup("_allproc"); if (addr == 0) { warnx("kvm_nlist(_allproc): %s", kvm_geterr(kvm)); return (NULL); } kvm_read(kvm, addr, &paddr, sizeof(paddr)); dumppcb = kgdb_lookup("_dumppcb"); if (dumppcb == 0) { warnx("kvm_nlist(_dumppcb): %s", kvm_geterr(kvm)); return (NULL); } addr = kgdb_lookup("_dumptid"); if (addr != 0) kvm_read(kvm, addr, &dumptid, sizeof(dumptid)); else dumptid = -1; addr = kgdb_lookup("_stopped_cpus"); if (addr != 0) kvm_read(kvm, addr, &stopped_cpus, sizeof(stopped_cpus)); else stopped_cpus = 0; stoppcbs = kgdb_lookup("_stoppcbs"); while (paddr != 0) { if (kvm_read(kvm, paddr, &p, sizeof(p)) != sizeof(p)) { warnx("kvm_read: %s", kvm_geterr(kvm)); break; } addr = (uintptr_t)TAILQ_FIRST(&p.p_threads); while (addr != 0) { if (kvm_read(kvm, addr, &td, sizeof(td)) != sizeof(td)) { warnx("kvm_read: %s", kvm_geterr(kvm)); break; } kt = malloc(sizeof(*kt)); kt->next = first; kt->kaddr = addr; if (td.td_tid == dumptid) kt->pcb = dumppcb; else if (td.td_state == TDS_RUNNING && ((1 << td.td_oncpu) & stopped_cpus) && stoppcbs != 0) kt->pcb = (uintptr_t) stoppcbs + sizeof(struct pcb) * td.td_oncpu; else kt->pcb = (uintptr_t)td.td_pcb; kt->kstack = td.td_kstack; kt->tid = td.td_tid; kt->pid = p.p_pid; kt->paddr = paddr; kt->cpu = td.td_oncpu; first = kt; addr = (uintptr_t)TAILQ_NEXT(&td, td_plist); } paddr = (uintptr_t)LIST_NEXT(&p, p_list); } curkthr = kgdb_thr_lookup_tid(dumptid); if (curkthr == NULL) curkthr = first; return (first); } struct kthr * kgdb_thr_lookup_tid(int tid) { struct kthr *kt; kt = first; while (kt != NULL && kt->tid != tid) kt = kt->next; return (kt); } struct kthr * kgdb_thr_lookup_taddr(uintptr_t taddr) { struct kthr *kt; kt = first; while (kt != NULL && kt->kaddr != taddr) kt = kt->next; return (kt); } struct kthr * kgdb_thr_lookup_pid(int pid) { struct kthr *kt; kt = first; while (kt != NULL && kt->pid != pid) kt = kt->next; return (kt); } struct kthr * kgdb_thr_lookup_paddr(uintptr_t paddr) { struct kthr *kt; kt = first; while (kt != NULL && kt->paddr != paddr) kt = kt->next; return (kt); } struct kthr * kgdb_thr_next(struct kthr *kt) { return (kt->next); } struct kthr * kgdb_thr_select(struct kthr *kt) { struct kthr *pcur; pcur = curkthr; curkthr = kt; return (pcur); } char * kgdb_thr_extra_thread_info(int tid) { char comm[MAXCOMLEN + 1]; char td_name[MAXCOMLEN + 1]; struct kthr *kt; struct proc *p; struct thread *t; static char buf[64]; kt = kgdb_thr_lookup_tid(tid); if (kt == NULL) return (NULL); snprintf(buf, sizeof(buf), "PID=%d", kt->pid); p = (struct proc *)kt->paddr; if (kvm_read(kvm, (uintptr_t)&p->p_comm[0], &comm, sizeof(comm)) != sizeof(comm)) return (buf); strlcat(buf, ": ", sizeof(buf)); strlcat(buf, comm, sizeof(buf)); t = (struct thread *)kt->kaddr; if (kvm_read(kvm, (uintptr_t)&t->td_name[0], &td_name, sizeof(td_name)) == sizeof(td_name) && strcmp(comm, td_name) != 0) { strlcat(buf, "/", sizeof(buf)); strlcat(buf, td_name, sizeof(buf)); } return (buf); } Index: user/imp/tbemd/lib/libusb/libusb.h =================================================================== --- user/imp/tbemd/lib/libusb/libusb.h (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb.h (revision 203850) @@ -1,389 +1,387 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2009 Sylvestre Gallon. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __LIBUSB_H__ #define __LIBUSB_H__ #include #include - - #ifdef __cplusplus extern "C" { #endif #if 0 } /* indent fix */ #endif /* libusb enums */ enum libusb_class_code { LIBUSB_CLASS_PER_INTERFACE = 0, LIBUSB_CLASS_AUDIO = 1, LIBUSB_CLASS_COMM = 2, LIBUSB_CLASS_HID = 3, LIBUSB_CLASS_PTP = 6, LIBUSB_CLASS_PRINTER = 7, LIBUSB_CLASS_MASS_STORAGE = 8, LIBUSB_CLASS_HUB = 9, LIBUSB_CLASS_DATA = 10, LIBUSB_CLASS_VENDOR_SPEC = 0xff, }; enum libusb_descriptor_type { LIBUSB_DT_DEVICE = 0x01, LIBUSB_DT_CONFIG = 0x02, LIBUSB_DT_STRING = 0x03, LIBUSB_DT_INTERFACE = 0x04, LIBUSB_DT_ENDPOINT = 0x05, LIBUSB_DT_HID = 0x21, LIBUSB_DT_REPORT = 0x22, LIBUSB_DT_PHYSICAL = 0x23, LIBUSB_DT_HUB = 0x29, }; #define LIBUSB_DT_DEVICE_SIZE 18 #define LIBUSB_DT_CONFIG_SIZE 9 #define LIBUSB_DT_INTERFACE_SIZE 9 #define LIBUSB_DT_ENDPOINT_SIZE 7 #define LIBUSB_DT_ENDPOINT_AUDIO_SIZE 9 #define LIBUSB_DT_HUB_NONVAR_SIZE 7 #define LIBUSB_ENDPOINT_ADDRESS_MASK 0x0f #define LIBUSB_ENDPOINT_DIR_MASK 0x80 enum libusb_endpoint_direction { LIBUSB_ENDPOINT_IN = 0x80, LIBUSB_ENDPOINT_OUT = 0x00, }; #define LIBUSB_TRANSFER_TYPE_MASK 0x03 enum libusb_transfer_type { LIBUSB_TRANSFER_TYPE_CONTROL = 0, LIBUSB_TRANSFER_TYPE_ISOCHRONOUS = 1, LIBUSB_TRANSFER_TYPE_BULK = 2, LIBUSB_TRANSFER_TYPE_INTERRUPT = 3, }; enum libusb_standard_request { LIBUSB_REQUEST_GET_STATUS = 0x00, LIBUSB_REQUEST_CLEAR_FEATURE = 0x01, LIBUSB_REQUEST_SET_FEATURE = 0x03, LIBUSB_REQUEST_SET_ADDRESS = 0x05, LIBUSB_REQUEST_GET_DESCRIPTOR = 0x06, LIBUSB_REQUEST_SET_DESCRIPTOR = 0x07, LIBUSB_REQUEST_GET_CONFIGURATION = 0x08, LIBUSB_REQUEST_SET_CONFIGURATION = 0x09, LIBUSB_REQUEST_GET_INTERFACE = 0x0A, LIBUSB_REQUEST_SET_INTERFACE = 0x0B, LIBUSB_REQUEST_SYNCH_FRAME = 0x0C, }; enum libusb_request_type { LIBUSB_REQUEST_TYPE_STANDARD = (0x00 << 5), LIBUSB_REQUEST_TYPE_CLASS = (0x01 << 5), LIBUSB_REQUEST_TYPE_VENDOR = (0x02 << 5), LIBUSB_REQUEST_TYPE_RESERVED = (0x03 << 5), }; enum libusb_request_recipient { LIBUSB_RECIPIENT_DEVICE = 0x00, LIBUSB_RECIPIENT_INTERFACE = 0x01, LIBUSB_RECIPIENT_ENDPOINT = 0x02, LIBUSB_RECIPIENT_OTHER = 0x03, }; #define LIBUSB_ISO_SYNC_TYPE_MASK 0x0C enum libusb_iso_sync_type { LIBUSB_ISO_SYNC_TYPE_NONE = 0, LIBUSB_ISO_SYNC_TYPE_ASYNC = 1, LIBUSB_ISO_SYNC_TYPE_ADAPTIVE = 2, LIBUSB_ISO_SYNC_TYPE_SYNC = 3, }; #define LIBUSB_ISO_USAGE_TYPE_MASK 0x30 enum libusb_iso_usage_type { LIBUSB_ISO_USAGE_TYPE_DATA = 0, LIBUSB_ISO_USAGE_TYPE_FEEDBACK = 1, LIBUSB_ISO_USAGE_TYPE_IMPLICIT = 2, }; enum libusb_error { LIBUSB_SUCCESS = 0, LIBUSB_ERROR_IO = -1, LIBUSB_ERROR_INVALID_PARAM = -2, LIBUSB_ERROR_ACCESS = -3, LIBUSB_ERROR_NO_DEVICE = -4, LIBUSB_ERROR_NOT_FOUND = -5, LIBUSB_ERROR_BUSY = -6, LIBUSB_ERROR_TIMEOUT = -7, LIBUSB_ERROR_OVERFLOW = -8, LIBUSB_ERROR_PIPE = -9, LIBUSB_ERROR_INTERRUPTED = -10, LIBUSB_ERROR_NO_MEM = -11, LIBUSB_ERROR_NOT_SUPPORTED = -12, LIBUSB_ERROR_OTHER = -99, }; enum libusb_transfer_status { LIBUSB_TRANSFER_COMPLETED, LIBUSB_TRANSFER_ERROR, LIBUSB_TRANSFER_TIMED_OUT, LIBUSB_TRANSFER_CANCELLED, LIBUSB_TRANSFER_STALL, LIBUSB_TRANSFER_NO_DEVICE, LIBUSB_TRANSFER_OVERFLOW, }; enum libusb_transfer_flags { LIBUSB_TRANSFER_SHORT_NOT_OK = 1 << 0, LIBUSB_TRANSFER_FREE_BUFFER = 1 << 1, LIBUSB_TRANSFER_FREE_TRANSFER = 1 << 2, }; enum libusb_debug_level { LIBUSB_DEBUG_NO=0, LIBUSB_DEBUG_FUNCTION=1, LIBUSB_DEBUG_TRANSFER=2, }; /* libusb structures */ struct libusb_context; struct libusb_device; struct libusb_transfer; struct libusb20_device; struct libusb_pollfd { int fd; short events; }; typedef struct libusb_context libusb_context; typedef struct libusb_device libusb_device; typedef struct libusb20_device libusb_device_handle; typedef struct libusb_pollfd libusb_pollfd; typedef void (*libusb_pollfd_added_cb) (int fd, short events, void *user_data); typedef void (*libusb_pollfd_removed_cb) (int fd, void *user_data); typedef struct libusb_device_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t bcdUSB; uint8_t bDeviceClass; uint8_t bDeviceSubClass; uint8_t bDeviceProtocol; uint8_t bMaxPacketSize0; uint16_t idVendor; uint16_t idProduct; uint16_t bcdDevice; uint8_t iManufacturer; uint8_t iProduct; uint8_t iSerialNumber; uint8_t bNumConfigurations; } libusb_device_descriptor; typedef struct libusb_endpoint_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bEndpointAddress; uint8_t bmAttributes; uint16_t wMaxPacketSize; uint8_t bInterval; uint8_t bRefresh; uint8_t bSynchAddress; uint8_t *extra; int extra_length; } libusb_endpoint_descriptor __aligned(sizeof(void *)); typedef struct libusb_interface_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bInterfaceNumber; uint8_t bAlternateSetting; uint8_t bNumEndpoints; uint8_t bInterfaceClass; uint8_t bInterfaceSubClass; uint8_t bInterfaceProtocol; uint8_t iInterface; struct libusb_endpoint_descriptor *endpoint; uint8_t *extra; int extra_length; } libusb_interface_descriptor __aligned(sizeof(void *)); typedef struct libusb_interface { struct libusb_interface_descriptor *altsetting; int num_altsetting; } libusb_interface __aligned(sizeof(void *)); typedef struct libusb_config_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t wTotalLength; uint8_t bNumInterfaces; uint8_t bConfigurationValue; uint8_t iConfiguration; uint8_t bmAttributes; uint8_t MaxPower; struct libusb_interface *interface; uint8_t *extra; int extra_length; } libusb_config_descriptor __aligned(sizeof(void *)); typedef struct libusb_control_setup { uint8_t bmRequestType; uint8_t bRequest; uint16_t wValue; uint16_t wIndex; uint16_t wLength; } libusb_control_setup; #define LIBUSB_CONTROL_SETUP_SIZE 8 /* bytes */ typedef struct libusb_iso_packet_descriptor { uint32_t length; uint32_t actual_length; enum libusb_transfer_status status; } libusb_iso_packet_descriptor __aligned(sizeof(void *)); typedef void (*libusb_transfer_cb_fn) (struct libusb_transfer *transfer); typedef struct libusb_transfer { libusb_device_handle *dev_handle; uint8_t flags; uint32_t endpoint; uint8_t type; uint32_t timeout; enum libusb_transfer_status status; int length; int actual_length; libusb_transfer_cb_fn callback; void *user_data; uint8_t *buffer; void *os_priv; int num_iso_packets; struct libusb_iso_packet_descriptor iso_packet_desc[0]; } libusb_transfer __aligned(sizeof(void *)); /* Library initialisation */ void libusb_set_debug(libusb_context * ctx, int level); int libusb_init(libusb_context ** context); void libusb_exit(struct libusb_context *ctx); /* Device handling and enumeration */ ssize_t libusb_get_device_list(libusb_context * ctx, libusb_device *** list); void libusb_free_device_list(libusb_device ** list, int unref_devices); uint8_t libusb_get_bus_number(libusb_device * dev); uint8_t libusb_get_device_address(libusb_device * dev); int libusb_clear_halt(libusb_device_handle *devh, uint8_t endpoint); int libusb_get_max_packet_size(libusb_device * dev, uint8_t endpoint); libusb_device *libusb_ref_device(libusb_device * dev); void libusb_unref_device(libusb_device * dev); int libusb_open(libusb_device * dev, libusb_device_handle ** devh); libusb_device_handle *libusb_open_device_with_vid_pid(libusb_context * ctx, uint16_t vendor_id, uint16_t product_id); void libusb_close(libusb_device_handle * devh); libusb_device *libusb_get_device(libusb_device_handle * devh); int libusb_get_configuration(libusb_device_handle * devh, int *config); int libusb_set_configuration(libusb_device_handle * devh, int configuration); int libusb_claim_interface(libusb_device_handle * devh, int interface_number); int libusb_release_interface(libusb_device_handle * devh, int interface_number); int libusb_reset_device(libusb_device_handle * devh); int libusb_kernel_driver_active(libusb_device_handle * devh, int interface); int libusb_detach_kernel_driver(libusb_device_handle * devh, int interface); int libusb_attach_kernel_driver(libusb_device_handle * devh, int interface); int libusb_set_interface_alt_setting(libusb_device_handle * devh, int interface_number, int alternate_setting); /* USB Descriptors */ int libusb_get_device_descriptor(libusb_device * dev, struct libusb_device_descriptor *desc); int libusb_get_active_config_descriptor(libusb_device * dev, struct libusb_config_descriptor **config); int libusb_get_config_descriptor(libusb_device * dev, uint8_t config_index, struct libusb_config_descriptor **config); int libusb_get_config_descriptor_by_value(libusb_device * dev, uint8_t bConfigurationValue, struct libusb_config_descriptor **config); void libusb_free_config_descriptor(struct libusb_config_descriptor *config); int libusb_get_string_descriptor_ascii(libusb_device_handle * devh, uint8_t desc_index, uint8_t *data, int length); int libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type, uint8_t desc_index, uint8_t *data, int length); /* Asynchronous device I/O */ struct libusb_transfer *libusb_alloc_transfer(int iso_packets); void libusb_free_transfer(struct libusb_transfer *transfer); int libusb_submit_transfer(struct libusb_transfer *transfer); int libusb_cancel_transfer(struct libusb_transfer *transfer); uint8_t *libusb_get_iso_packet_buffer(struct libusb_transfer *transfer, uint32_t index); uint8_t *libusb_get_iso_packet_buffer_simple(struct libusb_transfer *transfer, uint32_t index); void libusb_set_iso_packet_lengths(struct libusb_transfer *transfer, uint32_t length); uint8_t *libusb_control_transfer_get_data(struct libusb_transfer *transfer); struct libusb_control_setup *libusb_control_transfer_get_setup(struct libusb_transfer *transfer); void libusb_fill_control_setup(uint8_t *buf, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint16_t wLength); void libusb_fill_control_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t *buf, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout); void libusb_fill_bulk_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout); void libusb_fill_interrupt_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout); void libusb_fill_iso_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, int npacket, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout); /* Polling and timing */ int libusb_try_lock_events(libusb_context * ctx); void libusb_lock_events(libusb_context * ctx); void libusb_unlock_events(libusb_context * ctx); int libusb_event_handling_ok(libusb_context * ctx); int libusb_event_handler_active(libusb_context * ctx); void libusb_lock_event_waiters(libusb_context * ctx); void libusb_unlock_event_waiters(libusb_context * ctx); int libusb_wait_for_event(libusb_context * ctx, struct timeval *tv); int libusb_handle_events_timeout(libusb_context * ctx, struct timeval *tv); int libusb_handle_events(libusb_context * ctx); int libusb_handle_events_locked(libusb_context * ctx, struct timeval *tv); int libusb_get_next_timeout(libusb_context * ctx, struct timeval *tv); void libusb_set_pollfd_notifiers(libusb_context * ctx, libusb_pollfd_added_cb added_cb, libusb_pollfd_removed_cb removed_cb, void *user_data); struct libusb_pollfd **libusb_get_pollfds(libusb_context * ctx); /* Synchronous device I/O */ int libusb_control_transfer(libusb_device_handle * devh, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint8_t *data, uint16_t wLength, uint32_t timeout); int libusb_bulk_transfer(libusb_device_handle * devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, uint32_t timeout); int libusb_interrupt_transfer(libusb_device_handle * devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, uint32_t timeout); /* Byte-order */ uint16_t libusb_cpu_to_le16(uint16_t x); uint16_t libusb_le16_to_cpu(uint16_t x); #if 0 { /* indent fix */ #endif #ifdef __cplusplus } #endif #endif /* __LIBUSB_H__ */ Index: user/imp/tbemd/lib/libusb/libusb10.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb10.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb10.c (revision 203850) @@ -1,1361 +1,1362 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2009 Sylvestre Gallon. All rights reserved. * Copyright (c) 2009 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include +#include +#include + #include -#include -#include -#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" #include "libusb.h" #include "libusb10.h" static pthread_mutex_t default_context_lock = PTHREAD_MUTEX_INITIALIZER; struct libusb_context *usbi_default_context = NULL; /* Prototypes */ static struct libusb20_transfer *libusb10_get_transfer(struct libusb20_device *, uint8_t, uint8_t); static int libusb10_get_maxframe(struct libusb20_device *, libusb_transfer *); static int libusb10_get_buffsize(struct libusb20_device *, libusb_transfer *); static int libusb10_convert_error(uint8_t status); static void libusb10_complete_transfer(struct libusb20_transfer *, struct libusb_super_transfer *, int); static void libusb10_isoc_proxy(struct libusb20_transfer *); static void libusb10_bulk_intr_proxy(struct libusb20_transfer *); static void libusb10_ctrl_proxy(struct libusb20_transfer *); static void libusb10_submit_transfer_sub(struct libusb20_device *, uint8_t); /* Library initialisation / deinitialisation */ void libusb_set_debug(libusb_context *ctx, int level) { ctx = GET_CONTEXT(ctx); if (ctx) ctx->debug = level; } int libusb_init(libusb_context **context) { struct libusb_context *ctx; char *debug; int flag; int ret; ctx = malloc(sizeof(*ctx)); if (!ctx) return (LIBUSB_ERROR_INVALID_PARAM); memset(ctx, 0, sizeof(*ctx)); debug = getenv("LIBUSB_DEBUG"); if (debug != NULL) { ctx->debug = atoi(debug); if (ctx->debug != 0) ctx->debug_fixed = 1; } TAILQ_INIT(&ctx->pollfds); TAILQ_INIT(&ctx->tr_done); pthread_mutex_init(&ctx->ctx_lock, NULL); pthread_cond_init(&ctx->ctx_cond, NULL); ctx->ctx_handler = NO_THREAD; ret = pipe(ctx->ctrl_pipe); if (ret < 0) { pthread_mutex_destroy(&ctx->ctx_lock); pthread_cond_destroy(&ctx->ctx_cond); free(ctx); return (LIBUSB_ERROR_OTHER); } /* set non-blocking mode on the control pipe to avoid deadlock */ flag = 1; ret = fcntl(ctx->ctrl_pipe[0], O_NONBLOCK, &flag); assert(ret != -1 && "Couldn't set O_NONBLOCK for ctx->ctrl_pipe[0]"); flag = 1; ret = fcntl(ctx->ctrl_pipe[1], O_NONBLOCK, &flag); assert(ret != -1 && "Couldn't set O_NONBLOCK for ctx->ctrl_pipe[1]"); libusb10_add_pollfd(ctx, &ctx->ctx_poll, NULL, ctx->ctrl_pipe[0], POLLIN); pthread_mutex_lock(&default_context_lock); if (usbi_default_context == NULL) { usbi_default_context = ctx; } pthread_mutex_unlock(&default_context_lock); if (context) *context = ctx; DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_init complete"); return (0); } void libusb_exit(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); if (ctx == NULL) return; /* XXX cleanup devices */ libusb10_remove_pollfd(ctx, &ctx->ctx_poll); close(ctx->ctrl_pipe[0]); close(ctx->ctrl_pipe[1]); pthread_mutex_destroy(&ctx->ctx_lock); pthread_cond_destroy(&ctx->ctx_cond); pthread_mutex_lock(&default_context_lock); if (ctx == usbi_default_context) { usbi_default_context = NULL; } pthread_mutex_unlock(&default_context_lock); free(ctx); } /* Device handling and initialisation. */ ssize_t libusb_get_device_list(libusb_context *ctx, libusb_device ***list) { struct libusb20_backend *usb_backend; struct libusb20_device *pdev; struct libusb_device *dev; int i; ctx = GET_CONTEXT(ctx); if (ctx == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (list == NULL) return (LIBUSB_ERROR_INVALID_PARAM); usb_backend = libusb20_be_alloc_default(); if (usb_backend == NULL) return (LIBUSB_ERROR_NO_MEM); /* figure out how many USB devices are present */ pdev = NULL; i = 0; while ((pdev = libusb20_be_device_foreach(usb_backend, pdev))) i++; /* allocate device pointer list */ *list = malloc((i + 1) * sizeof(void *)); if (*list == NULL) { libusb20_be_free(usb_backend); return (LIBUSB_ERROR_NO_MEM); } /* create libusb v1.0 compliant devices */ i = 0; while ((pdev = libusb20_be_device_foreach(usb_backend, NULL))) { dev = malloc(sizeof(*dev)); if (dev == NULL) { while (i != 0) { libusb_unref_device((*list)[i - 1]); i--; } free(*list); *list = NULL; libusb20_be_free(usb_backend); return (LIBUSB_ERROR_NO_MEM); } /* get device into libUSB v1.0 list */ libusb20_be_dequeue_device(usb_backend, pdev); memset(dev, 0, sizeof(*dev)); /* init transfer queues */ TAILQ_INIT(&dev->tr_head); /* set context we belong to */ dev->ctx = ctx; /* link together the two structures */ dev->os_priv = pdev; pdev->privLuData = dev; (*list)[i] = libusb_ref_device(dev); i++; } (*list)[i] = NULL; libusb20_be_free(usb_backend); return (i); } void libusb_free_device_list(libusb_device **list, int unref_devices) { int i; if (list == NULL) return; /* be NULL safe */ if (unref_devices) { for (i = 0; list[i] != NULL; i++) libusb_unref_device(list[i]); } free(list); } uint8_t libusb_get_bus_number(libusb_device *dev) { if (dev == NULL) return (0); /* should not happen */ return (libusb20_dev_get_bus_number(dev->os_priv)); } uint8_t libusb_get_device_address(libusb_device *dev) { if (dev == NULL) return (0); /* should not happen */ return (libusb20_dev_get_address(dev->os_priv)); } int libusb_get_max_packet_size(libusb_device *dev, uint8_t endpoint) { struct libusb_config_descriptor *pdconf; struct libusb_interface *pinf; struct libusb_interface_descriptor *pdinf; struct libusb_endpoint_descriptor *pdend; int i; int j; int k; int ret; if (dev == NULL) return (LIBUSB_ERROR_NO_DEVICE); ret = libusb_get_active_config_descriptor(dev, &pdconf); if (ret < 0) return (ret); ret = LIBUSB_ERROR_NOT_FOUND; for (i = 0; i < pdconf->bNumInterfaces; i++) { pinf = &pdconf->interface[i]; for (j = 0; j < pinf->num_altsetting; j++) { pdinf = &pinf->altsetting[j]; for (k = 0; k < pdinf->bNumEndpoints; k++) { pdend = &pdinf->endpoint[k]; if (pdend->bEndpointAddress == endpoint) { ret = pdend->wMaxPacketSize; goto out; } } } } out: libusb_free_config_descriptor(pdconf); return (ret); } libusb_device * libusb_ref_device(libusb_device *dev) { if (dev == NULL) return (NULL); /* be NULL safe */ CTX_LOCK(dev->ctx); dev->refcnt++; CTX_UNLOCK(dev->ctx); return (dev); } void libusb_unref_device(libusb_device *dev) { if (dev == NULL) return; /* be NULL safe */ CTX_LOCK(dev->ctx); dev->refcnt--; CTX_UNLOCK(dev->ctx); if (dev->refcnt == 0) { libusb20_dev_free(dev->os_priv); free(dev); } } int libusb_open(libusb_device *dev, libusb_device_handle **devh) { libusb_context *ctx = dev->ctx; struct libusb20_device *pdev = dev->os_priv; uint8_t dummy; int err; if (devh == NULL) return (LIBUSB_ERROR_INVALID_PARAM); /* set default device handle value */ *devh = NULL; dev = libusb_ref_device(dev); if (dev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); err = libusb20_dev_open(pdev, 16 * 4 /* number of endpoints */ ); if (err) { libusb_unref_device(dev); return (LIBUSB_ERROR_NO_MEM); } libusb10_add_pollfd(ctx, &dev->dev_poll, pdev, libusb20_dev_get_fd(pdev), POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM); /* make sure our event loop detects the new device */ dummy = 0; err = write(ctx->ctrl_pipe[1], &dummy, sizeof(dummy)); if (err < sizeof(dummy)) { /* ignore error, if any */ DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_open write failed!"); } *devh = pdev; return (0); } libusb_device_handle * libusb_open_device_with_vid_pid(libusb_context *ctx, uint16_t vendor_id, uint16_t product_id) { struct libusb_device **devs; struct libusb20_device *pdev; struct LIBUSB20_DEVICE_DESC_DECODED *pdesc; int i; int j; ctx = GET_CONTEXT(ctx); if (ctx == NULL) return (NULL); /* be NULL safe */ DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_open_device_width_vid_pid enter"); if ((i = libusb_get_device_list(ctx, &devs)) < 0) return (NULL); for (j = 0; j < i; j++) { pdev = devs[j]->os_priv; pdesc = libusb20_dev_get_device_desc(pdev); /* * NOTE: The USB library will automatically swap the * fields in the device descriptor to be of host * endian type! */ if (pdesc->idVendor == vendor_id && pdesc->idProduct == product_id) { if (libusb_open(devs[j], &pdev) < 0) pdev = NULL; break; } } if (j == i) pdev = NULL; libusb_free_device_list(devs, 1); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_open_device_width_vid_pid leave"); return (pdev); } void libusb_close(struct libusb20_device *pdev) { libusb_context *ctx; struct libusb_device *dev; uint8_t dummy; int err; if (pdev == NULL) return; /* be NULL safe */ dev = libusb_get_device(pdev); ctx = dev->ctx; libusb10_remove_pollfd(ctx, &dev->dev_poll); libusb20_dev_close(pdev); /* unref will free the "pdev" when the refcount reaches zero */ libusb_unref_device(dev); /* make sure our event loop detects the closed device */ dummy = 0; err = write(ctx->ctrl_pipe[1], &dummy, sizeof(dummy)); if (err < sizeof(dummy)) { /* ignore error, if any */ DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_close write failed!"); } } libusb_device * libusb_get_device(struct libusb20_device *pdev) { if (pdev == NULL) return (NULL); return ((libusb_device *)pdev->privLuData); } int libusb_get_configuration(struct libusb20_device *pdev, int *config) { struct libusb20_config *pconf; if (pdev == NULL || config == NULL) return (LIBUSB_ERROR_INVALID_PARAM); pconf = libusb20_dev_alloc_config(pdev, libusb20_dev_get_config_index(pdev)); if (pconf == NULL) return (LIBUSB_ERROR_NO_MEM); *config = pconf->desc.bConfigurationValue; free(pconf); return (0); } int libusb_set_configuration(struct libusb20_device *pdev, int configuration) { struct libusb20_config *pconf; struct libusb_device *dev; int err; uint8_t i; dev = libusb_get_device(pdev); if (dev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (configuration < 1) { /* unconfigure */ i = 255; } else { for (i = 0; i != 255; i++) { uint8_t found; pconf = libusb20_dev_alloc_config(pdev, i); if (pconf == NULL) return (LIBUSB_ERROR_INVALID_PARAM); found = (pconf->desc.bConfigurationValue == configuration); free(pconf); if (found) goto set_config; } return (LIBUSB_ERROR_INVALID_PARAM); } set_config: libusb10_cancel_all_transfer(dev); libusb10_remove_pollfd(dev->ctx, &dev->dev_poll); err = libusb20_dev_set_config_index(pdev, i); libusb10_add_pollfd(dev->ctx, &dev->dev_poll, pdev, libusb20_dev_get_fd(pdev), POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM); return (err ? LIBUSB_ERROR_INVALID_PARAM : 0); } int libusb_claim_interface(struct libusb20_device *pdev, int interface_number) { libusb_device *dev; int err = 0; dev = libusb_get_device(pdev); if (dev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (interface_number < 0 || interface_number > 31) return (LIBUSB_ERROR_INVALID_PARAM); CTX_LOCK(dev->ctx); if (dev->claimed_interfaces & (1 << interface_number)) err = LIBUSB_ERROR_BUSY; if (!err) dev->claimed_interfaces |= (1 << interface_number); CTX_UNLOCK(dev->ctx); return (err); } int libusb_release_interface(struct libusb20_device *pdev, int interface_number) { libusb_device *dev; int err = 0; dev = libusb_get_device(pdev); if (dev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (interface_number < 0 || interface_number > 31) return (LIBUSB_ERROR_INVALID_PARAM); CTX_LOCK(dev->ctx); if (!(dev->claimed_interfaces & (1 << interface_number))) err = LIBUSB_ERROR_NOT_FOUND; if (!err) dev->claimed_interfaces &= ~(1 << interface_number); CTX_UNLOCK(dev->ctx); return (err); } int libusb_set_interface_alt_setting(struct libusb20_device *pdev, int interface_number, int alternate_setting) { libusb_device *dev; int err = 0; dev = libusb_get_device(pdev); if (dev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (interface_number < 0 || interface_number > 31) return (LIBUSB_ERROR_INVALID_PARAM); CTX_LOCK(dev->ctx); if (!(dev->claimed_interfaces & (1 << interface_number))) err = LIBUSB_ERROR_NOT_FOUND; CTX_UNLOCK(dev->ctx); if (err) return (err); libusb10_cancel_all_transfer(dev); libusb10_remove_pollfd(dev->ctx, &dev->dev_poll); err = libusb20_dev_set_alt_index(pdev, interface_number, alternate_setting); libusb10_add_pollfd(dev->ctx, &dev->dev_poll, pdev, libusb20_dev_get_fd(pdev), POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM); return (err ? LIBUSB_ERROR_OTHER : 0); } static struct libusb20_transfer * libusb10_get_transfer(struct libusb20_device *pdev, uint8_t endpoint, uint8_t index) { index &= 1; /* double buffering */ index |= (endpoint & LIBUSB20_ENDPOINT_ADDRESS_MASK) * 4; if (endpoint & LIBUSB20_ENDPOINT_DIR_MASK) { /* this is an IN endpoint */ index |= 2; } return (libusb20_tr_get_pointer(pdev, index)); } int libusb_clear_halt(struct libusb20_device *pdev, uint8_t endpoint) { struct libusb20_transfer *xfer; struct libusb_device *dev; int err; xfer = libusb10_get_transfer(pdev, endpoint, 0); if (xfer == NULL) return (LIBUSB_ERROR_INVALID_PARAM); dev = libusb_get_device(pdev); CTX_LOCK(dev->ctx); err = libusb20_tr_open(xfer, 0, 0, endpoint); CTX_UNLOCK(dev->ctx); if (err != 0 && err != LIBUSB20_ERROR_BUSY) return (LIBUSB_ERROR_OTHER); libusb20_tr_clear_stall_sync(xfer); /* check if we opened the transfer */ if (err == 0) { CTX_LOCK(dev->ctx); libusb20_tr_close(xfer); CTX_UNLOCK(dev->ctx); } return (0); /* success */ } int libusb_reset_device(struct libusb20_device *pdev) { libusb_device *dev; int err; dev = libusb_get_device(pdev); if (dev == NULL) return (LIBUSB20_ERROR_INVALID_PARAM); libusb10_cancel_all_transfer(dev); libusb10_remove_pollfd(dev->ctx, &dev->dev_poll); err = libusb20_dev_reset(pdev); libusb10_add_pollfd(dev->ctx, &dev->dev_poll, pdev, libusb20_dev_get_fd(pdev), POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM); return (err ? LIBUSB_ERROR_OTHER : 0); } int libusb_kernel_driver_active(struct libusb20_device *pdev, int interface) { if (pdev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); return (libusb20_dev_kernel_driver_active( pdev, interface)); } int libusb_detach_kernel_driver(struct libusb20_device *pdev, int interface) { int err; if (pdev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); err = libusb20_dev_detach_kernel_driver( pdev, interface); return (err ? LIBUSB20_ERROR_OTHER : 0); } int libusb_attach_kernel_driver(struct libusb20_device *pdev, int interface) { if (pdev == NULL) return (LIBUSB_ERROR_INVALID_PARAM); /* stub - currently not supported by libusb20 */ return (0); } /* Asynchronous device I/O */ struct libusb_transfer * libusb_alloc_transfer(int iso_packets) { struct libusb_transfer *uxfer; struct libusb_super_transfer *sxfer; int len; len = sizeof(struct libusb_transfer) + sizeof(struct libusb_super_transfer) + (iso_packets * sizeof(libusb_iso_packet_descriptor)); sxfer = malloc(len); if (sxfer == NULL) return (NULL); memset(sxfer, 0, len); uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); /* set default value */ uxfer->num_iso_packets = iso_packets; return (uxfer); } void libusb_free_transfer(struct libusb_transfer *uxfer) { struct libusb_super_transfer *sxfer; if (uxfer == NULL) return; /* be NULL safe */ sxfer = (struct libusb_super_transfer *)( (uint8_t *)uxfer - sizeof(*sxfer)); free(sxfer); } static int libusb10_get_maxframe(struct libusb20_device *pdev, libusb_transfer *xfer) { int ret; int usb_speed; usb_speed = libusb20_dev_get_speed(pdev); switch (xfer->type) { case LIBUSB_TRANSFER_TYPE_ISOCHRONOUS: switch (usb_speed) { case LIBUSB20_SPEED_LOW: case LIBUSB20_SPEED_FULL: ret = 60 * 1; break; default: ret = 60 * 8; break; } break; case LIBUSB_TRANSFER_TYPE_CONTROL: ret = 2; break; default: ret = 1; break; } return (ret); } static int libusb10_get_buffsize(struct libusb20_device *pdev, libusb_transfer *xfer) { int ret; int usb_speed; usb_speed = libusb20_dev_get_speed(pdev); switch (xfer->type) { case LIBUSB_TRANSFER_TYPE_ISOCHRONOUS: ret = 0; /* kernel will auto-select */ break; case LIBUSB_TRANSFER_TYPE_CONTROL: ret = 1024; break; default: switch (usb_speed) { case LIBUSB20_SPEED_LOW: ret = 256; break; case LIBUSB20_SPEED_FULL: ret = 4096; break; default: ret = 16384; break; } break; } return (ret); } static int libusb10_convert_error(uint8_t status) { ; /* indent fix */ switch (status) { case LIBUSB20_TRANSFER_START: case LIBUSB20_TRANSFER_COMPLETED: return (LIBUSB_TRANSFER_COMPLETED); case LIBUSB20_TRANSFER_OVERFLOW: return (LIBUSB_TRANSFER_OVERFLOW); case LIBUSB20_TRANSFER_NO_DEVICE: return (LIBUSB_TRANSFER_NO_DEVICE); case LIBUSB20_TRANSFER_STALL: return (LIBUSB_TRANSFER_STALL); case LIBUSB20_TRANSFER_CANCELLED: return (LIBUSB_TRANSFER_CANCELLED); case LIBUSB20_TRANSFER_TIMED_OUT: return (LIBUSB_TRANSFER_TIMED_OUT); default: return (LIBUSB_TRANSFER_ERROR); } } /* This function must be called locked */ static void libusb10_complete_transfer(struct libusb20_transfer *pxfer, struct libusb_super_transfer *sxfer, int status) { struct libusb_transfer *uxfer; struct libusb_device *dev; uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); if (pxfer != NULL) libusb20_tr_set_priv_sc1(pxfer, NULL); /* set transfer status */ uxfer->status = status; /* update super transfer state */ sxfer->state = LIBUSB_SUPER_XFER_ST_NONE; dev = libusb_get_device(uxfer->dev_handle); TAILQ_INSERT_TAIL(&dev->ctx->tr_done, sxfer, entry); } /* This function must be called locked */ static void libusb10_isoc_proxy(struct libusb20_transfer *pxfer) { struct libusb_super_transfer *sxfer; struct libusb_transfer *uxfer; uint32_t actlen; uint16_t iso_packets; uint16_t i; uint8_t status; uint8_t flags; status = libusb20_tr_get_status(pxfer); sxfer = libusb20_tr_get_priv_sc1(pxfer); actlen = libusb20_tr_get_actual_length(pxfer); iso_packets = libusb20_tr_get_max_frames(pxfer); if (sxfer == NULL) return; /* cancelled - nothing to do */ uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); if (iso_packets > uxfer->num_iso_packets) iso_packets = uxfer->num_iso_packets; if (iso_packets == 0) return; /* nothing to do */ /* make sure that the number of ISOCHRONOUS packets is valid */ uxfer->num_iso_packets = iso_packets; flags = uxfer->flags; switch (status) { case LIBUSB20_TRANSFER_COMPLETED: /* update actual length */ uxfer->actual_length = actlen; for (i = 0; i != iso_packets; i++) { uxfer->iso_packet_desc[i].actual_length = libusb20_tr_get_length(pxfer, i); } libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_COMPLETED); break; case LIBUSB20_TRANSFER_START: /* setup length(s) */ actlen = 0; for (i = 0; i != iso_packets; i++) { libusb20_tr_setup_isoc(pxfer, &uxfer->buffer[actlen], uxfer->iso_packet_desc[i].length, i); actlen += uxfer->iso_packet_desc[i].length; } /* no remainder */ sxfer->rem_len = 0; libusb20_tr_set_total_frames(pxfer, iso_packets); libusb20_tr_submit(pxfer); /* fork another USB transfer, if any */ libusb10_submit_transfer_sub(libusb20_tr_get_priv_sc0(pxfer), uxfer->endpoint); break; default: libusb10_complete_transfer(pxfer, sxfer, libusb10_convert_error(status)); break; } } /* This function must be called locked */ static void libusb10_bulk_intr_proxy(struct libusb20_transfer *pxfer) { struct libusb_super_transfer *sxfer; struct libusb_transfer *uxfer; uint32_t max_bulk; uint32_t actlen; uint8_t status; uint8_t flags; status = libusb20_tr_get_status(pxfer); sxfer = libusb20_tr_get_priv_sc1(pxfer); max_bulk = libusb20_tr_get_max_total_length(pxfer); actlen = libusb20_tr_get_actual_length(pxfer); if (sxfer == NULL) return; /* cancelled - nothing to do */ uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); flags = uxfer->flags; switch (status) { case LIBUSB20_TRANSFER_COMPLETED: uxfer->actual_length += actlen; /* check for short packet */ if (sxfer->last_len != actlen) { if (flags & LIBUSB_TRANSFER_SHORT_NOT_OK) { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_ERROR); } else { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_COMPLETED); } break; } /* check for end of data */ if (sxfer->rem_len == 0) { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_COMPLETED); break; } /* FALLTHROUGH */ case LIBUSB20_TRANSFER_START: if (max_bulk > sxfer->rem_len) { max_bulk = sxfer->rem_len; } /* setup new BULK or INTERRUPT transaction */ libusb20_tr_setup_bulk(pxfer, sxfer->curr_data, max_bulk, uxfer->timeout); /* update counters */ sxfer->last_len = max_bulk; sxfer->curr_data += max_bulk; sxfer->rem_len -= max_bulk; libusb20_tr_submit(pxfer); /* check if we can fork another USB transfer */ if (sxfer->rem_len == 0) libusb10_submit_transfer_sub(libusb20_tr_get_priv_sc0(pxfer), uxfer->endpoint); break; default: libusb10_complete_transfer(pxfer, sxfer, libusb10_convert_error(status)); break; } } /* This function must be called locked */ static void libusb10_ctrl_proxy(struct libusb20_transfer *pxfer) { struct libusb_super_transfer *sxfer; struct libusb_transfer *uxfer; uint32_t max_bulk; uint32_t actlen; uint8_t status; uint8_t flags; status = libusb20_tr_get_status(pxfer); sxfer = libusb20_tr_get_priv_sc1(pxfer); max_bulk = libusb20_tr_get_max_total_length(pxfer); actlen = libusb20_tr_get_actual_length(pxfer); if (sxfer == NULL) return; /* cancelled - nothing to do */ uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); flags = uxfer->flags; switch (status) { case LIBUSB20_TRANSFER_COMPLETED: uxfer->actual_length += actlen; /* subtract length of SETUP packet, if any */ actlen -= libusb20_tr_get_length(pxfer, 0); /* check for short packet */ if (sxfer->last_len != actlen) { if (flags & LIBUSB_TRANSFER_SHORT_NOT_OK) { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_ERROR); } else { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_COMPLETED); } break; } /* check for end of data */ if (sxfer->rem_len == 0) { libusb10_complete_transfer(pxfer, sxfer, LIBUSB_TRANSFER_COMPLETED); break; } /* FALLTHROUGH */ case LIBUSB20_TRANSFER_START: if (max_bulk > sxfer->rem_len) { max_bulk = sxfer->rem_len; } /* setup new CONTROL transaction */ if (status == LIBUSB20_TRANSFER_COMPLETED) { /* next fragment - don't send SETUP packet */ libusb20_tr_set_length(pxfer, 0, 0); } else { /* first fragment - send SETUP packet */ libusb20_tr_set_length(pxfer, 8, 0); libusb20_tr_set_buffer(pxfer, uxfer->buffer, 0); } if (max_bulk != 0) { libusb20_tr_set_length(pxfer, max_bulk, 1); libusb20_tr_set_buffer(pxfer, sxfer->curr_data, 1); libusb20_tr_set_total_frames(pxfer, 2); } else { libusb20_tr_set_total_frames(pxfer, 1); } /* update counters */ sxfer->last_len = max_bulk; sxfer->curr_data += max_bulk; sxfer->rem_len -= max_bulk; libusb20_tr_submit(pxfer); /* check if we can fork another USB transfer */ if (sxfer->rem_len == 0) libusb10_submit_transfer_sub(libusb20_tr_get_priv_sc0(pxfer), uxfer->endpoint); break; default: libusb10_complete_transfer(pxfer, sxfer, libusb10_convert_error(status)); break; } } /* The following function must be called locked */ static void libusb10_submit_transfer_sub(struct libusb20_device *pdev, uint8_t endpoint) { struct libusb20_transfer *pxfer0; struct libusb20_transfer *pxfer1; struct libusb_super_transfer *sxfer; struct libusb_transfer *uxfer; struct libusb_device *dev; int err; int buffsize; int maxframe; int temp; uint8_t dummy; dev = libusb_get_device(pdev); pxfer0 = libusb10_get_transfer(pdev, endpoint, 0); pxfer1 = libusb10_get_transfer(pdev, endpoint, 1); if (pxfer0 == NULL || pxfer1 == NULL) return; /* shouldn't happen */ temp = 0; if (libusb20_tr_pending(pxfer0)) temp |= 1; if (libusb20_tr_pending(pxfer1)) temp |= 2; switch (temp) { case 3: /* wait till one of the transfers complete */ return; case 2: sxfer = libusb20_tr_get_priv_sc1(pxfer1); if (sxfer == NULL) return; /* cancelling */ if (sxfer->rem_len) return; /* cannot queue another one */ /* swap transfers */ pxfer1 = pxfer0; break; case 1: sxfer = libusb20_tr_get_priv_sc1(pxfer0); if (sxfer == NULL) return; /* cancelling */ if (sxfer->rem_len) return; /* cannot queue another one */ /* swap transfers */ pxfer0 = pxfer1; break; default: break; } /* find next transfer on same endpoint */ TAILQ_FOREACH(sxfer, &dev->tr_head, entry) { uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); if (uxfer->endpoint == endpoint) { TAILQ_REMOVE(&dev->tr_head, sxfer, entry); sxfer->entry.tqe_prev = NULL; goto found; } } return; /* success */ found: libusb20_tr_set_priv_sc0(pxfer0, pdev); libusb20_tr_set_priv_sc1(pxfer0, sxfer); /* reset super transfer state */ sxfer->rem_len = uxfer->length; sxfer->curr_data = uxfer->buffer; uxfer->actual_length = 0; switch (uxfer->type) { case LIBUSB_TRANSFER_TYPE_ISOCHRONOUS: libusb20_tr_set_callback(pxfer0, libusb10_isoc_proxy); break; case LIBUSB_TRANSFER_TYPE_BULK: case LIBUSB_TRANSFER_TYPE_INTERRUPT: libusb20_tr_set_callback(pxfer0, libusb10_bulk_intr_proxy); break; case LIBUSB_TRANSFER_TYPE_CONTROL: libusb20_tr_set_callback(pxfer0, libusb10_ctrl_proxy); if (sxfer->rem_len < 8) goto failure; /* remove SETUP packet from data */ sxfer->rem_len -= 8; sxfer->curr_data += 8; break; default: goto failure; } buffsize = libusb10_get_buffsize(pdev, uxfer); maxframe = libusb10_get_maxframe(pdev, uxfer); /* make sure the transfer is opened */ err = libusb20_tr_open(pxfer0, buffsize, maxframe, endpoint); if (err && (err != LIBUSB20_ERROR_BUSY)) { goto failure; } libusb20_tr_start(pxfer0); return; failure: libusb10_complete_transfer(pxfer0, sxfer, LIBUSB_TRANSFER_ERROR); /* make sure our event loop spins the done handler */ dummy = 0; write(dev->ctx->ctrl_pipe[1], &dummy, sizeof(dummy)); } /* The following function must be called unlocked */ int libusb_submit_transfer(struct libusb_transfer *uxfer) { struct libusb20_transfer *pxfer0; struct libusb20_transfer *pxfer1; struct libusb_super_transfer *sxfer; struct libusb_device *dev; uint32_t endpoint; int err; if (uxfer == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if (uxfer->dev_handle == NULL) return (LIBUSB_ERROR_INVALID_PARAM); endpoint = uxfer->endpoint; if (endpoint > 255) return (LIBUSB_ERROR_INVALID_PARAM); dev = libusb_get_device(uxfer->dev_handle); DPRINTF(dev->ctx, LIBUSB_DEBUG_FUNCTION, "libusb_submit_transfer enter"); sxfer = (struct libusb_super_transfer *)( (uint8_t *)uxfer - sizeof(*sxfer)); CTX_LOCK(dev->ctx); pxfer0 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 0); pxfer1 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 1); if (pxfer0 == NULL || pxfer1 == NULL) { err = LIBUSB_ERROR_OTHER; } else if ((sxfer->entry.tqe_prev != NULL) || (libusb20_tr_get_priv_sc1(pxfer0) == sxfer) || (libusb20_tr_get_priv_sc1(pxfer1) == sxfer)) { err = LIBUSB_ERROR_BUSY; } else { /* set pending state */ sxfer->state = LIBUSB_SUPER_XFER_ST_PEND; /* insert transfer into transfer head list */ TAILQ_INSERT_TAIL(&dev->tr_head, sxfer, entry); /* start work transfers */ libusb10_submit_transfer_sub( uxfer->dev_handle, endpoint); err = 0; /* success */ } CTX_UNLOCK(dev->ctx); DPRINTF(dev->ctx, LIBUSB_DEBUG_FUNCTION, "libusb_submit_transfer leave %d", err); return (err); } /* Asynchronous transfer cancel */ int libusb_cancel_transfer(struct libusb_transfer *uxfer) { struct libusb20_transfer *pxfer0; struct libusb20_transfer *pxfer1; struct libusb_super_transfer *sxfer; struct libusb_device *dev; uint32_t endpoint; int retval; if (uxfer == NULL) return (LIBUSB_ERROR_INVALID_PARAM); /* check if not initialised */ if (uxfer->dev_handle == NULL) return (LIBUSB_ERROR_NOT_FOUND); endpoint = uxfer->endpoint; if (endpoint > 255) return (LIBUSB_ERROR_INVALID_PARAM); dev = libusb_get_device(uxfer->dev_handle); DPRINTF(dev->ctx, LIBUSB_DEBUG_FUNCTION, "libusb_cancel_transfer enter"); sxfer = (struct libusb_super_transfer *)( (uint8_t *)uxfer - sizeof(*sxfer)); retval = 0; CTX_LOCK(dev->ctx); pxfer0 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 0); pxfer1 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 1); if (sxfer->state != LIBUSB_SUPER_XFER_ST_PEND) { /* only update the transfer status */ uxfer->status = LIBUSB_TRANSFER_CANCELLED; retval = LIBUSB_ERROR_NOT_FOUND; } else if (sxfer->entry.tqe_prev != NULL) { /* we are lucky - transfer is on a queue */ TAILQ_REMOVE(&dev->tr_head, sxfer, entry); sxfer->entry.tqe_prev = NULL; libusb10_complete_transfer(NULL, sxfer, LIBUSB_TRANSFER_CANCELLED); } else if (pxfer0 == NULL || pxfer1 == NULL) { /* not started */ retval = LIBUSB_ERROR_NOT_FOUND; } else if (libusb20_tr_get_priv_sc1(pxfer0) == sxfer) { libusb10_complete_transfer(pxfer0, sxfer, LIBUSB_TRANSFER_CANCELLED); libusb20_tr_stop(pxfer0); /* make sure the queue doesn't stall */ libusb10_submit_transfer_sub( uxfer->dev_handle, endpoint); } else if (libusb20_tr_get_priv_sc1(pxfer1) == sxfer) { libusb10_complete_transfer(pxfer1, sxfer, LIBUSB_TRANSFER_CANCELLED); libusb20_tr_stop(pxfer1); /* make sure the queue doesn't stall */ libusb10_submit_transfer_sub( uxfer->dev_handle, endpoint); } else { /* not started */ retval = LIBUSB_ERROR_NOT_FOUND; } CTX_UNLOCK(dev->ctx); DPRINTF(dev->ctx, LIBUSB_DEBUG_FUNCTION, "libusb_cancel_transfer leave"); return (retval); } UNEXPORTED void libusb10_cancel_all_transfer(libusb_device *dev) { /* TODO */ } uint16_t libusb_cpu_to_le16(uint16_t x) { return (htole16(x)); } uint16_t libusb_le16_to_cpu(uint16_t x) { return (le16toh(x)); } Index: user/imp/tbemd/lib/libusb/libusb10_desc.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb10_desc.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb10_desc.c (revision 203850) @@ -1,317 +1,318 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2009 Sylvestre Gallon. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include -#include #include + +#include +#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" #include "libusb.h" #include "libusb10.h" #define N_ALIGN(n) (-((-(n)) & (-8UL))) /* USB descriptors */ int libusb_get_device_descriptor(libusb_device *dev, struct libusb_device_descriptor *desc) { struct LIBUSB20_DEVICE_DESC_DECODED *pdesc; struct libusb20_device *pdev; if ((dev == NULL) || (desc == NULL)) return (LIBUSB_ERROR_INVALID_PARAM); pdev = dev->os_priv; pdesc = libusb20_dev_get_device_desc(pdev); desc->bLength = pdesc->bLength; desc->bDescriptorType = pdesc->bDescriptorType; desc->bcdUSB = pdesc->bcdUSB; desc->bDeviceClass = pdesc->bDeviceClass; desc->bDeviceSubClass = pdesc->bDeviceSubClass; desc->bDeviceProtocol = pdesc->bDeviceProtocol; desc->bMaxPacketSize0 = pdesc->bMaxPacketSize0; desc->idVendor = pdesc->idVendor; desc->idProduct = pdesc->idProduct; desc->bcdDevice = pdesc->bcdDevice; desc->iManufacturer = pdesc->iManufacturer; desc->iProduct = pdesc->iProduct; desc->iSerialNumber = pdesc->iSerialNumber; desc->bNumConfigurations = pdesc->bNumConfigurations; return (0); } int libusb_get_active_config_descriptor(libusb_device *dev, struct libusb_config_descriptor **config) { struct libusb20_device *pdev; uint8_t config_index; pdev = dev->os_priv; config_index = libusb20_dev_get_config_index(pdev); return (libusb_get_config_descriptor(dev, config_index, config)); } int libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index, struct libusb_config_descriptor **config) { struct libusb20_device *pdev; struct libusb20_config *pconf; struct libusb20_interface *pinf; struct libusb20_endpoint *pend; struct libusb_config_descriptor *pconfd; struct libusb_interface_descriptor *ifd; struct libusb_endpoint_descriptor *endd; uint8_t *pextra; uint16_t nextra; uint8_t nif; uint8_t nep; uint8_t nalt; uint8_t i; uint8_t j; uint8_t k; if (dev == NULL || config == NULL) return (LIBUSB_ERROR_INVALID_PARAM); *config = NULL; pdev = dev->os_priv; pconf = libusb20_dev_alloc_config(pdev, config_index); if (pconf == NULL) return (LIBUSB_ERROR_NOT_FOUND); nalt = nif = pconf->num_interface; nep = 0; nextra = N_ALIGN(pconf->extra.len); for (i = 0; i < nif; i++) { pinf = pconf->interface + i; nextra += N_ALIGN(pinf->extra.len); nep += pinf->num_endpoints; k = pinf->num_endpoints; pend = pinf->endpoints; while (k--) { nextra += N_ALIGN(pend->extra.len); pend++; } j = pinf->num_altsetting; nalt += pinf->num_altsetting; pinf = pinf->altsetting; while (j--) { nextra += N_ALIGN(pinf->extra.len); nep += pinf->num_endpoints; k = pinf->num_endpoints; pend = pinf->endpoints; while (k--) { nextra += N_ALIGN(pend->extra.len); pend++; } pinf++; } } nextra = nextra + (1 * sizeof(libusb_config_descriptor)) + (nif * sizeof(libusb_interface)) + (nalt * sizeof(libusb_interface_descriptor)) + (nep * sizeof(libusb_endpoint_descriptor)); nextra = N_ALIGN(nextra); pconfd = malloc(nextra); if (pconfd == NULL) { free(pconf); return (LIBUSB_ERROR_NO_MEM); } /* make sure memory is initialised */ memset(pconfd, 0, nextra); pconfd->interface = (libusb_interface *) (pconfd + 1); ifd = (libusb_interface_descriptor *) (pconfd->interface + nif); endd = (libusb_endpoint_descriptor *) (ifd + nalt); pextra = (uint8_t *)(endd + nep); /* fill in config descriptor */ pconfd->bLength = pconf->desc.bLength; pconfd->bDescriptorType = pconf->desc.bDescriptorType; pconfd->wTotalLength = pconf->desc.wTotalLength; pconfd->bNumInterfaces = pconf->desc.bNumInterfaces; pconfd->bConfigurationValue = pconf->desc.bConfigurationValue; pconfd->iConfiguration = pconf->desc.iConfiguration; pconfd->bmAttributes = pconf->desc.bmAttributes; pconfd->MaxPower = pconf->desc.bMaxPower; if (pconf->extra.len != 0) { pconfd->extra_length = pconf->extra.len; pconfd->extra = pextra; memcpy(pextra, pconf->extra.ptr, pconfd->extra_length); pextra += N_ALIGN(pconfd->extra_length); } /* setup all interface and endpoint pointers */ for (i = 0; i < nif; i++) { pconfd->interface[i].altsetting = ifd; ifd->endpoint = endd; endd += pconf->interface[i].num_endpoints; ifd++; for (j = 0; j < pconf->interface[i].num_altsetting; j++) { ifd->endpoint = endd; endd += pconf->interface[i].altsetting[j].num_endpoints; ifd++; } } /* fill in all interface and endpoint data */ for (i = 0; i < nif; i++) { pinf = &pconf->interface[i]; pconfd->interface[i].num_altsetting = pinf->num_altsetting + 1; for (j = 0; j < pconfd->interface[i].num_altsetting; j++) { if (j != 0) pinf = &pconf->interface[i].altsetting[j - 1]; ifd = &pconfd->interface[i].altsetting[j]; ifd->bLength = pinf->desc.bLength; ifd->bDescriptorType = pinf->desc.bDescriptorType; ifd->bInterfaceNumber = pinf->desc.bInterfaceNumber; ifd->bAlternateSetting = pinf->desc.bAlternateSetting; ifd->bNumEndpoints = pinf->desc.bNumEndpoints; ifd->bInterfaceClass = pinf->desc.bInterfaceClass; ifd->bInterfaceSubClass = pinf->desc.bInterfaceSubClass; ifd->bInterfaceProtocol = pinf->desc.bInterfaceProtocol; ifd->iInterface = pinf->desc.iInterface; if (pinf->extra.len != 0) { ifd->extra_length = pinf->extra.len; ifd->extra = pextra; memcpy(pextra, pinf->extra.ptr, pinf->extra.len); pextra += N_ALIGN(pinf->extra.len); } for (k = 0; k < pinf->num_endpoints; k++) { pend = &pinf->endpoints[k]; endd = &ifd->endpoint[k]; endd->bLength = pend->desc.bLength; endd->bDescriptorType = pend->desc.bDescriptorType; endd->bEndpointAddress = pend->desc.bEndpointAddress; endd->bmAttributes = pend->desc.bmAttributes; endd->wMaxPacketSize = pend->desc.wMaxPacketSize; endd->bInterval = pend->desc.bInterval; endd->bRefresh = pend->desc.bRefresh; endd->bSynchAddress = pend->desc.bSynchAddress; if (pend->extra.len != 0) { endd->extra_length = pend->extra.len; endd->extra = pextra; memcpy(pextra, pend->extra.ptr, pend->extra.len); pextra += N_ALIGN(pend->extra.len); } } } } free(pconf); *config = pconfd; return (0); /* success */ } int libusb_get_config_descriptor_by_value(libusb_device *dev, uint8_t bConfigurationValue, struct libusb_config_descriptor **config) { struct LIBUSB20_DEVICE_DESC_DECODED *pdesc; struct libusb20_device *pdev; int i; int err; if (dev == NULL || config == NULL) return (LIBUSB_ERROR_INVALID_PARAM); pdev = dev->os_priv; pdesc = libusb20_dev_get_device_desc(pdev); for (i = 0; i < pdesc->bNumConfigurations; i++) { err = libusb_get_config_descriptor(dev, i, config); if (err) return (err); if ((*config)->bConfigurationValue == bConfigurationValue) return (0); /* success */ libusb_free_config_descriptor(*config); } *config = NULL; return (LIBUSB_ERROR_NOT_FOUND); } void libusb_free_config_descriptor(struct libusb_config_descriptor *config) { free(config); } int libusb_get_string_descriptor_ascii(libusb_device_handle *pdev, uint8_t desc_index, unsigned char *data, int length) { if (pdev == NULL || data == NULL || length < 1) return (LIBUSB20_ERROR_INVALID_PARAM); /* put some default data into the destination buffer */ data[0] = 0; if (libusb20_dev_req_string_simple_sync(pdev, desc_index, data, length) == 0) return (strlen(data)); return (LIBUSB_ERROR_OTHER); } int libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type, uint8_t desc_index, uint8_t *data, int length) { return (libusb_control_transfer(devh, LIBUSB_ENDPOINT_IN, LIBUSB_REQUEST_GET_DESCRIPTOR, (desc_type << 8) | desc_index, 0, data, length, 1000)); } Index: user/imp/tbemd/lib/libusb/libusb10_io.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb10_io.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb10_io.c (revision 203850) @@ -1,732 +1,733 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2009 Sylvestre Gallon. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include -#include -#include +#include + +#include #include #include +#include +#include #include -#include -#include +#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" #include "libusb.h" #include "libusb10.h" UNEXPORTED void libusb10_add_pollfd(libusb_context *ctx, struct libusb_super_pollfd *pollfd, struct libusb20_device *pdev, int fd, short events) { if (ctx == NULL) return; /* invalid */ if (pollfd->entry.tqe_prev != NULL) return; /* already queued */ if (fd < 0) return; /* invalid */ pollfd->pdev = pdev; pollfd->pollfd.fd = fd; pollfd->pollfd.events = events; CTX_LOCK(ctx); TAILQ_INSERT_TAIL(&ctx->pollfds, pollfd, entry); CTX_UNLOCK(ctx); if (ctx->fd_added_cb) ctx->fd_added_cb(fd, events, ctx->fd_cb_user_data); } UNEXPORTED void libusb10_remove_pollfd(libusb_context *ctx, struct libusb_super_pollfd *pollfd) { if (ctx == NULL) return; /* invalid */ if (pollfd->entry.tqe_prev == NULL) return; /* already dequeued */ CTX_LOCK(ctx); TAILQ_REMOVE(&ctx->pollfds, pollfd, entry); pollfd->entry.tqe_prev = NULL; CTX_UNLOCK(ctx); if (ctx->fd_removed_cb) ctx->fd_removed_cb(pollfd->pollfd.fd, ctx->fd_cb_user_data); } /* This function must be called locked */ static int libusb10_handle_events_sub(struct libusb_context *ctx, struct timeval *tv) { struct libusb_device *dev; struct libusb20_device **ppdev; struct libusb_super_pollfd *pfd; struct pollfd *fds; struct libusb_super_transfer *sxfer; struct libusb_transfer *uxfer; nfds_t nfds; int timeout; int i; int err; DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb10_handle_events_sub enter"); nfds = 0; i = 0; TAILQ_FOREACH(pfd, &ctx->pollfds, entry) nfds++; fds = alloca(sizeof(*fds) * nfds); if (fds == NULL) return (LIBUSB_ERROR_NO_MEM); ppdev = alloca(sizeof(*ppdev) * nfds); if (ppdev == NULL) return (LIBUSB_ERROR_NO_MEM); TAILQ_FOREACH(pfd, &ctx->pollfds, entry) { fds[i].fd = pfd->pollfd.fd; fds[i].events = pfd->pollfd.events; fds[i].revents = 0; ppdev[i] = pfd->pdev; if (pfd->pdev != NULL) libusb_get_device(pfd->pdev)->refcnt++; i++; } if (tv == NULL) timeout = -1; else timeout = (tv->tv_sec * 1000) + ((tv->tv_usec + 999) / 1000); CTX_UNLOCK(ctx); err = poll(fds, nfds, timeout); CTX_LOCK(ctx); if ((err == -1) && (errno == EINTR)) err = LIBUSB_ERROR_INTERRUPTED; else if (err < 0) err = LIBUSB_ERROR_IO; if (err < 1) { for (i = 0; i != nfds; i++) { if (ppdev[i] != NULL) { CTX_UNLOCK(ctx); libusb_unref_device(libusb_get_device(ppdev[i])); CTX_LOCK(ctx); } } goto do_done; } for (i = 0; i != nfds; i++) { if (ppdev[i] != NULL) { dev = libusb_get_device(ppdev[i]); if (fds[i].revents == 0) err = 0; /* nothing to do */ else err = libusb20_dev_process(ppdev[i]); if (err) { /* cancel all transfers - device is gone */ libusb10_cancel_all_transfer(dev); /* remove USB device from polling loop */ libusb10_remove_pollfd(dev->ctx, &dev->dev_poll); } CTX_UNLOCK(ctx); libusb_unref_device(dev); CTX_LOCK(ctx); } else { uint8_t dummy; while (1) { if (read(fds[i].fd, &dummy, 1) != 1) break; } } } err = 0; do_done: /* Do all done callbacks */ while ((sxfer = TAILQ_FIRST(&ctx->tr_done))) { TAILQ_REMOVE(&ctx->tr_done, sxfer, entry); sxfer->entry.tqe_prev = NULL; ctx->tr_done_ref++; CTX_UNLOCK(ctx); uxfer = (struct libusb_transfer *)( ((uint8_t *)sxfer) + sizeof(*sxfer)); if (uxfer->callback != NULL) (uxfer->callback) (uxfer); if (uxfer->flags & LIBUSB_TRANSFER_FREE_BUFFER) free(uxfer->buffer); if (uxfer->flags & LIBUSB_TRANSFER_FREE_TRANSFER) libusb_free_transfer(uxfer); CTX_LOCK(ctx); ctx->tr_done_ref--; ctx->tr_done_gen++; } /* Wakeup other waiters */ pthread_cond_broadcast(&ctx->ctx_cond); return (err); } /* Polling and timing */ int libusb_try_lock_events(libusb_context *ctx) { int err; ctx = GET_CONTEXT(ctx); if (ctx == NULL) return (1); err = CTX_TRYLOCK(ctx); if (err) return (1); err = (ctx->ctx_handler != NO_THREAD); if (err) CTX_UNLOCK(ctx); else ctx->ctx_handler = pthread_self(); return (err); } void libusb_lock_events(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); CTX_LOCK(ctx); if (ctx->ctx_handler == NO_THREAD) ctx->ctx_handler = pthread_self(); } void libusb_unlock_events(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); if (ctx->ctx_handler == pthread_self()) { ctx->ctx_handler = NO_THREAD; pthread_cond_broadcast(&ctx->ctx_cond); } CTX_UNLOCK(ctx); } int libusb_event_handling_ok(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); return (ctx->ctx_handler == pthread_self()); } int libusb_event_handler_active(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); return (ctx->ctx_handler != NO_THREAD); } void libusb_lock_event_waiters(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); CTX_LOCK(ctx); } void libusb_unlock_event_waiters(libusb_context *ctx) { ctx = GET_CONTEXT(ctx); CTX_UNLOCK(ctx); } int libusb_wait_for_event(libusb_context *ctx, struct timeval *tv) { struct timespec ts; int err; ctx = GET_CONTEXT(ctx); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_wait_for_event enter"); if (tv == NULL) { pthread_cond_wait(&ctx->ctx_cond, &ctx->ctx_lock); return (0); } err = clock_gettime(CLOCK_REALTIME, &ts); if (err < 0) return (LIBUSB_ERROR_OTHER); ts.tv_sec = tv->tv_sec; ts.tv_nsec = tv->tv_usec * 1000; if (ts.tv_nsec >= 1000000000) { ts.tv_nsec -= 1000000000; ts.tv_sec++; } err = pthread_cond_timedwait(&ctx->ctx_cond, &ctx->ctx_lock, &ts); if (err == ETIMEDOUT) return (1); return (0); } int libusb_handle_events_timeout(libusb_context *ctx, struct timeval *tv) { int err; ctx = GET_CONTEXT(ctx); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_handle_events_timeout enter"); libusb_lock_events(ctx); err = libusb_handle_events_locked(ctx, tv); libusb_unlock_events(ctx); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_handle_events_timeout leave"); return (err); } int libusb_handle_events(libusb_context *ctx) { return (libusb_handle_events_timeout(ctx, NULL)); } int libusb_handle_events_locked(libusb_context *ctx, struct timeval *tv) { int err; ctx = GET_CONTEXT(ctx); if (libusb_event_handling_ok(ctx)) { err = libusb10_handle_events_sub(ctx, tv); } else { libusb_wait_for_event(ctx, tv); err = 0; } return (err); } int libusb_get_next_timeout(libusb_context *ctx, struct timeval *tv) { /* all timeouts are currently being done by the kernel */ timerclear(tv); return (0); } void libusb_set_pollfd_notifiers(libusb_context *ctx, libusb_pollfd_added_cb added_cb, libusb_pollfd_removed_cb removed_cb, void *user_data) { ctx = GET_CONTEXT(ctx); ctx->fd_added_cb = added_cb; ctx->fd_removed_cb = removed_cb; ctx->fd_cb_user_data = user_data; } struct libusb_pollfd ** libusb_get_pollfds(libusb_context *ctx) { struct libusb_super_pollfd *pollfd; libusb_pollfd **ret; int i; ctx = GET_CONTEXT(ctx); CTX_LOCK(ctx); i = 0; TAILQ_FOREACH(pollfd, &ctx->pollfds, entry) i++; ret = calloc(i + 1, sizeof(struct libusb_pollfd *)); if (ret == NULL) goto done; i = 0; TAILQ_FOREACH(pollfd, &ctx->pollfds, entry) ret[i++] = &pollfd->pollfd; ret[i] = NULL; done: CTX_UNLOCK(ctx); return (ret); } /* Synchronous device I/O */ int libusb_control_transfer(libusb_device_handle *devh, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint8_t *data, uint16_t wLength, unsigned int timeout) { struct LIBUSB20_CONTROL_SETUP_DECODED req; int err; uint16_t actlen; if (devh == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if ((wLength != 0) && (data == NULL)) return (LIBUSB_ERROR_INVALID_PARAM); LIBUSB20_INIT(LIBUSB20_CONTROL_SETUP, &req); req.bmRequestType = bmRequestType; req.bRequest = bRequest; req.wValue = wValue; req.wIndex = wIndex; req.wLength = wLength; err = libusb20_dev_request_sync(devh, &req, data, &actlen, timeout, 0); if (err == LIBUSB20_ERROR_PIPE) return (LIBUSB_ERROR_PIPE); else if (err == LIBUSB20_ERROR_TIMEOUT) return (LIBUSB_ERROR_TIMEOUT); else if (err) return (LIBUSB_ERROR_NO_DEVICE); return (actlen); } static void libusb10_do_transfer_cb(struct libusb_transfer *transfer) { libusb_context *ctx; int *pdone; ctx = GET_CONTEXT(NULL); DPRINTF(ctx, LIBUSB_DEBUG_TRANSFER, "sync I/O done"); pdone = transfer->user_data; *pdone = 1; } /* * TODO: Replace the following function. Allocating and freeing on a * per-transfer basis is slow. --HPS */ static int libusb10_do_transfer(libusb_device_handle *devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, unsigned int timeout, int type) { libusb_context *ctx; struct libusb_transfer *xfer; volatile int complet; int ret; if (devh == NULL) return (LIBUSB_ERROR_INVALID_PARAM); if ((length != 0) && (data == NULL)) return (LIBUSB_ERROR_INVALID_PARAM); xfer = libusb_alloc_transfer(0); if (xfer == NULL) return (LIBUSB_ERROR_NO_MEM); ctx = libusb_get_device(devh)->ctx; xfer->dev_handle = devh; xfer->endpoint = endpoint; xfer->type = type; xfer->timeout = timeout; xfer->buffer = data; xfer->length = length; xfer->user_data = (void *)&complet; xfer->callback = libusb10_do_transfer_cb; complet = 0; if ((ret = libusb_submit_transfer(xfer)) < 0) { libusb_free_transfer(xfer); return (ret); } while (complet == 0) { if ((ret = libusb_handle_events(ctx)) < 0) { libusb_cancel_transfer(xfer); usleep(1000); /* nice it */ } } *transferred = xfer->actual_length; switch (xfer->status) { case LIBUSB_TRANSFER_COMPLETED: ret = 0; break; case LIBUSB_TRANSFER_TIMED_OUT: ret = LIBUSB_ERROR_TIMEOUT; break; case LIBUSB_TRANSFER_OVERFLOW: ret = LIBUSB_ERROR_OVERFLOW; break; case LIBUSB_TRANSFER_STALL: ret = LIBUSB_ERROR_PIPE; break; case LIBUSB_TRANSFER_NO_DEVICE: ret = LIBUSB_ERROR_NO_DEVICE; break; default: ret = LIBUSB_ERROR_OTHER; break; } libusb_free_transfer(xfer); return (ret); } int libusb_bulk_transfer(libusb_device_handle *devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, unsigned int timeout) { libusb_context *ctx; int ret; ctx = GET_CONTEXT(NULL); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_bulk_transfer enter"); ret = libusb10_do_transfer(devh, endpoint, data, length, transferred, timeout, LIBUSB_TRANSFER_TYPE_BULK); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_bulk_transfer leave"); return (ret); } int libusb_interrupt_transfer(libusb_device_handle *devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, unsigned int timeout) { libusb_context *ctx; int ret; ctx = GET_CONTEXT(NULL); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_interrupt_transfer enter"); ret = libusb10_do_transfer(devh, endpoint, data, length, transferred, timeout, LIBUSB_TRANSFER_TYPE_INTERRUPT); DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_interrupt_transfer leave"); return (ret); } uint8_t * libusb_get_iso_packet_buffer(struct libusb_transfer *transfer, uint32_t index) { uint8_t *ptr; uint32_t n; if (transfer->num_iso_packets < 0) return (NULL); if (index >= (uint32_t)transfer->num_iso_packets) return (NULL); ptr = transfer->buffer; if (ptr == NULL) return (NULL); for (n = 0; n != index; n++) { ptr += transfer->iso_packet_desc[n].length; } return (ptr); } uint8_t * libusb_get_iso_packet_buffer_simple(struct libusb_transfer *transfer, uint32_t index) { uint8_t *ptr; if (transfer->num_iso_packets < 0) return (NULL); if (index >= (uint32_t)transfer->num_iso_packets) return (NULL); ptr = transfer->buffer; if (ptr == NULL) return (NULL); ptr += transfer->iso_packet_desc[0].length * index; return (ptr); } void libusb_set_iso_packet_lengths(struct libusb_transfer *transfer, uint32_t length) { int n; if (transfer->num_iso_packets < 0) return; for (n = 0; n != transfer->num_iso_packets; n++) transfer->iso_packet_desc[n].length = length; } uint8_t * libusb_control_transfer_get_data(struct libusb_transfer *transfer) { if (transfer->buffer == NULL) return (NULL); return (transfer->buffer + LIBUSB_CONTROL_SETUP_SIZE); } struct libusb_control_setup * libusb_control_transfer_get_setup(struct libusb_transfer *transfer) { return ((struct libusb_control_setup *)transfer->buffer); } void libusb_fill_control_setup(uint8_t *buf, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint16_t wLength) { struct libusb_control_setup *req = (struct libusb_control_setup *)buf; /* The alignment is OK for all fields below. */ req->bmRequestType = bmRequestType; req->bRequest = bRequest; req->wValue = htole16(wValue); req->wIndex = htole16(wIndex); req->wLength = htole16(wLength); } void libusb_fill_control_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t *buf, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout) { struct libusb_control_setup *setup = (struct libusb_control_setup *)buf; transfer->dev_handle = devh; transfer->endpoint = 0; transfer->type = LIBUSB_TRANSFER_TYPE_CONTROL; transfer->timeout = timeout; transfer->buffer = buf; if (setup != NULL) transfer->length = LIBUSB_CONTROL_SETUP_SIZE + le16toh(setup->wLength); else transfer->length = 0; transfer->user_data = user_data; transfer->callback = callback; } void libusb_fill_bulk_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout) { transfer->dev_handle = devh; transfer->endpoint = endpoint; transfer->type = LIBUSB_TRANSFER_TYPE_BULK; transfer->timeout = timeout; transfer->buffer = buf; transfer->length = length; transfer->user_data = user_data; transfer->callback = callback; } void libusb_fill_interrupt_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout) { transfer->dev_handle = devh; transfer->endpoint = endpoint; transfer->type = LIBUSB_TRANSFER_TYPE_INTERRUPT; transfer->timeout = timeout; transfer->buffer = buf; transfer->length = length; transfer->user_data = user_data; transfer->callback = callback; } void libusb_fill_iso_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, int npacket, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout) { transfer->dev_handle = devh; transfer->endpoint = endpoint; transfer->type = LIBUSB_TRANSFER_TYPE_ISOCHRONOUS; transfer->timeout = timeout; transfer->buffer = buf; transfer->length = length; transfer->num_iso_packets = npacket; transfer->user_data = user_data; transfer->callback = callback; } Index: user/imp/tbemd/lib/libusb/libusb20.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb20.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb20.c (revision 203850) @@ -1,1225 +1,1226 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008-2009 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include + +#include +#include #include #include #include -#include -#include -#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" static int dummy_int(void) { return (LIBUSB20_ERROR_NOT_SUPPORTED); } static void dummy_void(void) { return; } static void dummy_callback(struct libusb20_transfer *xfer) { ; /* style fix */ switch (libusb20_tr_get_status(xfer)) { case LIBUSB20_TRANSFER_START: libusb20_tr_submit(xfer); break; default: /* complete or error */ break; } return; } #define dummy_get_config_desc_full (void *)dummy_int #define dummy_get_config_index (void *)dummy_int #define dummy_set_config_index (void *)dummy_int #define dummy_set_alt_index (void *)dummy_int #define dummy_reset_device (void *)dummy_int #define dummy_check_connected (void *)dummy_int #define dummy_set_power_mode (void *)dummy_int #define dummy_get_power_mode (void *)dummy_int #define dummy_kernel_driver_active (void *)dummy_int #define dummy_detach_kernel_driver (void *)dummy_int #define dummy_do_request_sync (void *)dummy_int #define dummy_tr_open (void *)dummy_int #define dummy_tr_close (void *)dummy_int #define dummy_tr_clear_stall_sync (void *)dummy_int #define dummy_process (void *)dummy_int #define dummy_dev_info (void *)dummy_int #define dummy_dev_get_iface_driver (void *)dummy_int #define dummy_tr_submit (void *)dummy_void #define dummy_tr_cancel_async (void *)dummy_void static const struct libusb20_device_methods libusb20_dummy_methods = { LIBUSB20_DEVICE(LIBUSB20_DECLARE, dummy) }; void libusb20_tr_callback_wrapper(struct libusb20_transfer *xfer) { ; /* style fix */ repeat: if (!xfer->is_pending) { xfer->status = LIBUSB20_TRANSFER_START; } else { xfer->is_pending = 0; } xfer->callback(xfer); if (xfer->is_restart) { xfer->is_restart = 0; goto repeat; } if (xfer->is_draining && (!xfer->is_pending)) { xfer->is_draining = 0; xfer->status = LIBUSB20_TRANSFER_DRAINED; xfer->callback(xfer); } return; } int libusb20_tr_close(struct libusb20_transfer *xfer) { int error; if (!xfer->is_opened) { return (LIBUSB20_ERROR_OTHER); } error = xfer->pdev->methods->tr_close(xfer); if (xfer->pLength) { free(xfer->pLength); } if (xfer->ppBuffer) { free(xfer->ppBuffer); } /* reset variable fields in case the transfer is opened again */ xfer->priv_sc0 = 0; xfer->priv_sc1 = 0; xfer->is_opened = 0; xfer->is_pending = 0; xfer->is_cancel = 0; xfer->is_draining = 0; xfer->is_restart = 0; xfer->status = 0; xfer->flags = 0; xfer->nFrames = 0; xfer->aFrames = 0; xfer->timeout = 0; xfer->maxFrames = 0; xfer->maxTotalLength = 0; xfer->maxPacketLen = 0; return (error); } int libusb20_tr_open(struct libusb20_transfer *xfer, uint32_t MaxBufSize, uint32_t MaxFrameCount, uint8_t ep_no) { uint32_t size; int error; if (xfer->is_opened) { return (LIBUSB20_ERROR_BUSY); } if (MaxFrameCount == 0) { return (LIBUSB20_ERROR_INVALID_PARAM); } xfer->maxFrames = MaxFrameCount; size = MaxFrameCount * sizeof(xfer->pLength[0]); xfer->pLength = malloc(size); if (xfer->pLength == NULL) { return (LIBUSB20_ERROR_NO_MEM); } memset(xfer->pLength, 0, size); size = MaxFrameCount * sizeof(xfer->ppBuffer[0]); xfer->ppBuffer = malloc(size); if (xfer->ppBuffer == NULL) { free(xfer->pLength); return (LIBUSB20_ERROR_NO_MEM); } memset(xfer->ppBuffer, 0, size); error = xfer->pdev->methods->tr_open(xfer, MaxBufSize, MaxFrameCount, ep_no); if (error) { free(xfer->ppBuffer); free(xfer->pLength); } else { xfer->is_opened = 1; } return (error); } struct libusb20_transfer * libusb20_tr_get_pointer(struct libusb20_device *pdev, uint16_t trIndex) { if (trIndex >= pdev->nTransfer) { return (NULL); } return (pdev->pTransfer + trIndex); } uint32_t libusb20_tr_get_actual_frames(struct libusb20_transfer *xfer) { return (xfer->aFrames); } uint16_t libusb20_tr_get_time_complete(struct libusb20_transfer *xfer) { return (xfer->timeComplete); } uint32_t libusb20_tr_get_actual_length(struct libusb20_transfer *xfer) { uint32_t x; uint32_t actlen = 0; for (x = 0; x != xfer->aFrames; x++) { actlen += xfer->pLength[x]; } return (actlen); } uint32_t libusb20_tr_get_max_frames(struct libusb20_transfer *xfer) { return (xfer->maxFrames); } uint32_t libusb20_tr_get_max_packet_length(struct libusb20_transfer *xfer) { /* * Special Case NOTE: If the packet multiplier is non-zero for * High Speed USB, the value returned is equal to * "wMaxPacketSize * multiplier" ! */ return (xfer->maxPacketLen); } uint32_t libusb20_tr_get_max_total_length(struct libusb20_transfer *xfer) { return (xfer->maxTotalLength); } uint8_t libusb20_tr_get_status(struct libusb20_transfer *xfer) { return (xfer->status); } uint8_t libusb20_tr_pending(struct libusb20_transfer *xfer) { return (xfer->is_pending); } void * libusb20_tr_get_priv_sc0(struct libusb20_transfer *xfer) { return (xfer->priv_sc0); } void * libusb20_tr_get_priv_sc1(struct libusb20_transfer *xfer) { return (xfer->priv_sc1); } void libusb20_tr_stop(struct libusb20_transfer *xfer) { if (!xfer->is_opened) { /* transfer is not opened */ return; } if (!xfer->is_pending) { /* transfer not pending */ return; } if (xfer->is_cancel) { /* already cancelling */ return; } xfer->is_cancel = 1; /* we are cancelling */ xfer->pdev->methods->tr_cancel_async(xfer); return; } void libusb20_tr_drain(struct libusb20_transfer *xfer) { if (!xfer->is_opened) { /* transfer is not opened */ return; } /* make sure that we are cancelling */ libusb20_tr_stop(xfer); if (xfer->is_pending) { xfer->is_draining = 1; } return; } void libusb20_tr_clear_stall_sync(struct libusb20_transfer *xfer) { xfer->pdev->methods->tr_clear_stall_sync(xfer); return; } void libusb20_tr_set_buffer(struct libusb20_transfer *xfer, void *buffer, uint16_t frIndex) { xfer->ppBuffer[frIndex] = buffer; return; } void libusb20_tr_set_callback(struct libusb20_transfer *xfer, libusb20_tr_callback_t *cb) { xfer->callback = cb; return; } void libusb20_tr_set_flags(struct libusb20_transfer *xfer, uint8_t flags) { xfer->flags = flags; return; } uint32_t libusb20_tr_get_length(struct libusb20_transfer *xfer, uint16_t frIndex) { return (xfer->pLength[frIndex]); } void libusb20_tr_set_length(struct libusb20_transfer *xfer, uint32_t length, uint16_t frIndex) { xfer->pLength[frIndex] = length; return; } void libusb20_tr_set_priv_sc0(struct libusb20_transfer *xfer, void *sc0) { xfer->priv_sc0 = sc0; return; } void libusb20_tr_set_priv_sc1(struct libusb20_transfer *xfer, void *sc1) { xfer->priv_sc1 = sc1; return; } void libusb20_tr_set_timeout(struct libusb20_transfer *xfer, uint32_t timeout) { xfer->timeout = timeout; return; } void libusb20_tr_set_total_frames(struct libusb20_transfer *xfer, uint32_t nFrames) { if (nFrames > xfer->maxFrames) { /* should not happen */ nFrames = xfer->maxFrames; } xfer->nFrames = nFrames; return; } void libusb20_tr_setup_bulk(struct libusb20_transfer *xfer, void *pBuf, uint32_t length, uint32_t timeout) { xfer->ppBuffer[0] = pBuf; xfer->pLength[0] = length; xfer->timeout = timeout; xfer->nFrames = 1; return; } void libusb20_tr_setup_control(struct libusb20_transfer *xfer, void *psetup, void *pBuf, uint32_t timeout) { uint16_t len; xfer->ppBuffer[0] = psetup; xfer->pLength[0] = 8; /* fixed */ xfer->timeout = timeout; len = ((uint8_t *)psetup)[6] | (((uint8_t *)psetup)[7] << 8); if (len != 0) { xfer->nFrames = 2; xfer->ppBuffer[1] = pBuf; xfer->pLength[1] = len; } else { xfer->nFrames = 1; } return; } void libusb20_tr_setup_intr(struct libusb20_transfer *xfer, void *pBuf, uint32_t length, uint32_t timeout) { xfer->ppBuffer[0] = pBuf; xfer->pLength[0] = length; xfer->timeout = timeout; xfer->nFrames = 1; return; } void libusb20_tr_setup_isoc(struct libusb20_transfer *xfer, void *pBuf, uint32_t length, uint16_t frIndex) { if (frIndex >= xfer->maxFrames) { /* should not happen */ return; } xfer->ppBuffer[frIndex] = pBuf; xfer->pLength[frIndex] = length; return; } uint8_t libusb20_tr_bulk_intr_sync(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t *pactlen, uint32_t timeout) { struct libusb20_device *pdev = xfer->pdev; uint32_t transfer_max; uint32_t transfer_act; uint8_t retval; /* set some sensible default value */ if (pactlen != NULL) *pactlen = 0; /* check for error condition */ if (libusb20_tr_pending(xfer)) return (LIBUSB20_ERROR_OTHER); do { /* compute maximum transfer length */ transfer_max = libusb20_tr_get_max_total_length(xfer); if (transfer_max > length) transfer_max = length; /* setup bulk or interrupt transfer */ libusb20_tr_setup_bulk(xfer, pbuf, transfer_max, timeout); /* start the transfer */ libusb20_tr_start(xfer); /* wait for transfer completion */ while (libusb20_dev_process(pdev) == 0) { if (libusb20_tr_pending(xfer) == 0) break; libusb20_dev_wait_process(pdev, -1); } transfer_act = libusb20_tr_get_actual_length(xfer); /* update actual length, if any */ if (pactlen != NULL) pactlen[0] += transfer_act; /* check transfer status */ retval = libusb20_tr_get_status(xfer); if (retval) break; /* check for short transfer */ if (transfer_act != transfer_max) break; /* update buffer pointer and length */ pbuf = ((uint8_t *)pbuf) + transfer_max; length = length - transfer_max; } while (length != 0); return (retval); } void libusb20_tr_submit(struct libusb20_transfer *xfer) { if (!xfer->is_opened) { /* transfer is not opened */ return; } if (xfer->is_pending) { /* should not happen */ return; } xfer->is_pending = 1; /* we are pending */ xfer->is_cancel = 0; /* not cancelling */ xfer->is_restart = 0; /* not restarting */ xfer->pdev->methods->tr_submit(xfer); return; } void libusb20_tr_start(struct libusb20_transfer *xfer) { if (!xfer->is_opened) { /* transfer is not opened */ return; } if (xfer->is_pending) { if (xfer->is_cancel) { /* cancelling - restart */ xfer->is_restart = 1; } /* transfer not pending */ return; } /* get into the callback */ libusb20_tr_callback_wrapper(xfer); return; } /* USB device operations */ int libusb20_dev_close(struct libusb20_device *pdev) { struct libusb20_transfer *xfer; uint16_t x; int error = 0; if (!pdev->is_opened) { return (LIBUSB20_ERROR_OTHER); } for (x = 0; x != pdev->nTransfer; x++) { xfer = pdev->pTransfer + x; if (!xfer->is_opened) { /* transfer is not opened */ continue; } libusb20_tr_drain(xfer); libusb20_tr_close(xfer); } if (pdev->pTransfer != NULL) { free(pdev->pTransfer); pdev->pTransfer = NULL; } error = pdev->beMethods->close_device(pdev); pdev->methods = &libusb20_dummy_methods; pdev->is_opened = 0; /* * The following variable is only used by the libusb v0.1 * compat layer: */ pdev->claimed_interface = 0; return (error); } int libusb20_dev_detach_kernel_driver(struct libusb20_device *pdev, uint8_t ifaceIndex) { int error; error = pdev->methods->detach_kernel_driver(pdev, ifaceIndex); return (error); } struct LIBUSB20_DEVICE_DESC_DECODED * libusb20_dev_get_device_desc(struct libusb20_device *pdev) { return (&(pdev->ddesc)); } int libusb20_dev_get_fd(struct libusb20_device *pdev) { return (pdev->file); } int libusb20_dev_kernel_driver_active(struct libusb20_device *pdev, uint8_t ifaceIndex) { int error; error = pdev->methods->kernel_driver_active(pdev, ifaceIndex); return (error); } int libusb20_dev_open(struct libusb20_device *pdev, uint16_t nTransferMax) { struct libusb20_transfer *xfer; uint32_t size; uint16_t x; int error; if (pdev->is_opened) { return (LIBUSB20_ERROR_BUSY); } if (nTransferMax >= 256) { return (LIBUSB20_ERROR_INVALID_PARAM); } else if (nTransferMax != 0) { size = sizeof(pdev->pTransfer[0]) * nTransferMax; pdev->pTransfer = malloc(size); if (pdev->pTransfer == NULL) { return (LIBUSB20_ERROR_NO_MEM); } memset(pdev->pTransfer, 0, size); } /* initialise all transfers */ for (x = 0; x != nTransferMax; x++) { xfer = pdev->pTransfer + x; xfer->pdev = pdev; xfer->trIndex = x; xfer->callback = &dummy_callback; } /* set "nTransfer" early */ pdev->nTransfer = nTransferMax; error = pdev->beMethods->open_device(pdev, nTransferMax); if (error) { if (pdev->pTransfer != NULL) { free(pdev->pTransfer); pdev->pTransfer = NULL; } pdev->file = -1; pdev->file_ctrl = -1; pdev->nTransfer = 0; } else { pdev->is_opened = 1; } return (error); } int libusb20_dev_reset(struct libusb20_device *pdev) { int error; error = pdev->methods->reset_device(pdev); return (error); } int libusb20_dev_check_connected(struct libusb20_device *pdev) { int error; error = pdev->methods->check_connected(pdev); return (error); } int libusb20_dev_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode) { int error; error = pdev->methods->set_power_mode(pdev, power_mode); return (error); } uint8_t libusb20_dev_get_power_mode(struct libusb20_device *pdev) { int error; uint8_t power_mode; error = pdev->methods->get_power_mode(pdev, &power_mode); if (error) power_mode = LIBUSB20_POWER_ON; /* fake power mode */ return (power_mode); } int libusb20_dev_set_alt_index(struct libusb20_device *pdev, uint8_t ifaceIndex, uint8_t altIndex) { int error; error = pdev->methods->set_alt_index(pdev, ifaceIndex, altIndex); return (error); } int libusb20_dev_set_config_index(struct libusb20_device *pdev, uint8_t configIndex) { int error; error = pdev->methods->set_config_index(pdev, configIndex); return (error); } int libusb20_dev_request_sync(struct libusb20_device *pdev, struct LIBUSB20_CONTROL_SETUP_DECODED *setup, void *data, uint16_t *pactlen, uint32_t timeout, uint8_t flags) { int error; error = pdev->methods->do_request_sync(pdev, setup, data, pactlen, timeout, flags); return (error); } int libusb20_dev_req_string_sync(struct libusb20_device *pdev, uint8_t str_index, uint16_t langid, void *ptr, uint16_t len) { struct LIBUSB20_CONTROL_SETUP_DECODED req; int error; /* make sure memory is initialised */ memset(ptr, 0, len); if (len < 4) { /* invalid length */ return (LIBUSB20_ERROR_INVALID_PARAM); } LIBUSB20_INIT(LIBUSB20_CONTROL_SETUP, &req); /* * We need to read the USB string in two steps else some USB * devices will complain. */ req.bmRequestType = LIBUSB20_REQUEST_TYPE_STANDARD | LIBUSB20_RECIPIENT_DEVICE | LIBUSB20_ENDPOINT_IN; req.bRequest = LIBUSB20_REQUEST_GET_DESCRIPTOR; req.wValue = (LIBUSB20_DT_STRING << 8) | str_index; req.wIndex = langid; req.wLength = 4; /* bytes */ error = libusb20_dev_request_sync(pdev, &req, ptr, NULL, 1000, LIBUSB20_TRANSFER_SINGLE_SHORT_NOT_OK); if (error) { return (error); } req.wLength = *(uint8_t *)ptr; /* bytes */ if (req.wLength > len) { /* partial string read */ req.wLength = len; } error = libusb20_dev_request_sync(pdev, &req, ptr, NULL, 1000, LIBUSB20_TRANSFER_SINGLE_SHORT_NOT_OK); if (error) { return (error); } if (((uint8_t *)ptr)[1] != LIBUSB20_DT_STRING) { return (LIBUSB20_ERROR_OTHER); } return (0); /* success */ } int libusb20_dev_req_string_simple_sync(struct libusb20_device *pdev, uint8_t str_index, void *ptr, uint16_t len) { char *buf; int error; uint16_t langid; uint16_t n; uint16_t i; uint16_t c; uint8_t temp[255]; uint8_t swap; /* the following code derives from the FreeBSD USB kernel */ if ((len < 1) || (ptr == NULL)) { /* too short buffer */ return (LIBUSB20_ERROR_INVALID_PARAM); } error = libusb20_dev_req_string_sync(pdev, 0, 0, temp, sizeof(temp)); if (error < 0) { *(uint8_t *)ptr = 0; /* zero terminate */ return (error); } langid = temp[2] | (temp[3] << 8); error = libusb20_dev_req_string_sync(pdev, str_index, langid, temp, sizeof(temp)); if (error < 0) { *(uint8_t *)ptr = 0; /* zero terminate */ return (error); } if (temp[0] < 2) { /* string length is too short */ *(uint8_t *)ptr = 0; /* zero terminate */ return (LIBUSB20_ERROR_OTHER); } /* reserve one byte for terminating zero */ len--; /* find maximum length */ n = (temp[0] / 2) - 1; if (n > len) { n = len; } /* reset swap state */ swap = 3; /* setup output buffer pointer */ buf = ptr; /* convert and filter */ for (i = 0; (i != n); i++) { c = temp[(2 * i) + 2] | (temp[(2 * i) + 3] << 8); /* convert from Unicode, handle buggy strings */ if (((c & 0xff00) == 0) && (swap & 1)) { /* Little Endian, default */ *buf = c; swap = 1; } else if (((c & 0x00ff) == 0) && (swap & 2)) { /* Big Endian */ *buf = c >> 8; swap = 2; } else { /* skip invalid character */ continue; } /* * Filter by default - we don't allow greater and less than * signs because they might confuse the dmesg printouts! */ if ((*buf == '<') || (*buf == '>') || (!isprint(*buf))) { /* skip invalid character */ continue; } buf++; } *buf = 0; /* zero terminate string */ return (0); } struct libusb20_config * libusb20_dev_alloc_config(struct libusb20_device *pdev, uint8_t configIndex) { struct libusb20_config *retval = NULL; uint8_t *ptr; uint16_t len; uint8_t do_close; int error; if (!pdev->is_opened) { error = libusb20_dev_open(pdev, 0); if (error) { return (NULL); } do_close = 1; } else { do_close = 0; } error = pdev->methods->get_config_desc_full(pdev, &ptr, &len, configIndex); if (error) { goto done; } /* parse new config descriptor */ retval = libusb20_parse_config_desc(ptr); /* free config descriptor */ free(ptr); done: if (do_close) { error = libusb20_dev_close(pdev); } return (retval); } struct libusb20_device * libusb20_dev_alloc(void) { struct libusb20_device *pdev; pdev = malloc(sizeof(*pdev)); if (pdev == NULL) { return (NULL); } memset(pdev, 0, sizeof(*pdev)); pdev->file = -1; pdev->file_ctrl = -1; pdev->methods = &libusb20_dummy_methods; return (pdev); } uint8_t libusb20_dev_get_config_index(struct libusb20_device *pdev) { int error; uint8_t cfg_index; uint8_t do_close; if (!pdev->is_opened) { error = libusb20_dev_open(pdev, 0); if (error == 0) { do_close = 1; } else { do_close = 0; } } else { do_close = 0; } error = pdev->methods->get_config_index(pdev, &cfg_index); if (error) { cfg_index = 0 - 1; /* current config index */ } if (do_close) { if (libusb20_dev_close(pdev)) { /* ignore */ } } return (cfg_index); } uint8_t libusb20_dev_get_mode(struct libusb20_device *pdev) { return (pdev->usb_mode); } uint8_t libusb20_dev_get_speed(struct libusb20_device *pdev) { return (pdev->usb_speed); } /* if this function returns an error, the device is gone */ int libusb20_dev_process(struct libusb20_device *pdev) { int error; error = pdev->methods->process(pdev); return (error); } void libusb20_dev_wait_process(struct libusb20_device *pdev, int timeout) { struct pollfd pfd[1]; if (!pdev->is_opened) { return; } pfd[0].fd = pdev->file; pfd[0].events = (POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM); pfd[0].revents = 0; if (poll(pfd, 1, timeout)) { /* ignore any error */ } return; } void libusb20_dev_free(struct libusb20_device *pdev) { if (pdev == NULL) { /* be NULL safe */ return; } if (pdev->is_opened) { if (libusb20_dev_close(pdev)) { /* ignore any errors */ } } free(pdev); return; } int libusb20_dev_get_info(struct libusb20_device *pdev, struct usb_device_info *pinfo) { if (pinfo == NULL) return (LIBUSB20_ERROR_INVALID_PARAM); return (pdev->beMethods->dev_get_info(pdev, pinfo)); } const char * libusb20_dev_get_backend_name(struct libusb20_device *pdev) { return (pdev->beMethods->get_backend_name()); } const char * libusb20_dev_get_desc(struct libusb20_device *pdev) { return (pdev->usb_desc); } void libusb20_dev_set_debug(struct libusb20_device *pdev, int debug) { pdev->debug = debug; return; } int libusb20_dev_get_debug(struct libusb20_device *pdev) { return (pdev->debug); } uint8_t libusb20_dev_get_address(struct libusb20_device *pdev) { return (pdev->device_address); } uint8_t libusb20_dev_get_bus_number(struct libusb20_device *pdev) { return (pdev->bus_number); } int libusb20_dev_get_iface_desc(struct libusb20_device *pdev, uint8_t iface_index, char *buf, uint8_t len) { if ((buf == NULL) || (len == 0)) return (LIBUSB20_ERROR_INVALID_PARAM); return (pdev->beMethods->dev_get_iface_desc( pdev, iface_index, buf, len)); } /* USB backend operations */ int libusb20_be_get_dev_quirk(struct libusb20_backend *pbe, uint16_t quirk_index, struct libusb20_quirk *pq) { return (pbe->methods->root_get_dev_quirk(pbe, quirk_index, pq)); } int libusb20_be_get_quirk_name(struct libusb20_backend *pbe, uint16_t quirk_index, struct libusb20_quirk *pq) { return (pbe->methods->root_get_quirk_name(pbe, quirk_index, pq)); } int libusb20_be_add_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq) { return (pbe->methods->root_add_dev_quirk(pbe, pq)); } int libusb20_be_remove_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq) { return (pbe->methods->root_remove_dev_quirk(pbe, pq)); } int libusb20_be_set_template(struct libusb20_backend *pbe, int temp) { return (pbe->methods->root_set_template(pbe, temp)); } int libusb20_be_get_template(struct libusb20_backend *pbe, int *ptemp) { int temp; if (ptemp == NULL) ptemp = &temp; return (pbe->methods->root_get_template(pbe, ptemp)); } struct libusb20_device * libusb20_be_device_foreach(struct libusb20_backend *pbe, struct libusb20_device *pdev) { if (pbe == NULL) { pdev = NULL; } else if (pdev == NULL) { pdev = TAILQ_FIRST(&(pbe->usb_devs)); } else { pdev = TAILQ_NEXT(pdev, dev_entry); } return (pdev); } struct libusb20_backend * libusb20_be_alloc(const struct libusb20_backend_methods *methods) { struct libusb20_backend *pbe; pbe = malloc(sizeof(*pbe)); if (pbe == NULL) { return (NULL); } memset(pbe, 0, sizeof(*pbe)); TAILQ_INIT(&(pbe->usb_devs)); pbe->methods = methods; /* set backend methods */ /* do the initial device scan */ if (pbe->methods->init_backend) { pbe->methods->init_backend(pbe); } return (pbe); } struct libusb20_backend * libusb20_be_alloc_linux(void) { struct libusb20_backend *pbe; #ifdef __linux__ pbe = libusb20_be_alloc(&libusb20_linux_backend); #else pbe = NULL; #endif return (pbe); } struct libusb20_backend * libusb20_be_alloc_ugen20(void) { struct libusb20_backend *pbe; #ifdef __FreeBSD__ pbe = libusb20_be_alloc(&libusb20_ugen20_backend); #else pbe = NULL; #endif return (pbe); } struct libusb20_backend * libusb20_be_alloc_default(void) { struct libusb20_backend *pbe; pbe = libusb20_be_alloc_linux(); if (pbe) { return (pbe); } pbe = libusb20_be_alloc_ugen20(); if (pbe) { return (pbe); } return (NULL); /* no backend found */ } void libusb20_be_free(struct libusb20_backend *pbe) { struct libusb20_device *pdev; if (pbe == NULL) { /* be NULL safe */ return; } while ((pdev = libusb20_be_device_foreach(pbe, NULL))) { libusb20_be_dequeue_device(pbe, pdev); libusb20_dev_free(pdev); } if (pbe->methods->exit_backend) { pbe->methods->exit_backend(pbe); } /* free backend */ free(pbe); } void libusb20_be_enqueue_device(struct libusb20_backend *pbe, struct libusb20_device *pdev) { pdev->beMethods = pbe->methods; /* copy backend methods */ TAILQ_INSERT_TAIL(&(pbe->usb_devs), pdev, dev_entry); } void libusb20_be_dequeue_device(struct libusb20_backend *pbe, struct libusb20_device *pdev) { TAILQ_REMOVE(&(pbe->usb_devs), pdev, dev_entry); } Index: user/imp/tbemd/lib/libusb/libusb20.h =================================================================== --- user/imp/tbemd/lib/libusb/libusb20.h (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb20.h (revision 203850) @@ -1,301 +1,301 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008-2009 Hans Petter Selasky. All rights reserved. * Copyright (c) 2007-2008 Daniel Drake. All rights reserved. * Copyright (c) 2001 Johannes Erdfelt. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _LIBUSB20_H_ #define _LIBUSB20_H_ -#include -#include -#include - +#include #include #include -#include + +#include +#include +#include #ifdef __cplusplus extern "C" { #endif #if 0 }; /* style */ #endif /** \ingroup misc * Error codes. Most libusb20 functions return 0 on success or one of * these codes on failure. */ enum libusb20_error { /** Success (no error) */ LIBUSB20_SUCCESS = 0, /** Input/output error */ LIBUSB20_ERROR_IO = -1, /** Invalid parameter */ LIBUSB20_ERROR_INVALID_PARAM = -2, /** Access denied (insufficient permissions) */ LIBUSB20_ERROR_ACCESS = -3, /** No such device (it may have been disconnected) */ LIBUSB20_ERROR_NO_DEVICE = -4, /** Entity not found */ LIBUSB20_ERROR_NOT_FOUND = -5, /** Resource busy */ LIBUSB20_ERROR_BUSY = -6, /** Operation timed out */ LIBUSB20_ERROR_TIMEOUT = -7, /** Overflow */ LIBUSB20_ERROR_OVERFLOW = -8, /** Pipe error */ LIBUSB20_ERROR_PIPE = -9, /** System call interrupted (perhaps due to signal) */ LIBUSB20_ERROR_INTERRUPTED = -10, /** Insufficient memory */ LIBUSB20_ERROR_NO_MEM = -11, /** Operation not supported or unimplemented on this platform */ LIBUSB20_ERROR_NOT_SUPPORTED = -12, /** Other error */ LIBUSB20_ERROR_OTHER = -99, }; /** \ingroup asyncio * libusb20_tr_get_status() values */ enum libusb20_transfer_status { /** Transfer completed without error. Note that this does not * indicate that the entire amount of requested data was * transferred. */ LIBUSB20_TRANSFER_COMPLETED, /** Callback code to start transfer */ LIBUSB20_TRANSFER_START, /** Drain complete callback code */ LIBUSB20_TRANSFER_DRAINED, /** Transfer failed */ LIBUSB20_TRANSFER_ERROR, /** Transfer timed out */ LIBUSB20_TRANSFER_TIMED_OUT, /** Transfer was cancelled */ LIBUSB20_TRANSFER_CANCELLED, /** For bulk/interrupt endpoints: halt condition detected * (endpoint stalled). For control endpoints: control request * not supported. */ LIBUSB20_TRANSFER_STALL, /** Device was disconnected */ LIBUSB20_TRANSFER_NO_DEVICE, /** Device sent more data than requested */ LIBUSB20_TRANSFER_OVERFLOW, }; /** \ingroup asyncio * libusb20_tr_set_flags() values */ enum libusb20_transfer_flags { /** Report a short frame as error */ LIBUSB20_TRANSFER_SINGLE_SHORT_NOT_OK = 0x0001, /** Multiple short frames are not allowed */ LIBUSB20_TRANSFER_MULTI_SHORT_NOT_OK = 0x0002, /** All transmitted frames are short terminated */ LIBUSB20_TRANSFER_FORCE_SHORT = 0x0004, /** Will do a clear-stall before xfer */ LIBUSB20_TRANSFER_DO_CLEAR_STALL = 0x0008, }; /** \ingroup misc * libusb20_dev_get_mode() values */ enum libusb20_device_mode { LIBUSB20_MODE_HOST, /* default */ LIBUSB20_MODE_DEVICE, }; /** \ingroup misc * libusb20_dev_get_speed() values */ enum { LIBUSB20_SPEED_UNKNOWN, /* default */ LIBUSB20_SPEED_LOW, LIBUSB20_SPEED_FULL, LIBUSB20_SPEED_HIGH, LIBUSB20_SPEED_VARIABLE, LIBUSB20_SPEED_SUPER, }; /** \ingroup misc * libusb20_dev_set_power() values */ enum { LIBUSB20_POWER_OFF, LIBUSB20_POWER_ON, LIBUSB20_POWER_SAVE, LIBUSB20_POWER_SUSPEND, LIBUSB20_POWER_RESUME, }; struct usb_device_info; struct libusb20_transfer; struct libusb20_backend; struct libusb20_backend_methods; struct libusb20_device; struct libusb20_device_methods; struct libusb20_config; struct LIBUSB20_CONTROL_SETUP_DECODED; struct LIBUSB20_DEVICE_DESC_DECODED; typedef void (libusb20_tr_callback_t)(struct libusb20_transfer *xfer); struct libusb20_quirk { uint16_t vid; /* vendor ID */ uint16_t pid; /* product ID */ uint16_t bcdDeviceLow; /* low revision value, inclusive */ uint16_t bcdDeviceHigh; /* high revision value, inclusive */ uint16_t reserved[2]; /* for the future */ /* quirk name, UQ_XXX, including terminating zero */ char quirkname[64 - 12]; }; /* USB transfer operations */ int libusb20_tr_close(struct libusb20_transfer *xfer); int libusb20_tr_open(struct libusb20_transfer *xfer, uint32_t max_buf_size, uint32_t max_frame_count, uint8_t ep_no); struct libusb20_transfer *libusb20_tr_get_pointer(struct libusb20_device *pdev, uint16_t tr_index); uint16_t libusb20_tr_get_time_complete(struct libusb20_transfer *xfer); uint32_t libusb20_tr_get_actual_frames(struct libusb20_transfer *xfer); uint32_t libusb20_tr_get_actual_length(struct libusb20_transfer *xfer); uint32_t libusb20_tr_get_max_frames(struct libusb20_transfer *xfer); uint32_t libusb20_tr_get_max_packet_length(struct libusb20_transfer *xfer); uint32_t libusb20_tr_get_max_total_length(struct libusb20_transfer *xfer); uint8_t libusb20_tr_get_status(struct libusb20_transfer *xfer); uint8_t libusb20_tr_pending(struct libusb20_transfer *xfer); void libusb20_tr_callback_wrapper(struct libusb20_transfer *xfer); void libusb20_tr_clear_stall_sync(struct libusb20_transfer *xfer); void libusb20_tr_drain(struct libusb20_transfer *xfer); void libusb20_tr_set_buffer(struct libusb20_transfer *xfer, void *buffer, uint16_t fr_index); void libusb20_tr_set_callback(struct libusb20_transfer *xfer, libusb20_tr_callback_t *cb); void libusb20_tr_set_flags(struct libusb20_transfer *xfer, uint8_t flags); uint32_t libusb20_tr_get_length(struct libusb20_transfer *xfer, uint16_t fr_index); void libusb20_tr_set_length(struct libusb20_transfer *xfer, uint32_t length, uint16_t fr_index); void libusb20_tr_set_priv_sc0(struct libusb20_transfer *xfer, void *sc0); void libusb20_tr_set_priv_sc1(struct libusb20_transfer *xfer, void *sc1); void libusb20_tr_set_timeout(struct libusb20_transfer *xfer, uint32_t timeout); void libusb20_tr_set_total_frames(struct libusb20_transfer *xfer, uint32_t nFrames); void libusb20_tr_setup_bulk(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t timeout); void libusb20_tr_setup_control(struct libusb20_transfer *xfer, void *psetup, void *pbuf, uint32_t timeout); void libusb20_tr_setup_intr(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t timeout); void libusb20_tr_setup_isoc(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint16_t fr_index); uint8_t libusb20_tr_bulk_intr_sync(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t *pactlen, uint32_t timeout); void libusb20_tr_start(struct libusb20_transfer *xfer); void libusb20_tr_stop(struct libusb20_transfer *xfer); void libusb20_tr_submit(struct libusb20_transfer *xfer); void *libusb20_tr_get_priv_sc0(struct libusb20_transfer *xfer); void *libusb20_tr_get_priv_sc1(struct libusb20_transfer *xfer); /* USB device operations */ const char *libusb20_dev_get_backend_name(struct libusb20_device *pdev); const char *libusb20_dev_get_desc(struct libusb20_device *pdev); int libusb20_dev_close(struct libusb20_device *pdev); int libusb20_dev_detach_kernel_driver(struct libusb20_device *pdev, uint8_t iface_index); int libusb20_dev_set_config_index(struct libusb20_device *pdev, uint8_t configIndex); int libusb20_dev_get_debug(struct libusb20_device *pdev); int libusb20_dev_get_fd(struct libusb20_device *pdev); int libusb20_dev_kernel_driver_active(struct libusb20_device *pdev, uint8_t iface_index); int libusb20_dev_open(struct libusb20_device *pdev, uint16_t transfer_max); int libusb20_dev_process(struct libusb20_device *pdev); int libusb20_dev_request_sync(struct libusb20_device *pdev, struct LIBUSB20_CONTROL_SETUP_DECODED *setup, void *data, uint16_t *pactlen, uint32_t timeout, uint8_t flags); int libusb20_dev_req_string_sync(struct libusb20_device *pdev, uint8_t index, uint16_t langid, void *ptr, uint16_t len); int libusb20_dev_req_string_simple_sync(struct libusb20_device *pdev, uint8_t index, void *ptr, uint16_t len); int libusb20_dev_reset(struct libusb20_device *pdev); int libusb20_dev_check_connected(struct libusb20_device *pdev); int libusb20_dev_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode); uint8_t libusb20_dev_get_power_mode(struct libusb20_device *pdev); int libusb20_dev_set_alt_index(struct libusb20_device *pdev, uint8_t iface_index, uint8_t alt_index); int libusb20_dev_get_info(struct libusb20_device *pdev, struct usb_device_info *pinfo); int libusb20_dev_get_iface_desc(struct libusb20_device *pdev, uint8_t iface_index, char *buf, uint8_t len); struct LIBUSB20_DEVICE_DESC_DECODED *libusb20_dev_get_device_desc(struct libusb20_device *pdev); struct libusb20_config *libusb20_dev_alloc_config(struct libusb20_device *pdev, uint8_t config_index); struct libusb20_device *libusb20_dev_alloc(void); uint8_t libusb20_dev_get_address(struct libusb20_device *pdev); uint8_t libusb20_dev_get_bus_number(struct libusb20_device *pdev); uint8_t libusb20_dev_get_mode(struct libusb20_device *pdev); uint8_t libusb20_dev_get_speed(struct libusb20_device *pdev); uint8_t libusb20_dev_get_config_index(struct libusb20_device *pdev); void libusb20_dev_free(struct libusb20_device *pdev); void libusb20_dev_set_debug(struct libusb20_device *pdev, int debug); void libusb20_dev_wait_process(struct libusb20_device *pdev, int timeout); /* USB global operations */ int libusb20_be_get_dev_quirk(struct libusb20_backend *pbe, uint16_t index, struct libusb20_quirk *pq); int libusb20_be_get_quirk_name(struct libusb20_backend *pbe, uint16_t index, struct libusb20_quirk *pq); int libusb20_be_add_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq); int libusb20_be_remove_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq); int libusb20_be_get_template(struct libusb20_backend *pbe, int *ptemp); int libusb20_be_set_template(struct libusb20_backend *pbe, int temp); /* USB backend operations */ struct libusb20_backend *libusb20_be_alloc(const struct libusb20_backend_methods *methods); struct libusb20_backend *libusb20_be_alloc_default(void); struct libusb20_backend *libusb20_be_alloc_freebsd(void); struct libusb20_backend *libusb20_be_alloc_linux(void); struct libusb20_backend *libusb20_be_alloc_ugen20(void); struct libusb20_device *libusb20_be_device_foreach(struct libusb20_backend *pbe, struct libusb20_device *pdev); void libusb20_be_dequeue_device(struct libusb20_backend *pbe, struct libusb20_device *pdev); void libusb20_be_enqueue_device(struct libusb20_backend *pbe, struct libusb20_device *pdev); void libusb20_be_free(struct libusb20_backend *pbe); #if 0 { /* style */ #endif #ifdef __cplusplus } #endif #endif /* _LIBUSB20_H_ */ Index: user/imp/tbemd/lib/libusb/libusb20_compat01.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb20_compat01.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb20_compat01.c (revision 203850) @@ -1,927 +1,927 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file contains the emulation layer for LibUSB v0.1 from sourceforge. */ #include -#include -#include #include +#include +#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" #include "usb.h" /* * The two following macros were taken from the original LibUSB v0.1 * for sake of compatibility: */ #define LIST_ADD(begin, ent) \ do { \ if (begin) { \ ent->next = begin; \ ent->next->prev = ent; \ } else { \ ent->next = NULL; \ } \ ent->prev = NULL; \ begin = ent; \ } while(0) #define LIST_DEL(begin, ent) \ do { \ if (ent->prev) { \ ent->prev->next = ent->next; \ } else { \ begin = ent->next; \ } \ if (ent->next) { \ ent->next->prev = ent->prev; \ } \ ent->prev = NULL; \ ent->next = NULL; \ } while (0) struct usb_bus *usb_busses = NULL; static struct usb_bus usb_global_bus = { .dirname = {"/dev/usb"}, .root_dev = NULL, .devices = NULL, }; static struct libusb20_backend *usb_backend = NULL; struct usb_parse_state { struct { struct libusb20_endpoint *currep; struct libusb20_interface *currifc; struct libusb20_config *currcfg; struct libusb20_me_struct *currextra; } a; struct { struct usb_config_descriptor *currcfg; struct usb_interface_descriptor *currifc; struct usb_endpoint_descriptor *currep; struct usb_interface *currifcw; uint8_t *currextra; } b; uint8_t preparse; }; static struct libusb20_transfer * usb_get_transfer_by_ep_no(usb_dev_handle * dev, uint8_t ep_no) { struct libusb20_device *pdev = (void *)dev; struct libusb20_transfer *xfer; int err; uint32_t bufsize; uint8_t x; uint8_t speed; x = (ep_no & LIBUSB20_ENDPOINT_ADDRESS_MASK) * 2; if (ep_no & LIBUSB20_ENDPOINT_DIR_MASK) { /* this is an IN endpoint */ x |= 1; } speed = libusb20_dev_get_speed(pdev); /* select a sensible buffer size */ if (speed == LIBUSB20_SPEED_LOW) { bufsize = 256; } else if (speed == LIBUSB20_SPEED_FULL) { bufsize = 4096; } else { bufsize = 16384; } xfer = libusb20_tr_get_pointer(pdev, x); if (xfer == NULL) return (xfer); err = libusb20_tr_open(xfer, bufsize, 1, ep_no); if (err == LIBUSB20_ERROR_BUSY) { /* already opened */ return (xfer); } else if (err) { return (NULL); } /* success */ return (xfer); } usb_dev_handle * usb_open(struct usb_device *dev) { int err; err = libusb20_dev_open(dev->dev, 16 * 2); if (err == LIBUSB20_ERROR_BUSY) { /* * Workaround buggy USB applications which open the USB * device multiple times: */ return (dev->dev); } if (err) return (NULL); /* * Dequeue USB device from backend queue so that it does not get * freed when the backend is re-scanned: */ libusb20_be_dequeue_device(usb_backend, dev->dev); return (dev->dev); } int usb_close(usb_dev_handle * udev) { struct usb_device *dev; int err; err = libusb20_dev_close((void *)udev); if (err) return (-1); if (usb_backend != NULL) { /* * Enqueue USB device to backend queue so that it gets freed * when the backend is re-scanned: */ libusb20_be_enqueue_device(usb_backend, (void *)udev); } else { /* * The backend is gone. Free device data so that we * don't start leaking memory! */ dev = usb_device(udev); libusb20_dev_free((void *)udev); LIST_DEL(usb_global_bus.devices, dev); free(dev); } return (0); } int usb_get_string(usb_dev_handle * dev, int strindex, int langid, char *buf, size_t buflen) { int err; err = libusb20_dev_req_string_sync((void *)dev, strindex, langid, buf, buflen); if (err) return (-1); return (0); } int usb_get_string_simple(usb_dev_handle * dev, int strindex, char *buf, size_t buflen) { int err; err = libusb20_dev_req_string_simple_sync((void *)dev, strindex, buf, buflen); if (err) return (-1); return (strlen(buf)); } int usb_get_descriptor_by_endpoint(usb_dev_handle * udev, int ep, uint8_t type, uint8_t ep_index, void *buf, int size) { memset(buf, 0, size); return (usb_control_msg(udev, ep | USB_ENDPOINT_IN, USB_REQ_GET_DESCRIPTOR, (type << 8) + ep_index, 0, buf, size, 1000)); } int usb_get_descriptor(usb_dev_handle * udev, uint8_t type, uint8_t desc_index, void *buf, int size) { memset(buf, 0, size); return (usb_control_msg(udev, USB_ENDPOINT_IN, USB_REQ_GET_DESCRIPTOR, (type << 8) + desc_index, 0, buf, size, 1000)); } int usb_parse_descriptor(uint8_t *source, char *description, void *dest) { uint8_t *sp = source; uint8_t *dp = dest; uint16_t w; uint32_t d; char *cp; for (cp = description; *cp; cp++) { switch (*cp) { case 'b': /* 8-bit byte */ *dp++ = *sp++; break; /* * 16-bit word, convert from little endian to CPU */ case 'w': w = (sp[1] << 8) | sp[0]; sp += 2; /* Align to word boundary */ dp += ((dp - (uint8_t *)0) & 1); *((uint16_t *)dp) = w; dp += 2; break; /* * 32-bit dword, convert from little endian to CPU */ case 'd': d = (sp[3] << 24) | (sp[2] << 16) | (sp[1] << 8) | sp[0]; sp += 4; /* Align to word boundary */ dp += ((dp - (uint8_t *)0) & 1); /* Align to double word boundary */ dp += ((dp - (uint8_t *)0) & 2); *((uint32_t *)dp) = d; dp += 4; break; } } return (sp - source); } static void usb_parse_extra(struct usb_parse_state *ps, uint8_t **pptr, int *plen) { void *ptr; uint16_t len; ptr = ps->a.currextra->ptr; len = ps->a.currextra->len; if (ps->preparse == 0) { memcpy(ps->b.currextra, ptr, len); *pptr = ps->b.currextra; *plen = len; } ps->b.currextra += len; return; } static void usb_parse_endpoint(struct usb_parse_state *ps) { struct usb_endpoint_descriptor *bep; struct libusb20_endpoint *aep; aep = ps->a.currep; bep = ps->b.currep++; if (ps->preparse == 0) { /* copy descriptor fields */ bep->bLength = aep->desc.bLength; bep->bDescriptorType = aep->desc.bDescriptorType; bep->bEndpointAddress = aep->desc.bEndpointAddress; bep->bmAttributes = aep->desc.bmAttributes; bep->wMaxPacketSize = aep->desc.wMaxPacketSize; bep->bInterval = aep->desc.bInterval; bep->bRefresh = aep->desc.bRefresh; bep->bSynchAddress = aep->desc.bSynchAddress; } ps->a.currextra = &aep->extra; usb_parse_extra(ps, &bep->extra, &bep->extralen); return; } static void usb_parse_iface_sub(struct usb_parse_state *ps) { struct libusb20_interface *aifc; struct usb_interface_descriptor *bifc; uint8_t x; aifc = ps->a.currifc; bifc = ps->b.currifc++; if (ps->preparse == 0) { /* copy descriptor fields */ bifc->bLength = aifc->desc.bLength; bifc->bDescriptorType = aifc->desc.bDescriptorType; bifc->bInterfaceNumber = aifc->desc.bInterfaceNumber; bifc->bAlternateSetting = aifc->desc.bAlternateSetting; bifc->bNumEndpoints = aifc->num_endpoints; bifc->bInterfaceClass = aifc->desc.bInterfaceClass; bifc->bInterfaceSubClass = aifc->desc.bInterfaceSubClass; bifc->bInterfaceProtocol = aifc->desc.bInterfaceProtocol; bifc->iInterface = aifc->desc.iInterface; bifc->endpoint = ps->b.currep; } for (x = 0; x != aifc->num_endpoints; x++) { ps->a.currep = aifc->endpoints + x; usb_parse_endpoint(ps); } ps->a.currextra = &aifc->extra; usb_parse_extra(ps, &bifc->extra, &bifc->extralen); return; } static void usb_parse_iface(struct usb_parse_state *ps) { struct libusb20_interface *aifc; struct usb_interface *bifc; uint8_t x; aifc = ps->a.currifc; bifc = ps->b.currifcw++; if (ps->preparse == 0) { /* initialise interface wrapper */ bifc->altsetting = ps->b.currifc; bifc->num_altsetting = aifc->num_altsetting + 1; } usb_parse_iface_sub(ps); for (x = 0; x != aifc->num_altsetting; x++) { ps->a.currifc = aifc->altsetting + x; usb_parse_iface_sub(ps); } return; } static void usb_parse_config(struct usb_parse_state *ps) { struct libusb20_config *acfg; struct usb_config_descriptor *bcfg; uint8_t x; acfg = ps->a.currcfg; bcfg = ps->b.currcfg; if (ps->preparse == 0) { /* initialise config wrapper */ bcfg->bLength = acfg->desc.bLength; bcfg->bDescriptorType = acfg->desc.bDescriptorType; bcfg->wTotalLength = acfg->desc.wTotalLength; bcfg->bNumInterfaces = acfg->num_interface; bcfg->bConfigurationValue = acfg->desc.bConfigurationValue; bcfg->iConfiguration = acfg->desc.iConfiguration; bcfg->bmAttributes = acfg->desc.bmAttributes; bcfg->MaxPower = acfg->desc.bMaxPower; bcfg->interface = ps->b.currifcw; } for (x = 0; x != acfg->num_interface; x++) { ps->a.currifc = acfg->interface + x; usb_parse_iface(ps); } ps->a.currextra = &acfg->extra; usb_parse_extra(ps, &bcfg->extra, &bcfg->extralen); return; } int usb_parse_configuration(struct usb_config_descriptor *config, uint8_t *buffer) { struct usb_parse_state ps; uint8_t *ptr; uint32_t a; uint32_t b; uint32_t c; uint32_t d; if ((buffer == NULL) || (config == NULL)) { return (-1); } memset(&ps, 0, sizeof(ps)); ps.a.currcfg = libusb20_parse_config_desc(buffer); ps.b.currcfg = config; if (ps.a.currcfg == NULL) { /* could not parse config or out of memory */ return (-1); } /* do the pre-parse */ ps.preparse = 1; usb_parse_config(&ps); a = ((uint8_t *)(ps.b.currifcw) - ((uint8_t *)0)); b = ((uint8_t *)(ps.b.currifc) - ((uint8_t *)0)); c = ((uint8_t *)(ps.b.currep) - ((uint8_t *)0)); d = ((uint8_t *)(ps.b.currextra) - ((uint8_t *)0)); /* allocate memory for our configuration */ ptr = malloc(a + b + c + d); /* "currifcw" must be first, hence this pointer is freed */ ps.b.currifcw = (void *)(ptr); ps.b.currifc = (void *)(ptr + a); ps.b.currep = (void *)(ptr + a + b); ps.b.currextra = (void *)(ptr + a + b + c); /* generate a libusb v0.1 compatible structure */ ps.preparse = 0; usb_parse_config(&ps); /* free config structure */ free(ps.a.currcfg); return (0); /* success */ } void usb_destroy_configuration(struct usb_device *dev) { uint8_t c; if (dev->config == NULL) { return; } for (c = 0; c != dev->descriptor.bNumConfigurations; c++) { struct usb_config_descriptor *cf = &dev->config[c]; if (cf->interface != NULL) { free(cf->interface); cf->interface = NULL; } } free(dev->config); dev->config = NULL; return; } void usb_fetch_and_parse_descriptors(usb_dev_handle * udev) { struct usb_device *dev; struct libusb20_device *pdev; uint8_t *ptr; int error; uint32_t size; uint16_t len; uint8_t x; if (udev == NULL) { /* be NULL safe */ return; } dev = usb_device(udev); pdev = (void *)udev; if (dev->descriptor.bNumConfigurations == 0) { /* invalid device */ return; } size = dev->descriptor.bNumConfigurations * sizeof(struct usb_config_descriptor); dev->config = malloc(size); if (dev->config == NULL) { /* out of memory */ return; } memset(dev->config, 0, size); for (x = 0; x != dev->descriptor.bNumConfigurations; x++) { error = (pdev->methods->get_config_desc_full) ( pdev, &ptr, &len, x); if (error) { usb_destroy_configuration(dev); return; } usb_parse_configuration(dev->config + x, ptr); /* free config buffer */ free(ptr); } return; } static int usb_std_io(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout, int is_intr) { struct libusb20_transfer *xfer; uint32_t temp; uint32_t maxsize; uint32_t actlen; char *oldbytes; xfer = usb_get_transfer_by_ep_no(dev, ep); if (xfer == NULL) return (-1); if (libusb20_tr_pending(xfer)) { /* there is already a transfer ongoing */ return (-1); } maxsize = libusb20_tr_get_max_total_length(xfer); oldbytes = bytes; /* * We allow transferring zero bytes which is the same * equivalent to a zero length USB packet. */ do { temp = size; if (temp > maxsize) { /* find maximum possible length */ temp = maxsize; } if (is_intr) libusb20_tr_setup_intr(xfer, bytes, temp, timeout); else libusb20_tr_setup_bulk(xfer, bytes, temp, timeout); libusb20_tr_start(xfer); while (1) { if (libusb20_dev_process((void *)dev) != 0) { /* device detached */ return (-1); } if (libusb20_tr_pending(xfer) == 0) { /* transfer complete */ break; } /* wait for USB event from kernel */ libusb20_dev_wait_process((void *)dev, -1); } switch (libusb20_tr_get_status(xfer)) { case 0: /* success */ break; case LIBUSB20_TRANSFER_TIMED_OUT: /* transfer timeout */ return (-ETIMEDOUT); default: /* other transfer error */ return (-ENXIO); } actlen = libusb20_tr_get_actual_length(xfer); bytes += actlen; size -= actlen; if (actlen != temp) { /* short transfer */ break; } } while (size > 0); return (bytes - oldbytes); } int usb_bulk_write(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout) { return (usb_std_io(dev, ep & ~USB_ENDPOINT_DIR_MASK, bytes, size, timeout, 0)); } int usb_bulk_read(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout) { return (usb_std_io(dev, ep | USB_ENDPOINT_DIR_MASK, bytes, size, timeout, 0)); } int usb_interrupt_write(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout) { return (usb_std_io(dev, ep & ~USB_ENDPOINT_DIR_MASK, bytes, size, timeout, 1)); } int usb_interrupt_read(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout) { return (usb_std_io(dev, ep | USB_ENDPOINT_DIR_MASK, bytes, size, timeout, 1)); } int usb_control_msg(usb_dev_handle * dev, int requesttype, int request, int value, int wIndex, char *bytes, int size, int timeout) { struct LIBUSB20_CONTROL_SETUP_DECODED req; int err; uint16_t actlen; LIBUSB20_INIT(LIBUSB20_CONTROL_SETUP, &req); req.bmRequestType = requesttype; req.bRequest = request; req.wValue = value; req.wIndex = wIndex; req.wLength = size; err = libusb20_dev_request_sync((void *)dev, &req, bytes, &actlen, timeout, 0); if (err) return (-1); return (actlen); } int usb_set_configuration(usb_dev_handle * udev, int bConfigurationValue) { struct usb_device *dev; int err; uint8_t i; /* * Need to translate from "bConfigurationValue" to * configuration index: */ if (bConfigurationValue == 0) { /* unconfigure */ i = 255; } else { /* lookup configuration index */ dev = usb_device(udev); /* check if the configuration array is not there */ if (dev->config == NULL) { return (-1); } for (i = 0;; i++) { if (i == dev->descriptor.bNumConfigurations) { /* "bConfigurationValue" not found */ return (-1); } if ((dev->config + i)->bConfigurationValue == bConfigurationValue) { break; } } } err = libusb20_dev_set_config_index((void *)udev, i); if (err) return (-1); return (0); } int usb_claim_interface(usb_dev_handle * dev, int interface) { struct libusb20_device *pdev = (void *)dev; pdev->claimed_interface = interface; return (0); } int usb_release_interface(usb_dev_handle * dev, int interface) { /* do nothing */ return (0); } int usb_set_altinterface(usb_dev_handle * dev, int alternate) { struct libusb20_device *pdev = (void *)dev; int err; uint8_t iface; iface = pdev->claimed_interface; err = libusb20_dev_set_alt_index((void *)dev, iface, alternate); if (err) return (-1); return (0); } int usb_resetep(usb_dev_handle * dev, unsigned int ep) { /* emulate an endpoint reset through clear-STALL */ return (usb_clear_halt(dev, ep)); } int usb_clear_halt(usb_dev_handle * dev, unsigned int ep) { struct libusb20_transfer *xfer; xfer = usb_get_transfer_by_ep_no(dev, ep); if (xfer == NULL) return (-1); libusb20_tr_clear_stall_sync(xfer); return (0); } int usb_reset(usb_dev_handle * dev) { int err; err = libusb20_dev_reset((void *)dev); if (err) return (-1); /* * Be compatible with LibUSB from sourceforge and close the * handle after reset! */ return (usb_close(dev)); } const char * usb_strerror(void) { /* TODO */ return ("Unknown error"); } void usb_init(void) { /* nothing to do */ return; } void usb_set_debug(int level) { /* use kernel UGEN debugging if you need to see what is going on */ return; } int usb_find_busses(void) { usb_busses = &usb_global_bus; return (0); } int usb_find_devices(void) { struct libusb20_device *pdev; struct usb_device *udev; struct LIBUSB20_DEVICE_DESC_DECODED *ddesc; int devnum; int err; /* cleanup after last device search */ /* close all opened devices, if any */ while ((pdev = libusb20_be_device_foreach(usb_backend, NULL))) { udev = pdev->privLuData; libusb20_be_dequeue_device(usb_backend, pdev); libusb20_dev_free(pdev); if (udev != NULL) { LIST_DEL(usb_global_bus.devices, udev); free(udev); } } /* free old USB backend, if any */ libusb20_be_free(usb_backend); /* do a new backend device search */ usb_backend = libusb20_be_alloc_default(); if (usb_backend == NULL) { return (-1); } /* iterate all devices */ devnum = 1; pdev = NULL; while ((pdev = libusb20_be_device_foreach(usb_backend, pdev))) { udev = malloc(sizeof(*udev)); if (udev == NULL) break; memset(udev, 0, sizeof(*udev)); udev->bus = &usb_global_bus; snprintf(udev->filename, sizeof(udev->filename), "/dev/ugen%u.%u", libusb20_dev_get_bus_number(pdev), libusb20_dev_get_address(pdev)); ddesc = libusb20_dev_get_device_desc(pdev); udev->descriptor.bLength = sizeof(udev->descriptor); udev->descriptor.bDescriptorType = ddesc->bDescriptorType; udev->descriptor.bcdUSB = ddesc->bcdUSB; udev->descriptor.bDeviceClass = ddesc->bDeviceClass; udev->descriptor.bDeviceSubClass = ddesc->bDeviceSubClass; udev->descriptor.bDeviceProtocol = ddesc->bDeviceProtocol; udev->descriptor.bMaxPacketSize0 = ddesc->bMaxPacketSize0; udev->descriptor.idVendor = ddesc->idVendor; udev->descriptor.idProduct = ddesc->idProduct; udev->descriptor.bcdDevice = ddesc->bcdDevice; udev->descriptor.iManufacturer = ddesc->iManufacturer; udev->descriptor.iProduct = ddesc->iProduct; udev->descriptor.iSerialNumber = ddesc->iSerialNumber; udev->descriptor.bNumConfigurations = ddesc->bNumConfigurations; if (udev->descriptor.bNumConfigurations > USB_MAXCONFIG) { /* truncate number of configurations */ udev->descriptor.bNumConfigurations = USB_MAXCONFIG; } udev->devnum = devnum++; /* link together the two structures */ udev->dev = pdev; pdev->privLuData = udev; err = libusb20_dev_open(pdev, 0); if (err == 0) { /* XXX get all config descriptors by default */ usb_fetch_and_parse_descriptors((void *)pdev); libusb20_dev_close(pdev); } LIST_ADD(usb_global_bus.devices, udev); } return (0); /* success */ } struct usb_device * usb_device(usb_dev_handle * dev) { struct libusb20_device *pdev; pdev = (void *)dev; return (pdev->privLuData); } struct usb_bus * usb_get_busses(void) { return (usb_busses); } Index: user/imp/tbemd/lib/libusb/libusb20_desc.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb20_desc.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb20_desc.c (revision 203850) @@ -1,787 +1,788 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ +#include + #include #include #include -#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" static const uint32_t libusb20_me_encode_empty[2]; /* dummy */ LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_DEVICE_DESC); LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_ENDPOINT_DESC); LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_INTERFACE_DESC); LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_CONFIG_DESC); LIBUSB20_MAKE_STRUCT_FORMAT(LIBUSB20_CONTROL_SETUP); /*------------------------------------------------------------------------* * libusb20_parse_config_desc * * Return values: * NULL: Out of memory. * Else: A valid config structure pointer which must be passed to "free()" *------------------------------------------------------------------------*/ struct libusb20_config * libusb20_parse_config_desc(const void *config_desc) { struct libusb20_config *lub_config; struct libusb20_interface *lub_interface; struct libusb20_interface *lub_alt_interface; struct libusb20_interface *last_if; struct libusb20_endpoint *lub_endpoint; struct libusb20_endpoint *last_ep; struct libusb20_me_struct pcdesc; const uint8_t *ptr; uint32_t size; uint16_t niface_no_alt; uint16_t niface; uint16_t nendpoint; uint8_t iface_no; ptr = config_desc; if (ptr[1] != LIBUSB20_DT_CONFIG) { return (NULL); /* not config descriptor */ } /* * The first "bInterfaceNumber" should never have the value 0xff. * Then it is corrupt. */ niface_no_alt = 0; nendpoint = 0; niface = 0; iface_no = 0 - 1; ptr = NULL; /* get "wTotalLength" and setup "pcdesc" */ pcdesc.ptr = LIBUSB20_ADD_BYTES(config_desc, 0); pcdesc.len = ((const uint8_t *)config_desc)[2] | (((const uint8_t *)config_desc)[3] << 8); pcdesc.type = LIBUSB20_ME_IS_RAW; /* descriptor pre-scan */ while ((ptr = libusb20_desc_foreach(&pcdesc, ptr))) { if (ptr[1] == LIBUSB20_DT_ENDPOINT) { nendpoint++; } else if ((ptr[1] == LIBUSB20_DT_INTERFACE) && (ptr[0] >= 4)) { niface++; /* check "bInterfaceNumber" */ if (ptr[2] != iface_no) { iface_no = ptr[2]; niface_no_alt++; } } } /* sanity checking */ if (niface >= 256) { return (NULL); /* corrupt */ } if (nendpoint >= 256) { return (NULL); /* corrupt */ } size = sizeof(*lub_config) + (niface * sizeof(*lub_interface)) + (nendpoint * sizeof(*lub_endpoint)) + pcdesc.len; lub_config = malloc(size); if (lub_config == NULL) { return (NULL); /* out of memory */ } /* make sure memory is initialised */ memset(lub_config, 0, size); lub_interface = (void *)(lub_config + 1); lub_alt_interface = (void *)(lub_interface + niface_no_alt); lub_endpoint = (void *)(lub_interface + niface); /* * Make a copy of the config descriptor, so that the caller can free * the inital config descriptor pointer! */ ptr = (void *)(lub_endpoint + nendpoint); memcpy(LIBUSB20_ADD_BYTES(ptr, 0), config_desc, pcdesc.len); pcdesc.ptr = LIBUSB20_ADD_BYTES(ptr, 0); config_desc = LIBUSB20_ADD_BYTES(ptr, 0); /* init config structure */ ptr = config_desc; LIBUSB20_INIT(LIBUSB20_CONFIG_DESC, &lub_config->desc); if (libusb20_me_decode(ptr, ptr[0], &lub_config->desc)) { /* ignore */ } lub_config->num_interface = 0; lub_config->interface = lub_interface; lub_config->extra.ptr = LIBUSB20_ADD_BYTES(ptr, ptr[0]); lub_config->extra.len = -ptr[0]; lub_config->extra.type = LIBUSB20_ME_IS_RAW; /* reset states */ niface = 0; iface_no = 0 - 1; ptr = NULL; lub_interface--; lub_endpoint--; last_if = NULL; last_ep = NULL; /* descriptor pre-scan */ while ((ptr = libusb20_desc_foreach(&pcdesc, ptr))) { if (ptr[1] == LIBUSB20_DT_ENDPOINT) { if (last_if) { lub_endpoint++; last_ep = lub_endpoint; last_if->num_endpoints++; LIBUSB20_INIT(LIBUSB20_ENDPOINT_DESC, &last_ep->desc); if (libusb20_me_decode(ptr, ptr[0], &last_ep->desc)) { /* ignore */ } last_ep->extra.ptr = LIBUSB20_ADD_BYTES(ptr, ptr[0]); last_ep->extra.len = 0; last_ep->extra.type = LIBUSB20_ME_IS_RAW; } else { lub_config->extra.len += ptr[0]; } } else if ((ptr[1] == LIBUSB20_DT_INTERFACE) && (ptr[0] >= 4)) { if (ptr[2] != iface_no) { /* new interface */ iface_no = ptr[2]; lub_interface++; lub_config->num_interface++; last_if = lub_interface; niface++; } else { /* one more alternate setting */ lub_interface->num_altsetting++; last_if = lub_alt_interface; lub_alt_interface++; } LIBUSB20_INIT(LIBUSB20_INTERFACE_DESC, &last_if->desc); if (libusb20_me_decode(ptr, ptr[0], &last_if->desc)) { /* ignore */ } /* * Sometimes USB devices have corrupt interface * descriptors and we need to overwrite the provided * interface number! */ last_if->desc.bInterfaceNumber = niface - 1; last_if->extra.ptr = LIBUSB20_ADD_BYTES(ptr, ptr[0]); last_if->extra.len = 0; last_if->extra.type = LIBUSB20_ME_IS_RAW; last_if->endpoints = lub_endpoint + 1; last_if->altsetting = lub_alt_interface; last_if->num_altsetting = 0; last_if->num_endpoints = 0; last_ep = NULL; } else { /* unknown descriptor */ if (last_if) { if (last_ep) { last_ep->extra.len += ptr[0]; } else { last_if->extra.len += ptr[0]; } } else { lub_config->extra.len += ptr[0]; } } } return (lub_config); } /*------------------------------------------------------------------------* * libusb20_desc_foreach * * Safe traversal of USB descriptors. * * Return values: * NULL: End of descriptors * Else: Pointer to next descriptor *------------------------------------------------------------------------*/ const uint8_t * libusb20_desc_foreach(const struct libusb20_me_struct *pdesc, const uint8_t *psubdesc) { const uint8_t *start; const uint8_t *end; const uint8_t *desc_next; /* be NULL safe */ if (pdesc == NULL) return (NULL); start = (const uint8_t *)pdesc->ptr; end = LIBUSB20_ADD_BYTES(start, pdesc->len); /* get start of next descriptor */ if (psubdesc == NULL) psubdesc = start; else psubdesc = psubdesc + psubdesc[0]; /* check that the next USB descriptor is within the range */ if ((psubdesc < start) || (psubdesc >= end)) return (NULL); /* out of range, or EOD */ /* check start of the second next USB descriptor, if any */ desc_next = psubdesc + psubdesc[0]; if ((desc_next < start) || (desc_next > end)) return (NULL); /* out of range */ /* check minimum descriptor length */ if (psubdesc[0] < 3) return (NULL); /* too short descriptor */ return (psubdesc); /* return start of next descriptor */ } /*------------------------------------------------------------------------* * libusb20_me_get_1 - safety wrapper to read out one byte *------------------------------------------------------------------------*/ uint8_t libusb20_me_get_1(const struct libusb20_me_struct *ie, uint16_t offset) { if (offset < ie->len) { return (*((uint8_t *)LIBUSB20_ADD_BYTES(ie->ptr, offset))); } return (0); } /*------------------------------------------------------------------------* * libusb20_me_get_2 - safety wrapper to read out one word *------------------------------------------------------------------------*/ uint16_t libusb20_me_get_2(const struct libusb20_me_struct *ie, uint16_t offset) { return (libusb20_me_get_1(ie, offset) | (libusb20_me_get_1(ie, offset + 1) << 8)); } /*------------------------------------------------------------------------* * libusb20_me_encode - encode a message structure * * Description of parameters: * "len" - maximum length of output buffer * "ptr" - pointer to output buffer. If NULL, no data will be written * "pd" - source structure * * Return values: * 0..65535 - Number of bytes used, limited by the "len" input parameter. *------------------------------------------------------------------------*/ uint16_t libusb20_me_encode(void *ptr, uint16_t len, const void *pd) { const uint8_t *pf; /* pointer to format data */ uint8_t *buf; /* pointer to output buffer */ uint32_t pd_offset; /* decoded structure offset */ uint16_t len_old; /* old length */ uint16_t pd_count; /* decoded element count */ uint8_t me; /* message element */ /* initialise */ len_old = len; buf = ptr; pd_offset = sizeof(void *); pf = (*((struct libusb20_me_format *const *)pd))->format; /* scan */ while (1) { /* get information element */ me = (pf[0]) & LIBUSB20_ME_MASK; pd_count = pf[1] | (pf[2] << 8); pf += 3; /* encode the message element */ switch (me) { case LIBUSB20_ME_INT8: while (pd_count--) { uint8_t temp; if (len < 1) /* overflow */ goto done; if (buf) { temp = *((const uint8_t *) LIBUSB20_ADD_BYTES(pd, pd_offset)); buf[0] = temp; buf += 1; } pd_offset += 1; len -= 1; } break; case LIBUSB20_ME_INT16: pd_offset = -((-pd_offset) & ~1); /* align */ while (pd_count--) { uint16_t temp; if (len < 2) /* overflow */ goto done; if (buf) { temp = *((const uint16_t *) LIBUSB20_ADD_BYTES(pd, pd_offset)); buf[1] = (temp >> 8) & 0xFF; buf[0] = temp & 0xFF; buf += 2; } pd_offset += 2; len -= 2; } break; case LIBUSB20_ME_INT32: pd_offset = -((-pd_offset) & ~3); /* align */ while (pd_count--) { uint32_t temp; if (len < 4) /* overflow */ goto done; if (buf) { temp = *((const uint32_t *) LIBUSB20_ADD_BYTES(pd, pd_offset)); buf[3] = (temp >> 24) & 0xFF; buf[2] = (temp >> 16) & 0xFF; buf[1] = (temp >> 8) & 0xFF; buf[0] = temp & 0xFF; buf += 4; } pd_offset += 4; len -= 4; } break; case LIBUSB20_ME_INT64: pd_offset = -((-pd_offset) & ~7); /* align */ while (pd_count--) { uint64_t temp; if (len < 8) /* overflow */ goto done; if (buf) { temp = *((const uint64_t *) LIBUSB20_ADD_BYTES(pd, pd_offset)); buf[7] = (temp >> 56) & 0xFF; buf[6] = (temp >> 48) & 0xFF; buf[5] = (temp >> 40) & 0xFF; buf[4] = (temp >> 32) & 0xFF; buf[3] = (temp >> 24) & 0xFF; buf[2] = (temp >> 16) & 0xFF; buf[1] = (temp >> 8) & 0xFF; buf[0] = temp & 0xFF; buf += 8; } pd_offset += 8; len -= 8; } break; case LIBUSB20_ME_STRUCT: pd_offset = -((-pd_offset) & ~(LIBUSB20_ME_STRUCT_ALIGN - 1)); /* align */ while (pd_count--) { void *src_ptr; uint16_t src_len; struct libusb20_me_struct *ps; ps = LIBUSB20_ADD_BYTES(pd, pd_offset); switch (ps->type) { case LIBUSB20_ME_IS_RAW: src_len = ps->len; src_ptr = ps->ptr; break; case LIBUSB20_ME_IS_ENCODED: if (ps->len == 0) { /* * Length is encoded * in the data itself * and should be * correct: */ ps->len = 0 - 1; } src_len = libusb20_me_get_1(pd, 0); src_ptr = LIBUSB20_ADD_BYTES(ps->ptr, 1); if (src_len == 0xFF) { /* length is escaped */ src_len = libusb20_me_get_2(pd, 1); src_ptr = LIBUSB20_ADD_BYTES(ps->ptr, 3); } break; case LIBUSB20_ME_IS_DECODED: /* reserve 3 length bytes */ src_len = libusb20_me_encode(NULL, 0 - 1 - 3, ps->ptr); src_ptr = NULL; break; default: /* empty structure */ src_len = 0; src_ptr = NULL; break; } if (src_len > 0xFE) { if (src_len > (uint16_t)(0 - 1 - 3)) /* overflow */ goto done; if (len < (src_len + 3)) /* overflow */ goto done; if (buf) { buf[0] = 0xFF; buf[1] = (src_len & 0xFF); buf[2] = (src_len >> 8) & 0xFF; buf += 3; } len -= (src_len + 3); } else { if (len < (src_len + 1)) /* overflow */ goto done; if (buf) { buf[0] = (src_len & 0xFF); buf += 1; } len -= (src_len + 1); } /* check for buffer and non-zero length */ if (buf && src_len) { if (ps->type == LIBUSB20_ME_IS_DECODED) { /* * Repeat encode * procedure - we have * room for the * complete structure: */ uint16_t dummy; dummy = libusb20_me_encode(buf, 0 - 1 - 3, ps->ptr); } else { bcopy(src_ptr, buf, src_len); } buf += src_len; } pd_offset += sizeof(struct libusb20_me_struct); } break; default: goto done; } } done: return (len_old - len); } /*------------------------------------------------------------------------* * libusb20_me_decode - decode a message into a decoded structure * * Description of parameters: * "ptr" - message pointer * "len" - message length * "pd" - pointer to decoded structure * * Returns: * "0..65535" - number of bytes decoded, limited by "len" *------------------------------------------------------------------------*/ uint16_t libusb20_me_decode(const void *ptr, uint16_t len, void *pd) { const uint8_t *pf; /* pointer to format data */ const uint8_t *buf; /* pointer to input buffer */ uint32_t pd_offset; /* decoded structure offset */ uint16_t len_old; /* old length */ uint16_t pd_count; /* decoded element count */ uint8_t me; /* message element */ /* initialise */ len_old = len; buf = ptr; pd_offset = sizeof(void *); pf = (*((struct libusb20_me_format **)pd))->format; /* scan */ while (1) { /* get information element */ me = (pf[0]) & LIBUSB20_ME_MASK; pd_count = pf[1] | (pf[2] << 8); pf += 3; /* decode the message element by type */ switch (me) { case LIBUSB20_ME_INT8: while (pd_count--) { uint8_t temp; if (len < 1) { len = 0; temp = 0; } else { len -= 1; temp = buf[0]; buf++; } *((uint8_t *)LIBUSB20_ADD_BYTES(pd, pd_offset)) = temp; pd_offset += 1; } break; case LIBUSB20_ME_INT16: pd_offset = -((-pd_offset) & ~1); /* align */ while (pd_count--) { uint16_t temp; if (len < 2) { len = 0; temp = 0; } else { len -= 2; temp = buf[1] << 8; temp |= buf[0]; buf += 2; } *((uint16_t *)LIBUSB20_ADD_BYTES(pd, pd_offset)) = temp; pd_offset += 2; } break; case LIBUSB20_ME_INT32: pd_offset = -((-pd_offset) & ~3); /* align */ while (pd_count--) { uint32_t temp; if (len < 4) { len = 0; temp = 0; } else { len -= 4; temp = buf[3] << 24; temp |= buf[2] << 16; temp |= buf[1] << 8; temp |= buf[0]; buf += 4; } *((uint32_t *)LIBUSB20_ADD_BYTES(pd, pd_offset)) = temp; pd_offset += 4; } break; case LIBUSB20_ME_INT64: pd_offset = -((-pd_offset) & ~7); /* align */ while (pd_count--) { uint64_t temp; if (len < 8) { len = 0; temp = 0; } else { len -= 8; temp = ((uint64_t)buf[7]) << 56; temp |= ((uint64_t)buf[6]) << 48; temp |= ((uint64_t)buf[5]) << 40; temp |= ((uint64_t)buf[4]) << 32; temp |= buf[3] << 24; temp |= buf[2] << 16; temp |= buf[1] << 8; temp |= buf[0]; buf += 8; } *((uint64_t *)LIBUSB20_ADD_BYTES(pd, pd_offset)) = temp; pd_offset += 8; } break; case LIBUSB20_ME_STRUCT: pd_offset = -((-pd_offset) & ~(LIBUSB20_ME_STRUCT_ALIGN - 1)); /* align */ while (pd_count--) { uint16_t temp; uint16_t dummy; struct libusb20_me_struct *ps; ps = LIBUSB20_ADD_BYTES(pd, pd_offset); if (ps->type == LIBUSB20_ME_IS_ENCODED) { /* * Pre-store a de-constified * pointer to the raw * structure: */ ps->ptr = LIBUSB20_ADD_BYTES(buf, 0); /* * Get the correct number of * length bytes: */ if (len != 0) { if (buf[0] == 0xFF) { ps->len = 3; } else { ps->len = 1; } } else { ps->len = 0; } } /* get the structure length */ if (len != 0) { if (buf[0] == 0xFF) { if (len < 3) { len = 0; temp = 0; } else { len -= 3; temp = buf[1] | (buf[2] << 8); buf += 3; } } else { len -= 1; temp = buf[0]; buf += 1; } } else { len = 0; temp = 0; } /* check for invalid length */ if (temp > len) { len = 0; temp = 0; } /* check wanted structure type */ switch (ps->type) { case LIBUSB20_ME_IS_ENCODED: /* check for zero length */ if (temp == 0) { /* * The pointer must * be valid: */ ps->ptr = LIBUSB20_ADD_BYTES( libusb20_me_encode_empty, 0); ps->len = 1; } else { ps->len += temp; } break; case LIBUSB20_ME_IS_RAW: /* update length and pointer */ ps->len = temp; ps->ptr = LIBUSB20_ADD_BYTES(buf, 0); break; case LIBUSB20_ME_IS_EMPTY: case LIBUSB20_ME_IS_DECODED: /* check for non-zero length */ if (temp != 0) { /* update type */ ps->type = LIBUSB20_ME_IS_DECODED; ps->len = 0; /* * Recursivly decode * the next structure */ dummy = libusb20_me_decode(buf, temp, ps->ptr); } else { /* update type */ ps->type = LIBUSB20_ME_IS_EMPTY; ps->len = 0; } break; default: /* * nothing to do - should * not happen */ ps->ptr = NULL; ps->len = 0; break; } buf += temp; len -= temp; pd_offset += sizeof(struct libusb20_me_struct); } break; default: goto done; } } done: return (len_old - len); } Index: user/imp/tbemd/lib/libusb/libusb20_ugen20.c =================================================================== --- user/imp/tbemd/lib/libusb/libusb20_ugen20.c (revision 203849) +++ user/imp/tbemd/lib/libusb/libusb20_ugen20.c (revision 203850) @@ -1,1014 +1,1014 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include +#include +#include #include #include -#include #include -#include -#include +#include #include "libusb20.h" #include "libusb20_desc.h" #include "libusb20_int.h" #include #include #include static libusb20_init_backend_t ugen20_init_backend; static libusb20_open_device_t ugen20_open_device; static libusb20_close_device_t ugen20_close_device; static libusb20_get_backend_name_t ugen20_get_backend_name; static libusb20_exit_backend_t ugen20_exit_backend; static libusb20_dev_get_iface_desc_t ugen20_dev_get_iface_desc; static libusb20_dev_get_info_t ugen20_dev_get_info; static libusb20_root_get_dev_quirk_t ugen20_root_get_dev_quirk; static libusb20_root_get_quirk_name_t ugen20_root_get_quirk_name; static libusb20_root_add_dev_quirk_t ugen20_root_add_dev_quirk; static libusb20_root_remove_dev_quirk_t ugen20_root_remove_dev_quirk; static libusb20_root_set_template_t ugen20_root_set_template; static libusb20_root_get_template_t ugen20_root_get_template; const struct libusb20_backend_methods libusb20_ugen20_backend = { LIBUSB20_BACKEND(LIBUSB20_DECLARE, ugen20) }; /* USB device specific */ static libusb20_get_config_desc_full_t ugen20_get_config_desc_full; static libusb20_get_config_index_t ugen20_get_config_index; static libusb20_set_config_index_t ugen20_set_config_index; static libusb20_set_alt_index_t ugen20_set_alt_index; static libusb20_reset_device_t ugen20_reset_device; static libusb20_check_connected_t ugen20_check_connected; static libusb20_set_power_mode_t ugen20_set_power_mode; static libusb20_get_power_mode_t ugen20_get_power_mode; static libusb20_kernel_driver_active_t ugen20_kernel_driver_active; static libusb20_detach_kernel_driver_t ugen20_detach_kernel_driver; static libusb20_do_request_sync_t ugen20_do_request_sync; static libusb20_process_t ugen20_process; /* USB transfer specific */ static libusb20_tr_open_t ugen20_tr_open; static libusb20_tr_close_t ugen20_tr_close; static libusb20_tr_clear_stall_sync_t ugen20_tr_clear_stall_sync; static libusb20_tr_submit_t ugen20_tr_submit; static libusb20_tr_cancel_async_t ugen20_tr_cancel_async; static const struct libusb20_device_methods libusb20_ugen20_device_methods = { LIBUSB20_DEVICE(LIBUSB20_DECLARE, ugen20) }; static const char * ugen20_get_backend_name(void) { return ("FreeBSD UGEN 2.0"); } static uint32_t ugen20_path_convert_one(const char **pp) { const char *ptr; uint32_t temp = 0; ptr = *pp; while ((*ptr >= '0') && (*ptr <= '9')) { temp *= 10; temp += (*ptr - '0'); if (temp >= 1000000) { /* catch overflow early */ return (0 - 1); } ptr++; } if (*ptr == '.') { /* skip dot */ ptr++; } *pp = ptr; return (temp); } static int ugen20_enumerate(struct libusb20_device *pdev, const char *id) { const char *tmp = id; struct usb_device_descriptor ddesc; struct usb_device_info devinfo; uint32_t plugtime; char buf[64]; int f; int error; pdev->bus_number = ugen20_path_convert_one(&tmp); pdev->device_address = ugen20_path_convert_one(&tmp); snprintf(buf, sizeof(buf), "/dev/" USB_GENERIC_NAME "%u.%u", pdev->bus_number, pdev->device_address); f = open(buf, O_RDWR); if (f < 0) { return (LIBUSB20_ERROR_OTHER); } if (ioctl(f, USB_GET_PLUGTIME, &plugtime)) { error = LIBUSB20_ERROR_OTHER; goto done; } /* store when the device was plugged */ pdev->session_data.plugtime = plugtime; if (ioctl(f, USB_GET_DEVICE_DESC, &ddesc)) { error = LIBUSB20_ERROR_OTHER; goto done; } LIBUSB20_INIT(LIBUSB20_DEVICE_DESC, &(pdev->ddesc)); libusb20_me_decode(&ddesc, sizeof(ddesc), &(pdev->ddesc)); if (pdev->ddesc.bNumConfigurations == 0) { error = LIBUSB20_ERROR_OTHER; goto done; } else if (pdev->ddesc.bNumConfigurations >= 8) { error = LIBUSB20_ERROR_OTHER; goto done; } if (ioctl(f, USB_GET_DEVICEINFO, &devinfo)) { error = LIBUSB20_ERROR_OTHER; goto done; } switch (devinfo.udi_mode) { case USB_MODE_DEVICE: pdev->usb_mode = LIBUSB20_MODE_DEVICE; break; default: pdev->usb_mode = LIBUSB20_MODE_HOST; break; } switch (devinfo.udi_speed) { case USB_SPEED_LOW: pdev->usb_speed = LIBUSB20_SPEED_LOW; break; case USB_SPEED_FULL: pdev->usb_speed = LIBUSB20_SPEED_FULL; break; case USB_SPEED_HIGH: pdev->usb_speed = LIBUSB20_SPEED_HIGH; break; case USB_SPEED_VARIABLE: pdev->usb_speed = LIBUSB20_SPEED_VARIABLE; break; case USB_SPEED_SUPER: pdev->usb_speed = LIBUSB20_SPEED_SUPER; break; default: pdev->usb_speed = LIBUSB20_SPEED_UNKNOWN; break; } /* generate a nice description for printout */ snprintf(pdev->usb_desc, sizeof(pdev->usb_desc), USB_GENERIC_NAME "%u.%u: <%s %s> at usbus%u", pdev->bus_number, pdev->device_address, devinfo.udi_product, devinfo.udi_vendor, pdev->bus_number); error = 0; done: close(f); return (error); } struct ugen20_urd_state { struct usb_read_dir urd; uint32_t nparsed; int f; uint8_t *ptr; const char *src; const char *dst; uint8_t buf[256]; uint8_t dummy_zero[1]; }; static int ugen20_readdir(struct ugen20_urd_state *st) { ; /* style fix */ repeat: if (st->ptr == NULL) { st->urd.urd_startentry += st->nparsed; st->urd.urd_data = st->buf; st->urd.urd_maxlen = sizeof(st->buf); st->nparsed = 0; if (ioctl(st->f, USB_READ_DIR, &st->urd)) { return (EINVAL); } st->ptr = st->buf; } if (st->ptr[0] == 0) { if (st->nparsed) { st->ptr = NULL; goto repeat; } else { return (ENXIO); } } st->src = (void *)(st->ptr + 1); st->dst = st->src + strlen(st->src) + 1; st->ptr = st->ptr + st->ptr[0]; st->nparsed++; if ((st->ptr < st->buf) || (st->ptr > st->dummy_zero)) { /* invalid entry */ return (EINVAL); } return (0); } static int ugen20_init_backend(struct libusb20_backend *pbe) { struct ugen20_urd_state state; struct libusb20_device *pdev; memset(&state, 0, sizeof(state)); state.f = open("/dev/" USB_DEVICE_NAME, O_RDONLY); if (state.f < 0) return (LIBUSB20_ERROR_OTHER); while (ugen20_readdir(&state) == 0) { if ((state.src[0] != 'u') || (state.src[1] != 'g') || (state.src[2] != 'e') || (state.src[3] != 'n')) { continue; } pdev = libusb20_dev_alloc(); if (pdev == NULL) { continue; } if (ugen20_enumerate(pdev, state.src + 4)) { libusb20_dev_free(pdev); continue; } /* put the device on the backend list */ libusb20_be_enqueue_device(pbe, pdev); } close(state.f); return (0); /* success */ } static void ugen20_tr_release(struct libusb20_device *pdev) { struct usb_fs_uninit fs_uninit; if (pdev->nTransfer == 0) { return; } /* release all pending USB transfers */ if (pdev->privBeData != NULL) { memset(&fs_uninit, 0, sizeof(fs_uninit)); if (ioctl(pdev->file, USB_FS_UNINIT, &fs_uninit)) { /* ignore any errors of this kind */ } } return; } static int ugen20_tr_renew(struct libusb20_device *pdev) { struct usb_fs_init fs_init; struct usb_fs_endpoint *pfse; int error; uint32_t size; uint16_t nMaxTransfer; nMaxTransfer = pdev->nTransfer; error = 0; if (nMaxTransfer == 0) { goto done; } size = nMaxTransfer * sizeof(*pfse); if (pdev->privBeData == NULL) { pfse = malloc(size); if (pfse == NULL) { error = LIBUSB20_ERROR_NO_MEM; goto done; } pdev->privBeData = pfse; } /* reset endpoint data */ memset(pdev->privBeData, 0, size); memset(&fs_init, 0, sizeof(fs_init)); fs_init.pEndpoints = pdev->privBeData; fs_init.ep_index_max = nMaxTransfer; if (ioctl(pdev->file, USB_FS_INIT, &fs_init)) { error = LIBUSB20_ERROR_OTHER; goto done; } done: return (error); } static int ugen20_open_device(struct libusb20_device *pdev, uint16_t nMaxTransfer) { uint32_t plugtime; char buf[64]; int f; int g; int error; snprintf(buf, sizeof(buf), "/dev/" USB_GENERIC_NAME "%u.%u", pdev->bus_number, pdev->device_address); /* * We need two file handles, one for the control endpoint and one * for BULK, INTERRUPT and ISOCHRONOUS transactions due to optimised * kernel locking. */ g = open(buf, O_RDWR); if (g < 0) { return (LIBUSB20_ERROR_NO_DEVICE); } f = open(buf, O_RDWR); if (f < 0) { close(g); return (LIBUSB20_ERROR_NO_DEVICE); } if (ioctl(f, USB_GET_PLUGTIME, &plugtime)) { error = LIBUSB20_ERROR_OTHER; goto done; } /* check that the correct device is still plugged */ if (pdev->session_data.plugtime != plugtime) { error = LIBUSB20_ERROR_NO_DEVICE; goto done; } /* need to set this before "tr_renew()" */ pdev->file = f; pdev->file_ctrl = g; /* renew all USB transfers */ error = ugen20_tr_renew(pdev); if (error) { goto done; } /* set methods */ pdev->methods = &libusb20_ugen20_device_methods; done: if (error) { if (pdev->privBeData) { /* cleanup after "tr_renew()" */ free(pdev->privBeData); pdev->privBeData = NULL; } pdev->file = -1; pdev->file_ctrl = -1; close(f); close(g); } return (error); } static int ugen20_close_device(struct libusb20_device *pdev) { struct usb_fs_uninit fs_uninit; if (pdev->privBeData) { memset(&fs_uninit, 0, sizeof(fs_uninit)); if (ioctl(pdev->file, USB_FS_UNINIT, &fs_uninit)) { /* ignore this error */ } free(pdev->privBeData); } pdev->nTransfer = 0; pdev->privBeData = NULL; close(pdev->file); close(pdev->file_ctrl); pdev->file = -1; pdev->file_ctrl = -1; return (0); /* success */ } static void ugen20_exit_backend(struct libusb20_backend *pbe) { return; /* nothing to do */ } static int ugen20_get_config_desc_full(struct libusb20_device *pdev, uint8_t **ppbuf, uint16_t *plen, uint8_t cfg_index) { struct usb_gen_descriptor gen_desc; struct usb_config_descriptor cdesc; uint8_t *ptr; uint16_t len; int error; /* make sure memory is initialised */ memset(&cdesc, 0, sizeof(cdesc)); memset(&gen_desc, 0, sizeof(gen_desc)); gen_desc.ugd_data = &cdesc; gen_desc.ugd_maxlen = sizeof(cdesc); gen_desc.ugd_config_index = cfg_index; error = ioctl(pdev->file_ctrl, USB_GET_FULL_DESC, &gen_desc); if (error) { return (LIBUSB20_ERROR_OTHER); } len = UGETW(cdesc.wTotalLength); if (len < sizeof(cdesc)) { /* corrupt descriptor */ return (LIBUSB20_ERROR_OTHER); } ptr = malloc(len); if (!ptr) { return (LIBUSB20_ERROR_NO_MEM); } /* make sure memory is initialised */ memset(ptr, 0, len); gen_desc.ugd_data = ptr; gen_desc.ugd_maxlen = len; error = ioctl(pdev->file_ctrl, USB_GET_FULL_DESC, &gen_desc); if (error) { free(ptr); return (LIBUSB20_ERROR_OTHER); } /* make sure that the device doesn't fool us */ memcpy(ptr, &cdesc, sizeof(cdesc)); *ppbuf = ptr; *plen = len; return (0); /* success */ } static int ugen20_get_config_index(struct libusb20_device *pdev, uint8_t *pindex) { int temp; if (ioctl(pdev->file_ctrl, USB_GET_CONFIG, &temp)) { return (LIBUSB20_ERROR_OTHER); } *pindex = temp; return (0); } static int ugen20_set_config_index(struct libusb20_device *pdev, uint8_t cfg_index) { int temp = cfg_index; /* release all active USB transfers */ ugen20_tr_release(pdev); if (ioctl(pdev->file_ctrl, USB_SET_CONFIG, &temp)) { return (LIBUSB20_ERROR_OTHER); } return (ugen20_tr_renew(pdev)); } static int ugen20_set_alt_index(struct libusb20_device *pdev, uint8_t iface_index, uint8_t alt_index) { struct usb_alt_interface alt_iface; memset(&alt_iface, 0, sizeof(alt_iface)); alt_iface.uai_interface_index = iface_index; alt_iface.uai_alt_index = alt_index; /* release all active USB transfers */ ugen20_tr_release(pdev); if (ioctl(pdev->file_ctrl, USB_SET_ALTINTERFACE, &alt_iface)) { return (LIBUSB20_ERROR_OTHER); } return (ugen20_tr_renew(pdev)); } static int ugen20_reset_device(struct libusb20_device *pdev) { int temp = 0; /* release all active USB transfers */ ugen20_tr_release(pdev); if (ioctl(pdev->file_ctrl, USB_DEVICEENUMERATE, &temp)) { return (LIBUSB20_ERROR_OTHER); } return (ugen20_tr_renew(pdev)); } static int ugen20_check_connected(struct libusb20_device *pdev) { uint32_t plugtime; int error = 0; if (ioctl(pdev->file_ctrl, USB_GET_PLUGTIME, &plugtime)) { error = LIBUSB20_ERROR_NO_DEVICE; goto done; } if (pdev->session_data.plugtime != plugtime) { error = LIBUSB20_ERROR_NO_DEVICE; goto done; } done: return (error); } static int ugen20_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode) { int temp; switch (power_mode) { case LIBUSB20_POWER_OFF: temp = USB_POWER_MODE_OFF; break; case LIBUSB20_POWER_ON: temp = USB_POWER_MODE_ON; break; case LIBUSB20_POWER_SAVE: temp = USB_POWER_MODE_SAVE; break; case LIBUSB20_POWER_SUSPEND: temp = USB_POWER_MODE_SUSPEND; break; case LIBUSB20_POWER_RESUME: temp = USB_POWER_MODE_RESUME; break; default: return (LIBUSB20_ERROR_INVALID_PARAM); } if (ioctl(pdev->file_ctrl, USB_SET_POWER_MODE, &temp)) { return (LIBUSB20_ERROR_OTHER); } return (0); } static int ugen20_get_power_mode(struct libusb20_device *pdev, uint8_t *power_mode) { int temp; if (ioctl(pdev->file_ctrl, USB_GET_POWER_MODE, &temp)) { return (LIBUSB20_ERROR_OTHER); } switch (temp) { case USB_POWER_MODE_OFF: temp = LIBUSB20_POWER_OFF; break; case USB_POWER_MODE_ON: temp = LIBUSB20_POWER_ON; break; case USB_POWER_MODE_SAVE: temp = LIBUSB20_POWER_SAVE; break; case USB_POWER_MODE_SUSPEND: temp = LIBUSB20_POWER_SUSPEND; break; case USB_POWER_MODE_RESUME: temp = LIBUSB20_POWER_RESUME; break; default: temp = LIBUSB20_POWER_ON; break; } *power_mode = temp; return (0); /* success */ } static int ugen20_kernel_driver_active(struct libusb20_device *pdev, uint8_t iface_index) { int temp = iface_index; if (ioctl(pdev->file_ctrl, USB_IFACE_DRIVER_ACTIVE, &temp)) { return (LIBUSB20_ERROR_OTHER); } return (0); /* kernel driver is active */ } static int ugen20_detach_kernel_driver(struct libusb20_device *pdev, uint8_t iface_index) { int temp = iface_index; if (ioctl(pdev->file_ctrl, USB_IFACE_DRIVER_DETACH, &temp)) { return (LIBUSB20_ERROR_OTHER); } return (0); /* kernel driver is active */ } static int ugen20_do_request_sync(struct libusb20_device *pdev, struct LIBUSB20_CONTROL_SETUP_DECODED *setup, void *data, uint16_t *pactlen, uint32_t timeout, uint8_t flags) { struct usb_ctl_request req; memset(&req, 0, sizeof(req)); req.ucr_data = data; if (!(flags & LIBUSB20_TRANSFER_SINGLE_SHORT_NOT_OK)) { req.ucr_flags |= USB_SHORT_XFER_OK; } if (libusb20_me_encode(&req.ucr_request, sizeof(req.ucr_request), setup)) { /* ignore */ } if (ioctl(pdev->file_ctrl, USB_DO_REQUEST, &req)) { return (LIBUSB20_ERROR_OTHER); } if (pactlen) { /* get actual length */ *pactlen = req.ucr_actlen; } return (0); /* kernel driver is active */ } static int ugen20_process(struct libusb20_device *pdev) { struct usb_fs_complete temp; struct usb_fs_endpoint *fsep; struct libusb20_transfer *xfer; while (1) { if (ioctl(pdev->file, USB_FS_COMPLETE, &temp)) { if (errno == EBUSY) { break; } else { /* device detached */ return (LIBUSB20_ERROR_OTHER); } } fsep = pdev->privBeData; xfer = pdev->pTransfer; fsep += temp.ep_index; xfer += temp.ep_index; /* update transfer status */ if (fsep->status == 0) { xfer->aFrames = fsep->aFrames; xfer->timeComplete = fsep->isoc_time_complete; xfer->status = LIBUSB20_TRANSFER_COMPLETED; } else if (fsep->status == USB_ERR_CANCELLED) { xfer->aFrames = 0; xfer->timeComplete = 0; xfer->status = LIBUSB20_TRANSFER_CANCELLED; } else if (fsep->status == USB_ERR_STALLED) { xfer->aFrames = 0; xfer->timeComplete = 0; xfer->status = LIBUSB20_TRANSFER_STALL; } else if (fsep->status == USB_ERR_TIMEOUT) { xfer->aFrames = 0; xfer->timeComplete = 0; xfer->status = LIBUSB20_TRANSFER_TIMED_OUT; } else { xfer->aFrames = 0; xfer->timeComplete = 0; xfer->status = LIBUSB20_TRANSFER_ERROR; } libusb20_tr_callback_wrapper(xfer); } return (0); /* done */ } static int ugen20_tr_open(struct libusb20_transfer *xfer, uint32_t MaxBufSize, uint32_t MaxFrameCount, uint8_t ep_no) { struct usb_fs_open temp; struct usb_fs_endpoint *fsep; memset(&temp, 0, sizeof(temp)); fsep = xfer->pdev->privBeData; fsep += xfer->trIndex; temp.max_bufsize = MaxBufSize; temp.max_frames = MaxFrameCount; temp.ep_index = xfer->trIndex; temp.ep_no = ep_no; if (ioctl(xfer->pdev->file, USB_FS_OPEN, &temp)) { return (LIBUSB20_ERROR_INVALID_PARAM); } /* maximums might have changed - update */ xfer->maxFrames = temp.max_frames; /* "max_bufsize" should be multiple of "max_packet_length" */ xfer->maxTotalLength = temp.max_bufsize; xfer->maxPacketLen = temp.max_packet_length; /* setup buffer and length lists */ fsep->ppBuffer = xfer->ppBuffer;/* zero copy */ fsep->pLength = xfer->pLength; /* zero copy */ return (0); /* success */ } static int ugen20_tr_close(struct libusb20_transfer *xfer) { struct usb_fs_close temp; memset(&temp, 0, sizeof(temp)); temp.ep_index = xfer->trIndex; if (ioctl(xfer->pdev->file, USB_FS_CLOSE, &temp)) { return (LIBUSB20_ERROR_INVALID_PARAM); } return (0); /* success */ } static int ugen20_tr_clear_stall_sync(struct libusb20_transfer *xfer) { struct usb_fs_clear_stall_sync temp; memset(&temp, 0, sizeof(temp)); /* if the transfer is active, an error will be returned */ temp.ep_index = xfer->trIndex; if (ioctl(xfer->pdev->file, USB_FS_CLEAR_STALL_SYNC, &temp)) { return (LIBUSB20_ERROR_INVALID_PARAM); } return (0); /* success */ } static void ugen20_tr_submit(struct libusb20_transfer *xfer) { struct usb_fs_start temp; struct usb_fs_endpoint *fsep; memset(&temp, 0, sizeof(temp)); fsep = xfer->pdev->privBeData; fsep += xfer->trIndex; fsep->nFrames = xfer->nFrames; fsep->flags = 0; if (!(xfer->flags & LIBUSB20_TRANSFER_SINGLE_SHORT_NOT_OK)) { fsep->flags |= USB_FS_FLAG_SINGLE_SHORT_OK; } if (!(xfer->flags & LIBUSB20_TRANSFER_MULTI_SHORT_NOT_OK)) { fsep->flags |= USB_FS_FLAG_MULTI_SHORT_OK; } if (xfer->flags & LIBUSB20_TRANSFER_FORCE_SHORT) { fsep->flags |= USB_FS_FLAG_FORCE_SHORT; } if (xfer->flags & LIBUSB20_TRANSFER_DO_CLEAR_STALL) { fsep->flags |= USB_FS_FLAG_CLEAR_STALL; } /* NOTE: The "fsep->timeout" variable is 16-bit. */ if (xfer->timeout > 65535) fsep->timeout = 65535; else fsep->timeout = xfer->timeout; temp.ep_index = xfer->trIndex; if (ioctl(xfer->pdev->file, USB_FS_START, &temp)) { /* ignore any errors - should never happen */ } return; /* success */ } static void ugen20_tr_cancel_async(struct libusb20_transfer *xfer) { struct usb_fs_stop temp; memset(&temp, 0, sizeof(temp)); temp.ep_index = xfer->trIndex; if (ioctl(xfer->pdev->file, USB_FS_STOP, &temp)) { /* ignore any errors - should never happen */ } return; } static int ugen20_be_ioctl(uint32_t cmd, void *data) { int f; int error; f = open("/dev/" USB_DEVICE_NAME, O_RDONLY); if (f < 0) return (LIBUSB20_ERROR_OTHER); error = ioctl(f, cmd, data); if (error == -1) { if (errno == EPERM) { error = LIBUSB20_ERROR_ACCESS; } else { error = LIBUSB20_ERROR_OTHER; } } close(f); return (error); } static int ugen20_dev_get_iface_desc(struct libusb20_device *pdev, uint8_t iface_index, char *buf, uint8_t len) { struct usb_gen_descriptor ugd; memset(&ugd, 0, sizeof(ugd)); ugd.ugd_data = buf; ugd.ugd_maxlen = len; ugd.ugd_iface_index = iface_index; if (ioctl(pdev->file, USB_GET_IFACE_DRIVER, &ugd)) { return (LIBUSB20_ERROR_INVALID_PARAM); } return (0); } static int ugen20_dev_get_info(struct libusb20_device *pdev, struct usb_device_info *pinfo) { if (ioctl(pdev->file, USB_GET_DEVICEINFO, pinfo)) { return (LIBUSB20_ERROR_INVALID_PARAM); } return (0); } static int ugen20_root_get_dev_quirk(struct libusb20_backend *pbe, uint16_t quirk_index, struct libusb20_quirk *pq) { struct usb_gen_quirk q; int error; memset(&q, 0, sizeof(q)); q.index = quirk_index; error = ugen20_be_ioctl(USB_DEV_QUIRK_GET, &q); if (error) { if (errno == EINVAL) { return (LIBUSB20_ERROR_NOT_FOUND); } } else { pq->vid = q.vid; pq->pid = q.pid; pq->bcdDeviceLow = q.bcdDeviceLow; pq->bcdDeviceHigh = q.bcdDeviceHigh; strlcpy(pq->quirkname, q.quirkname, sizeof(pq->quirkname)); } return (error); } static int ugen20_root_get_quirk_name(struct libusb20_backend *pbe, uint16_t quirk_index, struct libusb20_quirk *pq) { struct usb_gen_quirk q; int error; memset(&q, 0, sizeof(q)); q.index = quirk_index; error = ugen20_be_ioctl(USB_QUIRK_NAME_GET, &q); if (error) { if (errno == EINVAL) { return (LIBUSB20_ERROR_NOT_FOUND); } } else { strlcpy(pq->quirkname, q.quirkname, sizeof(pq->quirkname)); } return (error); } static int ugen20_root_add_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq) { struct usb_gen_quirk q; int error; memset(&q, 0, sizeof(q)); q.vid = pq->vid; q.pid = pq->pid; q.bcdDeviceLow = pq->bcdDeviceLow; q.bcdDeviceHigh = pq->bcdDeviceHigh; strlcpy(q.quirkname, pq->quirkname, sizeof(q.quirkname)); error = ugen20_be_ioctl(USB_DEV_QUIRK_ADD, &q); if (error) { if (errno == ENOMEM) { return (LIBUSB20_ERROR_NO_MEM); } } return (error); } static int ugen20_root_remove_dev_quirk(struct libusb20_backend *pbe, struct libusb20_quirk *pq) { struct usb_gen_quirk q; int error; memset(&q, 0, sizeof(q)); q.vid = pq->vid; q.pid = pq->pid; q.bcdDeviceLow = pq->bcdDeviceLow; q.bcdDeviceHigh = pq->bcdDeviceHigh; strlcpy(q.quirkname, pq->quirkname, sizeof(q.quirkname)); error = ugen20_be_ioctl(USB_DEV_QUIRK_REMOVE, &q); if (error) { if (errno == EINVAL) { return (LIBUSB20_ERROR_NOT_FOUND); } } return (error); } static int ugen20_root_set_template(struct libusb20_backend *pbe, int temp) { return (ugen20_be_ioctl(USB_SET_TEMPLATE, &temp)); } static int ugen20_root_get_template(struct libusb20_backend *pbe, int *ptemp) { return (ugen20_be_ioctl(USB_GET_TEMPLATE, ptemp)); } Index: user/imp/tbemd/lib/libusb/usb.h =================================================================== --- user/imp/tbemd/lib/libusb/usb.h (revision 203849) +++ user/imp/tbemd/lib/libusb/usb.h (revision 203850) @@ -1,309 +1,310 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _LIBUSB20_COMPAT_01_H_ #define _LIBUSB20_COMPAT_01_H_ -#include -#include #include +#include + +#include /* USB interface class codes */ #define USB_CLASS_PER_INTERFACE 0 #define USB_CLASS_AUDIO 1 #define USB_CLASS_COMM 2 #define USB_CLASS_HID 3 #define USB_CLASS_PRINTER 7 #define USB_CLASS_PTP 6 #define USB_CLASS_MASS_STORAGE 8 #define USB_CLASS_HUB 9 #define USB_CLASS_DATA 10 #define USB_CLASS_VENDOR_SPEC 0xff /* USB descriptor types */ #define USB_DT_DEVICE 0x01 #define USB_DT_CONFIG 0x02 #define USB_DT_STRING 0x03 #define USB_DT_INTERFACE 0x04 #define USB_DT_ENDPOINT 0x05 #define USB_DT_HID 0x21 #define USB_DT_REPORT 0x22 #define USB_DT_PHYSICAL 0x23 #define USB_DT_HUB 0x29 /* USB descriptor type sizes */ #define USB_DT_DEVICE_SIZE 18 #define USB_DT_CONFIG_SIZE 9 #define USB_DT_INTERFACE_SIZE 9 #define USB_DT_ENDPOINT_SIZE 7 #define USB_DT_ENDPOINT_AUDIO_SIZE 9 #define USB_DT_HUB_NONVAR_SIZE 7 /* USB descriptor header */ struct usb_descriptor_header { uint8_t bLength; uint8_t bDescriptorType; }; /* USB string descriptor */ struct usb_string_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t wData[1]; }; /* USB HID descriptor */ struct usb_hid_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t bcdHID; uint8_t bCountryCode; uint8_t bNumDescriptors; /* uint8_t bReportDescriptorType; */ /* uint16_t wDescriptorLength; */ /* ... */ }; /* USB endpoint descriptor */ #define USB_MAXENDPOINTS 32 struct usb_endpoint_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bEndpointAddress; #define USB_ENDPOINT_ADDRESS_MASK 0x0f #define USB_ENDPOINT_DIR_MASK 0x80 uint8_t bmAttributes; #define USB_ENDPOINT_TYPE_MASK 0x03 #define USB_ENDPOINT_TYPE_CONTROL 0 #define USB_ENDPOINT_TYPE_ISOCHRONOUS 1 #define USB_ENDPOINT_TYPE_BULK 2 #define USB_ENDPOINT_TYPE_INTERRUPT 3 uint16_t wMaxPacketSize; uint8_t bInterval; uint8_t bRefresh; uint8_t bSynchAddress; uint8_t *extra; /* Extra descriptors */ int extralen; }; /* USB interface descriptor */ #define USB_MAXINTERFACES 32 struct usb_interface_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bInterfaceNumber; uint8_t bAlternateSetting; uint8_t bNumEndpoints; uint8_t bInterfaceClass; uint8_t bInterfaceSubClass; uint8_t bInterfaceProtocol; uint8_t iInterface; struct usb_endpoint_descriptor *endpoint; uint8_t *extra; /* Extra descriptors */ int extralen; }; #define USB_MAXALTSETTING 128 /* Hard limit */ struct usb_interface { struct usb_interface_descriptor *altsetting; int num_altsetting; }; /* USB configuration descriptor */ #define USB_MAXCONFIG 8 struct usb_config_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t wTotalLength; uint8_t bNumInterfaces; uint8_t bConfigurationValue; uint8_t iConfiguration; uint8_t bmAttributes; uint8_t MaxPower; struct usb_interface *interface; uint8_t *extra; /* Extra descriptors */ int extralen; }; /* USB device descriptor */ struct usb_device_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint16_t bcdUSB; uint8_t bDeviceClass; uint8_t bDeviceSubClass; uint8_t bDeviceProtocol; uint8_t bMaxPacketSize0; uint16_t idVendor; uint16_t idProduct; uint16_t bcdDevice; uint8_t iManufacturer; uint8_t iProduct; uint8_t iSerialNumber; uint8_t bNumConfigurations; }; /* USB setup packet */ struct usb_ctrl_setup { uint8_t bRequestType; #define USB_RECIP_DEVICE 0x00 #define USB_RECIP_INTERFACE 0x01 #define USB_RECIP_ENDPOINT 0x02 #define USB_RECIP_OTHER 0x03 #define USB_TYPE_STANDARD (0x00 << 5) #define USB_TYPE_CLASS (0x01 << 5) #define USB_TYPE_VENDOR (0x02 << 5) #define USB_TYPE_RESERVED (0x03 << 5) #define USB_ENDPOINT_IN 0x80 #define USB_ENDPOINT_OUT 0x00 uint8_t bRequest; #define USB_REQ_GET_STATUS 0x00 #define USB_REQ_CLEAR_FEATURE 0x01 #define USB_REQ_SET_FEATURE 0x03 #define USB_REQ_SET_ADDRESS 0x05 #define USB_REQ_GET_DESCRIPTOR 0x06 #define USB_REQ_SET_DESCRIPTOR 0x07 #define USB_REQ_GET_CONFIGURATION 0x08 #define USB_REQ_SET_CONFIGURATION 0x09 #define USB_REQ_GET_INTERFACE 0x0A #define USB_REQ_SET_INTERFACE 0x0B #define USB_REQ_SYNCH_FRAME 0x0C uint16_t wValue; uint16_t wIndex; uint16_t wLength; }; /* Error codes */ #define USB_ERROR_BEGIN 500000 /* Byte swapping */ #define USB_LE16_TO_CPU(x) le16toh(x) /* Data types */ struct usb_device; struct usb_bus; /* * To maintain compatibility with applications already built with libusb, * we must only add entries to the end of this structure. NEVER delete or * move members and only change types if you really know what you're doing. */ struct usb_device { struct usb_device *next; struct usb_device *prev; char filename[PATH_MAX + 1]; struct usb_bus *bus; struct usb_device_descriptor descriptor; struct usb_config_descriptor *config; void *dev; uint8_t devnum; uint8_t num_children; struct usb_device **children; }; struct usb_bus { struct usb_bus *next; struct usb_bus *prev; char dirname[PATH_MAX + 1]; struct usb_device *devices; uint32_t location; struct usb_device *root_dev; }; struct usb_dev_handle; typedef struct usb_dev_handle usb_dev_handle; /* Variables */ extern struct usb_bus *usb_busses; #ifdef __cplusplus extern "C" { #endif #if 0 } /* style */ #endif /* Function prototypes from "libusb20_compat01.c" */ usb_dev_handle *usb_open(struct usb_device *dev); int usb_close(usb_dev_handle * dev); int usb_get_string(usb_dev_handle * dev, int index, int langid, char *buf, size_t buflen); int usb_get_string_simple(usb_dev_handle * dev, int index, char *buf, size_t buflen); int usb_get_descriptor_by_endpoint(usb_dev_handle * udev, int ep, uint8_t type, uint8_t index, void *buf, int size); int usb_get_descriptor(usb_dev_handle * udev, uint8_t type, uint8_t index, void *buf, int size); int usb_parse_descriptor(uint8_t *source, char *description, void *dest); int usb_parse_configuration(struct usb_config_descriptor *config, uint8_t *buffer); void usb_destroy_configuration(struct usb_device *dev); void usb_fetch_and_parse_descriptors(usb_dev_handle * udev); int usb_bulk_write(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout); int usb_bulk_read(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout); int usb_interrupt_write(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout); int usb_interrupt_read(usb_dev_handle * dev, int ep, char *bytes, int size, int timeout); int usb_control_msg(usb_dev_handle * dev, int requesttype, int request, int value, int index, char *bytes, int size, int timeout); int usb_set_configuration(usb_dev_handle * dev, int configuration); int usb_claim_interface(usb_dev_handle * dev, int interface); int usb_release_interface(usb_dev_handle * dev, int interface); int usb_set_altinterface(usb_dev_handle * dev, int alternate); int usb_resetep(usb_dev_handle * dev, unsigned int ep); int usb_clear_halt(usb_dev_handle * dev, unsigned int ep); int usb_reset(usb_dev_handle * dev); const char *usb_strerror(void); void usb_init(void); void usb_set_debug(int level); int usb_find_busses(void); int usb_find_devices(void); struct usb_device *usb_device(usb_dev_handle * dev); struct usb_bus *usb_get_busses(void); #if 0 { /* style */ #endif #ifdef __cplusplus } #endif #endif /* _LIBUSB20_COMPAT01_H_ */ Index: user/imp/tbemd/sbin/growfs/growfs.c =================================================================== --- user/imp/tbemd/sbin/growfs/growfs.c (revision 203849) +++ user/imp/tbemd/sbin/growfs/growfs.c (revision 203850) @@ -1,2519 +1,2517 @@ /* * Copyright (c) 2000 Christoph Herrmann, Thomas-Henning von Kamptz * Copyright (c) 1980, 1989, 1993 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * Christoph Herrmann and Thomas-Henning von Kamptz, Munich and Frankfurt. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgment: * This product includes software developed by the University of * California, Berkeley and its contributors, as well as Christoph * Herrmann and Thomas-Henning von Kamptz. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $TSHeader: src/sbin/growfs/growfs.c,v 1.5 2000/12/12 19:31:00 tomsoft Exp $ * */ #ifndef lint static const char copyright[] = "@(#) Copyright (c) 2000 Christoph Herrmann, Thomas-Henning von Kamptz\n\ Copyright (c) 1980, 1989, 1993 The Regents of the University of California.\n\ All rights reserved.\n"; #endif /* not lint */ #include __FBSDID("$FreeBSD$"); /* ********************************************************** INCLUDES ***** */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "debug.h" /* *************************************************** GLOBALS & TYPES ***** */ #ifdef FS_DEBUG int _dbg_lvl_ = (DL_INFO); /* DL_TRC */ #endif /* FS_DEBUG */ static union { struct fs fs; char pad[SBLOCKSIZE]; } fsun1, fsun2; #define sblock fsun1.fs /* the new superblock */ #define osblock fsun2.fs /* the old superblock */ /* * Possible superblock locations ordered from most to least likely. */ static int sblock_try[] = SBLOCKSEARCH; static ufs2_daddr_t sblockloc; static union { struct cg cg; char pad[MAXBSIZE]; } cgun1, cgun2; #define acg cgun1.cg /* a cylinder cgroup (new) */ #define aocg cgun2.cg /* an old cylinder group */ static char ablk[MAXBSIZE]; /* a block */ static struct csum *fscs; /* cylinder summary */ union dinode { struct ufs1_dinode dp1; struct ufs2_dinode dp2; }; #define DIP(dp, field) \ ((sblock.fs_magic == FS_UFS1_MAGIC) ? \ (uint32_t)(dp)->dp1.field : (dp)->dp2.field) #define DIP_SET(dp, field, val) do { \ if (sblock.fs_magic == FS_UFS1_MAGIC) \ (dp)->dp1.field = (val); \ else \ (dp)->dp2.field = (val); \ } while (0) static ufs2_daddr_t inoblk; /* inode block address */ static char inobuf[MAXBSIZE]; /* inode block */ ino_t maxino; /* last valid inode */ static int unlabeled; /* unlabeled partition, e.g. vinum volume etc. */ /* * An array of elements of type struct gfs_bpp describes all blocks to * be relocated in order to free the space needed for the cylinder group * summary for all cylinder groups located in the first cylinder group. */ struct gfs_bpp { ufs2_daddr_t old; /* old block number */ ufs2_daddr_t new; /* new block number */ #define GFS_FL_FIRST 1 #define GFS_FL_LAST 2 unsigned int flags; /* special handling required */ int found; /* how many references were updated */ }; /* ******************************************************** PROTOTYPES ***** */ static void growfs(int, int, unsigned int); static void rdfs(ufs2_daddr_t, size_t, void *, int); static void wtfs(ufs2_daddr_t, size_t, void *, int, unsigned int); static ufs2_daddr_t alloc(void); static int charsperline(void); static void usage(void); static int isblock(struct fs *, unsigned char *, int); static void clrblock(struct fs *, unsigned char *, int); static void setblock(struct fs *, unsigned char *, int); static void initcg(int, time_t, int, unsigned int); static void updjcg(int, time_t, int, int, unsigned int); static void updcsloc(time_t, int, int, unsigned int); static struct disklabel *get_disklabel(int); static void return_disklabel(int, struct disklabel *, unsigned int); static union dinode *ginode(ino_t, int, int); static void frag_adjust(ufs2_daddr_t, int); static int cond_bl_upd(ufs2_daddr_t *, struct gfs_bpp *, int, int, unsigned int); static void updclst(int); static void updrefs(int, ino_t, struct gfs_bpp *, int, int, unsigned int); static void indirchk(ufs_lbn_t, ufs_lbn_t, ufs2_daddr_t, ufs_lbn_t, struct gfs_bpp *, int, int, unsigned int); static void get_dev_size(int, int *); /* ************************************************************ growfs ***** */ /* * Here we actually start growing the file system. We basically read the * cylinder summary from the first cylinder group as we want to update * this on the fly during our various operations. First we handle the * changes in the former last cylinder group. Afterwards we create all new * cylinder groups. Now we handle the cylinder group containing the * cylinder summary which might result in a relocation of the whole * structure. In the end we write back the updated cylinder summary, the * new superblock, and slightly patched versions of the super block * copies. */ static void growfs(int fsi, int fso, unsigned int Nflag) { DBG_FUNC("growfs") time_t utime; uint cylno; int i, j, width; char tmpbuf[100]; #ifdef FSIRAND static int randinit=0; DBG_ENTER; if (!randinit) { randinit = 1; srandomdev(); } #else /* not FSIRAND */ DBG_ENTER; #endif /* FSIRAND */ time(&utime); /* * Get the cylinder summary into the memory. */ fscs = (struct csum *)calloc((size_t)1, (size_t)sblock.fs_cssize); if(fscs == NULL) { errx(1, "calloc failed"); } for (i = 0; i < osblock.fs_cssize; i += osblock.fs_bsize) { rdfs(fsbtodb(&osblock, osblock.fs_csaddr + numfrags(&osblock, i)), (size_t)MIN(osblock.fs_cssize - i, osblock.fs_bsize), (void *)(((char *)fscs)+i), fsi); } #ifdef FS_DEBUG { struct csum *dbg_csp; int dbg_csc; char dbg_line[80]; dbg_csp=fscs; for(dbg_csc=0; dbg_csc= width) { printf("\n"); i = 0; } i += j; printf("%s", tmpbuf); fflush(stdout); } printf("\n"); /* * Do all needed changes in the first cylinder group. * allocate blocks in new location */ updcsloc(utime, fsi, fso, Nflag); /* * Now write the cylinder summary back to disk. */ for (i = 0; i < sblock.fs_cssize; i += sblock.fs_bsize) { wtfs(fsbtodb(&sblock, sblock.fs_csaddr + numfrags(&sblock, i)), (size_t)MIN(sblock.fs_cssize - i, sblock.fs_bsize), (void *)(((char *)fscs) + i), fso, Nflag); } DBG_PRINT0("fscs written\n"); #ifdef FS_DEBUG { struct csum *dbg_csp; int dbg_csc; char dbg_line[80]; dbg_csp=fscs; for(dbg_csc=0; dbg_csc sblock.fs_size) dmax = sblock.fs_size; dlower = cgsblock(&sblock, cylno) - cbase; dupper = cgdmin(&sblock, cylno) - cbase; if (cylno == 0) /* XXX fscs may be relocated */ dupper += howmany(sblock.fs_cssize, sblock.fs_fsize); cs = &fscs[cylno]; memset(&acg, 0, sblock.fs_cgsize); /* * Note that we do not set cg_initediblk at all. * In this extension of a previous filesystem * we have no inodes initialized for the cylinder * group at all. The first access to that cylinder * group will do the correct initialization. */ acg.cg_time = utime; acg.cg_magic = CG_MAGIC; acg.cg_cgx = cylno; acg.cg_niblk = sblock.fs_ipg; acg.cg_ndblk = dmax - cbase; if (sblock.fs_contigsumsize > 0) acg.cg_nclusterblks = acg.cg_ndblk / sblock.fs_frag; start = &acg.cg_space[0] - (u_char *)(&acg.cg_firstfield); if (sblock.fs_magic == FS_UFS2_MAGIC) { acg.cg_iusedoff = start; } else { acg.cg_old_ncyl = sblock.fs_old_cpg; acg.cg_old_time = acg.cg_time; acg.cg_time = 0; acg.cg_old_niblk = acg.cg_niblk; acg.cg_niblk = 0; acg.cg_old_btotoff = start; acg.cg_old_boff = acg.cg_old_btotoff + sblock.fs_old_cpg * sizeof(int32_t); acg.cg_iusedoff = acg.cg_old_boff + sblock.fs_old_cpg * sizeof(u_int16_t); } acg.cg_freeoff = acg.cg_iusedoff + howmany(sblock.fs_ipg, CHAR_BIT); acg.cg_nextfreeoff = acg.cg_freeoff + howmany(sblock.fs_fpg, CHAR_BIT); if (sblock.fs_contigsumsize > 0) { acg.cg_clustersumoff = roundup(acg.cg_nextfreeoff, sizeof(u_int32_t)); acg.cg_clustersumoff -= sizeof(u_int32_t); acg.cg_clusteroff = acg.cg_clustersumoff + (sblock.fs_contigsumsize + 1) * sizeof(u_int32_t); acg.cg_nextfreeoff = acg.cg_clusteroff + howmany(fragstoblks(&sblock, sblock.fs_fpg), CHAR_BIT); } if (acg.cg_nextfreeoff > (unsigned)sblock.fs_cgsize) { /* * This should never happen as we would have had that panic * already on file system creation */ errx(37, "panic: cylinder group too big"); } acg.cg_cs.cs_nifree += sblock.fs_ipg; if (cylno == 0) for (i = 0; i < ROOTINO; i++) { setbit(cg_inosused(&acg), i); acg.cg_cs.cs_nifree--; } /* - * XXX Newfs writes out two blocks of initialized inodes - * unconditionally. Should we check here to make sure that they - * were actually written? + * For the old file system, we have to initialize all the inodes. */ if (sblock.fs_magic == FS_UFS1_MAGIC) { bzero(iobuf, sblock.fs_bsize); - for (i = 2 * sblock.fs_frag; i < sblock.fs_ipg / INOPF(&sblock); + for (i = 0; i < sblock.fs_ipg / INOPF(&sblock); i += sblock.fs_frag) { dp1 = (struct ufs1_dinode *)iobuf; #ifdef FSIRAND for (j = 0; j < INOPB(&sblock); j++) { dp1->di_gen = random(); dp1++; } #endif wtfs(fsbtodb(&sblock, cgimin(&sblock, cylno) + i), sblock.fs_bsize, iobuf, fso, Nflag); } } if (cylno > 0) { /* * In cylno 0, beginning space is reserved * for boot and super blocks. */ for (d = 0; d < dlower; d += sblock.fs_frag) { blkno = d / sblock.fs_frag; setblock(&sblock, cg_blksfree(&acg), blkno); if (sblock.fs_contigsumsize > 0) setbit(cg_clustersfree(&acg), blkno); acg.cg_cs.cs_nbfree++; } sblock.fs_dsize += dlower; } sblock.fs_dsize += acg.cg_ndblk - dupper; if ((i = dupper % sblock.fs_frag)) { acg.cg_frsum[sblock.fs_frag - i]++; for (d = dupper + sblock.fs_frag - i; dupper < d; dupper++) { setbit(cg_blksfree(&acg), dupper); acg.cg_cs.cs_nffree++; } } for (d = dupper; d + sblock.fs_frag <= acg.cg_ndblk; d += sblock.fs_frag) { blkno = d / sblock.fs_frag; setblock(&sblock, cg_blksfree(&acg), blkno); if (sblock.fs_contigsumsize > 0) setbit(cg_clustersfree(&acg), blkno); acg.cg_cs.cs_nbfree++; } if (d < acg.cg_ndblk) { acg.cg_frsum[acg.cg_ndblk - d]++; for (; d < acg.cg_ndblk; d++) { setbit(cg_blksfree(&acg), d); acg.cg_cs.cs_nffree++; } } if (sblock.fs_contigsumsize > 0) { int32_t *sump = cg_clustersum(&acg); u_char *mapp = cg_clustersfree(&acg); int map = *mapp++; int bit = 1; int run = 0; for (i = 0; i < acg.cg_nclusterblks; i++) { if ((map & bit) != 0) run++; else if (run != 0) { if (run > sblock.fs_contigsumsize) run = sblock.fs_contigsumsize; sump[run]++; run = 0; } if ((i & (CHAR_BIT - 1)) != CHAR_BIT - 1) bit <<= 1; else { map = *mapp++; bit = 1; } } if (run != 0) { if (run > sblock.fs_contigsumsize) run = sblock.fs_contigsumsize; sump[run]++; } } sblock.fs_cstotal.cs_ndir += acg.cg_cs.cs_ndir; sblock.fs_cstotal.cs_nffree += acg.cg_cs.cs_nffree; sblock.fs_cstotal.cs_nbfree += acg.cg_cs.cs_nbfree; sblock.fs_cstotal.cs_nifree += acg.cg_cs.cs_nifree; *cs = acg.cg_cs; wtfs(fsbtodb(&sblock, cgtod(&sblock, cylno)), sblock.fs_bsize, (char *)&acg, fso, Nflag); DBG_DUMP_CG(&sblock, "new cg", &acg); DBG_LEAVE; return; } /* ******************************************************* frag_adjust ***** */ /* * Here we add or subtract (sign +1/-1) the available fragments in a given * block to or from the fragment statistics. By subtracting before and adding * after an operation on the free frag map we can easy update the fragment * statistic, which seems to be otherwise a rather complex operation. */ static void frag_adjust(ufs2_daddr_t frag, int sign) { DBG_FUNC("frag_adjust") int fragsize; int f; DBG_ENTER; fragsize=0; /* * Here frag only needs to point to any fragment in the block we want * to examine. */ for(f=rounddown(frag, sblock.fs_frag); fold != 0; f++) { src = *block; if (fragstoblks(&sblock, src) != f->old) continue; /* * The fragment is part of the block, so update. */ dst = blkstofrags(&sblock, f->new); fragnum = fragnum(&sblock, src); *block = dst + fragnum; f->found++; DBG_PRINT3("scg (%jd->%jd)[%d] reference updated\n", (intmax_t)f->old, (intmax_t)f->new, fragnum); /* * Copy the block back immediately. * * XXX If src is is from an indirect block we have * to implement copy on write here in case of * active snapshots. */ ibuf = malloc(sblock.fs_bsize); if (!ibuf) errx(1, "malloc failed"); src -= fragnum; rdfs(fsbtodb(&sblock, src), (size_t)sblock.fs_bsize, ibuf, fsi); wtfs(dst, (size_t)sblock.fs_bsize, ibuf, fso, Nflag); free(ibuf); /* * The same block can't be found again in this loop. */ return (1); } DBG_LEAVE; return (0); } /* ************************************************************ updjcg ***** */ /* * Here we do all needed work for the former last cylinder group. It has to be * changed in any case, even if the file system ended exactly on the end of * this group, as there is some slightly inconsistent handling of the number * of cylinders in the cylinder group. We start again by reading the cylinder * group from disk. If the last block was not fully available, we first handle * the missing fragments, then we handle all new full blocks in that file * system and finally we handle the new last fragmented block in the file * system. We again have to handle the fragment statistics rotational layout * tables and cluster summary during all those operations. */ static void updjcg(int cylno, time_t utime, int fsi, int fso, unsigned int Nflag) { DBG_FUNC("updjcg") ufs2_daddr_t cbase, dmax, dupper; struct csum *cs; int i,k; int j=0; DBG_ENTER; /* * Read the former last (joining) cylinder group from disk, and make * a copy. */ rdfs(fsbtodb(&osblock, cgtod(&osblock, cylno)), (size_t)osblock.fs_cgsize, (void *)&aocg, fsi); DBG_PRINT0("jcg read\n"); DBG_DUMP_CG(&sblock, "old joining cg", &aocg); memcpy((void *)&cgun1, (void *)&cgun2, sizeof(cgun2)); /* * If the cylinder group had already its new final size almost * nothing is to be done ... except: * For some reason the value of cg_ncyl in the last cylinder group has * to be zero instead of fs_cpg. As this is now no longer the last * cylinder group we have to change that value now to fs_cpg. */ if(cgbase(&osblock, cylno+1) == osblock.fs_size) { if (sblock.fs_magic == FS_UFS1_MAGIC) acg.cg_old_ncyl=sblock.fs_old_cpg; wtfs(fsbtodb(&sblock, cgtod(&sblock, cylno)), (size_t)sblock.fs_cgsize, (void *)&acg, fso, Nflag); DBG_PRINT0("jcg written\n"); DBG_DUMP_CG(&sblock, "new joining cg", &acg); DBG_LEAVE; return; } /* * Set up some variables needed later. */ cbase = cgbase(&sblock, cylno); dmax = cbase + sblock.fs_fpg; if (dmax > sblock.fs_size) dmax = sblock.fs_size; dupper = cgdmin(&sblock, cylno) - cbase; if (cylno == 0) { /* XXX fscs may be relocated */ dupper += howmany(sblock.fs_cssize, sblock.fs_fsize); } /* * Set pointer to the cylinder summary for our cylinder group. */ cs = fscs + cylno; /* * Touch the cylinder group, update all fields in the cylinder group as * needed, update the free space in the superblock. */ acg.cg_time = utime; if ((unsigned)cylno == sblock.fs_ncg - 1) { /* * This is still the last cylinder group. */ if (sblock.fs_magic == FS_UFS1_MAGIC) acg.cg_old_ncyl = sblock.fs_old_ncyl % sblock.fs_old_cpg; } else { acg.cg_old_ncyl = sblock.fs_old_cpg; } DBG_PRINT2("jcg dbg: %d %u", cylno, sblock.fs_ncg); #ifdef FS_DEBUG if (sblock.fs_magic == FS_UFS1_MAGIC) DBG_PRINT2("%d %u", acg.cg_old_ncyl, sblock.fs_old_cpg); #endif DBG_PRINT0("\n"); acg.cg_ndblk = dmax - cbase; sblock.fs_dsize += acg.cg_ndblk-aocg.cg_ndblk; if (sblock.fs_contigsumsize > 0) { acg.cg_nclusterblks = acg.cg_ndblk / sblock.fs_frag; } /* * Now we have to update the free fragment bitmap for our new free * space. There again we have to handle the fragmentation and also * the rotational layout tables and the cluster summary. This is * also done per fragment for the first new block if the old file * system end was not on a block boundary, per fragment for the new * last block if the new file system end is not on a block boundary, * and per block for all space in between. * * Handle the first new block here if it was partially available * before. */ if(osblock.fs_size % sblock.fs_frag) { if(roundup(osblock.fs_size, sblock.fs_frag)<=sblock.fs_size) { /* * The new space is enough to fill at least this * block */ j=0; for(i=roundup(osblock.fs_size-cbase, sblock.fs_frag)-1; i>=osblock.fs_size-cbase; i--) { setbit(cg_blksfree(&acg), i); acg.cg_cs.cs_nffree++; j++; } /* * Check if the fragment just created could join an * already existing fragment at the former end of the * file system. */ if(isblock(&sblock, cg_blksfree(&acg), ((osblock.fs_size - cgbase(&sblock, cylno))/ sblock.fs_frag))) { /* * The block is now completely available. */ DBG_PRINT0("block was\n"); acg.cg_frsum[osblock.fs_size%sblock.fs_frag]--; acg.cg_cs.cs_nbfree++; acg.cg_cs.cs_nffree-=sblock.fs_frag; k=rounddown(osblock.fs_size-cbase, sblock.fs_frag); updclst((osblock.fs_size-cbase)/sblock.fs_frag); } else { /* * Lets rejoin a possible partially growed * fragment. */ k=0; while(isset(cg_blksfree(&acg), i) && (i>=rounddown(osblock.fs_size-cbase, sblock.fs_frag))) { i--; k++; } if(k) { acg.cg_frsum[k]--; } acg.cg_frsum[k+j]++; } } else { /* * We only grow by some fragments within this last * block. */ for(i=sblock.fs_size-cbase-1; i>=osblock.fs_size-cbase; i--) { setbit(cg_blksfree(&acg), i); acg.cg_cs.cs_nffree++; j++; } /* * Lets rejoin a possible partially growed fragment. */ k=0; while(isset(cg_blksfree(&acg), i) && (i>=rounddown(osblock.fs_size-cbase, sblock.fs_frag))) { i--; k++; } if(k) { acg.cg_frsum[k]--; } acg.cg_frsum[k+j]++; } } /* * Handle all new complete blocks here. */ for(i=roundup(osblock.fs_size-cbase, sblock.fs_frag); i+sblock.fs_frag<=dmax-cbase; /* XXX <= or only < ? */ i+=sblock.fs_frag) { j = i / sblock.fs_frag; setblock(&sblock, cg_blksfree(&acg), j); updclst(j); acg.cg_cs.cs_nbfree++; } /* * Handle the last new block if there are stll some new fragments left. * Here we don't have to bother about the cluster summary or the even * the rotational layout table. */ if (i < (dmax - cbase)) { acg.cg_frsum[dmax - cbase - i]++; for (; i < dmax - cbase; i++) { setbit(cg_blksfree(&acg), i); acg.cg_cs.cs_nffree++; } } sblock.fs_cstotal.cs_nffree += (acg.cg_cs.cs_nffree - aocg.cg_cs.cs_nffree); sblock.fs_cstotal.cs_nbfree += (acg.cg_cs.cs_nbfree - aocg.cg_cs.cs_nbfree); /* * The following statistics are not changed here: * sblock.fs_cstotal.cs_ndir * sblock.fs_cstotal.cs_nifree * As the statistics for this cylinder group are ready, copy it to * the summary information array. */ *cs = acg.cg_cs; /* * Write the updated "joining" cylinder group back to disk. */ wtfs(fsbtodb(&sblock, cgtod(&sblock, cylno)), (size_t)sblock.fs_cgsize, (void *)&acg, fso, Nflag); DBG_PRINT0("jcg written\n"); DBG_DUMP_CG(&sblock, "new joining cg", &acg); DBG_LEAVE; return; } /* ********************************************************** updcsloc ***** */ /* * Here we update the location of the cylinder summary. We have two possible * ways of growing the cylinder summary. * (1) We can try to grow the summary in the current location, and relocate * possibly used blocks within the current cylinder group. * (2) Alternatively we can relocate the whole cylinder summary to the first * new completely empty cylinder group. Once the cylinder summary is no * longer in the beginning of the first cylinder group you should never * use a version of fsck which is not aware of the possibility to have * this structure in a non standard place. * Option (1) is considered to be less intrusive to the structure of the file- * system. So we try to stick to that whenever possible. If there is not enough * space in the cylinder group containing the cylinder summary we have to use * method (2). In case of active snapshots in the file system we probably can * completely avoid implementing copy on write if we stick to method (2) only. */ static void updcsloc(time_t utime, int fsi, int fso, unsigned int Nflag) { DBG_FUNC("updcsloc") struct csum *cs; int ocscg, ncscg; int blocks; ufs2_daddr_t cbase, dupper, odupper, d, f, g; int ind, inc; uint cylno; struct gfs_bpp *bp; int i, l; int lcs=0; int block; DBG_ENTER; if(howmany(sblock.fs_cssize, sblock.fs_fsize) == howmany(osblock.fs_cssize, osblock.fs_fsize)) { /* * No new fragment needed. */ DBG_LEAVE; return; } ocscg=dtog(&osblock, osblock.fs_csaddr); cs=fscs+ocscg; blocks = 1+howmany(sblock.fs_cssize, sblock.fs_bsize)- howmany(osblock.fs_cssize, osblock.fs_bsize); /* * Read original cylinder group from disk, and make a copy. * XXX If Nflag is set in some very rare cases we now miss * some changes done in updjcg by reading the unmodified * block from disk. */ rdfs(fsbtodb(&osblock, cgtod(&osblock, ocscg)), (size_t)osblock.fs_cgsize, (void *)&aocg, fsi); DBG_PRINT0("oscg read\n"); DBG_DUMP_CG(&sblock, "old summary cg", &aocg); memcpy((void *)&cgun1, (void *)&cgun2, sizeof(cgun2)); /* * Touch the cylinder group, set up local variables needed later * and update the superblock. */ acg.cg_time = utime; /* * XXX In the case of having active snapshots we may need much more * blocks for the copy on write. We need each block twice, and * also up to 8*3 blocks for indirect blocks for all possible * references. */ if(/*((int)sblock.fs_time&0x3)>0||*/ cs->cs_nbfree < blocks) { /* * There is not enough space in the old cylinder group to * relocate all blocks as needed, so we relocate the whole * cylinder group summary to a new group. We try to use the * first complete new cylinder group just created. Within the * cylinder group we align the area immediately after the * cylinder group information location in order to be as * close as possible to the original implementation of ffs. * * First we have to make sure we'll find enough space in the * new cylinder group. If not, then we currently give up. * We start with freeing everything which was used by the * fragments of the old cylinder summary in the current group. * Now we write back the group meta data, read in the needed * meta data from the new cylinder group, and start allocating * within that group. Here we can assume, the group to be * completely empty. Which makes the handling of fragments and * clusters a lot easier. */ DBG_TRC; if(sblock.fs_ncg-osblock.fs_ncg < 2) { errx(2, "panic: not enough space"); } /* * Point "d" to the first fragment not used by the cylinder * summary. */ d=osblock.fs_csaddr+(osblock.fs_cssize/osblock.fs_fsize); /* * Set up last cluster size ("lcs") already here. Calculate * the size for the trailing cluster just behind where "d" * points to. */ if(sblock.fs_contigsumsize > 0) { for(block=howmany(d%sblock.fs_fpg, sblock.fs_frag), lcs=0; lcs 0) { setbit(cg_clustersfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); if(lcs < sblock.fs_contigsumsize) { if(lcs) { cg_clustersum(&acg) [lcs]--; } lcs++; cg_clustersum(&acg)[lcs]++; } } } /* * Point "d" to the first fragment of the block before * the last incomplete block. */ d--; } DBG_PRINT1("d=%jd\n", (intmax_t)d); for(d=rounddown(d, sblock.fs_frag); d >= osblock.fs_csaddr; d-=sblock.fs_frag) { DBG_TRC; DBG_PRINT1("d=%jd\n", (intmax_t)d); setblock(&sblock, cg_blksfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); acg.cg_cs.cs_nbfree++; sblock.fs_cstotal.cs_nbfree++; if(sblock.fs_contigsumsize > 0) { setbit(cg_clustersfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); /* * The last cluster size is already set up. */ if(lcs < sblock.fs_contigsumsize) { if(lcs) { cg_clustersum(&acg)[lcs]--; } lcs++; cg_clustersum(&acg)[lcs]++; } } } *cs = acg.cg_cs; /* * Now write the former cylinder group containing the cylinder * summary back to disk. */ wtfs(fsbtodb(&sblock, cgtod(&sblock, ocscg)), (size_t)sblock.fs_cgsize, (void *)&acg, fso, Nflag); DBG_PRINT0("oscg written\n"); DBG_DUMP_CG(&sblock, "old summary cg", &acg); /* * Find the beginning of the new cylinder group containing the * cylinder summary. */ sblock.fs_csaddr=cgdmin(&sblock, osblock.fs_ncg); ncscg=dtog(&sblock, sblock.fs_csaddr); cs=fscs+ncscg; /* * If Nflag is specified, we would now read random data instead * of an empty cg structure from disk. So we can't simulate that * part for now. */ if(Nflag) { DBG_PRINT0("nscg update skipped\n"); DBG_LEAVE; return; } /* * Read the future cylinder group containing the cylinder * summary from disk, and make a copy. */ rdfs(fsbtodb(&sblock, cgtod(&sblock, ncscg)), (size_t)sblock.fs_cgsize, (void *)&aocg, fsi); DBG_PRINT0("nscg read\n"); DBG_DUMP_CG(&sblock, "new summary cg", &aocg); memcpy((void *)&cgun1, (void *)&cgun2, sizeof(cgun2)); /* * Allocate all complete blocks used by the new cylinder * summary. */ for(d=sblock.fs_csaddr; d+sblock.fs_frag <= sblock.fs_csaddr+(sblock.fs_cssize/sblock.fs_fsize); d+=sblock.fs_frag) { clrblock(&sblock, cg_blksfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); acg.cg_cs.cs_nbfree--; sblock.fs_cstotal.cs_nbfree--; if(sblock.fs_contigsumsize > 0) { clrbit(cg_clustersfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); } } /* * Allocate all fragments used by the cylinder summary in the * last block. */ if(d 0) { clrbit(cg_clustersfree(&acg), (d%sblock.fs_fpg)/sblock.fs_frag); } frag_adjust(d%sblock.fs_fpg, +1); } /* * XXX Handle the cluster statistics here in the case this * cylinder group is now almost full, and the remaining * space is less then the maximum cluster size. This is * probably not needed, as you would hardly find a file * system which has only MAXCSBUFS+FS_MAXCONTIG of free * space right behind the cylinder group information in * any new cylinder group. */ /* * Update our statistics in the cylinder summary. */ *cs = acg.cg_cs; /* * Write the new cylinder group containing the cylinder summary * back to disk. */ wtfs(fsbtodb(&sblock, cgtod(&sblock, ncscg)), (size_t)sblock.fs_cgsize, (void *)&acg, fso, Nflag); DBG_PRINT0("nscg written\n"); DBG_DUMP_CG(&sblock, "new summary cg", &acg); DBG_LEAVE; return; } /* * We have got enough of space in the current cylinder group, so we * can relocate just a few blocks, and let the summary information * grow in place where it is right now. */ DBG_TRC; cbase = cgbase(&osblock, ocscg); /* old and new are equal */ dupper = sblock.fs_csaddr - cbase + howmany(sblock.fs_cssize, sblock.fs_fsize); odupper = osblock.fs_csaddr - cbase + howmany(osblock.fs_cssize, osblock.fs_fsize); sblock.fs_dsize -= dupper-odupper; /* * Allocate the space for the array of blocks to be relocated. */ bp=(struct gfs_bpp *)malloc(((dupper-odupper)/sblock.fs_frag+2)* sizeof(struct gfs_bpp)); if(bp == NULL) { errx(1, "malloc failed"); } memset((char *)bp, 0, ((dupper-odupper)/sblock.fs_frag+2)* sizeof(struct gfs_bpp)); /* * Lock all new frags needed for the cylinder group summary. This is * done per fragment in the first and last block of the new required * area, and per block for all other blocks. * * Handle the first new block here (but only if some fragments where * already used for the cylinder summary). */ ind=0; frag_adjust(odupper, -1); for(d=odupper; ((d= dupper) { bp[ind].flags|=GFS_FL_LAST; } ind++; } } else { clrbit(cg_blksfree(&acg), d); acg.cg_cs.cs_nffree--; sblock.fs_cstotal.cs_nffree--; } /* * No cluster handling is needed here, as there was at least * one fragment in use by the cylinder summary in the old * file system. * No block-free counter handling here as this block was not * a free block. */ } frag_adjust(odupper, 1); /* * Handle all needed complete blocks here. */ for(; d+sblock.fs_frag<=dupper; d+=sblock.fs_frag) { DBG_PRINT1("scg block check loop d=%jd\n", (intmax_t)d); if(!isblock(&sblock, cg_blksfree(&acg), d/sblock.fs_frag)) { for(f=d; f 0) { clrbit(cg_clustersfree(&acg), d/sblock.fs_frag); for(lcs=0, l=(d/sblock.fs_frag)+1; lcs 0) { clrbit(cg_clustersfree(&acg), d/sblock.fs_frag); for(lcs=0, l=(d/sblock.fs_frag)+1; lcs%jd) block relocated\n", (intmax_t)bp[i].old, (intmax_t)bp[i].new); } /* * Now we have to update all references to any fragment which * belongs to any block relocated. We iterate now over all * cylinder groups, within those over all non zero length * inodes. */ for(cylno=0; cylno0 ; inc--) { updrefs(cylno, (ino_t)inc, bp, fsi, fso, Nflag); } } /* * All inodes are checked, now make sure the number of * references found make sense. */ for(i=0; isblock.fs_frag)) { warnx("error: %jd refs found for block %jd.", (intmax_t)bp[i].found, (intmax_t)bp[i].old); } } } /* * The following statistics are not changed here: * sblock.fs_cstotal.cs_ndir * sblock.fs_cstotal.cs_nifree * The following statistics were already updated on the fly: * sblock.fs_cstotal.cs_nffree * sblock.fs_cstotal.cs_nbfree * As the statistics for this cylinder group are ready, copy it to * the summary information array. */ *cs = acg.cg_cs; /* * Write summary cylinder group back to disk. */ wtfs(fsbtodb(&sblock, cgtod(&sblock, ocscg)), (size_t)sblock.fs_cgsize, (void *)&acg, fso, Nflag); DBG_PRINT0("scg written\n"); DBG_DUMP_CG(&sblock, "new summary cg", &acg); DBG_LEAVE; return; } /* ************************************************************** rdfs ***** */ /* * Here we read some block(s) from disk. */ static void rdfs(ufs2_daddr_t bno, size_t size, void *bf, int fsi) { DBG_FUNC("rdfs") ssize_t n; DBG_ENTER; if (bno < 0) { err(32, "rdfs: attempting to read negative block number"); } if (lseek(fsi, (off_t)bno * DEV_BSIZE, 0) < 0) { err(33, "rdfs: seek error: %jd", (intmax_t)bno); } n = read(fsi, bf, size); if (n != (ssize_t)size) { err(34, "rdfs: read error: %jd", (intmax_t)bno); } DBG_LEAVE; return; } /* ************************************************************** wtfs ***** */ /* * Here we write some block(s) to disk. */ static void wtfs(ufs2_daddr_t bno, size_t size, void *bf, int fso, unsigned int Nflag) { DBG_FUNC("wtfs") ssize_t n; DBG_ENTER; if (Nflag) { DBG_LEAVE; return; } if (lseek(fso, (off_t)bno * DEV_BSIZE, SEEK_SET) < 0) { err(35, "wtfs: seek error: %ld", (long)bno); } n = write(fso, bf, size); if (n != (ssize_t)size) { err(36, "wtfs: write error: %ld", (long)bno); } DBG_LEAVE; return; } /* ************************************************************* alloc ***** */ /* * Here we allocate a free block in the current cylinder group. It is assumed, * that acg contains the current cylinder group. As we may take a block from * somewhere in the file system we have to handle cluster summary here. */ static ufs2_daddr_t alloc(void) { DBG_FUNC("alloc") ufs2_daddr_t d, blkno; int lcs1, lcs2; int l; int csmin, csmax; int dlower, dupper, dmax; DBG_ENTER; if (acg.cg_magic != CG_MAGIC) { warnx("acg: bad magic number"); DBG_LEAVE; return (0); } if (acg.cg_cs.cs_nbfree == 0) { warnx("error: cylinder group ran out of space"); DBG_LEAVE; return (0); } /* * We start seeking for free blocks only from the space available after * the end of the new grown cylinder summary. Otherwise we allocate a * block here which we have to relocate a couple of seconds later again * again, and we are not prepared to to this anyway. */ blkno=-1; dlower=cgsblock(&sblock, acg.cg_cgx)-cgbase(&sblock, acg.cg_cgx); dupper=cgdmin(&sblock, acg.cg_cgx)-cgbase(&sblock, acg.cg_cgx); dmax=cgbase(&sblock, acg.cg_cgx)+sblock.fs_fpg; if (dmax > sblock.fs_size) { dmax = sblock.fs_size; } dmax-=cgbase(&sblock, acg.cg_cgx); /* retransform into cg */ csmin=sblock.fs_csaddr-cgbase(&sblock, acg.cg_cgx); csmax=csmin+howmany(sblock.fs_cssize, sblock.fs_fsize); DBG_PRINT3("seek range: dl=%d, du=%d, dm=%d\n", dlower, dupper, dmax); DBG_PRINT2("range cont: csmin=%d, csmax=%d\n", csmin, csmax); for(d=0; (d=csmin && d<=csmax) { continue; } if(isblock(&sblock, cg_blksfree(&acg), fragstoblks(&sblock, d))) { blkno = fragstoblks(&sblock, d);/* Yeah found a block */ break; } } for(d=dupper; (d=csmin && d<=csmax) { continue; } if(isblock(&sblock, cg_blksfree(&acg), fragstoblks(&sblock, d))) { blkno = fragstoblks(&sblock, d);/* Yeah found a block */ break; } } if(blkno==-1) { warnx("internal error: couldn't find promised block in cg"); DBG_LEAVE; return (0); } /* * This is needed if the block was found already in the first loop. */ d=blkstofrags(&sblock, blkno); clrblock(&sblock, cg_blksfree(&acg), blkno); if (sblock.fs_contigsumsize > 0) { /* * Handle the cluster allocation bitmap. */ clrbit(cg_clustersfree(&acg), blkno); /* * We possibly have split a cluster here, so we have to do * recalculate the sizes of the remaining cluster halves now, * and use them for updating the cluster summary information. * * Lets start with the blocks before our allocated block ... */ for(lcs1=0, l=blkno-1; lcs1fs_frag) { case 8: DBG_LEAVE; return (cp[h] == 0xff); case 4: mask = 0x0f << ((h & 0x1) << 2); DBG_LEAVE; return ((cp[h >> 1] & mask) == mask); case 2: mask = 0x03 << ((h & 0x3) << 1); DBG_LEAVE; return ((cp[h >> 2] & mask) == mask); case 1: mask = 0x01 << (h & 0x7); DBG_LEAVE; return ((cp[h >> 3] & mask) == mask); default: fprintf(stderr, "isblock bad fs_frag %d\n", fs->fs_frag); DBG_LEAVE; return (0); } } /* ********************************************************** clrblock ***** */ /* * Here we allocate a complete block in the block map. For more details again * please see the source of newfs(8), as this function is taken over almost * unchanged. */ static void clrblock(struct fs *fs, unsigned char *cp, int h) { DBG_FUNC("clrblock") DBG_ENTER; switch ((fs)->fs_frag) { case 8: cp[h] = 0; break; case 4: cp[h >> 1] &= ~(0x0f << ((h & 0x1) << 2)); break; case 2: cp[h >> 2] &= ~(0x03 << ((h & 0x3) << 1)); break; case 1: cp[h >> 3] &= ~(0x01 << (h & 0x7)); break; default: warnx("clrblock bad fs_frag %d", fs->fs_frag); break; } DBG_LEAVE; return; } /* ********************************************************** setblock ***** */ /* * Here we free a complete block in the free block map. For more details again * please see the source of newfs(8), as this function is taken over almost * unchanged. */ static void setblock(struct fs *fs, unsigned char *cp, int h) { DBG_FUNC("setblock") DBG_ENTER; switch (fs->fs_frag) { case 8: cp[h] = 0xff; break; case 4: cp[h >> 1] |= (0x0f << ((h & 0x1) << 2)); break; case 2: cp[h >> 2] |= (0x03 << ((h & 0x3) << 1)); break; case 1: cp[h >> 3] |= (0x01 << (h & 0x7)); break; default: warnx("setblock bad fs_frag %d", fs->fs_frag); break; } DBG_LEAVE; return; } /* ************************************************************ ginode ***** */ /* * This function provides access to an individual inode. We find out in which * block the requested inode is located, read it from disk if needed, and * return the pointer into that block. We maintain a cache of one block to * not read the same block again and again if we iterate linearly over all * inodes. */ static union dinode * ginode(ino_t inumber, int fsi, int cg) { DBG_FUNC("ginode") static ino_t startinum = 0; /* first inode in cached block */ DBG_ENTER; /* * The inumber passed in is relative to the cg, so use it here to see * if the inode has been allocated yet. */ if (isclr(cg_inosused(&aocg), inumber)) { DBG_LEAVE; return NULL; } /* * Now make the inumber relative to the entire inode space so it can * be sanity checked. */ inumber += (cg * sblock.fs_ipg); if (inumber < ROOTINO) { DBG_LEAVE; return NULL; } if (inumber > maxino) errx(8, "bad inode number %d to ginode", inumber); if (startinum == 0 || inumber < startinum || inumber >= startinum + INOPB(&sblock)) { inoblk = fsbtodb(&sblock, ino_to_fsba(&sblock, inumber)); rdfs(inoblk, (size_t)sblock.fs_bsize, inobuf, fsi); startinum = (inumber / INOPB(&sblock)) * INOPB(&sblock); } DBG_LEAVE; if (sblock.fs_magic == FS_UFS1_MAGIC) return (union dinode *)((uintptr_t)inobuf + (inumber % INOPB(&sblock)) * sizeof(struct ufs1_dinode)); return (union dinode *)((uintptr_t)inobuf + (inumber % INOPB(&sblock)) * sizeof(struct ufs2_dinode)); } /* ****************************************************** charsperline ***** */ /* * Figure out how many lines our current terminal has. For more details again * please see the source of newfs(8), as this function is taken over almost * unchanged. */ static int charsperline(void) { DBG_FUNC("charsperline") int columns; char *cp; struct winsize ws; DBG_ENTER; columns = 0; if (ioctl(0, TIOCGWINSZ, &ws) != -1) { columns = ws.ws_col; } if (columns == 0 && (cp = getenv("COLUMNS"))) { columns = atoi(cp); } if (columns == 0) { columns = 80; /* last resort */ } DBG_LEAVE; return columns; } /* ****************************************************** get_dev_size ***** */ /* * Get the size of the partition if we can't figure it out from the disklabel, * e.g. from vinum volumes. */ static void get_dev_size(int fd, int *size) { int sectorsize; off_t mediasize; if (ioctl(fd, DIOCGSECTORSIZE, §orsize) == -1) err(1,"DIOCGSECTORSIZE"); if (ioctl(fd, DIOCGMEDIASIZE, &mediasize) == -1) err(1,"DIOCGMEDIASIZE"); if (sectorsize <= 0) errx(1, "bogus sectorsize: %d", sectorsize); *size = mediasize / sectorsize; } /* ************************************************************** main ***** */ /* * growfs(8) is a utility which allows to increase the size of an existing * ufs file system. Currently this can only be done on unmounted file system. * It recognizes some command line options to specify the new desired size, * and it does some basic checkings. The old file system size is determined * and after some more checks like we can really access the new last block * on the disk etc. we calculate the new parameters for the superblock. After * having done this we just call growfs() which will do the work. Before * we finish the only thing left is to update the disklabel. * We still have to provide support for snapshots. Therefore we first have to * understand what data structures are always replicated in the snapshot on * creation, for all other blocks we touch during our procedure, we have to * keep the old blocks unchanged somewhere available for the snapshots. If we * are lucky, then we only have to handle our blocks to be relocated in that * way. * Also we have to consider in what order we actually update the critical * data structures of the file system to make sure, that in case of a disaster * fsck(8) is still able to restore any lost data. * The foreseen last step then will be to provide for growing even mounted * file systems. There we have to extend the mount() system call to provide * userland access to the file system locking facility. */ int main(int argc, char **argv) { DBG_FUNC("main") char *device, *special, *cp; int ch; unsigned int size=0; size_t len; unsigned int Nflag=0; int ExpertFlag=0; struct stat st; struct disklabel *lp; struct partition *pp; int i,fsi,fso; u_int32_t p_size; char reply[5]; #ifdef FSMAXSNAP int j; #endif /* FSMAXSNAP */ DBG_ENTER; while((ch=getopt(argc, argv, "Ns:vy")) != -1) { switch(ch) { case 'N': Nflag=1; break; case 's': size=(size_t)atol(optarg); if(size<1) { usage(); } break; case 'v': /* for compatibility to newfs */ break; case 'y': ExpertFlag=1; break; case '?': /* FALLTHROUGH */ default: usage(); } } argc -= optind; argv += optind; if(argc != 1) { usage(); } device=*argv; /* * Now try to guess the (raw)device name. */ if (0 == strrchr(device, '/')) { /* * No path prefix was given, so try in that order: * /dev/r%s * /dev/%s * /dev/vinum/r%s * /dev/vinum/%s. * * FreeBSD now doesn't distinguish between raw and block * devices any longer, but it should still work this way. */ len=strlen(device)+strlen(_PATH_DEV)+2+strlen("vinum/"); special=(char *)malloc(len); if(special == NULL) { errx(1, "malloc failed"); } snprintf(special, len, "%sr%s", _PATH_DEV, device); if (stat(special, &st) == -1) { snprintf(special, len, "%s%s", _PATH_DEV, device); if (stat(special, &st) == -1) { snprintf(special, len, "%svinum/r%s", _PATH_DEV, device); if (stat(special, &st) == -1) { /* For now this is the 'last resort' */ snprintf(special, len, "%svinum/%s", _PATH_DEV, device); } } } device = special; } /* * Try to access our devices for writing ... */ if (Nflag) { fso = -1; } else { fso = open(device, O_WRONLY); if (fso < 0) { err(1, "%s", device); } } /* * ... and reading. */ fsi = open(device, O_RDONLY); if (fsi < 0) { err(1, "%s", device); } /* * Try to read a label and guess the slice if not specified. This * code should guess the right thing and avoid to bother the user * with the task of specifying the option -v on vinum volumes. */ cp=device+strlen(device)-1; lp = get_disklabel(fsi); pp = NULL; if (lp != NULL) { if (isdigit(*cp)) { pp = &lp->d_partitions[2]; } else if (*cp>='a' && *cp<='h') { pp = &lp->d_partitions[*cp - 'a']; } else { errx(1, "unknown device"); } p_size = pp->p_size; } else { get_dev_size(fsi, &p_size); } /* * Check if that partition is suitable for growing a file system. */ if (p_size < 1) { errx(1, "partition is unavailable"); } /* * Read the current superblock, and take a backup. */ for (i = 0; sblock_try[i] != -1; i++) { sblockloc = sblock_try[i] / DEV_BSIZE; rdfs(sblockloc, (size_t)SBLOCKSIZE, (void *)&(osblock), fsi); if ((osblock.fs_magic == FS_UFS1_MAGIC || (osblock.fs_magic == FS_UFS2_MAGIC && osblock.fs_sblockloc == sblock_try[i])) && osblock.fs_bsize <= MAXBSIZE && osblock.fs_bsize >= (int32_t) sizeof(struct fs)) break; } if (sblock_try[i] == -1) { errx(1, "superblock not recognized"); } memcpy((void *)&fsun1, (void *)&fsun2, sizeof(fsun2)); maxino = sblock.fs_ncg * sblock.fs_ipg; DBG_OPEN("/tmp/growfs.debug"); /* already here we need a superblock */ DBG_DUMP_FS(&sblock, "old sblock"); /* * Determine size to grow to. Default to the full size specified in * the disk label. */ sblock.fs_size = dbtofsb(&osblock, p_size); if (size != 0) { if (size > p_size){ errx(1, "there is not enough space (%d < %d)", p_size, size); } sblock.fs_size = dbtofsb(&osblock, size); } /* * Are we really growing ? */ if(osblock.fs_size >= sblock.fs_size) { errx(1, "we are not growing (%jd->%jd)", (intmax_t)osblock.fs_size, (intmax_t)sblock.fs_size); } #ifdef FSMAXSNAP /* * Check if we find an active snapshot. */ if(ExpertFlag == 0) { for(j=0; j sblock.fs_old_ncyl * sblock.fs_old_spc) sblock.fs_old_ncyl++; } sblock.fs_ncg = howmany(sblock.fs_size, sblock.fs_fpg); maxino = sblock.fs_ncg * sblock.fs_ipg; if (sblock.fs_size % sblock.fs_fpg != 0 && sblock.fs_size % sblock.fs_fpg < cgdmin(&sblock, sblock.fs_ncg)) { /* * The space in the new last cylinder group is too small, * so revert back. */ sblock.fs_ncg--; if (sblock.fs_magic == FS_UFS1_MAGIC) sblock.fs_old_ncyl = sblock.fs_ncg * sblock.fs_old_cpg; printf("Warning: %jd sector(s) cannot be allocated.\n", (intmax_t)fsbtodb(&sblock, sblock.fs_size % sblock.fs_fpg)); sblock.fs_size = sblock.fs_ncg * sblock.fs_fpg; maxino -= sblock.fs_ipg; } /* * Update the space for the cylinder group summary information in the * respective cylinder group data area. */ sblock.fs_cssize = fragroundup(&sblock, sblock.fs_ncg * sizeof(struct csum)); if(osblock.fs_size >= sblock.fs_size) { errx(1, "not enough new space"); } DBG_PRINT0("sblock calculated\n"); /* * Ok, everything prepared, so now let's do the tricks. */ growfs(fsi, fso, Nflag); /* * Update the disk label. */ if (!unlabeled) { pp->p_fsize = sblock.fs_fsize; pp->p_frag = sblock.fs_frag; pp->p_cpg = sblock.fs_fpg; return_disklabel(fso, lp, Nflag); DBG_PRINT0("label rewritten\n"); } close(fsi); if(fso>-1) close(fso); DBG_CLOSE; DBG_LEAVE; return 0; } /* ************************************************** return_disklabel ***** */ /* * Write the updated disklabel back to disk. */ static void return_disklabel(int fd, struct disklabel *lp, unsigned int Nflag) { DBG_FUNC("return_disklabel") u_short sum; u_short *ptr; DBG_ENTER; if(!lp) { DBG_LEAVE; return; } if(!Nflag) { lp->d_checksum=0; sum = 0; ptr=(u_short *)lp; /* * recalculate checksum */ while(ptr < (u_short *)&lp->d_partitions[lp->d_npartitions]) { sum ^= *ptr++; } lp->d_checksum=sum; if (ioctl(fd, DIOCWDINFO, (char *)lp) < 0) { errx(1, "DIOCWDINFO failed"); } } free(lp); DBG_LEAVE; return ; } /* ***************************************************** get_disklabel ***** */ /* * Read the disklabel from disk. */ static struct disklabel * get_disklabel(int fd) { DBG_FUNC("get_disklabel") static struct disklabel *lab; DBG_ENTER; lab=(struct disklabel *)malloc(sizeof(struct disklabel)); if (!lab) errx(1, "malloc failed"); if (!ioctl(fd, DIOCGDINFO, (char *)lab)) return (lab); unlabeled++; DBG_LEAVE; return (NULL); } /* ************************************************************* usage ***** */ /* * Dump a line of usage. */ static void usage(void) { DBG_FUNC("usage") DBG_ENTER; fprintf(stderr, "usage: growfs [-Ny] [-s size] special\n"); DBG_LEAVE; exit(1); } /* *********************************************************** updclst ***** */ /* * This updates most parameters and the bitmap related to cluster. We have to * assume that sblock, osblock, acg are set up. */ static void updclst(int block) { DBG_FUNC("updclst") static int lcs=0; DBG_ENTER; if(sblock.fs_contigsumsize < 1) { /* no clustering */ return; } /* * update cluster allocation map */ setbit(cg_clustersfree(&acg), block); /* * update cluster summary table */ if(!lcs) { /* * calculate size for the trailing cluster */ for(block--; lcs 0 && i < NIADDR; i++) { iptr = DIP(ino, di_ib[i]); if (iptr == 0) continue; if (cond_bl_upd(&iptr, bp, fsi, fso, Nflag)) { DIP_SET(ino, di_ib[i], iptr); inodeupdated++; } indirchk(blksperindir, lbn, iptr, numblks, bp, fsi, fso, Nflag); blksperindir *= NINDIR(&sblock); lbn += blksperindir; len -= blksperindir; DBG_PRINT1("scg indirect_%d blocks checked\n", i + 1); } if (inodeupdated) wtfs(inoblk, sblock.fs_bsize, inobuf, fso, Nflag); DBG_LEAVE; return; } /* * Recursively check all the indirect blocks. */ static void indirchk(ufs_lbn_t blksperindir, ufs_lbn_t lbn, ufs2_daddr_t blkno, ufs_lbn_t lastlbn, struct gfs_bpp *bp, int fsi, int fso, unsigned int Nflag) { DBG_FUNC("indirchk") void *ibuf; int i, last; ufs2_daddr_t iptr; DBG_ENTER; /* read in the indirect block. */ ibuf = malloc(sblock.fs_bsize); if (!ibuf) errx(1, "malloc failed"); rdfs(fsbtodb(&sblock, blkno), (size_t)sblock.fs_bsize, ibuf, fsi); last = howmany(lastlbn - lbn, blksperindir) < NINDIR(&sblock) ? howmany(lastlbn - lbn, blksperindir) : NINDIR(&sblock); for (i = 0; i < last; i++) { if (sblock.fs_magic == FS_UFS1_MAGIC) iptr = ((ufs1_daddr_t *)ibuf)[i]; else iptr = ((ufs2_daddr_t *)ibuf)[i]; if (iptr == 0) continue; if (cond_bl_upd(&iptr, bp, fsi, fso, Nflag)) { if (sblock.fs_magic == FS_UFS1_MAGIC) ((ufs1_daddr_t *)ibuf)[i] = iptr; else ((ufs2_daddr_t *)ibuf)[i] = iptr; } if (blksperindir == 1) continue; indirchk(blksperindir / NINDIR(&sblock), lbn + blksperindir * i, iptr, lastlbn, bp, fsi, fso, Nflag); } free(ibuf); DBG_LEAVE; return; } Index: user/imp/tbemd/sbin/restore/tape.c =================================================================== --- user/imp/tbemd/sbin/restore/tape.c (revision 203849) +++ user/imp/tbemd/sbin/restore/tape.c (revision 203850) @@ -1,1746 +1,1746 @@ /* * Copyright (c) 1983, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef lint #if 0 static char sccsid[] = "@(#)tape.c 8.9 (Berkeley) 5/1/95"; #endif #endif /* not lint */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "restore.h" #include "extern.h" static long fssize = MAXBSIZE; static int mt = -1; static int pipein = 0; static int pipecmdin = 0; static FILE *popenfp = NULL; static char *magtape; static int blkcnt; static int numtrec; static char *tapebuf; static union u_spcl endoftapemark; static long byteslide = 0; static long blksread; /* blocks read since last header */ static int64_t tapeaddr = 0; /* current TP_BSIZE tape record */ static long tapesread; static jmp_buf restart; static int gettingfile = 0; /* restart has a valid frame */ static char *host = NULL; static int readmapflag; static int ofile; static char *map; static char lnkbuf[MAXPATHLEN + 1]; static int pathlen; int Bcvt; /* Swap Bytes */ int oldinofmt; /* FreeBSD 1 inode format needs cvt */ #define FLUSHTAPEBUF() blkcnt = ntrec + 1 char *namespace_names[] = EXTATTR_NAMESPACE_NAMES; static void accthdr(struct s_spcl *); static int checksum(int *); static void findinode(struct s_spcl *); static void findtapeblksize(void); static char *setupextattr(int); static void xtrattr(char *, long); static void set_extattr_link(char *, void *, int); static void set_extattr_fd(int, char *, void *, int); static int gethead(struct s_spcl *); static void readtape(char *); static void setdumpnum(void); static u_long swabl(u_long); static u_char *swablong(u_char *, int); static u_char *swabshort(u_char *, int); static void terminateinput(void); static void xtrfile(char *, long); static void xtrlnkfile(char *, long); static void xtrlnkskip(char *, long); static void xtrmap(char *, long); static void xtrmapskip(char *, long); static void xtrskip(char *, long); /* * Set up an input source */ void setinput(char *source, int ispipecommand) { FLUSHTAPEBUF(); if (bflag) newtapebuf(ntrec); else newtapebuf(NTREC > HIGHDENSITYTREC ? NTREC : HIGHDENSITYTREC); terminal = stdin; if (ispipecommand) pipecmdin++; else #ifdef RRESTORE if (strchr(source, ':')) { host = source; source = strchr(host, ':'); *source++ = '\0'; if (rmthost(host) == 0) done(1); } else #endif if (strcmp(source, "-") == 0) { /* * Since input is coming from a pipe we must establish * our own connection to the terminal. */ terminal = fopen(_PATH_TTY, "r"); if (terminal == NULL) { (void)fprintf(stderr, "cannot open %s: %s\n", _PATH_TTY, strerror(errno)); terminal = fopen(_PATH_DEVNULL, "r"); if (terminal == NULL) { (void)fprintf(stderr, "cannot open %s: %s\n", _PATH_DEVNULL, strerror(errno)); done(1); } } pipein++; } setuid(getuid()); /* no longer need or want root privileges */ magtape = strdup(source); if (magtape == NULL) { fprintf(stderr, "Cannot allocate space for magtape buffer\n"); done(1); } } void newtapebuf(long size) { static int tapebufsize = -1; ntrec = size; if (size <= tapebufsize) return; if (tapebuf != NULL) free(tapebuf - TP_BSIZE); tapebuf = malloc((size+1) * TP_BSIZE); if (tapebuf == NULL) { fprintf(stderr, "Cannot allocate space for tape buffer\n"); done(1); } tapebuf += TP_BSIZE; tapebufsize = size; } /* * Verify that the tape drive can be accessed and * that it actually is a dump tape. */ void setup(void) { int i, j, *ip; struct stat stbuf; vprintf(stdout, "Verify tape and initialize maps\n"); if (pipecmdin) { if (setenv("RESTORE_VOLUME", "1", 1) == -1) { fprintf(stderr, "Cannot set $RESTORE_VOLUME: %s\n", strerror(errno)); done(1); } popenfp = popen(magtape, "r"); mt = popenfp ? fileno(popenfp) : -1; } else #ifdef RRESTORE if (host) mt = rmtopen(magtape, 0); else #endif if (pipein) mt = 0; else mt = open(magtape, O_RDONLY, 0); if (mt < 0) { fprintf(stderr, "%s: %s\n", magtape, strerror(errno)); done(1); } volno = 1; setdumpnum(); FLUSHTAPEBUF(); - if (!pipein && !bflag) + if (!pipein && !pipecmdin && !bflag) findtapeblksize(); if (gethead(&spcl) == FAIL) { fprintf(stderr, "Tape is not a dump tape\n"); done(1); } if (pipein) { endoftapemark.s_spcl.c_magic = FS_UFS2_MAGIC; endoftapemark.s_spcl.c_type = TS_END; ip = (int *)&endoftapemark; j = sizeof(union u_spcl) / sizeof(int); i = 0; do i += *ip++; while (--j); endoftapemark.s_spcl.c_checksum = CHECKSUM - i; } if (vflag || command == 't') printdumpinfo(); dumptime = _time64_to_time(spcl.c_ddate); dumpdate = _time64_to_time(spcl.c_date); if (stat(".", &stbuf) < 0) { fprintf(stderr, "cannot stat .: %s\n", strerror(errno)); done(1); } if (stbuf.st_blksize > 0 && stbuf.st_blksize < TP_BSIZE ) fssize = TP_BSIZE; if (stbuf.st_blksize >= TP_BSIZE && stbuf.st_blksize <= MAXBSIZE) fssize = stbuf.st_blksize; if (((fssize - 1) & fssize) != 0) { fprintf(stderr, "bad block size %ld\n", fssize); done(1); } if (spcl.c_volume != 1) { fprintf(stderr, "Tape is not volume 1 of the dump\n"); done(1); } if (gethead(&spcl) == FAIL) { dprintf(stdout, "header read failed at %ld blocks\n", blksread); panic("no header after volume mark!\n"); } findinode(&spcl); if (spcl.c_type != TS_CLRI) { fprintf(stderr, "Cannot find file removal list\n"); done(1); } maxino = (spcl.c_count * TP_BSIZE * NBBY) + 1; dprintf(stdout, "maxino = %d\n", maxino); map = calloc((unsigned)1, (unsigned)howmany(maxino, NBBY)); if (map == NULL) panic("no memory for active inode map\n"); usedinomap = map; curfile.action = USING; getfile(xtrmap, xtrmapskip, xtrmapskip); if (spcl.c_type != TS_BITS) { fprintf(stderr, "Cannot find file dump list\n"); done(1); } map = calloc((unsigned)1, (unsigned)howmany(maxino, NBBY)); if (map == (char *)NULL) panic("no memory for file dump list\n"); dumpmap = map; curfile.action = USING; getfile(xtrmap, xtrmapskip, xtrmapskip); /* * If there may be whiteout entries on the tape, pretend that the * whiteout inode exists, so that the whiteout entries can be * extracted. */ SETINO(WINO, dumpmap); /* 'r' restores don't call getvol() for tape 1, so mark it as read. */ if (command == 'r') tapesread = 1; } /* * Prompt user to load a new dump volume. * "Nextvol" is the next suggested volume to use. * This suggested volume is enforced when doing full * or incremental restores, but can be overridden by * the user when only extracting a subset of the files. */ void getvol(long nextvol) { int64_t prevtapea; long i, newvol, savecnt; union u_spcl tmpspcl; # define tmpbuf tmpspcl.s_spcl char buf[TP_BSIZE]; if (nextvol == 1) { tapesread = 0; gettingfile = 0; } prevtapea = tapeaddr; savecnt = blksread; if (pipein) { if (nextvol != 1) { panic("Changing volumes on pipe input?\n"); /* Avoid looping if we couldn't ask the user. */ if (yflag || ferror(terminal) || feof(terminal)) done(1); } if (volno == 1) return; goto gethdr; } again: if (pipein) done(1); /* pipes do not get a second chance */ if (command == 'R' || command == 'r' || curfile.action != SKIP) newvol = nextvol; else newvol = 0; while (newvol <= 0) { if (tapesread == 0) { fprintf(stderr, "%s%s%s%s%s%s%s", "You have not read any tapes yet.\n", "If you are extracting just a few files,", " start with the last volume\n", "and work towards the first; restore", " can quickly skip tapes that\n", "have no further files to extract.", " Otherwise, begin with volume 1.\n"); } else { fprintf(stderr, "You have read volumes"); strcpy(buf, ": "); for (i = 0; i < 32; i++) if (tapesread & (1 << i)) { fprintf(stderr, "%s%ld", buf, i + 1); strcpy(buf, ", "); } fprintf(stderr, "\n"); } do { fprintf(stderr, "Specify next volume #: "); (void) fflush(stderr); if (fgets(buf, BUFSIZ, terminal) == NULL) done(1); } while (buf[0] == '\n'); newvol = atoi(buf); if (newvol <= 0) { fprintf(stderr, "Volume numbers are positive numerics\n"); } } if (newvol == volno) { tapesread |= 1 << (volno - 1); return; } closemt(); fprintf(stderr, "Mount tape volume %ld\n", newvol); fprintf(stderr, "Enter ``none'' if there are no more tapes\n"); fprintf(stderr, "otherwise enter tape name (default: %s) ", magtape); (void) fflush(stderr); if (fgets(buf, BUFSIZ, terminal) == NULL) done(1); if (!strcmp(buf, "none\n")) { terminateinput(); return; } if (buf[0] != '\n') { (void) strcpy(magtape, buf); magtape[strlen(magtape) - 1] = '\0'; } if (pipecmdin) { char volno[sizeof("2147483647")]; (void)sprintf(volno, "%ld", newvol); if (setenv("RESTORE_VOLUME", volno, 1) == -1) { fprintf(stderr, "Cannot set $RESTORE_VOLUME: %s\n", strerror(errno)); done(1); } popenfp = popen(magtape, "r"); mt = popenfp ? fileno(popenfp) : -1; } else #ifdef RRESTORE if (host) mt = rmtopen(magtape, 0); else #endif mt = open(magtape, O_RDONLY, 0); if (mt == -1) { fprintf(stderr, "Cannot open %s\n", magtape); volno = -1; goto again; } gethdr: volno = newvol; setdumpnum(); FLUSHTAPEBUF(); if (gethead(&tmpbuf) == FAIL) { dprintf(stdout, "header read failed at %ld blocks\n", blksread); fprintf(stderr, "tape is not dump tape\n"); volno = 0; goto again; } if (tmpbuf.c_volume != volno) { fprintf(stderr, "Wrong volume (%jd)\n", (intmax_t)tmpbuf.c_volume); volno = 0; goto again; } if (_time64_to_time(tmpbuf.c_date) != dumpdate || _time64_to_time(tmpbuf.c_ddate) != dumptime) { time_t t = _time64_to_time(tmpbuf.c_date); fprintf(stderr, "Wrong dump date\n\tgot: %s", ctime(&t)); fprintf(stderr, "\twanted: %s", ctime(&dumpdate)); volno = 0; goto again; } tapesread |= 1 << (volno - 1); blksread = savecnt; /* * If continuing from the previous volume, skip over any * blocks read already at the end of the previous volume. * * If coming to this volume at random, skip to the beginning * of the next record. */ dprintf(stdout, "last rec %jd, tape starts with %jd\n", (intmax_t)prevtapea, (intmax_t)tmpbuf.c_tapea); if (tmpbuf.c_type == TS_TAPE) { if (curfile.action != USING) { /* * XXX Dump incorrectly sets c_count to 1 in the * volume header of the first tape, so ignore * c_count when volno == 1. */ if (volno != 1) for (i = tmpbuf.c_count; i > 0; i--) readtape(buf); } else if (tmpbuf.c_tapea <= prevtapea) { /* * Normally the value of c_tapea in the volume * header is the record number of the header itself. * However in the volume header following an EOT- * terminated tape, it is the record number of the * first continuation data block (dump bug?). * * The next record we want is `prevtapea + 1'. */ i = prevtapea + 1 - tmpbuf.c_tapea; dprintf(stderr, "Skipping %ld duplicate record%s.\n", i, i > 1 ? "s" : ""); while (--i >= 0) readtape(buf); } } if (curfile.action == USING) { if (volno == 1) panic("active file into volume 1\n"); return; } (void) gethead(&spcl); findinode(&spcl); if (gettingfile) { gettingfile = 0; longjmp(restart, 1); } } /* * Handle unexpected EOF. */ static void terminateinput(void) { if (gettingfile && curfile.action == USING) { printf("Warning: %s %s\n", "End-of-input encountered while extracting", curfile.name); } curfile.name = ""; curfile.action = UNKNOWN; curfile.mode = 0; curfile.ino = maxino; if (gettingfile) { gettingfile = 0; longjmp(restart, 1); } } /* * handle multiple dumps per tape by skipping forward to the * appropriate one. */ static void setdumpnum(void) { struct mtop tcom; if (dumpnum == 1 || volno != 1) return; if (pipein) { fprintf(stderr, "Cannot have multiple dumps on pipe input\n"); done(1); } tcom.mt_op = MTFSF; tcom.mt_count = dumpnum - 1; #ifdef RRESTORE if (host) rmtioctl(MTFSF, dumpnum - 1); else #endif if (!pipecmdin && ioctl(mt, MTIOCTOP, (char *)&tcom) < 0) fprintf(stderr, "ioctl MTFSF: %s\n", strerror(errno)); } void printdumpinfo(void) { time_t t; t = _time64_to_time(spcl.c_date); fprintf(stdout, "Dump date: %s", ctime(&t)); t = _time64_to_time(spcl.c_ddate); fprintf(stdout, "Dumped from: %s", (spcl.c_ddate == 0) ? "the epoch\n" : ctime(&t)); if (spcl.c_host[0] == '\0') return; fprintf(stderr, "Level %jd dump of %s on %s:%s\n", (intmax_t)spcl.c_level, spcl.c_filesys, spcl.c_host, spcl.c_dev); fprintf(stderr, "Label: %s\n", spcl.c_label); } int extractfile(char *name) { int flags; uid_t uid; gid_t gid; mode_t mode; int extsize; struct timeval mtimep[2], ctimep[2]; struct entry *ep; char *buf; curfile.name = name; curfile.action = USING; mtimep[0].tv_sec = curfile.atime_sec; mtimep[0].tv_usec = curfile.atime_nsec / 1000; mtimep[1].tv_sec = curfile.mtime_sec; mtimep[1].tv_usec = curfile.mtime_nsec / 1000; ctimep[0].tv_sec = curfile.atime_sec; ctimep[0].tv_usec = curfile.atime_nsec / 1000; ctimep[1].tv_sec = curfile.birthtime_sec; ctimep[1].tv_usec = curfile.birthtime_nsec / 1000; extsize = curfile.extsize; uid = getuid(); if (uid == 0) uid = curfile.uid; gid = curfile.gid; mode = curfile.mode; flags = curfile.file_flags; switch (mode & IFMT) { default: fprintf(stderr, "%s: unknown file mode 0%o\n", name, mode); skipfile(); return (FAIL); case IFSOCK: vprintf(stdout, "skipped socket %s\n", name); skipfile(); return (GOOD); case IFDIR: if (mflag) { ep = lookupname(name); if (ep == NULL || ep->e_flags & EXTRACT) panic("unextracted directory %s\n", name); skipfile(); return (GOOD); } vprintf(stdout, "extract file %s\n", name); return (genliteraldir(name, curfile.ino)); case IFLNK: lnkbuf[0] = '\0'; pathlen = 0; buf = setupextattr(extsize); getfile(xtrlnkfile, xtrattr, xtrlnkskip); if (pathlen == 0) { vprintf(stdout, "%s: zero length symbolic link (ignored)\n", name); return (GOOD); } if (linkit(lnkbuf, name, SYMLINK) == GOOD) { if (extsize > 0) set_extattr_link(name, buf, extsize); (void) lchown(name, uid, gid); (void) lchmod(name, mode); (void) lutimes(name, ctimep); (void) lutimes(name, mtimep); (void) lchflags(name, flags); return (GOOD); } return (FAIL); case IFIFO: vprintf(stdout, "extract fifo %s\n", name); if (Nflag) { skipfile(); return (GOOD); } if (uflag) (void) unlink(name); if (mkfifo(name, 0600) < 0) { fprintf(stderr, "%s: cannot create fifo: %s\n", name, strerror(errno)); skipfile(); return (FAIL); } if (extsize == 0) { skipfile(); } else { buf = setupextattr(extsize); getfile(xtrnull, xtrattr, xtrnull); set_extattr_file(name, buf, extsize); } (void) chown(name, uid, gid); (void) chmod(name, mode); (void) utimes(name, ctimep); (void) utimes(name, mtimep); (void) chflags(name, flags); return (GOOD); case IFCHR: case IFBLK: vprintf(stdout, "extract special file %s\n", name); if (Nflag) { skipfile(); return (GOOD); } if (uflag) (void) unlink(name); if (mknod(name, (mode & (IFCHR | IFBLK)) | 0600, (int)curfile.rdev) < 0) { fprintf(stderr, "%s: cannot create special file: %s\n", name, strerror(errno)); skipfile(); return (FAIL); } if (extsize == 0) { skipfile(); } else { buf = setupextattr(extsize); getfile(xtrnull, xtrattr, xtrnull); set_extattr_file(name, buf, extsize); } (void) chown(name, uid, gid); (void) chmod(name, mode); (void) utimes(name, ctimep); (void) utimes(name, mtimep); (void) chflags(name, flags); return (GOOD); case IFREG: vprintf(stdout, "extract file %s\n", name); if (Nflag) { skipfile(); return (GOOD); } if (uflag) (void) unlink(name); if ((ofile = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0600)) < 0) { fprintf(stderr, "%s: cannot create file: %s\n", name, strerror(errno)); skipfile(); return (FAIL); } buf = setupextattr(extsize); getfile(xtrfile, xtrattr, xtrskip); if (extsize > 0) set_extattr_fd(ofile, name, buf, extsize); (void) fchown(ofile, uid, gid); (void) fchmod(ofile, mode); (void) futimes(ofile, ctimep); (void) futimes(ofile, mtimep); (void) fchflags(ofile, flags); (void) close(ofile); return (GOOD); } /* NOTREACHED */ } /* * Set attributes for a file. */ void set_extattr_file(char *name, void *buf, int size) { struct extattr *eap, *eaend; vprintf(stdout, "Set attributes for %s:", name); eaend = buf + size; for (eap = buf; eap < eaend; eap = EXTATTR_NEXT(eap)) { /* * Make sure this entry is complete. */ if (EXTATTR_NEXT(eap) > eaend || eap->ea_length <= 0) { dprintf(stdout, "\n\t%scorrupted", eap == buf ? "" : "remainder "); break; } if (eap->ea_namespace == EXTATTR_NAMESPACE_EMPTY) continue; vprintf(stdout, "\n\t%s, (%d bytes), %*s", namespace_names[eap->ea_namespace], eap->ea_length, eap->ea_namelength, eap->ea_name); /* * First we try the general attribute setting interface. * However, some attributes can only be set by root or * by using special interfaces (for example, ACLs). */ if (extattr_set_file(name, eap->ea_namespace, eap->ea_name, EXTATTR_CONTENT(eap), EXTATTR_CONTENT_SIZE(eap)) != -1) { dprintf(stdout, " (set using extattr_set_file)"); continue; } /* * If the general interface refuses to set the attribute, * then we try all the specialized interfaces that we * know about. */ if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_ACCESS_EXTATTR_NAME)) { if (acl_set_file(name, ACL_TYPE_ACCESS, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_file)"); continue; } } if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_DEFAULT_EXTATTR_NAME)) { if (acl_set_file(name, ACL_TYPE_DEFAULT, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_file)"); continue; } } vprintf(stdout, " (unable to set)"); } vprintf(stdout, "\n"); } /* * Set attributes for a symbolic link. */ static void set_extattr_link(char *name, void *buf, int size) { struct extattr *eap, *eaend; vprintf(stdout, "Set attributes for %s:", name); eaend = buf + size; for (eap = buf; eap < eaend; eap = EXTATTR_NEXT(eap)) { /* * Make sure this entry is complete. */ if (EXTATTR_NEXT(eap) > eaend || eap->ea_length <= 0) { dprintf(stdout, "\n\t%scorrupted", eap == buf ? "" : "remainder "); break; } if (eap->ea_namespace == EXTATTR_NAMESPACE_EMPTY) continue; vprintf(stdout, "\n\t%s, (%d bytes), %*s", namespace_names[eap->ea_namespace], eap->ea_length, eap->ea_namelength, eap->ea_name); /* * First we try the general attribute setting interface. * However, some attributes can only be set by root or * by using special interfaces (for example, ACLs). */ if (extattr_set_link(name, eap->ea_namespace, eap->ea_name, EXTATTR_CONTENT(eap), EXTATTR_CONTENT_SIZE(eap)) != -1) { dprintf(stdout, " (set using extattr_set_link)"); continue; } /* * If the general interface refuses to set the attribute, * then we try all the specialized interfaces that we * know about. */ if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_ACCESS_EXTATTR_NAME)) { if (acl_set_link_np(name, ACL_TYPE_ACCESS, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_link_np)"); continue; } } if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_DEFAULT_EXTATTR_NAME)) { if (acl_set_link_np(name, ACL_TYPE_DEFAULT, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_link_np)"); continue; } } vprintf(stdout, " (unable to set)"); } vprintf(stdout, "\n"); } /* * Set attributes on a file descriptor. */ static void set_extattr_fd(int fd, char *name, void *buf, int size) { struct extattr *eap, *eaend; vprintf(stdout, "Set attributes for %s:", name); eaend = buf + size; for (eap = buf; eap < eaend; eap = EXTATTR_NEXT(eap)) { /* * Make sure this entry is complete. */ if (EXTATTR_NEXT(eap) > eaend || eap->ea_length <= 0) { dprintf(stdout, "\n\t%scorrupted", eap == buf ? "" : "remainder "); break; } if (eap->ea_namespace == EXTATTR_NAMESPACE_EMPTY) continue; vprintf(stdout, "\n\t%s, (%d bytes), %*s", namespace_names[eap->ea_namespace], eap->ea_length, eap->ea_namelength, eap->ea_name); /* * First we try the general attribute setting interface. * However, some attributes can only be set by root or * by using special interfaces (for example, ACLs). */ if (extattr_set_fd(fd, eap->ea_namespace, eap->ea_name, EXTATTR_CONTENT(eap), EXTATTR_CONTENT_SIZE(eap)) != -1) { dprintf(stdout, " (set using extattr_set_fd)"); continue; } /* * If the general interface refuses to set the attribute, * then we try all the specialized interfaces that we * know about. */ if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_ACCESS_EXTATTR_NAME)) { if (acl_set_fd(fd, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_fd)"); continue; } } if (eap->ea_namespace == EXTATTR_NAMESPACE_SYSTEM && !strcmp(eap->ea_name, POSIX1E_ACL_DEFAULT_EXTATTR_NAME)) { if (acl_set_file(name, ACL_TYPE_DEFAULT, EXTATTR_CONTENT(eap)) != -1) { dprintf(stdout, " (set using acl_set_file)"); continue; } } vprintf(stdout, " (unable to set)"); } vprintf(stdout, "\n"); } /* * skip over bit maps on the tape */ void skipmaps(void) { while (spcl.c_type == TS_BITS || spcl.c_type == TS_CLRI) skipfile(); } /* * skip over a file on the tape */ void skipfile(void) { curfile.action = SKIP; getfile(xtrnull, xtrnull, xtrnull); } /* * Extract a file from the tape. * When an allocated block is found it is passed to the fill function; * when an unallocated block (hole) is found, a zeroed buffer is passed * to the skip function. */ void getfile(void (*datafill)(char *, long), void (*attrfill)(char *, long), void (*skip)(char *, long)) { int i; off_t size; int curblk, attrsize; void (*fillit)(char *, long); static char clearedbuf[MAXBSIZE]; char buf[MAXBSIZE / TP_BSIZE][TP_BSIZE]; char junk[TP_BSIZE]; curblk = 0; size = spcl.c_size; attrsize = spcl.c_extsize; if (spcl.c_type == TS_END) panic("ran off end of tape\n"); if (spcl.c_magic != FS_UFS2_MAGIC) panic("not at beginning of a file\n"); if (!gettingfile && setjmp(restart) != 0) return; gettingfile++; fillit = datafill; if (size == 0 && attrsize > 0) { fillit = attrfill; size = attrsize; attrsize = 0; } loop: for (i = 0; i < spcl.c_count; i++) { if (!readmapflag && i > TP_NINDIR) { if (Dflag) { fprintf(stderr, "spcl.c_count = %jd\n", (intmax_t)spcl.c_count); break; } else panic("spcl.c_count = %jd\n", (intmax_t)spcl.c_count); } if (readmapflag || spcl.c_addr[i]) { readtape(&buf[curblk++][0]); if (curblk == fssize / TP_BSIZE) { (*fillit)((char *)buf, (long)(size > TP_BSIZE ? fssize : (curblk - 1) * TP_BSIZE + size)); curblk = 0; } } else { if (curblk > 0) { (*fillit)((char *)buf, (long)(size > TP_BSIZE ? curblk * TP_BSIZE : (curblk - 1) * TP_BSIZE + size)); curblk = 0; } (*skip)(clearedbuf, (long)(size > TP_BSIZE ? TP_BSIZE : size)); } if ((size -= TP_BSIZE) <= 0) { if (size > -TP_BSIZE && curblk > 0) { (*fillit)((char *)buf, (long)((curblk * TP_BSIZE) + size)); curblk = 0; } if (attrsize > 0) { fillit = attrfill; size = attrsize; attrsize = 0; continue; } if (spcl.c_count - i > 1) dprintf(stdout, "skipping %d junk block(s)\n", spcl.c_count - i - 1); for (i++; i < spcl.c_count; i++) { if (!readmapflag && i > TP_NINDIR) { if (Dflag) { fprintf(stderr, "spcl.c_count = %jd\n", (intmax_t)spcl.c_count); break; } else panic("spcl.c_count = %jd\n", (intmax_t)spcl.c_count); } if (readmapflag || spcl.c_addr[i]) readtape(junk); } break; } } if (gethead(&spcl) == GOOD && size > 0) { if (spcl.c_type == TS_ADDR) goto loop; dprintf(stdout, "Missing address (header) block for %s at %ld blocks\n", curfile.name, blksread); } if (curblk > 0) panic("getfile: lost data\n"); findinode(&spcl); gettingfile = 0; } /* * These variables are shared between the next two functions. */ static int extbufsize = 0; static char *extbuf; static int extloc; /* * Allocate a buffer into which to extract extended attributes. */ static char * setupextattr(int extsize) { extloc = 0; if (extsize <= extbufsize) return (extbuf); if (extbufsize > 0) free(extbuf); if ((extbuf = malloc(extsize)) != NULL) { extbufsize = extsize; return (extbuf); } extbufsize = 0; extbuf = NULL; fprintf(stderr, "Cannot extract %d bytes %s for inode %d, name %s\n", extsize, "of extended attributes", curfile.ino, curfile.name); return (NULL); } /* * Extract the next block of extended attributes. */ static void xtrattr(char *buf, long size) { if (extloc + size > extbufsize) panic("overrun attribute buffer\n"); memmove(&extbuf[extloc], buf, size); extloc += size; } /* * Write out the next block of a file. */ static void xtrfile(char *buf, long size) { if (Nflag) return; if (write(ofile, buf, (int) size) == -1) { fprintf(stderr, "write error extracting inode %d, name %s\nwrite: %s\n", curfile.ino, curfile.name, strerror(errno)); } } /* * Skip over a hole in a file. */ /* ARGSUSED */ static void xtrskip(char *buf, long size) { if (lseek(ofile, size, SEEK_CUR) == -1) { fprintf(stderr, "seek error extracting inode %d, name %s\nlseek: %s\n", curfile.ino, curfile.name, strerror(errno)); done(1); } } /* * Collect the next block of a symbolic link. */ static void xtrlnkfile(char *buf, long size) { pathlen += size; if (pathlen > MAXPATHLEN) { fprintf(stderr, "symbolic link name: %s->%s%s; too long %d\n", curfile.name, lnkbuf, buf, pathlen); done(1); } (void) strcat(lnkbuf, buf); } /* * Skip over a hole in a symbolic link (should never happen). */ /* ARGSUSED */ static void xtrlnkskip(char *buf, long size) { fprintf(stderr, "unallocated block in symbolic link %s\n", curfile.name); done(1); } /* * Collect the next block of a bit map. */ static void xtrmap(char *buf, long size) { memmove(map, buf, size); map += size; } /* * Skip over a hole in a bit map (should never happen). */ /* ARGSUSED */ static void xtrmapskip(char *buf, long size) { panic("hole in map\n"); map += size; } /* * Noop, when an extraction function is not needed. */ /* ARGSUSED */ void xtrnull(char *buf, long size) { return; } /* * Read TP_BSIZE blocks from the input. * Handle read errors, and end of media. */ static void readtape(char *buf) { long rd, newvol, i, oldnumtrec; int cnt, seek_failed; if (blkcnt + (byteslide > 0) < numtrec) { memmove(buf, &tapebuf[(blkcnt++ * TP_BSIZE) + byteslide], (long)TP_BSIZE); blksread++; tapeaddr++; return; } if (numtrec > 0) memmove(&tapebuf[-TP_BSIZE], &tapebuf[(numtrec-1) * TP_BSIZE], (long)TP_BSIZE); oldnumtrec = numtrec; for (i = 0; i < ntrec; i++) ((struct s_spcl *)&tapebuf[i * TP_BSIZE])->c_magic = 0; if (numtrec == 0) numtrec = ntrec; cnt = ntrec * TP_BSIZE; rd = 0; getmore: #ifdef RRESTORE if (host) i = rmtread(&tapebuf[rd], cnt); else #endif i = read(mt, &tapebuf[rd], cnt); /* * Check for mid-tape short read error. * If found, skip rest of buffer and start with the next. */ if (!pipein && !pipecmdin && numtrec < ntrec && i > 0) { dprintf(stdout, "mid-media short read error.\n"); numtrec = ntrec; } /* * Handle partial block read. */ if ((pipein || pipecmdin) && i == 0 && rd > 0) i = rd; else if (i > 0 && i != ntrec * TP_BSIZE) { if (pipein || pipecmdin) { rd += i; cnt -= i; if (cnt > 0) goto getmore; i = rd; } else { /* * Short read. Process the blocks read. */ if (i % TP_BSIZE != 0) vprintf(stdout, "partial block read: %ld should be %ld\n", i, ntrec * TP_BSIZE); numtrec = i / TP_BSIZE; } } /* * Handle read error. */ if (i < 0) { fprintf(stderr, "Tape read error while "); switch (curfile.action) { default: fprintf(stderr, "trying to set up tape\n"); break; case UNKNOWN: fprintf(stderr, "trying to resynchronize\n"); break; case USING: fprintf(stderr, "restoring %s\n", curfile.name); break; case SKIP: fprintf(stderr, "skipping over inode %d\n", curfile.ino); break; } if (!yflag && !reply("continue")) done(1); i = ntrec * TP_BSIZE; memset(tapebuf, 0, i); #ifdef RRESTORE if (host) seek_failed = (rmtseek(i, 1) < 0); else #endif seek_failed = (lseek(mt, i, SEEK_CUR) == (off_t)-1); if (seek_failed) { fprintf(stderr, "continuation failed: %s\n", strerror(errno)); done(1); } } /* * Handle end of tape. */ if (i == 0) { vprintf(stdout, "End-of-tape encountered\n"); if (!pipein) { newvol = volno + 1; volno = 0; numtrec = 0; getvol(newvol); readtape(buf); return; } if (rd % TP_BSIZE != 0) panic("partial block read: %ld should be %ld\n", rd, ntrec * TP_BSIZE); terminateinput(); memmove(&tapebuf[rd], &endoftapemark, (long)TP_BSIZE); } if (oldnumtrec == 0) blkcnt = 0; else blkcnt -= oldnumtrec; memmove(buf, &tapebuf[(blkcnt++ * TP_BSIZE) + byteslide], (long)TP_BSIZE); blksread++; tapeaddr++; } static void findtapeblksize(void) { long i; for (i = 0; i < ntrec; i++) ((struct s_spcl *)&tapebuf[i * TP_BSIZE])->c_magic = 0; blkcnt = 0; #ifdef RRESTORE if (host) i = rmtread(tapebuf, ntrec * TP_BSIZE); else #endif i = read(mt, tapebuf, ntrec * TP_BSIZE); if (i <= 0) { fprintf(stderr, "tape read error: %s\n", strerror(errno)); done(1); } if (i % TP_BSIZE != 0) { fprintf(stderr, "Tape block size (%ld) %s (%d)\n", i, "is not a multiple of dump block size", TP_BSIZE); done(1); } ntrec = i / TP_BSIZE; numtrec = ntrec; vprintf(stdout, "Tape block size is %ld\n", ntrec); } void closemt(void) { if (mt < 0) return; if (pipecmdin) { pclose(popenfp); popenfp = NULL; } else #ifdef RRESTORE if (host) rmtclose(); else #endif (void) close(mt); } /* * Read the next block from the tape. * If it is not any valid header, return an error. */ static int gethead(struct s_spcl *buf) { long i; readtape((char *)buf); if (buf->c_magic != FS_UFS2_MAGIC && buf->c_magic != NFS_MAGIC) { if (buf->c_magic == OFS_MAGIC) { fprintf(stderr, "Format of dump tape is too old. Must use\n"); fprintf(stderr, "a version of restore from before 2002.\n"); return (FAIL); } if (swabl(buf->c_magic) != FS_UFS2_MAGIC && buf->c_magic != NFS_MAGIC) { if (buf->c_magic == OFS_MAGIC) { fprintf(stderr, "Format of dump tape is too old. Must use\n"); fprintf(stderr, "a version of restore from before 2002.\n"); } return (FAIL); } if (!Bcvt) { vprintf(stdout, "Note: Doing Byte swapping\n"); Bcvt = 1; } } if (checksum((int *)buf) == FAIL) return (FAIL); if (Bcvt) { swabst((u_char *)"8l4s1q8l2q17l", (u_char *)buf); swabst((u_char *)"l",(u_char *) &buf->c_level); swabst((u_char *)"2l4q",(u_char *) &buf->c_flags); } readmapflag = 0; switch (buf->c_type) { case TS_CLRI: case TS_BITS: /* * Have to patch up missing information in bit map headers */ buf->c_size = buf->c_count * TP_BSIZE; if (buf->c_count > TP_NINDIR) readmapflag = 1; else for (i = 0; i < buf->c_count; i++) buf->c_addr[i]++; /* FALL THROUGH */ case TS_TAPE: if (buf->c_magic == NFS_MAGIC && (buf->c_flags & NFS_DR_NEWINODEFMT) == 0) oldinofmt = 1; /* FALL THROUGH */ case TS_END: buf->c_inumber = 0; /* FALL THROUGH */ case TS_ADDR: case TS_INODE: /* * For old dump tapes, have to copy up old fields to * new locations. */ if (buf->c_magic == NFS_MAGIC) { buf->c_tapea = buf->c_old_tapea; buf->c_firstrec = buf->c_old_firstrec; buf->c_date = _time32_to_time(buf->c_old_date); buf->c_ddate = _time32_to_time(buf->c_old_ddate); buf->c_atime = _time32_to_time(buf->c_old_atime); buf->c_mtime = _time32_to_time(buf->c_old_mtime); buf->c_birthtime = 0; buf->c_birthtimensec = 0; buf->c_extsize = 0; } break; default: panic("gethead: unknown inode type %d\n", buf->c_type); break; } if (dumpdate != 0 && _time64_to_time(buf->c_date) != dumpdate) fprintf(stderr, "Header with wrong dumpdate.\n"); /* * If we're restoring a filesystem with the old (FreeBSD 1) * format inodes, copy the uid/gid to the new location */ if (oldinofmt) { buf->c_uid = buf->c_spare1[1]; buf->c_gid = buf->c_spare1[2]; } buf->c_magic = FS_UFS2_MAGIC; tapeaddr = buf->c_tapea; if (dflag) accthdr(buf); return(GOOD); } /* * Check that a header is where it belongs and predict the next header */ static void accthdr(struct s_spcl *header) { static ino_t previno = 0x7fffffff; static int prevtype; static long predict; long blks, i; if (header->c_type == TS_TAPE) { fprintf(stderr, "Volume header "); if (header->c_firstrec) fprintf(stderr, "begins with record %jd", (intmax_t)header->c_firstrec); fprintf(stderr, "\n"); previno = 0x7fffffff; return; } if (previno == 0x7fffffff) goto newcalc; switch (prevtype) { case TS_BITS: fprintf(stderr, "Dumped inodes map header"); break; case TS_CLRI: fprintf(stderr, "Used inodes map header"); break; case TS_INODE: fprintf(stderr, "File header, ino %d", previno); break; case TS_ADDR: fprintf(stderr, "File continuation header, ino %d", previno); break; case TS_END: fprintf(stderr, "End of tape header"); break; } if (predict != blksread - 1) fprintf(stderr, "; predicted %ld blocks, got %ld blocks", predict, blksread - 1); fprintf(stderr, "\n"); newcalc: blks = 0; if (header->c_type != TS_END) for (i = 0; i < header->c_count; i++) if (readmapflag || header->c_addr[i] != 0) blks++; predict = blks; blksread = 0; prevtype = header->c_type; previno = header->c_inumber; } /* * Find an inode header. * Complain if had to skip. */ static void findinode(struct s_spcl *header) { static long skipcnt = 0; long i; char buf[TP_BSIZE]; int htype; curfile.name = ""; curfile.action = UNKNOWN; curfile.mode = 0; curfile.ino = 0; do { htype = header->c_type; switch (htype) { case TS_ADDR: /* * Skip up to the beginning of the next record */ for (i = 0; i < header->c_count; i++) if (header->c_addr[i]) readtape(buf); while (gethead(header) == FAIL || _time64_to_time(header->c_date) != dumpdate) { skipcnt++; if (Dflag) { byteslide++; if (byteslide < TP_BSIZE) { blkcnt--; blksread--; } else byteslide = 0; } } break; case TS_INODE: curfile.mode = header->c_mode; curfile.uid = header->c_uid; curfile.gid = header->c_gid; curfile.file_flags = header->c_file_flags; curfile.rdev = header->c_rdev; curfile.atime_sec = header->c_atime; curfile.atime_nsec = header->c_atimensec; curfile.mtime_sec = header->c_mtime; curfile.mtime_nsec = header->c_mtimensec; curfile.birthtime_sec = header->c_birthtime; curfile.birthtime_nsec = header->c_birthtimensec; curfile.extsize = header->c_extsize; curfile.size = header->c_size; curfile.ino = header->c_inumber; break; case TS_END: /* If we missed some tapes, get another volume. */ if (tapesread & (tapesread + 1)) { getvol(0); continue; } curfile.ino = maxino; break; case TS_CLRI: curfile.name = ""; break; case TS_BITS: curfile.name = ""; break; case TS_TAPE: if (Dflag) fprintf(stderr, "unexpected tape header\n"); else panic("unexpected tape header\n"); default: if (Dflag) fprintf(stderr, "unknown tape header type %d\n", spcl.c_type); else panic("unknown tape header type %d\n", spcl.c_type); while (gethead(header) == FAIL || _time64_to_time(header->c_date) != dumpdate) { skipcnt++; if (Dflag) { byteslide++; if (byteslide < TP_BSIZE) { blkcnt--; blksread--; } else byteslide = 0; } } } } while (htype == TS_ADDR); if (skipcnt > 0) fprintf(stderr, "resync restore, skipped %ld %s\n", skipcnt, Dflag ? "bytes" : "blocks"); skipcnt = 0; } static int checksum(int *buf) { int i, j; j = sizeof(union u_spcl) / sizeof(int); i = 0; if (!Bcvt) { do i += *buf++; while (--j); } else { /* What happens if we want to read restore tapes for a 16bit int machine??? */ do i += swabl(*buf++); while (--j); } if (i != CHECKSUM) { fprintf(stderr, "Checksum error %o, inode %d file %s\n", i, curfile.ino, curfile.name); return(FAIL); } return(GOOD); } #ifdef RRESTORE #include void msg(const char *fmt, ...) { va_list ap; va_start(ap, fmt); (void)vfprintf(stderr, fmt, ap); va_end(ap); } #endif /* RRESTORE */ static u_char * swabshort(u_char *sp, int n) { char c; while (--n >= 0) { c = sp[0]; sp[0] = sp[1]; sp[1] = c; sp += 2; } return (sp); } static u_char * swablong(u_char *sp, int n) { char c; while (--n >= 0) { c = sp[0]; sp[0] = sp[3]; sp[3] = c; c = sp[2]; sp[2] = sp[1]; sp[1] = c; sp += 4; } return (sp); } static u_char * swabquad(u_char *sp, int n) { char c; while (--n >= 0) { c = sp[0]; sp[0] = sp[7]; sp[7] = c; c = sp[1]; sp[1] = sp[6]; sp[6] = c; c = sp[2]; sp[2] = sp[5]; sp[5] = c; c = sp[3]; sp[3] = sp[4]; sp[4] = c; sp += 8; } return (sp); } void swabst(u_char *cp, u_char *sp) { int n = 0; while (*cp) { switch (*cp) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': n = (n * 10) + (*cp++ - '0'); continue; case 's': case 'w': case 'h': if (n == 0) n = 1; sp = swabshort(sp, n); break; case 'l': if (n == 0) n = 1; sp = swablong(sp, n); break; case 'q': if (n == 0) n = 1; sp = swabquad(sp, n); break; case 'b': if (n == 0) n = 1; sp += n; break; default: fprintf(stderr, "Unknown conversion character: %c\n", *cp); done(0); break; } cp++; n = 0; } } static u_long swabl(u_long x) { swabst((u_char *)"l", (u_char *)&x); return (x); } Index: user/imp/tbemd/share/man/man9/locking.9 =================================================================== --- user/imp/tbemd/share/man/man9/locking.9 (revision 203849) +++ user/imp/tbemd/share/man/man9/locking.9 (revision 203850) @@ -1,391 +1,366 @@ .\" Copyright (c) 2007 Julian Elischer (julian - freebsd org ) .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd February 10, 2010 +.Dd February 13, 2010 .Dt LOCKING 9 .Os .Sh NAME .Nm locking .Nd kernel synchronization primitives .Sh DESCRIPTION The .Em FreeBSD kernel is written to run across multiple CPUs and as such requires several different synchronization primitives to allow the developers to safely access and manipulate the many data types required. -.Pp -These include: -.Bl -enum -.It -Mutexes -.It -Spin mutexes -.It -Pool mutexes -.It -Shared/exclusive locks -.It -Reader/writer locks -.It -Read-mostly locks -.It -Counting semaphores -.It -Condition variables -.It -Sleep/wakeup -.It -Giant -.It -Lockmanager locks -.El -.Pp -The primitives interact and have a number of rules regarding how -they can and can not be combined. -Many of these rules are checked using the -.Xr witness 4 -code. -.Pp .Ss Mutexes -Mutexes are the most commonly used synchronization primitive in the kernel. +Mutexes (also called "sleep mutexes") are the most commonly used +synchronization primitive in the kernel. Thread acquires (locks) a mutex before accessing data shared with other threads (including interrupt threads), and releases (unlocks) it afterwards. If the mutex cannot be acquired, the thread requesting it will sleep. Mutexes fully support priority propagation. .Pp See .Xr mutex 9 for details. .Ss Spin mutexes Spin mutexes are variation of basic mutexes; the main difference between the two is that spin mutexes never sleep - instead, they spin, waiting for the thread holding the lock, which runs on another CPU, to release it. Differently from ordinary mutex, spin mutexes disable interrupts when acquired. Since disabling interrupts is expensive, they are also generally slower. -Spin mutexes should only be used to protect data shared with primary -(INTR_FILTER) interrupt code. -You -.Em must not -do anything that deschedules the thread while you -are holding a spin mutex. +Spin mutexes should be used only when neccessary, e.g. to protect data shared +with interrupt filter code (see +.Xr bus_setup_intr 9 +for details). .Ss Pool mutexes With most synchronisaton primitives, such as mutexes, programmer must provide a piece of allocated memory to hold the primitive. For example, a mutex may be embedded inside the structure it protects. Pool mutex is a variant of mutex without this requirement - to lock or unlock a pool mutex, one uses address of the structure being protected with it, not the mutex itself. Pool mutexes are seldom used. .Pp See .Xr mtx_pool 9 for details. .Ss Reader/writer locks Reader/writer locks allow shared access to protected data by multiple threads, or exclusive access by a single thread. The threads with shared access are known as .Em readers since they should only read the protected data. A thread with exclusive access is known as a .Em writer since it may modify protected data. .Pp Reader/writer locks can be treated as mutexes (see above and .Xr mutex 9 ) with shared/exclusive semantics. More specifically, regular mutexes can be considered to be equivalent to a write-lock on an .Em rw_lock. The .Em rw_lock locks have priority propagation like mutexes, but priority can be propagated only to an exclusive holder. This limitation comes from the fact that shared owners are anonymous. Another important property is that shared holders of .Em rw_lock can recurse, but exclusive locks are not allowed to recurse. This ability should not be used lightly and .Em may go away. .Pp See .Xr rwlock 9 for details. .Ss Read-mostly locks Mostly reader locks are similar to .Em reader/writer locks but optimized for very infrequent write locking. .Em Read-mostly locks implement full priority propagation by tracking shared owners using a lock user supplied .Em tracker data structure. .Pp See .Xr rmlock 9 for details. .Ss Shared/exclusive locks Shared/exclusive locks are similar to reader/writer locks; the main difference between them is that shared/exclusive locks may be held during unbounded sleep (and may thus perform an unbounded sleep). They are inherently less efficient than mutexes, reader/writer locks and read-mostly locks. They don't support priority propagation. They should be considered to be closely related to .Xr sleep 9 . In fact it could in some cases be considered a conditional sleep. .Pp See .Xr sx 9 for details. .Ss Counting semaphores Counting semaphores provide a mechanism for synchronizing access to a pool of resources. Unlike mutexes, semaphores do not have the concept of an owner, so they can be useful in situations where one thread needs to acquire a resource, and another thread needs to release it. They are largely deprecated. .Pp See .Xr sema 9 for details. .Ss Condition variables Condition variables are used in conjunction with mutexes to wait for conditions to occur. A thread must hold the mutex before calling the .Fn cv_wait* , functions. When a thread waits on a condition, the mutex is atomically released before the thread is blocked, then reacquired before the function call returns. .Pp See .Xr condvar 9 for details. .Ss Giant Giant is an instance of a mutex, with some special characteristics: .Bl -enum .It It is recursive. .It Drivers and filesystems can request that Giant be locked around them by not marking themselves MPSAFE. Note that infrastructure to do this is slowly going away as non-MPSAFE drivers either became properly locked or disappear. .It Giant must be locked first before other locks. .It It is OK to hold Giant while performing unbounded sleep; in such case, Giant will be dropped before sleeping and picked up after wakeup. .It There are places in the kernel that drop Giant and pick it back up again. Sleep locks will do this before sleeping. Parts of the network or VM code may do this as well, depending on the setting of a sysctl. This means that you cannot count on Giant keeping other code from running if your code sleeps, even if you want it to. .El .Ss Sleep/wakeup The functions .Fn tsleep , .Fn msleep , .Fn msleep_spin , .Fn pause , .Fn wakeup , and .Fn wakeup_one handle event-based thread blocking. If a thread must wait for an external event, it is put to sleep by .Fn tsleep , .Fn msleep , .Fn msleep_spin , or .Fn pause . Threads may also wait using one of the locking primitive sleep routines .Xr mtx_sleep 9 , .Xr rw_sleep 9 , or .Xr sx_sleep 9 . .Pp The parameter .Fa chan is an arbitrary address that uniquely identifies the event on which the thread is being put to sleep. All threads sleeping on a single .Fa chan are woken up later by .Fn wakeup , often called from inside an interrupt routine, to indicate that the resource the thread was blocking on is available now. .Pp Several of the sleep functions including .Fn msleep , .Fn msleep_spin , and the locking primitive sleep routines specify an additional lock parameter. The lock will be released before sleeping and reacquired before the sleep routine returns. If .Fa priority includes the .Dv PDROP flag, then the lock will not be reacquired before returning. The lock is used to ensure that a condition can be checked atomically, and that the current thread can be suspended without missing a change to the condition, or an associated wakeup. In addition, all of the sleep routines will fully drop the .Va Giant mutex (even if recursed) while the thread is suspended and will reacquire the .Va Giant mutex before the function returns. .Pp See .Xr sleep 9 for details. .Pp .Ss Lockmanager locks Shared/exclusive locks, used mostly in .Xr VFS 9 , in particular as a .Xr vnode 9 lock. They have features other lock types don't have, such as sleep timeout, writer starvation avoidance, draining, and interlock mutex, but this makes them complicated to implement; for this reason, they are deprecated. .Pp See .Xr lock 9 for details. .Sh INTERACTIONS +The primitives interact and have a number of rules regarding how +they can and can not be combined. +Many of these rules are checked using the +.Xr witness 4 +code. .Ss Bounded vs. unbounded sleep The following primitives perform bounded sleep: mutexes, pool mutexes, reader/writer locks and read-mostly locks. .Pp The following primitives block (perform unbounded sleep): shared/exclusive locks, counting semaphores, condition variables, sleep/wakeup and lockmanager locks. .Pp It is an error to do any operation that could result in any kind of sleep while holding spin mutex. .Pp As a general rule, it is an error to do any operation that could result in unbounded sleep while holding any primitive from the 'bounded sleep' group. For example, it is an error to try to acquire shared/exclusive lock while holding mutex, or to try to allocate memory with M_WAITOK while holding read-write lock. .Pp As a special case, it is possible to call .Fn sleep 9 or .Fn mtx_sleep 9 while holding a mutex. It will atomically drop the mutex and reacquire it as part of waking up. This is often however a bad idea because it generally relies on you having such a good knowledge of all the call graph above you and what assumptions it is making that there are a lot of ways to make hard-to-find mistakes. For example you must re-test all the assumptions you made before, all the way up the call graph to where you got the lock. You can not just assume that mtx_sleep can be inserted anywhere. If any caller above you has any mutex or rwlock, your sleep, will cause a panic. If the sleep only happens rarely it may be years before the bad code path is found. .Pp +It is an error to do any operation that could result in any kind of sleep when +running inside an interrupt filter. +.Pp It is an error to do any operation that could result in unbounded sleep when running inside an interrupt thread. .Ss Interaction table -The following table shows what you can and can not do if you hold -one of the synchronization primitives discussed here: -(someone who knows what they are talking about should write this table) +The following table shows what you can and can not do while holding +one of the synchronization primitives discussed: .Bl -column ".Ic xxxxxxxxxxxxxxxxxxx" ".Xr XXXXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXX" -offset indent .It Xo .Em "You have: You want:" Ta spin mtx Ta mutex Ta sx Ta rwlock Ta rmlock Ta sleep .Xc -.It Ic spin mtx Ta \&ok-1 Ta \&no Ta \&no Ta \&no Ta \&no Ta \&no-3 -.It Ic mutex Ta \&ok Ta \&ok-1 Ta \&no Ta \&ok Ta \&ok Ta \&no-3 -.It Ic sx Ta \&ok Ta \&ok Ta \&ok-2 Ta \&ok Ta \&ok Ta \&ok-4 -.It Ic rwlock Ta \&ok Ta \&ok Ta \&no Ta \&ok-2 Ta \&ok Ta \&no-3 -.It Ic rmlock Ta \&ok Ta \&ok Ta \&no Ta \&ok Ta \&ok-2 Ta \&no +.It spin mtx Ta \&ok-1 Ta \&no Ta \&no Ta \&no Ta \&no Ta \&no-3 +.It mutex Ta \&ok Ta \&ok-1 Ta \&no Ta \&ok Ta \&ok Ta \&no-3 +.It sx Ta \&ok Ta \&ok Ta \&ok-2 Ta \&ok Ta \&ok Ta \&ok-4 +.It rwlock Ta \&ok Ta \&ok Ta \&no Ta \&ok-2 Ta \&ok Ta \&no-3 +.It rmlock Ta \&ok Ta \&ok Ta \&no Ta \&ok Ta \&ok-2 Ta \&no .El .Pp .Em *1 Recursion is defined per lock. Lock order is important. .Pp .Em *2 readers can recurse though writers can not. Lock order is important. .Pp .Em *3 There are calls atomically release this primitive when going to sleep and reacquire it on wakeup (e.g. .Fn mtx_sleep , .Fn rw_sleep and .Fn msleep_spin ). .Pp .Em *4 Though one can sleep holding an sx lock, one can also use .Fn sx_sleep which atomically release this primitive when going to sleep and reacquire it on wakeup. .Ss Context mode table The next table shows what can be used in different contexts. At this time this is a rather easy to remember table. .Bl -column ".Ic Xxxxxxxxxxxxxxxxxxx" ".Xr XXXXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXXX" ".Xr XXXXXX" -offset indent .It Xo -.Em "Context:" Ta spin mtx Ta mutex Ta sx Ta rwlock Ta rmlock Ta sleep +.Em "Context:" Ta spin mtx Ta mutex Ta sx Ta rwlock Ta rmlock Ta sleep .Xc -.It interrupt: Ta \&ok Ta \&no Ta \&no Ta \&no Ta \&no Ta \&no -.It idle: Ta \&ok Ta \&no Ta \&no Ta \&no Ta \&no Ta \&no +.It interrupt filter: Ta \&ok Ta \&no Ta \&no Ta \&no Ta \&no Ta \&no +.It ithread: Ta \&ok Ta \&ok Ta \&no Ta \&ok Ta \&ok Ta \&no +.It callout: Ta \&ok Ta \&ok Ta \&no Ta \&ok Ta \&no Ta \&no +.It syscall: Ta \&ok Ta \&ok Ta \&ok Ta \&ok Ta \&ok Ta \&ok .El .Sh SEE ALSO .Xr condvar 9 , .Xr lock 9 , .Xr mtx_pool 9 , .Xr mutex 9 , .Xr rmlock 9 , .Xr rwlock 9 , .Xr sema 9 , .Xr sleep 9 , .Xr sx 9 , .Xr witness 9 , .Xr LOCK_PROFILING 9 .Sh HISTORY These functions appeared in .Bsx 4.1 through .Fx 7.0 .Sh BUGS There are too many locking primitives to choose from. Index: user/imp/tbemd/share/man/man9/vnode.9 =================================================================== --- user/imp/tbemd/share/man/man9/vnode.9 (revision 203849) +++ user/imp/tbemd/share/man/man9/vnode.9 (revision 203850) @@ -1,199 +1,199 @@ .\" Copyright (c) 1996 Doug Rabson .\" .\" All rights reserved. .\" .\" This program is free software. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd February 9, 2010 +.Dd February 13, 2010 .Os .Dt VNODE 9 .Sh NAME .Nm vnode .Nd internal representation of a file or directory .Sh SYNOPSIS .In sys/param.h .In sys/vnode.h .Sh DESCRIPTION The vnode is the focus of all file activity in .Ux . A vnode is described by .Vt "struct vnode" . There is a unique vnode allocated for each active file, each current directory, each mounted-on file, text file, and the root. .Pp Each vnode has three reference counts, .Va v_usecount , .Va v_holdcnt and .Va v_writecount . The first is the number of clients within the kernel which are using this vnode. This count is maintained by .Xr vref 9 , .Xr vrele 9 and .Xr vput 9 . The second is the number of clients within the kernel who veto the recycling of this vnode. This count is maintained by .Xr vhold 9 and .Xr vdrop 9 . When both the .Va v_usecount and the .Va v_holdcnt of a vnode reaches zero then the vnode will be put on the freelist and may be reused for another file, possibly in another file system. The transition to and from the freelist is handled by .Xr getnewvnode 9 , .Xr vfree 9 and .Xr vbusy 9 . The third is a count of the number of clients which are writing into the file. It is maintained by the .Xr open 2 and .Xr close 2 system calls. .Pp Any call which returns a vnode (e.g.\& .Xr vget 9 , .Xr VOP_LOOKUP 9 etc.) will increase the .Va v_usecount of the vnode by one. When the caller is finished with the vnode, it should release this reference by calling .Xr vrele 9 (or .Xr vput 9 if the vnode is locked). .Pp Other commonly used members of the vnode structure are .Va v_id which is used to maintain consistency in the name cache, .Va v_mount which points at the file system which owns the vnode, .Va v_type which contains the type of object the vnode represents and .Va v_data which is used by file systems to store file system specific data with the vnode. The .Va v_op field is used by the .Dv VOP_* macros to call functions in the file system which implement the vnode's functionality. .Sh VNODE TYPES .Bl -tag -width VSOCK .It Dv VNON No type. .It Dv VREG A regular file; may be with or without VM object backing. If you want to make sure this get a backing object, call .Xr vfs_object_create 9 . .It Dv VDIR A directory. .It Dv VBLK A block device; may be with or without VM object backing. If you want to make sure this get a backing object, call .Xr vfs_object_create 9 . .It Dv VCHR A character device. .It Dv VLNK A symbolic link. .It Dv VSOCK A socket. Advisory locking will not work on this. .It Dv VFIFO A FIFO (named pipe). Advisory locking will not work on this. .It Dv VBAD -An old style bad sector map +Indicates that the vnode has been reclaimed. .El .Sh IMPLEMENTATION NOTES VFIFO uses the "struct fileops" from .Pa /sys/kern/sys_pipe.c . VSOCK uses the "struct fileops" from .Pa /sys/kern/sys_socket.c . Everything else uses the one from .Pa /sys/kern/vfs_vnops.c . .Pp The VFIFO/VSOCK code, which is why "struct fileops" is used at all, is an artifact of an incomplete integration of the VFS code into the kernel. .Pp Calls to .Xr malloc 9 or .Xr free 9 when holding a .Nm interlock, will cause a LOR (Lock Order Reversal) due to the intertwining of VM Objects and Vnodes. .Sh SEE ALSO .Xr malloc 9 , .Xr VOP_ACCESS 9 , .Xr VOP_ACLCHECK 9 , .Xr VOP_ADVLOCK 9 , .Xr VOP_ATTRIB 9 , .Xr VOP_BWRITE 9 , .Xr VOP_CREATE 9 , .Xr VOP_FSYNC 9 , .Xr VOP_GETACL 9 , .Xr VOP_GETEXTATTR 9 , .Xr VOP_GETPAGES 9 , .Xr VOP_GETVOBJECT 9 , .Xr VOP_INACTIVE 9 , .Xr VOP_IOCTL 9 , .Xr VOP_LINK 9 , .Xr VOP_LISTEXTATTR 9 , .Xr VOP_LOCK 9 , .Xr VOP_LOOKUP 9 , .Xr VOP_OPENCLOSE 9 , .Xr VOP_PATHCONF 9 , .Xr VOP_PRINT 9 , .Xr VOP_RDWR 9 , .Xr VOP_READDIR 9 , .Xr VOP_READLINK 9 , .Xr VOP_REALLOCBLKS 9 , .Xr VOP_REMOVE 9 , .Xr VOP_RENAME 9 , .Xr VOP_REVOKE 9 , .Xr VOP_SETACL 9 , .Xr VOP_SETEXTATTR 9 , .Xr VOP_STRATEGY 9 , .Xr VOP_VPTOCNP 9 , .Xr VOP_VPTOFH 9 , .Xr VFS 9 .Sh AUTHORS This manual page was written by .An Doug Rabson . Index: user/imp/tbemd/sys/boot/sparc64/loader/main.c =================================================================== --- user/imp/tbemd/sys/boot/sparc64/loader/main.c (revision 203849) +++ user/imp/tbemd/sys/boot/sparc64/loader/main.c (revision 203850) @@ -1,895 +1,954 @@ /*- * Initial implementation: * Copyright (c) 2001 Robert Drehmel * All rights reserved. * * As long as the above copyright statement and this notice remain * unchanged, you can do what ever you want with this file. */ /*- * Copyright (c) 2008 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * FreeBSD/sparc64 kernel loader - machine dependent part * * - implements copyin and readin functions that map kernel * pages on demand. The machine independent code does not * know the size of the kernel early enough to pre-enter * TTEs and install just one 4MB mapping seemed to limiting * to me. */ #include -#include #include -#include +#include #include +#include #include #include #include +#include #include #include +#include +#include #include #include #include #include #include #include #include #include "bootstrap.h" #include "libofw.h" #include "dev_net.h" +#ifndef CTASSERT +#define CTASSERT(x) _CTASSERT(x, __LINE__) +#define _CTASSERT(x, y) __CTASSERT(x, y) +#define __CTASSERT(x, y) typedef char __assert ## y[(x) ? 1 : -1] +#endif + extern char bootprog_name[], bootprog_rev[], bootprog_date[], bootprog_maker[]; enum { HEAPVA = 0x800000, HEAPSZ = 0x1000000, LOADSZ = 0x1000000 /* for kernel and modules */ }; +/* At least Sun Fire V1280 require page sized allocations to be claimed. */ +CTASSERT(HEAPSZ % PAGE_SIZE == 0); + static struct mmu_ops { void (*tlb_init)(void); int (*mmu_mapin)(vm_offset_t va, vm_size_t len); } *mmu_ops; typedef void kernel_entry_t(vm_offset_t mdp, u_long o1, u_long o2, u_long o3, void *openfirmware); -static inline u_long dtlb_get_data_sun4u(int slot); -static void dtlb_enter_sun4u(u_long vpn, u_long data); +static inline u_long dtlb_get_data_sun4u(u_int); +static int dtlb_enter_sun4u(u_int, u_long data, vm_offset_t); static vm_offset_t dtlb_va_to_pa_sun4u(vm_offset_t); -static inline u_long itlb_get_data_sun4u(int slot); -static void itlb_enter_sun4u(u_long vpn, u_long data); +static inline u_long itlb_get_data_sun4u(u_int); +static int itlb_enter_sun4u(u_int, u_long data, vm_offset_t); static vm_offset_t itlb_va_to_pa_sun4u(vm_offset_t); static void itlb_relocate_locked0_sun4u(void); extern vm_offset_t md_load(char *, vm_offset_t *); static int sparc64_autoload(void); static ssize_t sparc64_readin(const int, vm_offset_t, const size_t); static ssize_t sparc64_copyin(const void *, vm_offset_t, size_t); static void sparc64_maphint(vm_offset_t, size_t); static vm_offset_t claim_virt(vm_offset_t, size_t, int); static vm_offset_t alloc_phys(size_t, int); static int map_phys(int, size_t, vm_offset_t, vm_offset_t); static void release_phys(vm_offset_t, u_int); static int __elfN(exec)(struct preloaded_file *); static int mmu_mapin_sun4u(vm_offset_t, vm_size_t); static int mmu_mapin_sun4v(vm_offset_t, vm_size_t); static vm_offset_t init_heap(void); +static phandle_t find_bsp_sun4u(phandle_t, uint32_t); +const char *cpu_cpuid_prop_sun4u(void); +uint32_t cpu_get_mid_sun4u(void); static void tlb_init_sun4u(void); static void tlb_init_sun4v(void); #ifdef LOADER_DEBUG typedef u_int64_t tte_t; static void pmap_print_tlb_sun4u(void); static void pmap_print_tte_sun4u(tte_t, tte_t); #endif static struct mmu_ops mmu_ops_sun4u = { tlb_init_sun4u, mmu_mapin_sun4u }; static struct mmu_ops mmu_ops_sun4v = { tlb_init_sun4v, mmu_mapin_sun4v }; /* sun4u */ struct tlb_entry *dtlb_store; struct tlb_entry *itlb_store; -int dtlb_slot; -int itlb_slot; +u_int dtlb_slot; +u_int itlb_slot; int cpu_impl; -static int dtlb_slot_max; -static int itlb_slot_max; +static u_int dtlb_slot_max; +static u_int itlb_slot_max; /* sun4v */ static struct tlb_entry *tlb_store; static int is_sun4v = 0; /* * no direct TLB access on sun4v * we somewhat arbitrarily declare enough * slots to cover a 4GB AS with 4MB pages */ #define SUN4V_TLB_SLOT_MAX (1 << 10) static vm_offset_t curkva = 0; static vm_offset_t heapva; static phandle_t root; /* * Machine dependent structures that the machine independent * loader part uses. */ struct devsw *devsw[] = { #ifdef LOADER_DISK_SUPPORT &ofwdisk, #endif #ifdef LOADER_NET_SUPPORT &netdev, #endif 0 }; struct arch_switch archsw; static struct file_format sparc64_elf = { __elfN(loadfile), __elfN(exec) }; struct file_format *file_formats[] = { &sparc64_elf, 0 }; struct fs_ops *file_system[] = { #ifdef LOADER_UFS_SUPPORT &ufs_fsops, #endif #ifdef LOADER_CD9660_SUPPORT &cd9660_fsops, #endif #ifdef LOADER_ZIP_SUPPORT &zipfs_fsops, #endif #ifdef LOADER_GZIP_SUPPORT &gzipfs_fsops, #endif #ifdef LOADER_BZIP2_SUPPORT &bzipfs_fsops, #endif #ifdef LOADER_NFS_SUPPORT &nfs_fsops, #endif #ifdef LOADER_TFTP_SUPPORT &tftp_fsops, #endif 0 }; struct netif_driver *netif_drivers[] = { #ifdef LOADER_NET_SUPPORT &ofwnet, #endif 0 }; extern struct console ofwconsole; struct console *consoles[] = { &ofwconsole, 0 }; #ifdef LOADER_DEBUG static int watch_phys_set_mask(vm_offset_t pa, u_long mask) { u_long lsucr; stxa(AA_DMMU_PWPR, ASI_DMMU, pa & (((2UL << 38) - 1) << 3)); lsucr = ldxa(0, ASI_LSU_CTL_REG); lsucr = ((lsucr | LSU_PW) & ~LSU_PM_MASK) | (mask << LSU_PM_SHIFT); stxa(0, ASI_LSU_CTL_REG, lsucr); return (0); } static int watch_phys_set(vm_offset_t pa, int sz) { u_long off; off = (u_long)pa & 7; /* Test for misaligned watch points. */ if (off + sz > 8) return (-1); return (watch_phys_set_mask(pa, ((1 << sz) - 1) << off)); } static int watch_virt_set_mask(vm_offset_t va, u_long mask) { u_long lsucr; stxa(AA_DMMU_VWPR, ASI_DMMU, va & (((2UL << 41) - 1) << 3)); lsucr = ldxa(0, ASI_LSU_CTL_REG); lsucr = ((lsucr | LSU_VW) & ~LSU_VM_MASK) | (mask << LSU_VM_SHIFT); stxa(0, ASI_LSU_CTL_REG, lsucr); return (0); } static int watch_virt_set(vm_offset_t va, int sz) { u_long off; off = (u_long)va & 7; /* Test for misaligned watch points. */ if (off + sz > 8) return (-1); return (watch_virt_set_mask(va, ((1 << sz) - 1) << off)); } #endif /* * archsw functions */ static int sparc64_autoload(void) { return (0); } static ssize_t sparc64_readin(const int fd, vm_offset_t va, const size_t len) { mmu_ops->mmu_mapin(va, len); return (read(fd, (void *)va, len)); } static ssize_t sparc64_copyin(const void *src, vm_offset_t dest, size_t len) { mmu_ops->mmu_mapin(dest, len); memcpy((void *)dest, src, len); return (len); } static void sparc64_maphint(vm_offset_t va, size_t len) { vm_paddr_t pa; vm_offset_t mva; size_t size; int i, free_excess = 0; if (!is_sun4v) return; if (tlb_store[va >> 22].te_pa != -1) return; /* round up to nearest 4MB page */ size = (len + PAGE_MASK_4M) & ~PAGE_MASK_4M; #if 0 pa = alloc_phys(PAGE_SIZE_256M, PAGE_SIZE_256M); if (pa != -1) free_excess = 1; else #endif pa = alloc_phys(size, PAGE_SIZE_256M); if (pa == -1) pa = alloc_phys(size, PAGE_SIZE_4M); if (pa == -1) panic("%s: out of memory", __func__); for (i = 0; i < size; i += PAGE_SIZE_4M) { mva = claim_virt(va + i, PAGE_SIZE_4M, 0); if (mva != (va + i)) panic("%s: can't claim virtual page " "(wanted %#lx, got %#lx)", __func__, va, mva); tlb_store[mva >> 22].te_pa = pa + i; if (map_phys(-1, PAGE_SIZE_4M, mva, pa + i) != 0) printf("%s: can't map physical page\n", __func__); } if (free_excess) release_phys(pa, PAGE_SIZE_256M); } /* * other MD functions */ static vm_offset_t claim_virt(vm_offset_t virt, size_t size, int align) { vm_offset_t mva; if (OF_call_method("claim", mmu, 3, 1, virt, size, align, &mva) == -1) return ((vm_offset_t)-1); return (mva); } static vm_offset_t alloc_phys(size_t size, int align) { cell_t phys_hi, phys_low; if (OF_call_method("claim", memory, 2, 2, size, align, &phys_low, &phys_hi) == -1) return ((vm_offset_t)-1); return ((vm_offset_t)phys_hi << 32 | phys_low); } static int map_phys(int mode, size_t size, vm_offset_t virt, vm_offset_t phys) { return (OF_call_method("map", mmu, 5, 0, (uint32_t)phys, (uint32_t)(phys >> 32), virt, size, mode)); } static void release_phys(vm_offset_t phys, u_int size) { (void)OF_call_method("release", memory, 3, 0, (uint32_t)phys, (uint32_t)(phys >> 32), size); } static int __elfN(exec)(struct preloaded_file *fp) { struct file_metadata *fmp; vm_offset_t mdp; Elf_Addr entry; Elf_Ehdr *e; int error; if ((fmp = file_findmetadata(fp, MODINFOMD_ELFHDR)) == 0) return (EFTYPE); e = (Elf_Ehdr *)&fmp->md_data; if ((error = md_load(fp->f_args, &mdp)) != 0) return (error); printf("jumping to kernel entry at %#lx.\n", e->e_entry); #ifdef LOADER_DEBUG pmap_print_tlb_sun4u(); #endif dev_cleanup(); entry = e->e_entry; OF_release((void *)heapva, HEAPSZ); ((kernel_entry_t *)entry)(mdp, 0, 0, 0, openfirmware); panic("%s: exec returned", __func__); } static inline u_long -dtlb_get_data_sun4u(int slot) +dtlb_get_data_sun4u(u_int slot) { /* * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ (void)ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG); return (ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG)); } static inline u_long -itlb_get_data_sun4u(int slot) +itlb_get_data_sun4u(u_int slot) { /* * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ (void)ldxa(TLB_DAR_SLOT(slot), ASI_ITLB_DATA_ACCESS_REG); return (ldxa(TLB_DAR_SLOT(slot), ASI_ITLB_DATA_ACCESS_REG)); } static vm_offset_t dtlb_va_to_pa_sun4u(vm_offset_t va) { u_long pstate, reg; int i; pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); for (i = 0; i < dtlb_slot_max; i++) { reg = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); if (TLB_TAR_VA(reg) != va) continue; reg = dtlb_get_data_sun4u(i); wrpr(pstate, pstate, 0); reg >>= TD_PA_SHIFT; if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) return (reg & TD_PA_CH_MASK); return (reg & TD_PA_SF_MASK); } wrpr(pstate, pstate, 0); return (-1); } static vm_offset_t itlb_va_to_pa_sun4u(vm_offset_t va) { u_long pstate, reg; int i; pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); for (i = 0; i < itlb_slot_max; i++) { reg = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); if (TLB_TAR_VA(reg) != va) continue; reg = itlb_get_data_sun4u(i); wrpr(pstate, pstate, 0); reg >>= TD_PA_SHIFT; if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) return (reg & TD_PA_CH_MASK); return (reg & TD_PA_SF_MASK); } wrpr(pstate, pstate, 0); return (-1); } -static void -dtlb_enter_sun4u(u_long vpn, u_long data) +static int +dtlb_enter_sun4u(u_int index, u_long data, vm_offset_t virt) { - u_long reg; - reg = rdpr(pstate); - wrpr(pstate, reg & ~PSTATE_IE, 0); - stxa(AA_DMMU_TAR, ASI_DMMU, - TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL)); - stxa(0, ASI_DTLB_DATA_IN_REG, data); - membar(Sync); - wrpr(pstate, reg, 0); + return (OF_call_method("SUNW,dtlb-load", mmu, 3, 0, index, data, + virt)); } -static void -itlb_enter_sun4u(u_long vpn, u_long data) +static int +itlb_enter_sun4u(u_int index, u_long data, vm_offset_t virt) { - u_long reg; - int i; - reg = rdpr(pstate); - wrpr(pstate, reg & ~PSTATE_IE, 0); - - if (cpu_impl == CPU_IMPL_ULTRASPARCIIIp) { - /* - * Search an unused slot != 0 and explicitly enter the data - * and tag there in order to avoid Cheetah+ erratum 34. - */ - for (i = 1; i < itlb_slot_max; i++) { - if ((itlb_get_data_sun4u(i) & TD_V) != 0) - continue; - - stxa(AA_IMMU_TAR, ASI_IMMU, - TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL)); - stxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, data); - flush(PROMBASE); - break; - } - wrpr(pstate, reg, 0); - if (i == itlb_slot_max) - panic("%s: could not find an unused slot", __func__); - return; - } - - stxa(AA_IMMU_TAR, ASI_IMMU, - TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL)); - stxa(0, ASI_ITLB_DATA_IN_REG, data); - flush(PROMBASE); - wrpr(pstate, reg, 0); + if (cpu_impl == CPU_IMPL_ULTRASPARCIIIp && index == 0 && + (data & TD_L) != 0) + panic("%s: won't enter locked TLB entry at index 0 on USIII+", + __func__); + return (OF_call_method("SUNW,itlb-load", mmu, 3, 0, index, data, + virt)); } static void itlb_relocate_locked0_sun4u(void) { u_long data, pstate, tag; int i; if (cpu_impl != CPU_IMPL_ULTRASPARCIIIp) return; pstate = rdpr(pstate); wrpr(pstate, pstate & ~PSTATE_IE, 0); data = itlb_get_data_sun4u(0); if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) { wrpr(pstate, pstate, 0); return; } /* Flush the mapping of slot 0. */ tag = ldxa(TLB_DAR_SLOT(0), ASI_ITLB_TAG_READ_REG); stxa(TLB_DEMAP_VA(TLB_TAR_VA(tag)) | TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, ASI_IMMU_DEMAP, 0); flush(0); /* The USIII-family ignores the address. */ /* * Search a replacement slot != 0 and enter the data and tag * that formerly were in slot 0. */ for (i = 1; i < itlb_slot_max; i++) { if ((itlb_get_data_sun4u(i) & TD_V) != 0) continue; stxa(AA_IMMU_TAR, ASI_IMMU, tag); stxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, data); flush(0); /* The USIII-family ignores the address. */ break; } wrpr(pstate, pstate, 0); if (i == itlb_slot_max) panic("%s: could not find a replacement slot", __func__); } static int mmu_mapin_sun4u(vm_offset_t va, vm_size_t len) { vm_offset_t pa, mva; u_long data; + u_int index; if (va + len > curkva) curkva = va + len; pa = (vm_offset_t)-1; len += va & PAGE_MASK_4M; va &= ~PAGE_MASK_4M; while (len) { if (dtlb_va_to_pa_sun4u(va) == (vm_offset_t)-1 || itlb_va_to_pa_sun4u(va) == (vm_offset_t)-1) { /* Allocate a physical page, claim the virtual area. */ if (pa == (vm_offset_t)-1) { pa = alloc_phys(PAGE_SIZE_4M, PAGE_SIZE_4M); if (pa == (vm_offset_t)-1) panic("%s: out of memory", __func__); mva = claim_virt(va, PAGE_SIZE_4M, 0); if (mva != va) panic("%s: can't claim virtual page " "(wanted %#lx, got %#lx)", __func__, va, mva); /* * The mappings may have changed, be paranoid. */ continue; } /* * Actually, we can only allocate two pages less at * most (depending on the kernel TSB size). */ if (dtlb_slot >= dtlb_slot_max) panic("%s: out of dtlb_slots", __func__); if (itlb_slot >= itlb_slot_max) panic("%s: out of itlb_slots", __func__); data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; dtlb_store[dtlb_slot].te_pa = pa; dtlb_store[dtlb_slot].te_va = va; + index = dtlb_slot_max - dtlb_slot - 1; + if (dtlb_enter_sun4u(index, data, va) < 0) + panic("%s: can't enter dTLB slot %d data " + "%#lx va %#lx", __func__, index, data, + va); + dtlb_slot++; itlb_store[itlb_slot].te_pa = pa; itlb_store[itlb_slot].te_va = va; - dtlb_slot++; + index = itlb_slot_max - itlb_slot - 1; + if (itlb_enter_sun4u(index, data, va) < 0) + panic("%s: can't enter iTLB slot %d data " + "%#lx va %#lxd", __func__, index, data, + va); itlb_slot++; - dtlb_enter_sun4u(va, data); - itlb_enter_sun4u(va, data); pa = (vm_offset_t)-1; } len -= len > PAGE_SIZE_4M ? PAGE_SIZE_4M : len; va += PAGE_SIZE_4M; } if (pa != (vm_offset_t)-1) release_phys(pa, PAGE_SIZE_4M); return (0); } static int mmu_mapin_sun4v(vm_offset_t va, vm_size_t len) { vm_offset_t pa, mva; if (va + len > curkva) curkva = va + len; pa = (vm_offset_t)-1; len += va & PAGE_MASK_4M; va &= ~PAGE_MASK_4M; while (len) { if ((va >> 22) > SUN4V_TLB_SLOT_MAX) panic("%s: trying to map more than 4GB", __func__); if (tlb_store[va >> 22].te_pa == -1) { /* Allocate a physical page, claim the virtual area */ if (pa == (vm_offset_t)-1) { pa = alloc_phys(PAGE_SIZE_4M, PAGE_SIZE_4M); if (pa == (vm_offset_t)-1) panic("%s: out of memory", __func__); mva = claim_virt(va, PAGE_SIZE_4M, 0); if (mva != va) panic("%s: can't claim virtual page " "(wanted %#lx, got %#lx)", __func__, va, mva); } tlb_store[va >> 22].te_pa = pa; if (map_phys(-1, PAGE_SIZE_4M, va, pa) == -1) printf("%s: can't map physical page\n", __func__); pa = (vm_offset_t)-1; } len -= len > PAGE_SIZE_4M ? PAGE_SIZE_4M : len; va += PAGE_SIZE_4M; } if (pa != (vm_offset_t)-1) release_phys(pa, PAGE_SIZE_4M); return (0); } static vm_offset_t init_heap(void) { /* There is no need for continuous physical heap memory. */ heapva = (vm_offset_t)OF_claim((void *)HEAPVA, HEAPSZ, 32); return (heapva); } +static phandle_t +find_bsp_sun4u(phandle_t node, uint32_t bspid) +{ + char type[sizeof("cpu")]; + phandle_t child; + uint32_t cpuid; + + for (; node > 0; node = OF_peer(node)) { + child = OF_child(node); + if (child > 0) { + child = find_bsp_sun4u(child, bspid); + if (child > 0) + return (child); + } else { + if (OF_getprop(node, "device_type", type, + sizeof(type)) <= 0) + continue; + if (strcmp(type, "cpu") != 0) + continue; + if (OF_getprop(node, cpu_cpuid_prop_sun4u(), &cpuid, + sizeof(cpuid)) <= 0) + continue; + if (cpuid == bspid) + return (node); + } + } + return (0); +} + +const char * +cpu_cpuid_prop_sun4u(void) +{ + + switch (cpu_impl) { + case CPU_IMPL_SPARC64: + case CPU_IMPL_ULTRASPARCI: + case CPU_IMPL_ULTRASPARCII: + case CPU_IMPL_ULTRASPARCIIi: + case CPU_IMPL_ULTRASPARCIIe: + return ("upa-portid"); + case CPU_IMPL_ULTRASPARCIII: + case CPU_IMPL_ULTRASPARCIIIp: + case CPU_IMPL_ULTRASPARCIIIi: + case CPU_IMPL_ULTRASPARCIIIip: + return ("portid"); + case CPU_IMPL_ULTRASPARCIV: + case CPU_IMPL_ULTRASPARCIVp: + return ("cpuid"); + default: + return (""); + } +} + +uint32_t +cpu_get_mid_sun4u(void) +{ + + switch (cpu_impl) { + case CPU_IMPL_SPARC64: + case CPU_IMPL_ULTRASPARCI: + case CPU_IMPL_ULTRASPARCII: + case CPU_IMPL_ULTRASPARCIIi: + case CPU_IMPL_ULTRASPARCIIe: + return (UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIII: + case CPU_IMPL_ULTRASPARCIIIp: + return (FIREPLANE_CR_GET_AID(ldxa(AA_FIREPLANE_CONFIG, + ASI_FIREPLANE_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIIIi: + case CPU_IMPL_ULTRASPARCIIIip: + return (JBUS_CR_GET_JID(ldxa(0, ASI_JBUS_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIV: + case CPU_IMPL_ULTRASPARCIVp: + return (INTR_ID_GET_ID(ldxa(AA_INTR_ID, ASI_INTR_ID))); + default: + return (0); + } +} + static void tlb_init_sun4u(void) { - phandle_t child; - char buf[128]; - u_int bootcpu; - u_int cpu; + phandle_t bsp; cpu_impl = VER_IMPL(rdpr(ver)); - bootcpu = UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG)); - for (child = OF_child(root); child != 0; child = OF_peer(child)) { - if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0) - continue; - if (strcmp(buf, "cpu") != 0) - continue; - if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ? - "upa-portid" : "portid", &cpu, sizeof(cpu)) <= 0) - continue; - if (cpu == bootcpu) - break; - } - if (cpu != bootcpu) + bsp = find_bsp_sun4u(OF_child(root), cpu_get_mid_sun4u()); + if (bsp == 0) panic("%s: no node for bootcpu?!?!", __func__); - if (OF_getprop(child, "#dtlb-entries", &dtlb_slot_max, + if (OF_getprop(bsp, "#dtlb-entries", &dtlb_slot_max, sizeof(dtlb_slot_max)) == -1 || - OF_getprop(child, "#itlb-entries", &itlb_slot_max, + OF_getprop(bsp, "#itlb-entries", &itlb_slot_max, sizeof(itlb_slot_max)) == -1) panic("%s: can't get TLB slot max.", __func__); if (cpu_impl == CPU_IMPL_ULTRASPARCIIIp) { #ifdef LOADER_DEBUG printf("pre fixup:\n"); pmap_print_tlb_sun4u(); #endif /* * Relocate the locked entry in it16 slot 0 (if existent) * as part of working around Cheetah+ erratum 34. */ itlb_relocate_locked0_sun4u(); #ifdef LOADER_DEBUG printf("post fixup:\n"); pmap_print_tlb_sun4u(); #endif } dtlb_store = malloc(dtlb_slot_max * sizeof(*dtlb_store)); itlb_store = malloc(itlb_slot_max * sizeof(*itlb_store)); if (dtlb_store == NULL || itlb_store == NULL) panic("%s: can't allocate TLB store", __func__); } static void tlb_init_sun4v(void) { tlb_store = malloc(SUN4V_TLB_SLOT_MAX * sizeof(*tlb_store)); memset(tlb_store, 0xFF, SUN4V_TLB_SLOT_MAX * sizeof(*tlb_store)); } int main(int (*openfirm)(void *)) { char bootpath[64]; char compatible[32]; struct devsw **dp; /* * Tell the Open Firmware functions where they find the OFW gate. */ OF_init(openfirm); archsw.arch_getdev = ofw_getdev; archsw.arch_copyin = sparc64_copyin; archsw.arch_copyout = ofw_copyout; archsw.arch_readin = sparc64_readin; archsw.arch_autoload = sparc64_autoload; archsw.arch_maphint = sparc64_maphint; - init_heap(); - setheap((void *)heapva, (void *)(heapva + HEAPSZ)); - /* * Probe for a console. */ cons_probe(); + + if (init_heap() == (vm_offset_t)-1) + panic("%s: can't claim heap", __func__); + setheap((void *)heapva, (void *)(heapva + HEAPSZ)); if ((root = OF_peer(0)) == -1) panic("%s: can't get root phandle", __func__); OF_getprop(root, "compatible", compatible, sizeof(compatible)); if (!strcmp(compatible, "sun4v")) { printf("\nBooting with sun4v support.\n"); mmu_ops = &mmu_ops_sun4v; is_sun4v = 1; } else { printf("\nBooting with sun4u support.\n"); mmu_ops = &mmu_ops_sun4u; } mmu_ops->tlb_init(); /* * Initialize devices. */ for (dp = devsw; *dp != 0; dp++) { if ((*dp)->dv_init != 0) (*dp)->dv_init(); } /* * Set up the current device. */ OF_getprop(chosen, "bootpath", bootpath, sizeof(bootpath)); /* * Sun compatible bootable CD-ROMs have a disk label placed * before the cd9660 data, with the actual filesystem being * in the first partition, while the other partitions contain * pseudo disk labels with embedded boot blocks for different * architectures, which may be followed by UFS filesystems. * The firmware will set the boot path to the partition it * boots from ('f' in the sun4u case), but we want the kernel * to be loaded from the cd9660 fs ('a'), so the boot path * needs to be altered. */ if (bootpath[strlen(bootpath) - 2] == ':' && bootpath[strlen(bootpath) - 1] == 'f') { bootpath[strlen(bootpath) - 1] = 'a'; printf("Boot path set to %s\n", bootpath); } env_setenv("currdev", EV_VOLATILE, bootpath, ofw_setcurrdev, env_nounset); env_setenv("loaddev", EV_VOLATILE, bootpath, env_noset, env_nounset); printf("\n"); printf("%s, Revision %s\n", bootprog_name, bootprog_rev); printf("(%s, %s)\n", bootprog_maker, bootprog_date); printf("bootpath=\"%s\"\n", bootpath); /* Give control to the machine independent loader code. */ interact(); return (1); } COMMAND_SET(reboot, "reboot", "reboot the system", command_reboot); static int command_reboot(int argc, char *argv[]) { int i; for (i = 0; devsw[i] != NULL; ++i) if (devsw[i]->dv_cleanup != NULL) (devsw[i]->dv_cleanup)(); printf("Rebooting...\n"); OF_exit(); } /* provide this for panic, as it's not in the startup code */ void exit(int code) { OF_exit(); } #ifdef LOADER_DEBUG static const char *const page_sizes[] = { " 8k", " 64k", "512k", " 4m" }; static void pmap_print_tte_sun4u(tte_t tag, tte_t tte) { printf("%s %s ", page_sizes[(tte >> TD_SIZE_SHIFT) & TD_SIZE_MASK], tag & TD_G ? "G" : " "); printf(tte & TD_W ? "W " : " "); printf(tte & TD_P ? "\e[33mP\e[0m " : " "); printf(tte & TD_E ? "E " : " "); printf(tte & TD_CV ? "CV " : " "); printf(tte & TD_CP ? "CP " : " "); printf(tte & TD_L ? "\e[32mL\e[0m " : " "); printf(tte & TD_IE ? "IE " : " "); printf(tte & TD_NFO ? "NFO " : " "); printf("pa=0x%lx va=0x%lx ctx=%ld\n", TD_PA(tte), TLB_TAR_VA(tag), TLB_TAR_CTX(tag)); } static void pmap_print_tlb_sun4u(void) { tte_t tag, tte; u_long pstate; int i; pstate = rdpr(pstate); for (i = 0; i < itlb_slot_max; i++) { wrpr(pstate, pstate & ~PSTATE_IE, 0); tte = itlb_get_data_sun4u(i); wrpr(pstate, pstate, 0); if (!(tte & TD_V)) continue; tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); printf("iTLB-%2u: ", i); pmap_print_tte_sun4u(tag, tte); } for (i = 0; i < dtlb_slot_max; i++) { wrpr(pstate, pstate & ~PSTATE_IE, 0); tte = dtlb_get_data_sun4u(i); wrpr(pstate, pstate, 0); if (!(tte & TD_V)) continue; tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); printf("dTLB-%2u: ", i); pmap_print_tte_sun4u(tag, tte); } } #endif Index: user/imp/tbemd/sys/dev/acpica/acpi_video.c =================================================================== --- user/imp/tbemd/sys/dev/acpica/acpi_video.c (revision 203849) +++ user/imp/tbemd/sys/dev/acpica/acpi_video.c (revision 203850) @@ -1,1031 +1,1031 @@ /*- * Copyright (c) 2002-2003 Taku YAMAMOTO * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: acpi_vid.c,v 1.4 2003/10/13 10:07:36 taku Exp $ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include /* ACPI video extension driver. */ struct acpi_video_output { ACPI_HANDLE handle; UINT32 adr; STAILQ_ENTRY(acpi_video_output) vo_next; struct { int num; STAILQ_ENTRY(acpi_video_output) next; } vo_unit; int vo_brightness; int vo_fullpower; int vo_economy; int vo_numlevels; int *vo_levels; struct sysctl_ctx_list vo_sysctl_ctx; struct sysctl_oid *vo_sysctl_tree; }; STAILQ_HEAD(acpi_video_output_queue, acpi_video_output); struct acpi_video_softc { device_t device; ACPI_HANDLE handle; struct acpi_video_output_queue vid_outputs; eventhandler_tag vid_pwr_evh; }; /* interfaces */ static int acpi_video_modevent(struct module*, int, void *); static void acpi_video_identify(driver_t *driver, device_t parent); static int acpi_video_probe(device_t); static int acpi_video_attach(device_t); static int acpi_video_detach(device_t); static int acpi_video_shutdown(device_t); static void acpi_video_notify_handler(ACPI_HANDLE, UINT32, void *); static void acpi_video_power_profile(void *); static void acpi_video_bind_outputs(struct acpi_video_softc *); static struct acpi_video_output *acpi_video_vo_init(UINT32); static void acpi_video_vo_bind(struct acpi_video_output *, ACPI_HANDLE); static void acpi_video_vo_destroy(struct acpi_video_output *); static int acpi_video_vo_check_level(struct acpi_video_output *, int); static void acpi_video_vo_notify_handler(ACPI_HANDLE, UINT32, void *); static int acpi_video_vo_active_sysctl(SYSCTL_HANDLER_ARGS); static int acpi_video_vo_bright_sysctl(SYSCTL_HANDLER_ARGS); static int acpi_video_vo_presets_sysctl(SYSCTL_HANDLER_ARGS); static int acpi_video_vo_levels_sysctl(SYSCTL_HANDLER_ARGS); /* operations */ static void vid_set_switch_policy(ACPI_HANDLE, UINT32); static int vid_enum_outputs(ACPI_HANDLE, void(*)(ACPI_HANDLE, UINT32, void *), void *); static int vo_get_brightness_levels(ACPI_HANDLE, int **); static int vo_get_brightness(ACPI_HANDLE); static void vo_set_brightness(ACPI_HANDLE, int); static UINT32 vo_get_device_status(ACPI_HANDLE); static UINT32 vo_get_graphics_state(ACPI_HANDLE); static void vo_set_device_state(ACPI_HANDLE, UINT32); /* events */ #define VID_NOTIFY_SWITCHED 0x80 #define VID_NOTIFY_REPROBE 0x81 #define VID_NOTIFY_INC_BRN 0x86 #define VID_NOTIFY_DEC_BRN 0x87 /* _DOS (Enable/Disable Output Switching) argument bits */ #define DOS_SWITCH_MASK 3 #define DOS_SWITCH_BY_OSPM 0 #define DOS_SWITCH_BY_BIOS 1 #define DOS_SWITCH_LOCKED 2 #define DOS_BRIGHTNESS_BY_BIOS (1 << 2) /* _DOD and subdev's _ADR */ #define DOD_DEVID_MASK 0x0f00 #define DOD_DEVID_MASK_FULL 0xffff #define DOD_DEVID_MASK_DISPIDX 0x000f #define DOD_DEVID_MASK_DISPPORT 0x00f0 #define DOD_DEVID_MONITOR 0x0100 #define DOD_DEVID_LCD 0x0110 #define DOD_DEVID_TV 0x0200 #define DOD_DEVID_EXT 0x0300 #define DOD_DEVID_INTDFP 0x0400 #define DOD_BIOS (1 << 16) #define DOD_NONVGA (1 << 17) #define DOD_HEAD_ID_SHIFT 18 #define DOD_HEAD_ID_BITS 3 #define DOD_HEAD_ID_MASK \ (((1 << DOD_HEAD_ID_BITS) - 1) << DOD_HEAD_ID_SHIFT) #define DOD_DEVID_SCHEME_STD (1 << 31) /* _BCL related constants */ #define BCL_FULLPOWER 0 #define BCL_ECONOMY 1 /* _DCS (Device Currrent Status) value bits and masks. */ #define DCS_EXISTS (1 << 0) #define DCS_ACTIVE (1 << 1) #define DCS_READY (1 << 2) #define DCS_FUNCTIONAL (1 << 3) #define DCS_ATTACHED (1 << 4) /* _DSS (Device Set Status) argument bits and masks. */ #define DSS_INACTIVE 0 #define DSS_ACTIVE (1 << 0) #define DSS_SETNEXT (1 << 30) #define DSS_COMMIT (1 << 31) static device_method_t acpi_video_methods[] = { DEVMETHOD(device_identify, acpi_video_identify), DEVMETHOD(device_probe, acpi_video_probe), DEVMETHOD(device_attach, acpi_video_attach), DEVMETHOD(device_detach, acpi_video_detach), DEVMETHOD(device_shutdown, acpi_video_shutdown), { 0, 0 } }; static driver_t acpi_video_driver = { "acpi_video", acpi_video_methods, sizeof(struct acpi_video_softc), }; static devclass_t acpi_video_devclass; DRIVER_MODULE(acpi_video, vgapci, acpi_video_driver, acpi_video_devclass, acpi_video_modevent, NULL); MODULE_DEPEND(acpi_video, acpi, 1, 1, 1); static struct sysctl_ctx_list acpi_video_sysctl_ctx; static struct sysctl_oid *acpi_video_sysctl_tree; static struct acpi_video_output_queue crt_units, tv_units, ext_units, lcd_units, other_units; /* * The 'video' lock protects the hierarchy of video output devices * (the video "bus"). The 'video_output' lock protects per-output * data is equivalent to a softc lock for each video output. */ ACPI_SERIAL_DECL(video, "ACPI video"); ACPI_SERIAL_DECL(video_output, "ACPI video output"); MALLOC_DEFINE(M_ACPIVIDEO, "acpivideo", "ACPI video extension"); static int acpi_video_modevent(struct module *mod __unused, int evt, void *cookie __unused) { int err; err = 0; switch (evt) { case MOD_LOAD: sysctl_ctx_init(&acpi_video_sysctl_ctx); STAILQ_INIT(&crt_units); STAILQ_INIT(&tv_units); STAILQ_INIT(&ext_units); STAILQ_INIT(&lcd_units); STAILQ_INIT(&other_units); break; case MOD_UNLOAD: sysctl_ctx_free(&acpi_video_sysctl_ctx); acpi_video_sysctl_tree = NULL; break; default: err = EINVAL; } return (err); } static void acpi_video_identify(driver_t *driver, device_t parent) { if (device_find_child(parent, "acpi_video", -1) == NULL) device_add_child(parent, "acpi_video", -1); } static int acpi_video_probe(device_t dev) { ACPI_HANDLE devh, h; ACPI_OBJECT_TYPE t_dos; devh = acpi_get_handle(dev); if (acpi_disabled("video") || ACPI_FAILURE(AcpiGetHandle(devh, "_DOD", &h)) || ACPI_FAILURE(AcpiGetHandle(devh, "_DOS", &h)) || ACPI_FAILURE(AcpiGetType(h, &t_dos)) || t_dos != ACPI_TYPE_METHOD) return (ENXIO); device_set_desc(dev, "ACPI video extension"); return (0); } static int acpi_video_attach(device_t dev) { struct acpi_softc *acpi_sc; struct acpi_video_softc *sc; sc = device_get_softc(dev); acpi_sc = devclass_get_softc(devclass_find("acpi"), 0); if (acpi_sc == NULL) return (ENXIO); ACPI_SERIAL_BEGIN(video); if (acpi_video_sysctl_tree == NULL) { acpi_video_sysctl_tree = SYSCTL_ADD_NODE(&acpi_video_sysctl_ctx, SYSCTL_CHILDREN(acpi_sc->acpi_sysctl_tree), OID_AUTO, "video", CTLFLAG_RD, 0, "video extension control"); } ACPI_SERIAL_END(video); sc->device = dev; sc->handle = acpi_get_handle(dev); STAILQ_INIT(&sc->vid_outputs); AcpiInstallNotifyHandler(sc->handle, ACPI_DEVICE_NOTIFY, acpi_video_notify_handler, sc); sc->vid_pwr_evh = EVENTHANDLER_REGISTER(power_profile_change, acpi_video_power_profile, sc, 0); ACPI_SERIAL_BEGIN(video); acpi_video_bind_outputs(sc); ACPI_SERIAL_END(video); /* * Notify the BIOS that we want to switch both active outputs and * brightness levels. */ vid_set_switch_policy(sc->handle, DOS_SWITCH_BY_OSPM | DOS_BRIGHTNESS_BY_BIOS); acpi_video_power_profile(sc); return (0); } static int acpi_video_detach(device_t dev) { struct acpi_video_softc *sc; struct acpi_video_output *vo, *vn; sc = device_get_softc(dev); vid_set_switch_policy(sc->handle, DOS_SWITCH_BY_BIOS); EVENTHANDLER_DEREGISTER(power_profile_change, sc->vid_pwr_evh); AcpiRemoveNotifyHandler(sc->handle, ACPI_DEVICE_NOTIFY, acpi_video_notify_handler); ACPI_SERIAL_BEGIN(video); STAILQ_FOREACH_SAFE(vo, &sc->vid_outputs, vo_next, vn) { acpi_video_vo_destroy(vo); } ACPI_SERIAL_END(video); return (0); } static int acpi_video_shutdown(device_t dev) { struct acpi_video_softc *sc; sc = device_get_softc(dev); vid_set_switch_policy(sc->handle, DOS_SWITCH_BY_BIOS); return (0); } static void acpi_video_notify_handler(ACPI_HANDLE handle, UINT32 notify, void *context) { struct acpi_video_softc *sc; struct acpi_video_output *vo, *vo_tmp; ACPI_HANDLE lasthand; UINT32 dcs, dss, dss_p; sc = (struct acpi_video_softc *)context; switch (notify) { case VID_NOTIFY_SWITCHED: dss_p = 0; lasthand = NULL; ACPI_SERIAL_BEGIN(video); ACPI_SERIAL_BEGIN(video_output); STAILQ_FOREACH(vo, &sc->vid_outputs, vo_next) { dss = vo_get_graphics_state(vo->handle); dcs = vo_get_device_status(vo->handle); if (!(dcs & DCS_READY)) dss = DSS_INACTIVE; if (((dcs & DCS_ACTIVE) && dss == DSS_INACTIVE) || (!(dcs & DCS_ACTIVE) && dss == DSS_ACTIVE)) { if (lasthand != NULL) vo_set_device_state(lasthand, dss_p); dss_p = dss; lasthand = vo->handle; } } if (lasthand != NULL) vo_set_device_state(lasthand, dss_p|DSS_COMMIT); ACPI_SERIAL_END(video_output); ACPI_SERIAL_END(video); break; case VID_NOTIFY_REPROBE: ACPI_SERIAL_BEGIN(video); STAILQ_FOREACH(vo, &sc->vid_outputs, vo_next) vo->handle = NULL; acpi_video_bind_outputs(sc); STAILQ_FOREACH_SAFE(vo, &sc->vid_outputs, vo_next, vo_tmp) { if (vo->handle == NULL) { STAILQ_REMOVE(&sc->vid_outputs, vo, acpi_video_output, vo_next); acpi_video_vo_destroy(vo); } } ACPI_SERIAL_END(video); break; default: device_printf(sc->device, "unknown notify event 0x%x\n", notify); } } static void acpi_video_power_profile(void *context) { int state; struct acpi_video_softc *sc; struct acpi_video_output *vo; sc = context; state = power_profile_get_state(); if (state != POWER_PROFILE_PERFORMANCE && state != POWER_PROFILE_ECONOMY) return; ACPI_SERIAL_BEGIN(video); ACPI_SERIAL_BEGIN(video_output); STAILQ_FOREACH(vo, &sc->vid_outputs, vo_next) { if (vo->vo_levels != NULL && vo->vo_brightness == -1) vo_set_brightness(vo->handle, state == POWER_PROFILE_ECONOMY ? vo->vo_economy : vo->vo_fullpower); } ACPI_SERIAL_END(video_output); ACPI_SERIAL_END(video); } static void acpi_video_bind_outputs_subr(ACPI_HANDLE handle, UINT32 adr, void *context) { struct acpi_video_softc *sc; struct acpi_video_output *vo; ACPI_SERIAL_ASSERT(video); sc = context; STAILQ_FOREACH(vo, &sc->vid_outputs, vo_next) { if (vo->adr == adr) { acpi_video_vo_bind(vo, handle); return; } } vo = acpi_video_vo_init(adr); if (vo != NULL) { acpi_video_vo_bind(vo, handle); STAILQ_INSERT_TAIL(&sc->vid_outputs, vo, vo_next); } } static void acpi_video_bind_outputs(struct acpi_video_softc *sc) { ACPI_SERIAL_ASSERT(video); vid_enum_outputs(sc->handle, acpi_video_bind_outputs_subr, sc); } static struct acpi_video_output * acpi_video_vo_init(UINT32 adr) { struct acpi_video_output *vn, *vo, *vp; int n, x; int display_index; int display_port; char name[8], env[32]; const char *type, *desc; struct acpi_video_output_queue *voqh; ACPI_SERIAL_ASSERT(video); display_index = adr & DOD_DEVID_MASK_DISPIDX; display_port = (adr & DOD_DEVID_MASK_DISPPORT) >> 4; switch (adr & DOD_DEVID_MASK) { case DOD_DEVID_MONITOR: if ((adr & DOD_DEVID_MASK_FULL) == DOD_DEVID_LCD) { /* DOD_DEVID_LCD is a common, backward compatible ID */ desc = "Internal/Integrated Digital Flat Panel"; type = "lcd"; voqh = &lcd_units; } else { desc = "VGA CRT or VESA Compatible Analog Monitor"; type = "crt"; voqh = &crt_units; } break; case DOD_DEVID_TV: desc = "TV/HDTV or Analog-Video Monitor"; type = "tv"; voqh = &tv_units; break; case DOD_DEVID_EXT: desc = "External Digital Monitor"; type = "ext"; voqh = &ext_units; break; case DOD_DEVID_INTDFP: desc = "Internal/Integrated Digital Flat Panel"; type = "lcd"; voqh = &lcd_units; break; default: desc = "unknown output"; type = "out"; voqh = &other_units; } n = 0; vn = vp = NULL; STAILQ_FOREACH(vn, voqh, vo_unit.next) { if (vn->vo_unit.num != n) break; vp = vn; n++; } snprintf(name, sizeof(name), "%s%d", type, n); vo = malloc(sizeof(*vo), M_ACPIVIDEO, M_NOWAIT); if (vo != NULL) { vo->handle = NULL; vo->adr = adr; vo->vo_unit.num = n; vo->vo_brightness = -1; vo->vo_fullpower = -1; /* TODO: override with tunables */ vo->vo_economy = -1; vo->vo_numlevels = 0; vo->vo_levels = NULL; snprintf(env, sizeof(env), "hw.acpi.video.%s.fullpower", name); if (getenv_int(env, &x)) vo->vo_fullpower = x; snprintf(env, sizeof(env), "hw.acpi.video.%s.economy", name); if (getenv_int(env, &x)) vo->vo_economy = x; sysctl_ctx_init(&vo->vo_sysctl_ctx); if (vp != NULL) STAILQ_INSERT_AFTER(voqh, vp, vo, vo_unit.next); else STAILQ_INSERT_TAIL(voqh, vo, vo_unit.next); if (acpi_video_sysctl_tree != NULL) vo->vo_sysctl_tree = SYSCTL_ADD_NODE(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(acpi_video_sysctl_tree), OID_AUTO, name, CTLFLAG_RD, 0, desc); if (vo->vo_sysctl_tree != NULL) { SYSCTL_ADD_PROC(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(vo->vo_sysctl_tree), OID_AUTO, "active", CTLTYPE_INT|CTLFLAG_RW, vo, 0, acpi_video_vo_active_sysctl, "I", "current activity of this device"); SYSCTL_ADD_PROC(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(vo->vo_sysctl_tree), OID_AUTO, "brightness", CTLTYPE_INT|CTLFLAG_RW, vo, 0, acpi_video_vo_bright_sysctl, "I", "current brightness level"); SYSCTL_ADD_PROC(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(vo->vo_sysctl_tree), OID_AUTO, "fullpower", CTLTYPE_INT|CTLFLAG_RW, vo, POWER_PROFILE_PERFORMANCE, acpi_video_vo_presets_sysctl, "I", "preset level for full power mode"); SYSCTL_ADD_PROC(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(vo->vo_sysctl_tree), OID_AUTO, "economy", CTLTYPE_INT|CTLFLAG_RW, vo, POWER_PROFILE_ECONOMY, acpi_video_vo_presets_sysctl, "I", "preset level for economy mode"); SYSCTL_ADD_PROC(&vo->vo_sysctl_ctx, SYSCTL_CHILDREN(vo->vo_sysctl_tree), OID_AUTO, "levels", CTLTYPE_OPAQUE|CTLFLAG_RD, vo, 0, acpi_video_vo_levels_sysctl, "I", "supported brightness levels"); } else printf("%s: sysctl node creation failed\n", type); } else printf("%s: softc allocation failed\n", type); if (bootverbose) { printf("found %s(%x)", desc, adr & DOD_DEVID_MASK_FULL); printf(", idx#%x", adr & DOD_DEVID_MASK_DISPIDX); printf(", port#%x", (adr & DOD_DEVID_MASK_DISPPORT) >> 4); if (adr & DOD_BIOS) printf(", detectable by BIOS"); if (adr & DOD_NONVGA) printf(" (Non-VGA output device whose power " "is related to the VGA device)"); printf(", head #%d\n", (adr & DOD_HEAD_ID_MASK) >> DOD_HEAD_ID_SHIFT); } return (vo); } static void acpi_video_vo_bind(struct acpi_video_output *vo, ACPI_HANDLE handle) { ACPI_SERIAL_BEGIN(video_output); if (vo->vo_levels != NULL) AcpiOsFree(vo->vo_levels); vo->handle = handle; vo->vo_numlevels = vo_get_brightness_levels(handle, &vo->vo_levels); if (vo->vo_numlevels >= 2) { if (vo->vo_fullpower == -1 || acpi_video_vo_check_level(vo, vo->vo_fullpower) != 0) /* XXX - can't deal with rebinding... */ vo->vo_fullpower = vo->vo_levels[BCL_FULLPOWER]; if (vo->vo_economy == -1 || acpi_video_vo_check_level(vo, vo->vo_economy) != 0) /* XXX - see above. */ vo->vo_economy = vo->vo_levels[BCL_ECONOMY]; } if (vo->vo_levels != NULL) AcpiInstallNotifyHandler(handle, ACPI_DEVICE_NOTIFY, acpi_video_vo_notify_handler, vo); ACPI_SERIAL_END(video_output); } static void acpi_video_vo_destroy(struct acpi_video_output *vo) { struct acpi_video_output_queue *voqh; ACPI_SERIAL_ASSERT(video); if (vo->vo_sysctl_tree != NULL) { vo->vo_sysctl_tree = NULL; sysctl_ctx_free(&vo->vo_sysctl_ctx); } if (vo->vo_levels != NULL) { AcpiRemoveNotifyHandler(vo->handle, ACPI_DEVICE_NOTIFY, acpi_video_vo_notify_handler); AcpiOsFree(vo->vo_levels); } switch (vo->adr & DOD_DEVID_MASK) { case DOD_DEVID_MONITOR: voqh = &crt_units; break; case DOD_DEVID_TV: voqh = &tv_units; break; case DOD_DEVID_EXT: voqh = &ext_units; break; case DOD_DEVID_INTDFP: voqh = &lcd_units; break; default: voqh = &other_units; } STAILQ_REMOVE(voqh, vo, acpi_video_output, vo_unit.next); free(vo, M_ACPIVIDEO); } static int acpi_video_vo_check_level(struct acpi_video_output *vo, int level) { int i; ACPI_SERIAL_ASSERT(video_output); if (vo->vo_levels == NULL) return (ENODEV); for (i = 0; i < vo->vo_numlevels; i++) if (vo->vo_levels[i] == level) return (0); return (EINVAL); } static void acpi_video_vo_notify_handler(ACPI_HANDLE handle, UINT32 notify, void *context) { struct acpi_video_output *vo; int i, j, level, new_level; vo = context; ACPI_SERIAL_BEGIN(video_output); - if (vo->handle == NULL) { + if (vo->handle != handle) { ACPI_SERIAL_END(video_output); return; } switch (notify) { case VID_NOTIFY_INC_BRN: case VID_NOTIFY_DEC_BRN: if (vo->vo_levels == NULL) break; - level = vo_get_brightness(vo->handle); + level = vo_get_brightness(handle); if (level < 0) break; new_level = level; for (i = 0; i < vo->vo_numlevels; i++) { j = vo->vo_levels[i]; if (notify == VID_NOTIFY_INC_BRN) { if (j > level && (j < new_level || level == new_level)) new_level = j; } else { if (j < level && (j > new_level || level == new_level)) new_level = j; } } if (new_level != level) { - vo_set_brightness(vo->handle, new_level); + vo_set_brightness(handle, new_level); vo->vo_brightness = new_level; } break; default: - printf("%s: unknown notify event 0x%x\n", - acpi_name(vo->handle), notify); + printf("unknown notify event 0x%x from %s\n", + notify, acpi_name(handle)); } ACPI_SERIAL_END(video_output); } /* ARGSUSED */ static int acpi_video_vo_active_sysctl(SYSCTL_HANDLER_ARGS) { struct acpi_video_output *vo; int state, err; vo = (struct acpi_video_output *)arg1; if (vo->handle == NULL) return (ENXIO); ACPI_SERIAL_BEGIN(video_output); state = (vo_get_device_status(vo->handle) & DCS_ACTIVE) ? 1 : 0; err = sysctl_handle_int(oidp, &state, 0, req); if (err != 0 || req->newptr == NULL) goto out; vo_set_device_state(vo->handle, DSS_COMMIT | (state ? DSS_ACTIVE : DSS_INACTIVE)); out: ACPI_SERIAL_END(video_output); return (err); } /* ARGSUSED */ static int acpi_video_vo_bright_sysctl(SYSCTL_HANDLER_ARGS) { struct acpi_video_output *vo; int level, preset, err; vo = (struct acpi_video_output *)arg1; ACPI_SERIAL_BEGIN(video_output); if (vo->handle == NULL) { err = ENXIO; goto out; } if (vo->vo_levels == NULL) { err = ENODEV; goto out; } preset = (power_profile_get_state() == POWER_PROFILE_ECONOMY) ? vo->vo_economy : vo->vo_fullpower; level = vo->vo_brightness; if (level == -1) level = preset; err = sysctl_handle_int(oidp, &level, 0, req); if (err != 0 || req->newptr == NULL) goto out; if (level < -1 || level > 100) { err = EINVAL; goto out; } if (level != -1 && (err = acpi_video_vo_check_level(vo, level))) goto out; vo->vo_brightness = level; vo_set_brightness(vo->handle, (level == -1) ? preset : level); out: ACPI_SERIAL_END(video_output); return (err); } static int acpi_video_vo_presets_sysctl(SYSCTL_HANDLER_ARGS) { struct acpi_video_output *vo; int i, level, *preset, err; err = 0; vo = (struct acpi_video_output *)arg1; ACPI_SERIAL_BEGIN(video_output); if (vo->handle == NULL) { err = ENXIO; goto out; } if (vo->vo_levels == NULL) { err = ENODEV; goto out; } preset = (arg2 == POWER_PROFILE_ECONOMY) ? &vo->vo_economy : &vo->vo_fullpower; level = *preset; err = sysctl_handle_int(oidp, &level, 0, req); if (err != 0 || req->newptr == NULL) goto out; if (level < -1 || level > 100) { err = EINVAL; goto out; } if (level == -1) { i = (arg2 == POWER_PROFILE_ECONOMY) ? BCL_ECONOMY : BCL_FULLPOWER; level = vo->vo_levels[i]; } else if ((err = acpi_video_vo_check_level(vo, level)) != 0) goto out; if (vo->vo_brightness == -1 && (power_profile_get_state() == arg2)) vo_set_brightness(vo->handle, level); *preset = level; out: ACPI_SERIAL_END(video_output); return (err); } /* ARGSUSED */ static int acpi_video_vo_levels_sysctl(SYSCTL_HANDLER_ARGS) { struct acpi_video_output *vo; int err; vo = (struct acpi_video_output *)arg1; ACPI_SERIAL_BEGIN(video_output); if (vo->vo_levels == NULL) { err = ENODEV; goto out; } if (req->newptr != NULL) { err = EPERM; goto out; } err = sysctl_handle_opaque(oidp, vo->vo_levels, vo->vo_numlevels * sizeof(*vo->vo_levels), req); out: ACPI_SERIAL_END(video_output); return (err); } static void vid_set_switch_policy(ACPI_HANDLE handle, UINT32 policy) { ACPI_STATUS status; status = acpi_SetInteger(handle, "_DOS", policy); if (ACPI_FAILURE(status)) printf("can't evaluate %s._DOS - %s\n", acpi_name(handle), AcpiFormatException(status)); } struct enum_callback_arg { void (*callback)(ACPI_HANDLE, UINT32, void *); void *context; ACPI_OBJECT *dod_pkg; int count; }; static ACPI_STATUS vid_enum_outputs_subr(ACPI_HANDLE handle, UINT32 level __unused, void *context, void **retp __unused) { ACPI_STATUS status; UINT32 adr, val; struct enum_callback_arg *argset; size_t i; ACPI_SERIAL_ASSERT(video); argset = context; status = acpi_GetInteger(handle, "_ADR", &adr); if (ACPI_FAILURE(status)) return (AE_OK); for (i = 0; i < argset->dod_pkg->Package.Count; i++) { if (acpi_PkgInt32(argset->dod_pkg, i, &val) == 0 && (val & DOD_DEVID_MASK_FULL) == adr) { argset->callback(handle, val, argset->context); argset->count++; } } return (AE_OK); } static int vid_enum_outputs(ACPI_HANDLE handle, void (*callback)(ACPI_HANDLE, UINT32, void *), void *context) { ACPI_STATUS status; ACPI_BUFFER dod_buf; ACPI_OBJECT *res; struct enum_callback_arg argset; ACPI_SERIAL_ASSERT(video); dod_buf.Length = ACPI_ALLOCATE_BUFFER; dod_buf.Pointer = NULL; status = AcpiEvaluateObject(handle, "_DOD", NULL, &dod_buf); if (ACPI_FAILURE(status)) { if (status != AE_NOT_FOUND) printf("can't evaluate %s._DOD - %s\n", acpi_name(handle), AcpiFormatException(status)); argset.count = -1; goto out; } res = (ACPI_OBJECT *)dod_buf.Pointer; if (!ACPI_PKG_VALID(res, 1)) { printf("evaluation of %s._DOD makes no sense\n", acpi_name(handle)); argset.count = -1; goto out; } if (callback == NULL) { argset.count = res->Package.Count; goto out; } argset.callback = callback; argset.context = context; argset.dod_pkg = res; argset.count = 0; status = AcpiWalkNamespace(ACPI_TYPE_DEVICE, handle, 1, vid_enum_outputs_subr, NULL, &argset, NULL); if (ACPI_FAILURE(status)) printf("failed walking down %s - %s\n", acpi_name(handle), AcpiFormatException(status)); out: if (dod_buf.Pointer != NULL) AcpiOsFree(dod_buf.Pointer); return (argset.count); } static int vo_get_brightness_levels(ACPI_HANDLE handle, int **levelp) { ACPI_STATUS status; ACPI_BUFFER bcl_buf; ACPI_OBJECT *res; int num, i, n, *levels; num = 0; bcl_buf.Length = ACPI_ALLOCATE_BUFFER; bcl_buf.Pointer = NULL; status = AcpiEvaluateObject(handle, "_BCL", NULL, &bcl_buf); if (ACPI_FAILURE(status)) { if (status != AE_NOT_FOUND) printf("can't evaluate %s._BCL - %s\n", acpi_name(handle), AcpiFormatException(status)); num = -1; goto out; } res = (ACPI_OBJECT *)bcl_buf.Pointer; if (!ACPI_PKG_VALID(res, 2)) { printf("evaluation of %s._BCL makes no sense\n", acpi_name(handle)); num = -1; goto out; } num = res->Package.Count; if (levelp == NULL) goto out; levels = AcpiOsAllocate(num * sizeof(*levels)); if (levels == NULL) { num = -1; goto out; } for (i = 0, n = 0; i < num; i++) if (acpi_PkgInt32(res, i, &levels[n]) == 0) n++; if (n < 2) { num = -1; AcpiOsFree(levels); } else { num = n; *levelp = levels; } out: if (bcl_buf.Pointer != NULL) AcpiOsFree(bcl_buf.Pointer); return (num); } static int vo_get_brightness(ACPI_HANDLE handle) { UINT32 level; ACPI_STATUS status; ACPI_SERIAL_ASSERT(video_output); status = acpi_GetInteger(handle, "_BQC", &level); if (ACPI_FAILURE(status)) { printf("can't evaluate %s._BQC - %s\n", acpi_name(handle), AcpiFormatException(status)); return (-1); } if (level > 100) return (-1); return (level); } static void vo_set_brightness(ACPI_HANDLE handle, int level) { ACPI_STATUS status; ACPI_SERIAL_ASSERT(video_output); status = acpi_SetInteger(handle, "_BCM", level); if (ACPI_FAILURE(status)) printf("can't evaluate %s._BCM - %s\n", acpi_name(handle), AcpiFormatException(status)); } static UINT32 vo_get_device_status(ACPI_HANDLE handle) { UINT32 dcs; ACPI_STATUS status; ACPI_SERIAL_ASSERT(video_output); dcs = 0; status = acpi_GetInteger(handle, "_DCS", &dcs); if (ACPI_FAILURE(status)) printf("can't evaluate %s._DCS - %s\n", acpi_name(handle), AcpiFormatException(status)); return (dcs); } static UINT32 vo_get_graphics_state(ACPI_HANDLE handle) { UINT32 dgs; ACPI_STATUS status; dgs = 0; status = acpi_GetInteger(handle, "_DGS", &dgs); if (ACPI_FAILURE(status)) printf("can't evaluate %s._DGS - %s\n", acpi_name(handle), AcpiFormatException(status)); return (dgs); } static void vo_set_device_state(ACPI_HANDLE handle, UINT32 state) { ACPI_STATUS status; ACPI_SERIAL_ASSERT(video_output); status = acpi_SetInteger(handle, "_DSS", state); if (ACPI_FAILURE(status)) printf("can't evaluate %s._DSS - %s\n", acpi_name(handle), AcpiFormatException(status)); } Index: user/imp/tbemd/sys/dev/cxgb/cxgb_sge.c =================================================================== --- user/imp/tbemd/sys/dev/cxgb/cxgb_sge.c (revision 203849) +++ user/imp/tbemd/sys/dev/cxgb/cxgb_sge.c (revision 203850) @@ -1,3837 +1,3839 @@ /************************************************************************** Copyright (c) 2007-2009, Chelsio Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Neither the name of the Chelsio Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include int txq_fills = 0; int multiq_tx_enable = 1; extern struct sysctl_oid_list sysctl__hw_cxgb_children; int cxgb_txq_buf_ring_size = TX_ETH_Q_SIZE; TUNABLE_INT("hw.cxgb.txq_mr_size", &cxgb_txq_buf_ring_size); SYSCTL_UINT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_size, 0, "size of per-queue mbuf ring"); static int cxgb_tx_coalesce_force = 0; TUNABLE_INT("hw.cxgb.tx_coalesce_force", &cxgb_tx_coalesce_force); SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_force, CTLFLAG_RW, &cxgb_tx_coalesce_force, 0, "coalesce small packets into a single work request regardless of ring state"); #define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1 #define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3)) #define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2 #define COALESCE_STOP_MIN TX_ETH_Q_SIZE>>5 #define TX_RECLAIM_DEFAULT TX_ETH_Q_SIZE>>5 #define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2 #define TX_RECLAIM_MIN TX_ETH_Q_SIZE>>6 static int cxgb_tx_coalesce_enable_start = COALESCE_START_DEFAULT; TUNABLE_INT("hw.cxgb.tx_coalesce_enable_start", &cxgb_tx_coalesce_enable_start); SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_start, CTLFLAG_RW, &cxgb_tx_coalesce_enable_start, 0, "coalesce enable threshold"); static int cxgb_tx_coalesce_enable_stop = COALESCE_STOP_DEFAULT; TUNABLE_INT("hw.cxgb.tx_coalesce_enable_stop", &cxgb_tx_coalesce_enable_stop); SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce_enable_stop, CTLFLAG_RW, &cxgb_tx_coalesce_enable_stop, 0, "coalesce disable threshold"); static int cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT; TUNABLE_INT("hw.cxgb.tx_reclaim_threshold", &cxgb_tx_reclaim_threshold); SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_reclaim_threshold, CTLFLAG_RW, &cxgb_tx_reclaim_threshold, 0, "tx cleaning minimum threshold"); /* * XXX don't re-enable this until TOE stops assuming * we have an m_ext */ static int recycle_enable = 0; int cxgb_ext_freed = 0; int cxgb_ext_inited = 0; int fl_q_size = 0; int jumbo_q_size = 0; extern int cxgb_use_16k_clusters; extern int nmbjumbo4; extern int nmbjumbo9; extern int nmbjumbo16; #define USE_GTS 0 #define SGE_RX_SM_BUF_SIZE 1536 #define SGE_RX_DROP_THRES 16 #define SGE_RX_COPY_THRES 128 /* * Period of the Tx buffer reclaim timer. This timer does not need to run * frequently as Tx buffers are usually reclaimed by new Tx packets. */ #define TX_RECLAIM_PERIOD (hz >> 1) /* * Values for sge_txq.flags */ enum { TXQ_RUNNING = 1 << 0, /* fetch engine is running */ TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ }; struct tx_desc { uint64_t flit[TX_DESC_FLITS]; } __packed; struct rx_desc { uint32_t addr_lo; uint32_t len_gen; uint32_t gen2; uint32_t addr_hi; } __packed; struct rsp_desc { /* response queue descriptor */ struct rss_header rss_hdr; uint32_t flags; uint32_t len_cq; uint8_t imm_data[47]; uint8_t intr_gen; } __packed; #define RX_SW_DESC_MAP_CREATED (1 << 0) #define TX_SW_DESC_MAP_CREATED (1 << 1) #define RX_SW_DESC_INUSE (1 << 3) #define TX_SW_DESC_MAPPED (1 << 4) #define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0) #define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP) #define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP) #define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) struct tx_sw_desc { /* SW state per Tx descriptor */ struct mbuf *m; bus_dmamap_t map; int flags; }; struct rx_sw_desc { /* SW state per Rx descriptor */ caddr_t rxsd_cl; struct mbuf *m; bus_dmamap_t map; int flags; }; struct txq_state { unsigned int compl; unsigned int gen; unsigned int pidx; }; struct refill_fl_cb_arg { int error; bus_dma_segment_t seg; int nseg; }; /* * Maps a number of flits to the number of Tx descriptors that can hold them. * The formula is * * desc = 1 + (flits - 2) / (WR_FLITS - 1). * * HW allows up to 4 descriptors to be combined into a WR. */ static uint8_t flit_desc_map[] = { 0, #if SGE_NUM_GENBITS == 1 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 #elif SGE_NUM_GENBITS == 2 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, #else # error "SGE_NUM_GENBITS must be 1 or 2" #endif }; #define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED) #define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock) #define TXQ_LOCK(qs) mtx_lock(&(qs)->lock) #define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock) #define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) +#define TXQ_RING_NEEDS_ENQUEUE(qs) \ + drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) #define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) #define TXQ_RING_DEQUEUE_COND(qs, func, arg) \ drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg) #define TXQ_RING_DEQUEUE(qs) \ drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr) int cxgb_debug = 0; static void sge_timer_cb(void *arg); static void sge_timer_reclaim(void *arg, int ncount); static void sge_txq_reclaim_handler(void *arg, int ncount); static void cxgb_start_locked(struct sge_qset *qs); /* * XXX need to cope with bursty scheduling by looking at a wider * window than we are now for determining the need for coalescing * */ static __inline uint64_t check_pkt_coalesce(struct sge_qset *qs) { struct adapter *sc; struct sge_txq *txq; uint8_t *fill; if (__predict_false(cxgb_tx_coalesce_force)) return (1); txq = &qs->txq[TXQ_ETH]; sc = qs->port->adapter; fill = &sc->tunq_fill[qs->idx]; if (cxgb_tx_coalesce_enable_start > COALESCE_START_MAX) cxgb_tx_coalesce_enable_start = COALESCE_START_MAX; if (cxgb_tx_coalesce_enable_stop < COALESCE_STOP_MIN) cxgb_tx_coalesce_enable_start = COALESCE_STOP_MIN; /* * if the hardware transmit queue is more than 1/8 full * we mark it as coalescing - we drop back from coalescing * when we go below 1/32 full and there are no packets enqueued, * this provides us with some degree of hysteresis */ if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) && TXQ_RING_EMPTY(qs) && (qs->coalescing == 0)) *fill = 0; else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start)) *fill = 1; return (sc->tunq_coalesce); } #ifdef __LP64__ static void set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo) { uint64_t wr_hilo; #if _BYTE_ORDER == _LITTLE_ENDIAN wr_hilo = wr_hi; wr_hilo |= (((uint64_t)wr_lo)<<32); #else wr_hilo = wr_lo; wr_hilo |= (((uint64_t)wr_hi)<<32); #endif wrp->wrh_hilo = wr_hilo; } #else static void set_wr_hdr(struct work_request_hdr *wrp, uint32_t wr_hi, uint32_t wr_lo) { wrp->wrh_hi = wr_hi; wmb(); wrp->wrh_lo = wr_lo; } #endif struct coalesce_info { int count; int nbytes; }; static int coalesce_check(struct mbuf *m, void *arg) { struct coalesce_info *ci = arg; int *count = &ci->count; int *nbytes = &ci->nbytes; if ((*nbytes == 0) || ((*nbytes + m->m_len <= 10500) && (*count < 7) && (m->m_next == NULL))) { *count += 1; *nbytes += m->m_len; return (1); } return (0); } static struct mbuf * cxgb_dequeue(struct sge_qset *qs) { struct mbuf *m, *m_head, *m_tail; struct coalesce_info ci; if (check_pkt_coalesce(qs) == 0) return TXQ_RING_DEQUEUE(qs); m_head = m_tail = NULL; ci.count = ci.nbytes = 0; do { m = TXQ_RING_DEQUEUE_COND(qs, coalesce_check, &ci); if (m_head == NULL) { m_tail = m_head = m; } else if (m != NULL) { m_tail->m_nextpkt = m; m_tail = m; } } while (m != NULL); if (ci.count > 7) panic("trying to coalesce %d packets in to one WR", ci.count); return (m_head); } /** * reclaim_completed_tx - reclaims completed Tx descriptors * @adapter: the adapter * @q: the Tx queue to reclaim completed descriptors from * * Reclaims Tx descriptors that the SGE has indicated it has processed, * and frees the associated buffers if possible. Called with the Tx * queue's lock held. */ static __inline int reclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue) { struct sge_txq *q = &qs->txq[queue]; int reclaim = desc_reclaimable(q); if ((cxgb_tx_reclaim_threshold > TX_RECLAIM_MAX) || (cxgb_tx_reclaim_threshold < TX_RECLAIM_MIN)) cxgb_tx_reclaim_threshold = TX_RECLAIM_DEFAULT; if (reclaim < reclaim_min) return (0); mtx_assert(&qs->lock, MA_OWNED); if (reclaim > 0) { t3_free_tx_desc(qs, reclaim, queue); q->cleaned += reclaim; q->in_use -= reclaim; } if (isset(&qs->txq_stopped, TXQ_ETH)) clrbit(&qs->txq_stopped, TXQ_ETH); return (reclaim); } /** * should_restart_tx - are there enough resources to restart a Tx queue? * @q: the Tx queue * * Checks if there are enough descriptors to restart a suspended Tx queue. */ static __inline int should_restart_tx(const struct sge_txq *q) { unsigned int r = q->processed - q->cleaned; return q->in_use - r < (q->size >> 1); } /** * t3_sge_init - initialize SGE * @adap: the adapter * @p: the SGE parameters * * Performs SGE initialization needed every time after a chip reset. * We do not initialize any of the queue sets here, instead the driver * top-level must request those individually. We also do not enable DMA * here, that should be done after the queues have been set up. */ void t3_sge_init(adapter_t *adap, struct sge_params *p) { u_int ctrl, ups; ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN | V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; #if SGE_NUM_GENBITS == 1 ctrl |= F_EGRGENCTRL; #endif if (adap->params.rev > 0) { if (!(adap->flags & (USING_MSIX | USING_MSI))) ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; } t3_write_reg(adap, A_SG_CONTROL, ctrl); t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | V_LORCQDRBTHRSH(512)); t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | V_TIMEOUT(200 * core_ticks_per_usec(adap))); t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, adap->params.rev < T3_REV_C ? 1000 : 500); t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); } /** * sgl_len - calculates the size of an SGL of the given capacity * @n: the number of SGL entries * * Calculates the number of flits needed for a scatter/gather list that * can hold the given number of entries. */ static __inline unsigned int sgl_len(unsigned int n) { return ((3 * n) / 2 + (n & 1)); } /** * get_imm_packet - return the next ingress packet buffer from a response * @resp: the response descriptor containing the packet data * * Return a packet containing the immediate data of the given response. */ static int get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m) { m->m_len = m->m_pkthdr.len = IMMED_PKT_SIZE; m->m_ext.ext_buf = NULL; m->m_ext.ext_type = 0; memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE); return (0); } static __inline u_int flits_to_desc(u_int n) { return (flit_desc_map[n]); } #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \ F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \ V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \ F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \ F_HIRCQPARITYERROR) #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR) #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \ F_RSPQDISABLED) /** * t3_sge_err_intr_handler - SGE async event interrupt handler * @adapter: the adapter * * Interrupt handler for SGE asynchronous (non-data) events. */ void t3_sge_err_intr_handler(adapter_t *adapter) { unsigned int v, status; status = t3_read_reg(adapter, A_SG_INT_CAUSE); if (status & SGE_PARERR) CH_ALERT(adapter, "SGE parity error (0x%x)\n", status & SGE_PARERR); if (status & SGE_FRAMINGERR) CH_ALERT(adapter, "SGE framing error (0x%x)\n", status & SGE_FRAMINGERR); if (status & F_RSPQCREDITOVERFOW) CH_ALERT(adapter, "SGE response queue credit overflow\n"); if (status & F_RSPQDISABLED) { v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); CH_ALERT(adapter, "packet delivered to disabled response queue (0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff); } t3_write_reg(adapter, A_SG_INT_CAUSE, status); if (status & SGE_FATALERR) t3_fatal_err(adapter); } void t3_sge_prep(adapter_t *adap, struct sge_params *p) { int i, nqsets; nqsets = min(SGE_QSETS, mp_ncpus*4); fl_q_size = min(nmbclusters/(3*nqsets), FL_Q_SIZE); while (!powerof2(fl_q_size)) fl_q_size--; #if __FreeBSD_version >= 700111 if (cxgb_use_16k_clusters) jumbo_q_size = min(nmbjumbo16/(3*nqsets), JUMBO_Q_SIZE); else jumbo_q_size = min(nmbjumbo9/(3*nqsets), JUMBO_Q_SIZE); #else jumbo_q_size = min(nmbjumbo4/(3*nqsets), JUMBO_Q_SIZE); #endif while (!powerof2(jumbo_q_size)) jumbo_q_size--; if (fl_q_size < (FL_Q_SIZE / 4) || jumbo_q_size < (JUMBO_Q_SIZE / 2)) device_printf(adap->dev, "Insufficient clusters and/or jumbo buffers.\n"); /* XXX Does ETHER_ALIGN need to be accounted for here? */ p->max_pkt_size = adap->sge.qs[0].fl[1].buf_size - sizeof(struct cpl_rx_data); for (i = 0; i < SGE_QSETS; ++i) { struct qset_params *q = p->qset + i; if (adap->params.nports > 2) { q->coalesce_usecs = 50; } else { #ifdef INVARIANTS q->coalesce_usecs = 10; #else q->coalesce_usecs = 5; #endif } q->polling = 0; q->rspq_size = RSPQ_Q_SIZE; q->fl_size = fl_q_size; q->jumbo_size = jumbo_q_size; q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; q->txq_size[TXQ_OFLD] = 1024; q->txq_size[TXQ_CTRL] = 256; q->cong_thres = 0; } } int t3_sge_alloc(adapter_t *sc) { /* The parent tag. */ if (bus_dma_tag_create( NULL, /* parent */ 1, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 0, /* flags */ NULL, NULL, /* lock, lockarg */ &sc->parent_dmat)) { device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); return (ENOMEM); } /* * DMA tag for normal sized RX frames */ if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); return (ENOMEM); } /* * DMA tag for jumbo sized RX frames. */ if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); return (ENOMEM); } /* * DMA tag for TX frames. */ if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, TX_MAX_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->tx_dmat)) { device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); return (ENOMEM); } return (0); } int t3_sge_free(struct adapter * sc) { if (sc->tx_dmat != NULL) bus_dma_tag_destroy(sc->tx_dmat); if (sc->rx_jumbo_dmat != NULL) bus_dma_tag_destroy(sc->rx_jumbo_dmat); if (sc->rx_dmat != NULL) bus_dma_tag_destroy(sc->rx_dmat); if (sc->parent_dmat != NULL) bus_dma_tag_destroy(sc->parent_dmat); return (0); } void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) { qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U); qs->rspq.polling = 0 /* p->polling */; } #if !defined(__i386__) && !defined(__amd64__) static void refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct refill_fl_cb_arg *cb_arg = arg; cb_arg->error = error; cb_arg->seg = segs[0]; cb_arg->nseg = nseg; } #endif /** * refill_fl - refill an SGE free-buffer list * @sc: the controller softc * @q: the free-list to refill * @n: the number of new buffers to allocate * * (Re)populate an SGE free-buffer list with up to @n new packet buffers. * The caller must assure that @n does not exceed the queue's capacity. */ static void refill_fl(adapter_t *sc, struct sge_fl *q, int n) { struct rx_sw_desc *sd = &q->sdesc[q->pidx]; struct rx_desc *d = &q->desc[q->pidx]; struct refill_fl_cb_arg cb_arg; struct mbuf *m; caddr_t cl; int err, count = 0; cb_arg.error = 0; while (n--) { /* * We only allocate a cluster, mbuf allocation happens after rx */ if (q->zone == zone_pack) { if ((m = m_getcl(M_NOWAIT, MT_NOINIT, M_PKTHDR)) == NULL) break; cl = m->m_ext.ext_buf; } else { if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL) break; if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { uma_zfree(q->zone, cl); break; } } if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); uma_zfree(q->zone, cl); goto done; } sd->flags |= RX_SW_DESC_MAP_CREATED; } #if !defined(__i386__) && !defined(__amd64__) err = bus_dmamap_load(q->entry_tag, sd->map, cl, q->buf_size, refill_fl_cb, &cb_arg, 0); if (err != 0 || cb_arg.error) { if (q->zone == zone_pack) uma_zfree(q->zone, cl); m_free(m); goto done; } #else cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)cl); #endif sd->flags |= RX_SW_DESC_INUSE; sd->rxsd_cl = cl; sd->m = m; d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); d->len_gen = htobe32(V_FLD_GEN1(q->gen)); d->gen2 = htobe32(V_FLD_GEN2(q->gen)); d++; sd++; if (++q->pidx == q->size) { q->pidx = 0; q->gen ^= 1; sd = q->sdesc; d = q->desc; } q->credits++; count++; } done: if (count) t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); } /** * free_rx_bufs - free the Rx buffers on an SGE free list * @sc: the controle softc * @q: the SGE free list to clean up * * Release the buffers on an SGE free-buffer Rx queue. HW fetching from * this queue should be stopped before calling this function. */ static void free_rx_bufs(adapter_t *sc, struct sge_fl *q) { u_int cidx = q->cidx; while (q->credits--) { struct rx_sw_desc *d = &q->sdesc[cidx]; if (d->flags & RX_SW_DESC_INUSE) { bus_dmamap_unload(q->entry_tag, d->map); bus_dmamap_destroy(q->entry_tag, d->map); if (q->zone == zone_pack) { m_init(d->m, zone_pack, MCLBYTES, M_NOWAIT, MT_DATA, M_EXT); uma_zfree(zone_pack, d->m); } else { m_init(d->m, zone_mbuf, MLEN, M_NOWAIT, MT_DATA, 0); uma_zfree(zone_mbuf, d->m); uma_zfree(q->zone, d->rxsd_cl); } } d->rxsd_cl = NULL; d->m = NULL; if (++cidx == q->size) cidx = 0; } } static __inline void __refill_fl(adapter_t *adap, struct sge_fl *fl) { refill_fl(adap, fl, min(16U, fl->size - fl->credits)); } static __inline void __refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max) { if ((fl->size - fl->credits) < max) refill_fl(adap, fl, min(max, fl->size - fl->credits)); } /** * recycle_rx_buf - recycle a receive buffer * @adapter: the adapter * @q: the SGE free list * @idx: index of buffer to recycle * * Recycles the specified buffer on the given free list by adding it at * the next available slot on the list. */ static void recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) { struct rx_desc *from = &q->desc[idx]; struct rx_desc *to = &q->desc[q->pidx]; q->sdesc[q->pidx] = q->sdesc[idx]; to->addr_lo = from->addr_lo; // already big endian to->addr_hi = from->addr_hi; // likewise wmb(); /* necessary ? */ to->len_gen = htobe32(V_FLD_GEN1(q->gen)); to->gen2 = htobe32(V_FLD_GEN2(q->gen)); q->credits++; if (++q->pidx == q->size) { q->pidx = 0; q->gen ^= 1; } t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); } static void alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { uint32_t *addr; addr = arg; *addr = segs[0].ds_addr; } static int alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, bus_addr_t *phys, void *desc, void *sdesc, bus_dma_tag_t *tag, bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) { size_t len = nelem * elem_size; void *s = NULL; void *p = NULL; int err; if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor tag\n"); return (ENOMEM); } if ((err = bus_dmamem_alloc(*tag, (void **)&p, BUS_DMA_NOWAIT, map)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor memory\n"); return (ENOMEM); } bus_dmamap_load(*tag, *map, p, len, alloc_ring_cb, phys, 0); bzero(p, len); *(void **)desc = p; if (sw_size) { len = nelem * sw_size; s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); *(void **)sdesc = s; } if (parent_entry_tag == NULL) return (0); if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, TX_MAX_SIZE, TX_MAX_SEGS, TX_MAX_SIZE, BUS_DMA_ALLOCNOW, NULL, NULL, entry_tag)) != 0) { device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); return (ENOMEM); } return (0); } static void sge_slow_intr_handler(void *arg, int ncount) { adapter_t *sc = arg; t3_slow_intr_handler(sc); } /** * sge_timer_cb - perform periodic maintenance of an SGE qset * @data: the SGE queue set to maintain * * Runs periodically from a timer to perform maintenance of an SGE queue * set. It performs two tasks: * * a) Cleans up any completed Tx descriptors that may still be pending. * Normal descriptor cleanup happens when new packets are added to a Tx * queue so this timer is relatively infrequent and does any cleanup only * if the Tx queue has not seen any new packets in a while. We make a * best effort attempt to reclaim descriptors, in that we don't wait * around if we cannot get a queue's lock (which most likely is because * someone else is queueing new packets and so will also handle the clean * up). Since control queues use immediate data exclusively we don't * bother cleaning them up here. * * b) Replenishes Rx queues that have run out due to memory shortage. * Normally new Rx buffers are added when existing ones are consumed but * when out of memory a queue can become empty. We try to add only a few * buffers here, the queue will be replenished fully as these new buffers * are used up if memory shortage has subsided. * * c) Return coalesced response queue credits in case a response queue is * starved. * * d) Ring doorbells for T304 tunnel queues since we have seen doorbell * fifo overflows and the FW doesn't implement any recovery scheme yet. */ static void sge_timer_cb(void *arg) { adapter_t *sc = arg; if ((sc->flags & USING_MSIX) == 0) { struct port_info *pi; struct sge_qset *qs; struct sge_txq *txq; int i, j; int reclaim_ofl, refill_rx; if (sc->open_device_map == 0) return; for (i = 0; i < sc->params.nports; i++) { pi = &sc->port[i]; for (j = 0; j < pi->nqsets; j++) { qs = &sc->sge.qs[pi->first_qset + j]; txq = &qs->txq[0]; reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || (qs->fl[1].credits < qs->fl[1].size)); if (reclaim_ofl || refill_rx) { taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task); break; } } } } if (sc->params.nports > 2) { int i; for_each_port(sc, i) { struct port_info *pi = &sc->port[i]; t3_write_reg(sc, A_SG_KDOORBELL, F_SELEGRCNTX | (FW_TUNNEL_SGEEC_START + pi->first_qset)); } } if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) && sc->open_device_map != 0) callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); } /* * This is meant to be a catch-all function to keep sge state private * to sge.c * */ int t3_sge_init_adapter(adapter_t *sc) { callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); return (0); } int t3_sge_reset_adapter(adapter_t *sc) { callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); return (0); } int t3_sge_init_port(struct port_info *pi) { TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); return (0); } /** * refill_rspq - replenish an SGE response queue * @adapter: the adapter * @q: the response queue to replenish * @credits: how many new responses to make available * * Replenishes a response queue by making the supplied number of responses * available to HW. */ static __inline void refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) { /* mbufs are allocated on demand when a rspq entry is processed. */ t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); } static void sge_txq_reclaim_handler(void *arg, int ncount) { struct sge_qset *qs = arg; int i; for (i = 0; i < 3; i++) reclaim_completed_tx(qs, 16, i); } static void sge_timer_reclaim(void *arg, int ncount) { struct port_info *pi = arg; int i, nqsets = pi->nqsets; adapter_t *sc = pi->adapter; struct sge_qset *qs; struct mtx *lock; KASSERT((sc->flags & USING_MSIX) == 0, ("can't call timer reclaim for msi-x")); for (i = 0; i < nqsets; i++) { qs = &sc->sge.qs[pi->first_qset + i]; reclaim_completed_tx(qs, 16, TXQ_OFLD); lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : &sc->sge.qs[0].rspq.lock; if (mtx_trylock(lock)) { /* XXX currently assume that we are *NOT* polling */ uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); if (qs->fl[0].credits < qs->fl[0].size - 16) __refill_fl(sc, &qs->fl[0]); if (qs->fl[1].credits < qs->fl[1].size - 16) __refill_fl(sc, &qs->fl[1]); if (status & (1 << qs->rspq.cntxt_id)) { if (qs->rspq.credits) { refill_rspq(sc, &qs->rspq, 1); qs->rspq.credits--; t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 1 << qs->rspq.cntxt_id); } } mtx_unlock(lock); } } } /** * init_qset_cntxt - initialize an SGE queue set context info * @qs: the queue set * @id: the queue set id * * Initializes the TIDs and context ids for the queues of a queue set. */ static void init_qset_cntxt(struct sge_qset *qs, u_int id) { qs->rspq.cntxt_id = id; qs->fl[0].cntxt_id = 2 * id; qs->fl[1].cntxt_id = 2 * id + 1; qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; mbufq_init(&qs->txq[TXQ_ETH].sendq); mbufq_init(&qs->txq[TXQ_OFLD].sendq); mbufq_init(&qs->txq[TXQ_CTRL].sendq); } static void txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) { txq->in_use += ndesc; /* * XXX we don't handle stopping of queue * presumably start handles this when we bump against the end */ txqs->gen = txq->gen; txq->unacked += ndesc; txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); txq->unacked &= 31; txqs->pidx = txq->pidx; txq->pidx += ndesc; #ifdef INVARIANTS if (((txqs->pidx > txq->cidx) && (txq->pidx < txqs->pidx) && (txq->pidx >= txq->cidx)) || ((txqs->pidx < txq->cidx) && (txq->pidx >= txq-> cidx)) || ((txqs->pidx < txq->cidx) && (txq->cidx < txqs->pidx))) panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", txqs->pidx, txq->pidx, txq->cidx); #endif if (txq->pidx >= txq->size) { txq->pidx -= txq->size; txq->gen ^= 1; } } /** * calc_tx_descs - calculate the number of Tx descriptors for a packet * @m: the packet mbufs * @nsegs: the number of segments * * Returns the number of Tx descriptors needed for the given Ethernet * packet. Ethernet packets require addition of WR and CPL headers. */ static __inline unsigned int calc_tx_descs(const struct mbuf *m, int nsegs) { unsigned int flits; if (m->m_pkthdr.len <= PIO_LEN) return 1; flits = sgl_len(nsegs) + 2; #ifdef TSO_SUPPORTED if (m->m_pkthdr.csum_flags & CSUM_TSO) flits++; #endif return flits_to_desc(flits); } static unsigned int busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs) { struct mbuf *m0; int err, pktlen, pass = 0; bus_dma_tag_t tag = txq->entry_tag; retry: err = 0; m0 = *m; pktlen = m0->m_pkthdr.len; #if defined(__i386__) || defined(__amd64__) if (busdma_map_sg_collapse(tag, txsd->map, m, segs, nsegs) == 0) { goto done; } else #endif err = bus_dmamap_load_mbuf_sg(tag, txsd->map, m0, segs, nsegs, 0); if (err == 0) { goto done; } if (err == EFBIG && pass == 0) { pass = 1; /* Too many segments, try to defrag */ m0 = m_defrag(m0, M_DONTWAIT); if (m0 == NULL) { m_freem(*m); *m = NULL; return (ENOBUFS); } *m = m0; goto retry; } else if (err == ENOMEM) { return (err); } if (err) { if (cxgb_debug) printf("map failure err=%d pktlen=%d\n", err, pktlen); m_freem(m0); *m = NULL; return (err); } done: #if !defined(__i386__) && !defined(__amd64__) bus_dmamap_sync(tag, txsd->map, BUS_DMASYNC_PREWRITE); #endif txsd->flags |= TX_SW_DESC_MAPPED; return (0); } /** * make_sgl - populate a scatter/gather list for a packet * @sgp: the SGL to populate * @segs: the packet dma segments * @nsegs: the number of segments * * Generates a scatter/gather list for the buffers that make up a packet * and returns the SGL size in 8-byte words. The caller must size the SGL * appropriately. */ static __inline void make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) { int i, idx; for (idx = 0, i = 0; i < nsegs; i++) { /* * firmware doesn't like empty segments */ if (segs[i].ds_len == 0) continue; if (i && idx == 0) ++sgp; sgp->len[idx] = htobe32(segs[i].ds_len); sgp->addr[idx] = htobe64(segs[i].ds_addr); idx ^= 1; } if (idx) { sgp->len[idx] = 0; sgp->addr[idx] = 0; } } /** * check_ring_tx_db - check and potentially ring a Tx queue's doorbell * @adap: the adapter * @q: the Tx queue * * Ring the doorbell if a Tx queue is asleep. There is a natural race, * where the HW is going to sleep just after we checked, however, * then the interrupt handler will detect the outstanding TX packet * and ring the doorbell for us. * * When GTS is disabled we unconditionally ring the doorbell. */ static __inline void check_ring_tx_db(adapter_t *adap, struct sge_txq *q) { #if USE_GTS clear_bit(TXQ_LAST_PKT_DB, &q->flags); if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { set_bit(TXQ_LAST_PKT_DB, &q->flags); #ifdef T3_TRACE T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", q->cntxt_id); #endif t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); } #else wmb(); /* write descriptors before telling HW */ t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); #endif } static __inline void wr_gen2(struct tx_desc *d, unsigned int gen) { #if SGE_NUM_GENBITS == 2 d->flit[TX_DESC_FLITS - 1] = htobe64(gen); #endif } /** * write_wr_hdr_sgl - write a WR header and, optionally, SGL * @ndesc: number of Tx descriptors spanned by the SGL * @txd: first Tx descriptor to be written * @txqs: txq state (generation and producer index) * @txq: the SGE Tx queue * @sgl: the SGL * @flits: number of flits to the start of the SGL in the first descriptor * @sgl_flits: the SGL size in flits * @wr_hi: top 32 bits of WR header based on WR type (big endian) * @wr_lo: low 32 bits of WR header based on WR type (big endian) * * Write a work request header and an associated SGL. If the SGL is * small enough to fit into one Tx descriptor it has already been written * and we just need to write the WR header. Otherwise we distribute the * SGL across the number of descriptors it spans. */ static void write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) { struct work_request_hdr *wrp = (struct work_request_hdr *)txd; struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; if (__predict_true(ndesc == 1)) { set_wr_hdr(wrp, htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | V_WR_SGLSFLT(flits)) | wr_hi, htonl(V_WR_LEN(flits + sgl_flits) | V_WR_GEN(txqs->gen)) | wr_lo); /* XXX gen? */ wr_gen2(txd, txqs->gen); } else { unsigned int ogen = txqs->gen; const uint64_t *fp = (const uint64_t *)sgl; struct work_request_hdr *wp = wrp; wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | V_WR_SGLSFLT(flits)) | wr_hi; while (sgl_flits) { unsigned int avail = WR_FLITS - flits; if (avail > sgl_flits) avail = sgl_flits; memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); sgl_flits -= avail; ndesc--; if (!sgl_flits) break; fp += avail; txd++; txsd++; if (++txqs->pidx == txq->size) { txqs->pidx = 0; txqs->gen ^= 1; txd = txq->desc; txsd = txq->sdesc; } /* * when the head of the mbuf chain * is freed all clusters will be freed * with it */ wrp = (struct work_request_hdr *)txd; wrp->wrh_hi = htonl(V_WR_DATATYPE(1) | V_WR_SGLSFLT(1)) | wr_hi; wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS, sgl_flits + 1)) | V_WR_GEN(txqs->gen)) | wr_lo; wr_gen2(txd, txqs->gen); flits = 1; } wrp->wrh_hi |= htonl(F_WR_EOP); wmb(); wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; wr_gen2((struct tx_desc *)wp, ogen); } } /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) #ifdef VLAN_SUPPORTED #define GET_VTAG(cntrl, m) \ do { \ if ((m)->m_flags & M_VLANTAG) \ cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \ } while (0) #else #define GET_VTAG(cntrl, m) #endif static int t3_encap(struct sge_qset *qs, struct mbuf **m) { adapter_t *sc; struct mbuf *m0; struct sge_txq *txq; struct txq_state txqs; struct port_info *pi; unsigned int ndesc, flits, cntrl, mlen; int err, nsegs, tso_info = 0; struct work_request_hdr *wrp; struct tx_sw_desc *txsd; struct sg_ent *sgp, *sgl; uint32_t wr_hi, wr_lo, sgl_flits; bus_dma_segment_t segs[TX_MAX_SEGS]; struct tx_desc *txd; pi = qs->port; sc = pi->adapter; txq = &qs->txq[TXQ_ETH]; txd = &txq->desc[txq->pidx]; txsd = &txq->sdesc[txq->pidx]; sgl = txq->txq_sgl; prefetch(txd); m0 = *m; DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); mtx_assert(&qs->lock, MA_OWNED); cntrl = V_TXPKT_INTF(pi->txpkt_intf); KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n")); #ifdef VLAN_SUPPORTED if (m0->m_nextpkt == NULL && m0->m_next != NULL && m0->m_pkthdr.csum_flags & (CSUM_TSO)) tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); #endif if (m0->m_nextpkt != NULL) { busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs); ndesc = 1; mlen = 0; } else { if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map, &m0, segs, &nsegs))) { if (cxgb_debug) printf("failed ... err=%d\n", err); return (err); } mlen = m0->m_pkthdr.len; ndesc = calc_tx_descs(m0, nsegs); } txq_prod(txq, ndesc, &txqs); KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs)); txsd->m = m0; if (m0->m_nextpkt != NULL) { struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; int i, fidx; if (nsegs > 7) panic("trying to coalesce %d packets in to one WR", nsegs); txq->txq_coalesced += nsegs; wrp = (struct work_request_hdr *)txd; flits = nsegs*2 + 1; for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) { struct cpl_tx_pkt_batch_entry *cbe; uint64_t flit; uint32_t *hflit = (uint32_t *)&flit; int cflags = m0->m_pkthdr.csum_flags; cntrl = V_TXPKT_INTF(pi->txpkt_intf); GET_VTAG(cntrl, m0); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); if (__predict_false(!(cflags & CSUM_IP))) cntrl |= F_TXPKT_IPCSUM_DIS; if (__predict_false(!(cflags & (CSUM_TCP | CSUM_UDP)))) cntrl |= F_TXPKT_L4CSUM_DIS; hflit[0] = htonl(cntrl); hflit[1] = htonl(segs[i].ds_len | 0x80000000); flit |= htobe64(1 << 24); cbe = &cpl_batch->pkt_entry[i]; cbe->cntrl = hflit[0]; cbe->len = hflit[1]; cbe->addr = htobe64(segs[i].ds_addr); } wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | V_WR_SGLSFLT(flits)) | htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); set_wr_hdr(wrp, wr_hi, wr_lo); wmb(); wr_gen2(txd, txqs.gen); check_ring_tx_db(sc, txq); return (0); } else if (tso_info) { int min_size = TCPPKTHDRSIZE, eth_type, tagged; struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; struct ip *ip; struct tcphdr *tcp; char *pkthdr; txd->flit[2] = 0; GET_VTAG(cntrl, m0); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); hdr->cntrl = htonl(cntrl); hdr->len = htonl(mlen | 0x80000000); DPRINTF("tso buf len=%d\n", mlen); tagged = m0->m_flags & M_VLANTAG; if (!tagged) min_size -= ETHER_VLAN_ENCAP_LEN; if (__predict_false(mlen < min_size)) { printf("mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%#x,flags=%#x", m0, mlen, m0->m_pkthdr.tso_segsz, m0->m_pkthdr.csum_flags, m0->m_flags); panic("tx tso packet too small"); } /* Make sure that ether, ip, tcp headers are all in m0 */ if (__predict_false(m0->m_len < min_size)) { m0 = m_pullup(m0, min_size); if (__predict_false(m0 == NULL)) { /* XXX panic probably an overreaction */ panic("couldn't fit header into mbuf"); } } pkthdr = m0->m_data; if (tagged) { eth_type = CPL_ETH_II_VLAN; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); } else { eth_type = CPL_ETH_II; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); } tcp = (struct tcphdr *)((uint8_t *)ip + sizeof(*ip)); tso_info |= V_LSO_ETH_TYPE(eth_type) | V_LSO_IPHDR_WORDS(ip->ip_hl) | V_LSO_TCPHDR_WORDS(tcp->th_off); hdr->lso_info = htonl(tso_info); if (__predict_false(mlen <= PIO_LEN)) { /* pkt not undersized but fits in PIO_LEN * Indicates a TSO bug at the higher levels. * */ DPRINTF("**5592 Fix** mbuf=%p,len=%d,tso_segsz=%d,csum_flags=%#x,flags=%#x", m0, mlen, m0->m_pkthdr.tso_segsz, m0->m_pkthdr.csum_flags, m0->m_flags); txsd->m = NULL; m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]); flits = (mlen + 7) / 8 + 3; wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | F_WR_SOP | F_WR_EOP | txqs.compl); wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); set_wr_hdr(&hdr->wr, wr_hi, wr_lo); wmb(); wr_gen2(txd, txqs.gen); check_ring_tx_db(sc, txq); return (0); } flits = 3; } else { struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; GET_VTAG(cntrl, m0); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) cntrl |= F_TXPKT_IPCSUM_DIS; if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)))) cntrl |= F_TXPKT_L4CSUM_DIS; cpl->cntrl = htonl(cntrl); cpl->len = htonl(mlen | 0x80000000); if (mlen <= PIO_LEN) { txsd->m = NULL; m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); flits = (mlen + 7) / 8 + 2; wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | F_WR_SOP | F_WR_EOP | txqs.compl); wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); set_wr_hdr(&cpl->wr, wr_hi, wr_lo); wmb(); wr_gen2(txd, txqs.gen); check_ring_tx_db(sc, txq); return (0); } flits = 2; } wrp = (struct work_request_hdr *)txd; sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; make_sgl(sgp, segs, nsegs); sgl_flits = sgl_len(nsegs); KASSERT(ndesc <= 4, ("ndesc too large %d", ndesc)); wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); wr_lo = htonl(V_WR_TID(txq->token)); write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); check_ring_tx_db(pi->adapter, txq); return (0); } void cxgb_tx_watchdog(void *arg) { struct sge_qset *qs = arg; struct sge_txq *txq = &qs->txq[TXQ_ETH]; if (qs->coalescing != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) && TXQ_RING_EMPTY(qs)) qs->coalescing = 0; else if (qs->coalescing == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start)) qs->coalescing = 1; if (TXQ_TRYLOCK(qs)) { qs->qs_flags |= QS_FLUSHING; cxgb_start_locked(qs); qs->qs_flags &= ~QS_FLUSHING; TXQ_UNLOCK(qs); } if (qs->port->ifp->if_drv_flags & IFF_DRV_RUNNING) callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog, qs, txq->txq_watchdog.c_cpu); } static void cxgb_tx_timeout(void *arg) { struct sge_qset *qs = arg; struct sge_txq *txq = &qs->txq[TXQ_ETH]; if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3))) qs->coalescing = 1; if (TXQ_TRYLOCK(qs)) { qs->qs_flags |= QS_TIMEOUT; cxgb_start_locked(qs); qs->qs_flags &= ~QS_TIMEOUT; TXQ_UNLOCK(qs); } } static void cxgb_start_locked(struct sge_qset *qs) { struct mbuf *m_head = NULL; struct sge_txq *txq = &qs->txq[TXQ_ETH]; int avail, txmax; int in_use_init = txq->in_use; struct port_info *pi = qs->port; struct ifnet *ifp = pi->ifp; avail = txq->size - txq->in_use - 4; txmax = min(TX_START_MAX_DESC, avail); if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT)) reclaim_completed_tx(qs, 0, TXQ_ETH); if (!pi->link_config.link_ok) { TXQ_RING_FLUSH(qs); return; } TXQ_LOCK_ASSERT(qs); while ((txq->in_use - in_use_init < txmax) && !TXQ_RING_EMPTY(qs) && (ifp->if_drv_flags & IFF_DRV_RUNNING) && pi->link_config.link_ok) { reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH); if ((m_head = cxgb_dequeue(qs)) == NULL) break; /* * Encapsulation can modify our pointer, and or make it * NULL on failure. In that event, we can't requeue. */ if (t3_encap(qs, &m_head) || m_head == NULL) break; /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); /* * We sent via PIO, no longer need a copy */ if (m_head->m_nextpkt == NULL && m_head->m_pkthdr.len <= PIO_LEN) m_freem(m_head); m_head = NULL; } if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 && pi->link_config.link_ok) callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, qs, txq->txq_timer.c_cpu); if (m_head != NULL) m_freem(m_head); } static int cxgb_transmit_locked(struct ifnet *ifp, struct sge_qset *qs, struct mbuf *m) { struct port_info *pi = qs->port; struct sge_txq *txq = &qs->txq[TXQ_ETH]; struct buf_ring *br = txq->txq_mr; int error, avail; avail = txq->size - txq->in_use; TXQ_LOCK_ASSERT(qs); /* * We can only do a direct transmit if the following are true: * - we aren't coalescing (ring < 3/4 full) * - the link is up -- checked in caller * - there are no packets enqueued already * - there is space in hardware transmit queue */ if (check_pkt_coalesce(qs) == 0 && - TXQ_RING_EMPTY(qs) && avail > 4) { + !TXQ_RING_NEEDS_ENQUEUE(qs) && avail > 4) { if (t3_encap(qs, &m)) { if (m != NULL && (error = drbr_enqueue(ifp, br, m)) != 0) return (error); } else { /* * We've bypassed the buf ring so we need to update * the stats directly */ txq->txq_direct_packets++; txq->txq_direct_bytes += m->m_pkthdr.len; /* ** Send a copy of the frame to the BPF ** listener and set the watchdog on. */ ETHER_BPF_MTAP(ifp, m); /* * We sent via PIO, no longer need a copy */ if (m->m_pkthdr.len <= PIO_LEN) m_freem(m); } } else if ((error = drbr_enqueue(ifp, br, m)) != 0) return (error); reclaim_completed_tx(qs, cxgb_tx_reclaim_threshold, TXQ_ETH); if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok && (!check_pkt_coalesce(qs) || (drbr_inuse(ifp, br) >= 7))) cxgb_start_locked(qs); else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer)) callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, qs, txq->txq_timer.c_cpu); return (0); } int cxgb_transmit(struct ifnet *ifp, struct mbuf *m) { struct sge_qset *qs; struct port_info *pi = ifp->if_softc; int error, qidx = pi->first_qset; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||(!pi->link_config.link_ok)) { m_freem(m); return (0); } if (m->m_flags & M_FLOWID) qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; qs = &pi->adapter->sge.qs[qidx]; if (TXQ_TRYLOCK(qs)) { /* XXX running */ error = cxgb_transmit_locked(ifp, qs, m); TXQ_UNLOCK(qs); } else error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); return (error); } void cxgb_start(struct ifnet *ifp) { struct port_info *pi = ifp->if_softc; struct sge_qset *qs = &pi->adapter->sge.qs[pi->first_qset]; if (!pi->link_config.link_ok) return; TXQ_LOCK(qs); cxgb_start_locked(qs); TXQ_UNLOCK(qs); } void cxgb_qflush(struct ifnet *ifp) { /* * flush any enqueued mbufs in the buf_rings * and in the transmit queues * no-op for now */ return; } /** * write_imm - write a packet into a Tx descriptor as immediate data * @d: the Tx descriptor to write * @m: the packet * @len: the length of packet data to write as immediate data * @gen: the generation bit value to write * * Writes a packet as immediate data into a Tx descriptor. The packet * contains a work request at its beginning. We must write the packet * carefully so the SGE doesn't read accidentally before it's written in * its entirety. */ static __inline void write_imm(struct tx_desc *d, struct mbuf *m, unsigned int len, unsigned int gen) { struct work_request_hdr *from = mtod(m, struct work_request_hdr *); struct work_request_hdr *to = (struct work_request_hdr *)d; uint32_t wr_hi, wr_lo; if (len > WR_LEN) panic("len too big %d\n", len); if (len < sizeof(*from)) panic("len too small %d", len); memcpy(&to[1], &from[1], len - sizeof(*from)); wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP | V_WR_BCNTLFLT(len & 7)); wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8)); set_wr_hdr(to, wr_hi, wr_lo); wmb(); wr_gen2(d, gen); /* * This check is a hack we should really fix the logic so * that this can't happen */ if (m->m_type != MT_DONTFREE) m_freem(m); } /** * check_desc_avail - check descriptor availability on a send queue * @adap: the adapter * @q: the TX queue * @m: the packet needing the descriptors * @ndesc: the number of Tx descriptors needed * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) * * Checks if the requested number of Tx descriptors is available on an * SGE send queue. If the queue is already suspended or not enough * descriptors are available the packet is queued for later transmission. * Must be called with the Tx queue locked. * * Returns 0 if enough descriptors are available, 1 if there aren't * enough descriptors and the packet has been queued, and 2 if the caller * needs to retry because there weren't enough descriptors at the * beginning of the call but some freed up in the mean time. */ static __inline int check_desc_avail(adapter_t *adap, struct sge_txq *q, struct mbuf *m, unsigned int ndesc, unsigned int qid) { /* * XXX We currently only use this for checking the control queue * the control queue is only used for binding qsets which happens * at init time so we are guaranteed enough descriptors */ if (__predict_false(!mbufq_empty(&q->sendq))) { addq_exit: mbufq_tail(&q->sendq, m); return 1; } if (__predict_false(q->size - q->in_use < ndesc)) { struct sge_qset *qs = txq_to_qset(q, qid); setbit(&qs->txq_stopped, qid); if (should_restart_tx(q) && test_and_clear_bit(qid, &qs->txq_stopped)) return 2; q->stops++; goto addq_exit; } return 0; } /** * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs * @q: the SGE control Tx queue * * This is a variant of reclaim_completed_tx() that is used for Tx queues * that send only immediate data (presently just the control queues) and * thus do not have any mbufs */ static __inline void reclaim_completed_tx_imm(struct sge_txq *q) { unsigned int reclaim = q->processed - q->cleaned; q->in_use -= reclaim; q->cleaned += reclaim; } static __inline int immediate(const struct mbuf *m) { return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; } /** * ctrl_xmit - send a packet through an SGE control Tx queue * @adap: the adapter * @q: the control queue * @m: the packet * * Send a packet through an SGE control Tx queue. Packets sent through * a control queue must fit entirely as immediate data in a single Tx * descriptor and have no page fragments. */ static int ctrl_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m) { int ret; struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *); struct sge_txq *q = &qs->txq[TXQ_CTRL]; if (__predict_false(!immediate(m))) { m_freem(m); return 0; } wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP); wrp->wrh_lo = htonl(V_WR_TID(q->token)); TXQ_LOCK(qs); again: reclaim_completed_tx_imm(q); ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); if (__predict_false(ret)) { if (ret == 1) { TXQ_UNLOCK(qs); return (ENOSPC); } goto again; } write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); q->in_use++; if (++q->pidx >= q->size) { q->pidx = 0; q->gen ^= 1; } TXQ_UNLOCK(qs); wmb(); t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); return (0); } /** * restart_ctrlq - restart a suspended control queue * @qs: the queue set cotaining the control queue * * Resumes transmission on a suspended Tx control queue. */ static void restart_ctrlq(void *data, int npending) { struct mbuf *m; struct sge_qset *qs = (struct sge_qset *)data; struct sge_txq *q = &qs->txq[TXQ_CTRL]; adapter_t *adap = qs->port->adapter; TXQ_LOCK(qs); again: reclaim_completed_tx_imm(q); while (q->in_use < q->size && (m = mbufq_dequeue(&q->sendq)) != NULL) { write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); if (++q->pidx >= q->size) { q->pidx = 0; q->gen ^= 1; } q->in_use++; } if (!mbufq_empty(&q->sendq)) { setbit(&qs->txq_stopped, TXQ_CTRL); if (should_restart_tx(q) && test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) goto again; q->stops++; } TXQ_UNLOCK(qs); t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); } /* * Send a management message through control queue 0 */ int t3_mgmt_tx(struct adapter *adap, struct mbuf *m) { return ctrl_xmit(adap, &adap->sge.qs[0], m); } /** * free_qset - free the resources of an SGE queue set * @sc: the controller owning the queue set * @q: the queue set * * Release the HW and SW resources associated with an SGE queue set, such * as HW contexts, packet buffers, and descriptor rings. Traffic to the * queue set must be quiesced prior to calling this. */ static void t3_free_qset(adapter_t *sc, struct sge_qset *q) { int i; reclaim_completed_tx(q, 0, TXQ_ETH); for (i = 0; i < SGE_TXQ_PER_SET; i++) { if (q->txq[i].txq_mr != NULL) buf_ring_free(q->txq[i].txq_mr, M_DEVBUF); if (q->txq[i].txq_ifq != NULL) { ifq_delete(q->txq[i].txq_ifq); free(q->txq[i].txq_ifq, M_DEVBUF); } } for (i = 0; i < SGE_RXQ_PER_SET; ++i) { if (q->fl[i].desc) { mtx_lock_spin(&sc->sge.reg_lock); t3_sge_disable_fl(sc, q->fl[i].cntxt_id); mtx_unlock_spin(&sc->sge.reg_lock); bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, q->fl[i].desc_map); bus_dma_tag_destroy(q->fl[i].desc_tag); bus_dma_tag_destroy(q->fl[i].entry_tag); } if (q->fl[i].sdesc) { free_rx_bufs(sc, &q->fl[i]); free(q->fl[i].sdesc, M_DEVBUF); } } mtx_unlock(&q->lock); MTX_DESTROY(&q->lock); for (i = 0; i < SGE_TXQ_PER_SET; i++) { if (q->txq[i].desc) { mtx_lock_spin(&sc->sge.reg_lock); t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); mtx_unlock_spin(&sc->sge.reg_lock); bus_dmamap_unload(q->txq[i].desc_tag, q->txq[i].desc_map); bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, q->txq[i].desc_map); bus_dma_tag_destroy(q->txq[i].desc_tag); bus_dma_tag_destroy(q->txq[i].entry_tag); } if (q->txq[i].sdesc) { free(q->txq[i].sdesc, M_DEVBUF); } } if (q->rspq.desc) { mtx_lock_spin(&sc->sge.reg_lock); t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); mtx_unlock_spin(&sc->sge.reg_lock); bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, q->rspq.desc_map); bus_dma_tag_destroy(q->rspq.desc_tag); MTX_DESTROY(&q->rspq.lock); } #ifdef LRO_SUPPORTED tcp_lro_free(&q->lro.ctrl); #endif bzero(q, sizeof(*q)); } /** * t3_free_sge_resources - free SGE resources * @sc: the adapter softc * * Frees resources used by the SGE queue sets. */ void t3_free_sge_resources(adapter_t *sc) { int i, nqsets; for (nqsets = i = 0; i < (sc)->params.nports; i++) nqsets += sc->port[i].nqsets; for (i = 0; i < nqsets; ++i) { TXQ_LOCK(&sc->sge.qs[i]); t3_free_qset(sc, &sc->sge.qs[i]); } } /** * t3_sge_start - enable SGE * @sc: the controller softc * * Enables the SGE for DMAs. This is the last step in starting packet * transfers. */ void t3_sge_start(adapter_t *sc) { t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); } /** * t3_sge_stop - disable SGE operation * @sc: the adapter * * Disables the DMA engine. This can be called in emeregencies (e.g., * from error interrupts) or from normal process context. In the latter * case it also disables any pending queue restart tasklets. Note that * if it is called in interrupt context it cannot disable the restart * tasklets as it cannot wait, however the tasklets will have no effect * since the doorbells are disabled and the driver will call this again * later from process context, at which time the tasklets will be stopped * if they are still running. */ void t3_sge_stop(adapter_t *sc) { int i, nqsets; t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); if (sc->tq == NULL) return; for (nqsets = i = 0; i < (sc)->params.nports; i++) nqsets += sc->port[i].nqsets; #ifdef notyet /* * * XXX */ for (i = 0; i < nqsets; ++i) { struct sge_qset *qs = &sc->sge.qs[i]; taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); } #endif } /** * t3_free_tx_desc - reclaims Tx descriptors and their buffers * @adapter: the adapter * @q: the Tx queue to reclaim descriptors from * @reclaimable: the number of descriptors to reclaim * @m_vec_size: maximum number of buffers to reclaim * @desc_reclaimed: returns the number of descriptors reclaimed * * Reclaims Tx descriptors from an SGE Tx queue and frees the associated * Tx buffers. Called with the Tx queue lock held. * * Returns number of buffers of reclaimed */ void t3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue) { struct tx_sw_desc *txsd; unsigned int cidx, mask; struct sge_txq *q = &qs->txq[queue]; #ifdef T3_TRACE T3_TRACE2(sc->tb[q->cntxt_id & 7], "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); #endif cidx = q->cidx; mask = q->size - 1; txsd = &q->sdesc[cidx]; mtx_assert(&qs->lock, MA_OWNED); while (reclaimable--) { prefetch(q->sdesc[(cidx + 1) & mask].m); prefetch(q->sdesc[(cidx + 2) & mask].m); if (txsd->m != NULL) { if (txsd->flags & TX_SW_DESC_MAPPED) { bus_dmamap_unload(q->entry_tag, txsd->map); txsd->flags &= ~TX_SW_DESC_MAPPED; } m_freem_list(txsd->m); txsd->m = NULL; } else q->txq_skipped++; ++txsd; if (++cidx == q->size) { cidx = 0; txsd = q->sdesc; } } q->cidx = cidx; } /** * is_new_response - check if a response is newly written * @r: the response descriptor * @q: the response queue * * Returns true if a response descriptor contains a yet unprocessed * response. */ static __inline int is_new_response(const struct rsp_desc *r, const struct sge_rspq *q) { return (r->intr_gen & F_RSPD_GEN2) == q->gen; } #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ #define NOMEM_INTR_DELAY 2500 /** * write_ofld_wr - write an offload work request * @adap: the adapter * @m: the packet to send * @q: the Tx queue * @pidx: index of the first Tx descriptor to write * @gen: the generation value to use * @ndesc: number of descriptors the packet will occupy * * Write an offload work request to send the supplied packet. The packet * data already carry the work request with most fields populated. */ static void write_ofld_wr(adapter_t *adap, struct mbuf *m, struct sge_txq *q, unsigned int pidx, unsigned int gen, unsigned int ndesc, bus_dma_segment_t *segs, unsigned int nsegs) { unsigned int sgl_flits, flits; struct work_request_hdr *from; struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; struct tx_desc *d = &q->desc[pidx]; struct txq_state txqs; if (immediate(m) && nsegs == 0) { write_imm(d, m, m->m_len, gen); return; } /* Only TX_DATA builds SGLs */ from = mtod(m, struct work_request_hdr *); memcpy(&d->flit[1], &from[1], m->m_len - sizeof(*from)); flits = m->m_len / 8; sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; make_sgl(sgp, segs, nsegs); sgl_flits = sgl_len(nsegs); txqs.gen = gen; txqs.pidx = pidx; txqs.compl = 0; write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, from->wrh_hi, from->wrh_lo); } /** * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet * @m: the packet * * Returns the number of Tx descriptors needed for the given offload * packet. These packets are already fully constructed. */ static __inline unsigned int calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) { unsigned int flits, cnt = 0; int ndescs; if (m->m_len <= WR_LEN && nsegs == 0) return (1); /* packet fits as immediate data */ /* * This needs to be re-visited for TOE */ cnt = nsegs; /* headers */ flits = m->m_len / 8; ndescs = flits_to_desc(flits + sgl_len(cnt)); return (ndescs); } /** * ofld_xmit - send a packet through an offload queue * @adap: the adapter * @q: the Tx offload queue * @m: the packet * * Send an offload packet through an SGE offload queue. */ static int ofld_xmit(adapter_t *adap, struct sge_qset *qs, struct mbuf *m) { int ret, nsegs; unsigned int ndesc; unsigned int pidx, gen; struct sge_txq *q = &qs->txq[TXQ_OFLD]; bus_dma_segment_t segs[TX_MAX_SEGS], *vsegs; struct tx_sw_desc *stx; nsegs = m_get_sgllen(m); vsegs = m_get_sgl(m); ndesc = calc_tx_descs_ofld(m, nsegs); busdma_map_sgl(vsegs, segs, nsegs); stx = &q->sdesc[q->pidx]; TXQ_LOCK(qs); again: reclaim_completed_tx(qs, 16, TXQ_OFLD); ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); if (__predict_false(ret)) { if (ret == 1) { printf("no ofld desc avail\n"); m_set_priority(m, ndesc); /* save for restart */ TXQ_UNLOCK(qs); return (EINTR); } goto again; } gen = q->gen; q->in_use += ndesc; pidx = q->pidx; q->pidx += ndesc; if (q->pidx >= q->size) { q->pidx -= q->size; q->gen ^= 1; } #ifdef T3_TRACE T3_TRACE5(adap->tb[q->cntxt_id & 7], "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", ndesc, pidx, skb->len, skb->len - skb->data_len, skb_shinfo(skb)->nr_frags); #endif TXQ_UNLOCK(qs); write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); check_ring_tx_db(adap, q); return (0); } /** * restart_offloadq - restart a suspended offload queue * @qs: the queue set cotaining the offload queue * * Resumes transmission on a suspended Tx offload queue. */ static void restart_offloadq(void *data, int npending) { struct mbuf *m; struct sge_qset *qs = data; struct sge_txq *q = &qs->txq[TXQ_OFLD]; adapter_t *adap = qs->port->adapter; bus_dma_segment_t segs[TX_MAX_SEGS]; struct tx_sw_desc *stx = &q->sdesc[q->pidx]; int nsegs, cleaned; TXQ_LOCK(qs); again: cleaned = reclaim_completed_tx(qs, 16, TXQ_OFLD); while ((m = mbufq_peek(&q->sendq)) != NULL) { unsigned int gen, pidx; unsigned int ndesc = m_get_priority(m); if (__predict_false(q->size - q->in_use < ndesc)) { setbit(&qs->txq_stopped, TXQ_OFLD); if (should_restart_tx(q) && test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) goto again; q->stops++; break; } gen = q->gen; q->in_use += ndesc; pidx = q->pidx; q->pidx += ndesc; if (q->pidx >= q->size) { q->pidx -= q->size; q->gen ^= 1; } (void)mbufq_dequeue(&q->sendq); busdma_map_mbufs(&m, q, stx, segs, &nsegs); TXQ_UNLOCK(qs); write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); TXQ_LOCK(qs); } #if USE_GTS set_bit(TXQ_RUNNING, &q->flags); set_bit(TXQ_LAST_PKT_DB, &q->flags); #endif TXQ_UNLOCK(qs); wmb(); t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); } /** * queue_set - return the queue set a packet should use * @m: the packet * * Maps a packet to the SGE queue set it should use. The desired queue * set is carried in bits 1-3 in the packet's priority. */ static __inline int queue_set(const struct mbuf *m) { return m_get_priority(m) >> 1; } /** * is_ctrl_pkt - return whether an offload packet is a control packet * @m: the packet * * Determines whether an offload packet should use an OFLD or a CTRL * Tx queue. This is indicated by bit 0 in the packet's priority. */ static __inline int is_ctrl_pkt(const struct mbuf *m) { return m_get_priority(m) & 1; } /** * t3_offload_tx - send an offload packet * @tdev: the offload device to send to * @m: the packet * * Sends an offload packet. We use the packet priority to select the * appropriate Tx queue as follows: bit 0 indicates whether the packet * should be sent as regular or control, bits 1-3 select the queue set. */ int t3_offload_tx(struct t3cdev *tdev, struct mbuf *m) { adapter_t *adap = tdev2adap(tdev); struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; if (__predict_false(is_ctrl_pkt(m))) return ctrl_xmit(adap, qs, m); return ofld_xmit(adap, qs, m); } /** * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts * @tdev: the offload device that will be receiving the packets * @q: the SGE response queue that assembled the bundle * @m: the partial bundle * @n: the number of packets in the bundle * * Delivers a (partial) bundle of Rx offload packets to an offload device. */ static __inline void deliver_partial_bundle(struct t3cdev *tdev, struct sge_rspq *q, struct mbuf *mbufs[], int n) { if (n) { q->offload_bundles++; cxgb_ofld_recv(tdev, mbufs, n); } } static __inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq, struct mbuf *m, struct mbuf *rx_gather[], unsigned int gather_idx) { rq->offload_pkts++; m->m_pkthdr.header = mtod(m, void *); rx_gather[gather_idx++] = m; if (gather_idx == RX_BUNDLE_SIZE) { cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE); gather_idx = 0; rq->offload_bundles++; } return (gather_idx); } static void restart_tx(struct sge_qset *qs) { struct adapter *sc = qs->port->adapter; if (isset(&qs->txq_stopped, TXQ_OFLD) && should_restart_tx(&qs->txq[TXQ_OFLD]) && test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { qs->txq[TXQ_OFLD].restarts++; DPRINTF("restarting TXQ_OFLD\n"); taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); } DPRINTF("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n", qs->txq_stopped, should_restart_tx(&qs->txq[TXQ_CTRL]), qs->txq[TXQ_CTRL].processed, qs->txq[TXQ_CTRL].cleaned, qs->txq[TXQ_CTRL].in_use); if (isset(&qs->txq_stopped, TXQ_CTRL) && should_restart_tx(&qs->txq[TXQ_CTRL]) && test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { qs->txq[TXQ_CTRL].restarts++; DPRINTF("restarting TXQ_CTRL\n"); taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); } } /** * t3_sge_alloc_qset - initialize an SGE queue set * @sc: the controller softc * @id: the queue set id * @nports: how many Ethernet ports will be using this queue set * @irq_vec_idx: the IRQ vector index for response queue interrupts * @p: configuration parameters for this queue set * @ntxq: number of Tx queues for the queue set * @pi: port info for queue set * * Allocate resources and initialize an SGE queue set. A queue set * comprises a response queue, two Rx free-buffer queues, and up to 3 * Tx queues. The Tx queues are assigned roles in the order Ethernet * queue, offload queue, and control queue. */ int t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, const struct qset_params *p, int ntxq, struct port_info *pi) { struct sge_qset *q = &sc->sge.qs[id]; int i, ret = 0; MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF); q->port = pi; for (i = 0; i < SGE_TXQ_PER_SET; i++) { if ((q->txq[i].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, M_DEVBUF, M_WAITOK, &q->lock)) == NULL) { device_printf(sc->dev, "failed to allocate mbuf ring\n"); goto err; } if ((q->txq[i].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF, M_NOWAIT|M_ZERO)) == NULL) { device_printf(sc->dev, "failed to allocate ifq\n"); goto err; } ifq_init(q->txq[i].txq_ifq, pi->ifp); callout_init(&q->txq[i].txq_timer, 1); callout_init(&q->txq[i].txq_watchdog, 1); q->txq[i].txq_timer.c_cpu = id % mp_ncpus; q->txq[i].txq_watchdog.c_cpu = id % mp_ncpus; } init_qset_cntxt(q, id); q->idx = id; if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, &q->fl[0].desc, &q->fl[0].sdesc, &q->fl[0].desc_tag, &q->fl[0].desc_map, sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { printf("error %d from alloc ring fl0\n", ret); goto err; } if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, &q->fl[1].desc, &q->fl[1].sdesc, &q->fl[1].desc_tag, &q->fl[1].desc_map, sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { printf("error %d from alloc ring fl1\n", ret); goto err; } if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, &q->rspq.phys_addr, &q->rspq.desc, NULL, &q->rspq.desc_tag, &q->rspq.desc_map, NULL, NULL)) != 0) { printf("error %d from alloc ring rspq\n", ret); goto err; } for (i = 0; i < ntxq; ++i) { size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); if ((ret = alloc_ring(sc, p->txq_size[i], sizeof(struct tx_desc), sz, &q->txq[i].phys_addr, &q->txq[i].desc, &q->txq[i].sdesc, &q->txq[i].desc_tag, &q->txq[i].desc_map, sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { printf("error %d from alloc ring tx %i\n", ret, i); goto err; } mbufq_init(&q->txq[i].sendq); q->txq[i].gen = 1; q->txq[i].size = p->txq_size[i]; } TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q); TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q); q->fl[0].gen = q->fl[1].gen = 1; q->fl[0].size = p->fl_size; q->fl[1].size = p->jumbo_size; q->rspq.gen = 1; q->rspq.cidx = 0; q->rspq.size = p->rspq_size; q->txq[TXQ_ETH].stop_thres = nports * flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); q->fl[0].buf_size = MCLBYTES; q->fl[0].zone = zone_pack; q->fl[0].type = EXT_PACKET; #if __FreeBSD_version > 800000 if (cxgb_use_16k_clusters) { q->fl[1].buf_size = MJUM16BYTES; q->fl[1].zone = zone_jumbo16; q->fl[1].type = EXT_JUMBO16; } else { q->fl[1].buf_size = MJUM9BYTES; q->fl[1].zone = zone_jumbo9; q->fl[1].type = EXT_JUMBO9; } #else q->fl[1].buf_size = MJUMPAGESIZE; q->fl[1].zone = zone_jumbop; q->fl[1].type = EXT_JUMBOP; #endif #ifdef LRO_SUPPORTED /* Allocate and setup the lro_ctrl structure */ q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO); ret = tcp_lro_init(&q->lro.ctrl); if (ret) { printf("error %d from tcp_lro_init\n", ret); goto err; } q->lro.ctrl.ifp = pi->ifp; #endif mtx_lock_spin(&sc->sge.reg_lock); ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, q->rspq.phys_addr, q->rspq.size, q->fl[0].buf_size, 1, 0); if (ret) { printf("error %d from t3_sge_init_rspcntxt\n", ret); goto err_unlock; } for (i = 0; i < SGE_RXQ_PER_SET; ++i) { ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, q->fl[i].phys_addr, q->fl[i].size, q->fl[i].buf_size, p->cong_thres, 1, 0); if (ret) { printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); goto err_unlock; } } ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, 1, 0); if (ret) { printf("error %d from t3_sge_init_ecntxt\n", ret); goto err_unlock; } if (ntxq > 1) { ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, USE_GTS, SGE_CNTXT_OFLD, id, q->txq[TXQ_OFLD].phys_addr, q->txq[TXQ_OFLD].size, 0, 1, 0); if (ret) { printf("error %d from t3_sge_init_ecntxt\n", ret); goto err_unlock; } } if (ntxq > 2) { ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, SGE_CNTXT_CTRL, id, q->txq[TXQ_CTRL].phys_addr, q->txq[TXQ_CTRL].size, q->txq[TXQ_CTRL].token, 1, 0); if (ret) { printf("error %d from t3_sge_init_ecntxt\n", ret); goto err_unlock; } } snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", device_get_unit(sc->dev), irq_vec_idx); MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); mtx_unlock_spin(&sc->sge.reg_lock); t3_update_qset_coalesce(q, p); q->port = pi; refill_fl(sc, &q->fl[0], q->fl[0].size); refill_fl(sc, &q->fl[1], q->fl[1].size); refill_rspq(sc, &q->rspq, q->rspq.size - 1); t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | V_NEWTIMER(q->rspq.holdoff_tmr)); return (0); err_unlock: mtx_unlock_spin(&sc->sge.reg_lock); err: TXQ_LOCK(q); t3_free_qset(sc, q); return (ret); } /* * Remove CPL_RX_PKT headers from the mbuf and reduce it to a regular mbuf with * ethernet data. Hardware assistance with various checksums and any vlan tag * will also be taken into account here. */ void t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad) { struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad); struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; struct ifnet *ifp = pi->ifp; DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff); if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) { m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); m->m_pkthdr.csum_data = 0xffff; } /* * XXX need to add VLAN support for 6.x */ #ifdef VLAN_SUPPORTED if (__predict_false(cpl->vlan_valid)) { m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); m->m_flags |= M_VLANTAG; } #endif m->m_pkthdr.rcvif = ifp; m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad; /* * adjust after conversion to mbuf chain */ m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); m->m_len -= (sizeof(*cpl) + ethpad); m->m_data += (sizeof(*cpl) + ethpad); } /** * get_packet - return the next ingress packet buffer from a free list * @adap: the adapter that received the packet * @drop_thres: # of remaining buffers before we start dropping packets * @qs: the qset that the SGE free list holding the packet belongs to * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain * @r: response descriptor * * Get the next packet from a free list and complete setup of the * sk_buff. If the packet is small we make a copy and recycle the * original buffer, otherwise we use the original buffer itself. If a * positive drop threshold is supplied packets are dropped and their * buffers recycled if (a) the number of remaining buffers is under the * threshold and the packet is too big to copy, or (b) the packet should * be copied but there is no memory for the copy. */ static int get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, struct t3_mbuf_hdr *mh, struct rsp_desc *r) { unsigned int len_cq = ntohl(r->len_cq); struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; int mask, cidx = fl->cidx; struct rx_sw_desc *sd = &fl->sdesc[cidx]; uint32_t len = G_RSPD_LEN(len_cq); uint32_t flags = M_EXT; uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags)); caddr_t cl; struct mbuf *m; int ret = 0; mask = fl->size - 1; prefetch(fl->sdesc[(cidx + 1) & mask].m); prefetch(fl->sdesc[(cidx + 2) & mask].m); prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl); prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl); fl->credits--; bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { if ((m = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) goto skip_recycle; cl = mtod(m, void *); memcpy(cl, sd->rxsd_cl, len); recycle_rx_buf(adap, fl, fl->cidx); m->m_pkthdr.len = m->m_len = len; m->m_flags = 0; mh->mh_head = mh->mh_tail = m; ret = 1; goto done; } else { skip_recycle: bus_dmamap_unload(fl->entry_tag, sd->map); cl = sd->rxsd_cl; m = sd->m; if ((sopeop == RSPQ_SOP_EOP) || (sopeop == RSPQ_SOP)) flags |= M_PKTHDR; m_init(m, fl->zone, fl->buf_size, M_NOWAIT, MT_DATA, flags); if (fl->zone == zone_pack) { /* * restore clobbered data pointer */ m->m_data = m->m_ext.ext_buf; } else { m_cljset(m, cl, fl->type); } m->m_len = len; } switch(sopeop) { case RSPQ_SOP_EOP: ret = 1; /* FALLTHROUGH */ case RSPQ_SOP: mh->mh_head = mh->mh_tail = m; m->m_pkthdr.len = len; break; case RSPQ_EOP: ret = 1; /* FALLTHROUGH */ case RSPQ_NSOP_NEOP: if (mh->mh_tail == NULL) { log(LOG_ERR, "discarding intermediate descriptor entry\n"); m_freem(m); break; } mh->mh_tail->m_next = m; mh->mh_tail = m; mh->mh_head->m_pkthdr.len += len; break; } if (cxgb_debug) printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len); done: if (++fl->cidx == fl->size) fl->cidx = 0; return (ret); } /** * handle_rsp_cntrl_info - handles control information in a response * @qs: the queue set corresponding to the response * @flags: the response control flags * * Handles the control information of an SGE response, such as GTS * indications and completion credits for the queue set's Tx queues. * HW coalesces credits, we don't do any extra SW coalescing. */ static __inline void handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) { unsigned int credits; #if USE_GTS if (flags & F_RSPD_TXQ0_GTS) clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); #endif credits = G_RSPD_TXQ0_CR(flags); if (credits) qs->txq[TXQ_ETH].processed += credits; credits = G_RSPD_TXQ2_CR(flags); if (credits) qs->txq[TXQ_CTRL].processed += credits; # if USE_GTS if (flags & F_RSPD_TXQ1_GTS) clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); # endif credits = G_RSPD_TXQ1_CR(flags); if (credits) qs->txq[TXQ_OFLD].processed += credits; } static void check_ring_db(adapter_t *adap, struct sge_qset *qs, unsigned int sleeping) { ; } /** * process_responses - process responses from an SGE response queue * @adap: the adapter * @qs: the queue set to which the response queue belongs * @budget: how many responses can be processed in this round * * Process responses from an SGE response queue up to the supplied budget. * Responses include received packets as well as credits and other events * for the queues that belong to the response queue's queue set. * A negative budget is effectively unlimited. * * Additionally choose the interrupt holdoff time for the next interrupt * on this queue. If the system is under memory shortage use a fairly * long delay to help recovery. */ static int process_responses(adapter_t *adap, struct sge_qset *qs, int budget) { struct sge_rspq *rspq = &qs->rspq; struct rsp_desc *r = &rspq->desc[rspq->cidx]; int budget_left = budget; unsigned int sleeping = 0; #ifdef LRO_SUPPORTED int lro_enabled = qs->lro.enabled; int skip_lro; struct lro_ctrl *lro_ctrl = &qs->lro.ctrl; #endif struct mbuf *offload_mbufs[RX_BUNDLE_SIZE]; int ngathered = 0; #ifdef DEBUG static int last_holdoff = 0; if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { printf("next_holdoff=%d\n", rspq->holdoff_tmr); last_holdoff = rspq->holdoff_tmr; } #endif rspq->next_holdoff = rspq->holdoff_tmr; while (__predict_true(budget_left && is_new_response(r, rspq))) { int eth, eop = 0, ethpad = 0; uint32_t flags = ntohl(r->flags); uint32_t rss_csum = *(const uint32_t *)r; uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); eth = (r->rss_hdr.opcode == CPL_RX_PKT); if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { struct mbuf *m; if (cxgb_debug) printf("async notification\n"); if (rspq->rspq_mh.mh_head == NULL) { rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); m = rspq->rspq_mh.mh_head; } else { m = m_gethdr(M_DONTWAIT, MT_DATA); } if (m == NULL) goto no_mem; memcpy(mtod(m, char *), r, AN_PKT_SIZE); m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; *mtod(m, char *) = CPL_ASYNC_NOTIF; rss_csum = htonl(CPL_ASYNC_NOTIF << 24); eop = 1; rspq->async_notif++; goto skip; } else if (flags & F_RSPD_IMM_DATA_VALID) { struct mbuf *m = NULL; DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx); if (rspq->rspq_mh.mh_head == NULL) rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA); else m = m_gethdr(M_DONTWAIT, MT_DATA); if (rspq->rspq_mh.mh_head == NULL && m == NULL) { no_mem: rspq->next_holdoff = NOMEM_INTR_DELAY; budget_left--; break; } get_imm_packet(adap, r, rspq->rspq_mh.mh_head); eop = 1; rspq->imm_data++; } else if (r->len_cq) { int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r); if (eop) { rspq->rspq_mh.mh_head->m_flags |= M_FLOWID; rspq->rspq_mh.mh_head->m_pkthdr.flowid = rss_hash; } ethpad = 2; } else { rspq->pure_rsps++; } skip: if (flags & RSPD_CTRL_MASK) { sleeping |= flags & RSPD_GTS_MASK; handle_rsp_cntrl_info(qs, flags); } r++; if (__predict_false(++rspq->cidx == rspq->size)) { rspq->cidx = 0; rspq->gen ^= 1; r = rspq->desc; } if (++rspq->credits >= (rspq->size / 4)) { refill_rspq(adap, rspq, rspq->credits); rspq->credits = 0; } if (!eth && eop) { rspq->rspq_mh.mh_head->m_pkthdr.csum_data = rss_csum; /* * XXX size mismatch */ m_set_priority(rspq->rspq_mh.mh_head, rss_hash); ngathered = rx_offload(&adap->tdev, rspq, rspq->rspq_mh.mh_head, offload_mbufs, ngathered); rspq->rspq_mh.mh_head = NULL; DPRINTF("received offload packet\n"); } else if (eth && eop) { struct mbuf *m = rspq->rspq_mh.mh_head; t3_rx_eth(adap, rspq, m, ethpad); #ifdef LRO_SUPPORTED /* * The T304 sends incoming packets on any qset. If LRO * is also enabled, we could end up sending packet up * lro_ctrl->ifp's input. That is incorrect. * * The mbuf's rcvif was derived from the cpl header and * is accurate. Skip LRO and just use that. */ skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif); if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro && (tcp_lro_rx(lro_ctrl, m, 0) == 0)) { /* successfully queue'd for LRO */ } else #endif { /* * LRO not enabled, packet unsuitable for LRO, * or unable to queue. Pass it up right now in * either case. */ struct ifnet *ifp = m->m_pkthdr.rcvif; (*ifp->if_input)(ifp, m); } rspq->rspq_mh.mh_head = NULL; } __refill_fl_lt(adap, &qs->fl[0], 32); __refill_fl_lt(adap, &qs->fl[1], 32); --budget_left; } deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered); #ifdef LRO_SUPPORTED /* Flush LRO */ while (!SLIST_EMPTY(&lro_ctrl->lro_active)) { struct lro_entry *queued = SLIST_FIRST(&lro_ctrl->lro_active); SLIST_REMOVE_HEAD(&lro_ctrl->lro_active, next); tcp_lro_flush(lro_ctrl, queued); } #endif if (sleeping) check_ring_db(adap, qs, sleeping); mb(); /* commit Tx queue processed updates */ if (__predict_false(qs->txq_stopped > 1)) restart_tx(qs); __refill_fl_lt(adap, &qs->fl[0], 512); __refill_fl_lt(adap, &qs->fl[1], 512); budget -= budget_left; return (budget); } /* * A helper function that processes responses and issues GTS. */ static __inline int process_responses_gts(adapter_t *adap, struct sge_rspq *rq) { int work; static int last_holdoff = 0; work = process_responses(adap, rspq_to_qset(rq), -1); if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { printf("next_holdoff=%d\n", rq->next_holdoff); last_holdoff = rq->next_holdoff; } t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); return (work); } /* * Interrupt handler for legacy INTx interrupts for T3B-based cards. * Handles data events from SGE response queues as well as error and other * async events as they all use the same interrupt pin. We use one SGE * response queue per port in this mode and protect all response queues with * queue 0's lock. */ void t3b_intr(void *data) { uint32_t i, map; adapter_t *adap = data; struct sge_rspq *q0 = &adap->sge.qs[0].rspq; t3_write_reg(adap, A_PL_CLI, 0); map = t3_read_reg(adap, A_SG_DATA_INTR); if (!map) return; if (__predict_false(map & F_ERRINTR)) taskqueue_enqueue(adap->tq, &adap->slow_intr_task); mtx_lock(&q0->lock); for_each_port(adap, i) if (map & (1 << i)) process_responses_gts(adap, &adap->sge.qs[i].rspq); mtx_unlock(&q0->lock); } /* * The MSI interrupt handler. This needs to handle data events from SGE * response queues as well as error and other async events as they all use * the same MSI vector. We use one SGE response queue per port in this mode * and protect all response queues with queue 0's lock. */ void t3_intr_msi(void *data) { adapter_t *adap = data; struct sge_rspq *q0 = &adap->sge.qs[0].rspq; int i, new_packets = 0; mtx_lock(&q0->lock); for_each_port(adap, i) if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) new_packets = 1; mtx_unlock(&q0->lock); if (new_packets == 0) taskqueue_enqueue(adap->tq, &adap->slow_intr_task); } void t3_intr_msix(void *data) { struct sge_qset *qs = data; adapter_t *adap = qs->port->adapter; struct sge_rspq *rspq = &qs->rspq; if (process_responses_gts(adap, rspq) == 0) rspq->unhandled_irqs++; } #define QDUMP_SBUF_SIZE 32 * 400 static int t3_dump_rspq(SYSCTL_HANDLER_ARGS) { struct sge_rspq *rspq; struct sge_qset *qs; int i, err, dump_end, idx; static int multiplier = 1; struct sbuf *sb; struct rsp_desc *rspd; uint32_t data[4]; rspq = arg1; qs = rspq_to_qset(rspq); if (rspq->rspq_dump_count == 0) return (0); if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { log(LOG_WARNING, "dump count is too large %d\n", rspq->rspq_dump_count); rspq->rspq_dump_count = 0; return (EINVAL); } if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { log(LOG_WARNING, "dump start of %d is greater than queue size\n", rspq->rspq_dump_start); rspq->rspq_dump_start = 0; return (EINVAL); } err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); if (err) return (err); retry_sbufops: sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f), ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); sbuf_printf(sb, " generation=%u CQ mode=%u FL threshold=%u\n", ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; for (i = rspq->rspq_dump_start; i < dump_end; i++) { idx = i & (RSPQ_Q_SIZE-1); rspd = &rspq->desc[idx]; sbuf_printf(sb, "\tidx=%04d opcode=%02x cpu_idx=%x hash_type=%x cq_idx=%x\n", idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); sbuf_printf(sb, "\trss_hash_val=%x flags=%08x len_cq=%x intr_gen=%x\n", rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), be32toh(rspd->len_cq), rspd->intr_gen); } if (sbuf_overflowed(sb)) { sbuf_delete(sb); multiplier++; goto retry_sbufops; } sbuf_finish(sb); err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); sbuf_delete(sb); return (err); } static int t3_dump_txq_eth(SYSCTL_HANDLER_ARGS) { struct sge_txq *txq; struct sge_qset *qs; int i, j, err, dump_end; static int multiplier = 1; struct sbuf *sb; struct tx_desc *txd; uint32_t *WR, wr_hi, wr_lo, gen; uint32_t data[4]; txq = arg1; qs = txq_to_qset(txq, TXQ_ETH); if (txq->txq_dump_count == 0) { return (0); } if (txq->txq_dump_count > TX_ETH_Q_SIZE) { log(LOG_WARNING, "dump count is too large %d\n", txq->txq_dump_count); txq->txq_dump_count = 1; return (EINVAL); } if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { log(LOG_WARNING, "dump start of %d is greater than queue size\n", txq->txq_dump_start); txq->txq_dump_start = 0; return (EINVAL); } err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); if (err) return (err); retry_sbufops: sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); sbuf_printf(sb, " \n credits=%u GTS=%u index=%u size=%u rspq#=%u cmdq#=%u\n", (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16), (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1)); sbuf_printf(sb, " TUN=%u TOE=%u generation%u uP token=%u valid=%u\n", ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1), ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1)); sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, txq->txq_dump_start, (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); dump_end = txq->txq_dump_start + txq->txq_dump_count; for (i = txq->txq_dump_start; i < dump_end; i++) { txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; WR = (uint32_t *)txd->flit; wr_hi = ntohl(WR[0]); wr_lo = ntohl(WR[1]); gen = G_WR_GEN(wr_lo); sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", wr_hi, wr_lo, gen); for (j = 2; j < 30; j += 4) sbuf_printf(sb, "\t%08x %08x %08x %08x \n", WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); } if (sbuf_overflowed(sb)) { sbuf_delete(sb); multiplier++; goto retry_sbufops; } sbuf_finish(sb); err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); sbuf_delete(sb); return (err); } static int t3_dump_txq_ctrl(SYSCTL_HANDLER_ARGS) { struct sge_txq *txq; struct sge_qset *qs; int i, j, err, dump_end; static int multiplier = 1; struct sbuf *sb; struct tx_desc *txd; uint32_t *WR, wr_hi, wr_lo, gen; txq = arg1; qs = txq_to_qset(txq, TXQ_CTRL); if (txq->txq_dump_count == 0) { return (0); } if (txq->txq_dump_count > 256) { log(LOG_WARNING, "dump count is too large %d\n", txq->txq_dump_count); txq->txq_dump_count = 1; return (EINVAL); } if (txq->txq_dump_start > 255) { log(LOG_WARNING, "dump start of %d is greater than queue size\n", txq->txq_dump_start); txq->txq_dump_start = 0; return (EINVAL); } retry_sbufops: sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN); sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, txq->txq_dump_start, (txq->txq_dump_start + txq->txq_dump_count) & 255); dump_end = txq->txq_dump_start + txq->txq_dump_count; for (i = txq->txq_dump_start; i < dump_end; i++) { txd = &txq->desc[i & (255)]; WR = (uint32_t *)txd->flit; wr_hi = ntohl(WR[0]); wr_lo = ntohl(WR[1]); gen = G_WR_GEN(wr_lo); sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n", wr_hi, wr_lo, gen); for (j = 2; j < 30; j += 4) sbuf_printf(sb, "\t%08x %08x %08x %08x \n", WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); } if (sbuf_overflowed(sb)) { sbuf_delete(sb); multiplier++; goto retry_sbufops; } sbuf_finish(sb); err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); sbuf_delete(sb); return (err); } static int t3_set_coalesce_usecs(SYSCTL_HANDLER_ARGS) { adapter_t *sc = arg1; struct qset_params *qsp = &sc->params.sge.qset[0]; int coalesce_usecs; struct sge_qset *qs; int i, j, err, nqsets = 0; struct mtx *lock; if ((sc->flags & FULL_INIT_DONE) == 0) return (ENXIO); coalesce_usecs = qsp->coalesce_usecs; err = sysctl_handle_int(oidp, &coalesce_usecs, arg2, req); if (err != 0) { return (err); } if (coalesce_usecs == qsp->coalesce_usecs) return (0); for (i = 0; i < sc->params.nports; i++) for (j = 0; j < sc->port[i].nqsets; j++) nqsets++; coalesce_usecs = max(1, coalesce_usecs); for (i = 0; i < nqsets; i++) { qs = &sc->sge.qs[i]; qsp = &sc->params.sge.qset[i]; qsp->coalesce_usecs = coalesce_usecs; lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : &sc->sge.qs[0].rspq.lock; mtx_lock(lock); t3_update_qset_coalesce(qs, qsp); t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | V_NEWTIMER(qs->rspq.holdoff_tmr)); mtx_unlock(lock); } return (0); } void t3_add_attach_sysctls(adapter_t *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; ctx = device_get_sysctl_ctx(sc->dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); /* random information */ SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, &sc->params.rev, 0, "chip model"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "port_types", CTLFLAG_RD, &sc->port_types, 0, "type of ports"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "enable_debug", CTLFLAG_RW, &cxgb_debug, 0, "enable verbose debugging output"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tunq_coalesce", CTLFLAG_RD, &sc->tunq_coalesce, "#tunneled packets freed"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "txq_overrun", CTLFLAG_RD, &txq_fills, 0, "#times txq overrun"); } static const char *rspq_name = "rspq"; static const char *txq_names[] = { "txq_eth", "txq_ofld", "txq_ctrl" }; static int sysctl_handle_macstat(SYSCTL_HANDLER_ARGS) { struct port_info *p = arg1; uint64_t *parg; if (!p) return (EINVAL); parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2); PORT_LOCK(p); t3_mac_update_stats(&p->mac); PORT_UNLOCK(p); return (sysctl_handle_quad(oidp, parg, 0, req)); } void t3_add_configured_sysctls(adapter_t *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; int i, j; ctx = device_get_sysctl_ctx(sc->dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal", CTLTYPE_INT|CTLFLAG_RW, sc, 0, t3_set_coalesce_usecs, "I", "interrupt coalescing timer (us)"); for (i = 0; i < sc->params.nports; i++) { struct port_info *pi = &sc->port[i]; struct sysctl_oid *poid; struct sysctl_oid_list *poidlist; struct mac_stats *mstats = &pi->mac.stats; snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); poid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, pi->namebuf, CTLFLAG_RD, NULL, "port statistics"); poidlist = SYSCTL_CHILDREN(poid); SYSCTL_ADD_INT(ctx, poidlist, OID_AUTO, "nqsets", CTLFLAG_RD, &pi->nqsets, 0, "#queue sets"); for (j = 0; j < pi->nqsets; j++) { struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; struct sysctl_oid *qspoid, *rspqpoid, *txqpoid, *ctrlqpoid, *lropoid; struct sysctl_oid_list *qspoidlist, *rspqpoidlist, *txqpoidlist, *ctrlqpoidlist, *lropoidlist; struct sge_txq *txq = &qs->txq[TXQ_ETH]; snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); qspoid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, qs->namebuf, CTLFLAG_RD, NULL, "qset statistics"); qspoidlist = SYSCTL_CHILDREN(qspoid); SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl0_empty", CTLFLAG_RD, &qs->fl[0].empty, 0, "freelist #0 empty"); SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "fl1_empty", CTLFLAG_RD, &qs->fl[1].empty, 0, "freelist #1 empty"); rspqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, rspq_name, CTLFLAG_RD, NULL, "rspq statistics"); rspqpoidlist = SYSCTL_CHILDREN(rspqpoid); txqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, txq_names[0], CTLFLAG_RD, NULL, "txq statistics"); txqpoidlist = SYSCTL_CHILDREN(txqpoid); ctrlqpoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, txq_names[2], CTLFLAG_RD, NULL, "ctrlq statistics"); ctrlqpoidlist = SYSCTL_CHILDREN(ctrlqpoid); lropoid = SYSCTL_ADD_NODE(ctx, qspoidlist, OID_AUTO, "lro_stats", CTLFLAG_RD, NULL, "LRO statistics"); lropoidlist = SYSCTL_CHILDREN(lropoid); SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "size", CTLFLAG_RD, &qs->rspq.size, 0, "#entries in response queue"); SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "cidx", CTLFLAG_RD, &qs->rspq.cidx, 0, "consumer index"); SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "credits", CTLFLAG_RD, &qs->rspq.credits, 0, "#credits"); SYSCTL_ADD_XLONG(ctx, rspqpoidlist, OID_AUTO, "phys_addr", CTLFLAG_RD, &qs->rspq.phys_addr, "physical_address_of the queue"); SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_start", CTLFLAG_RW, &qs->rspq.rspq_dump_start, 0, "start rspq dump entry"); SYSCTL_ADD_UINT(ctx, rspqpoidlist, OID_AUTO, "dump_count", CTLFLAG_RW, &qs->rspq.rspq_dump_count, 0, "#rspq entries to dump"); SYSCTL_ADD_PROC(ctx, rspqpoidlist, OID_AUTO, "qdump", CTLTYPE_STRING | CTLFLAG_RD, &qs->rspq, 0, t3_dump_rspq, "A", "dump of the response queue"); SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "dropped", CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_drops, 0, "#tunneled packets dropped"); SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "sendqlen", CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.qlen, 0, "#tunneled packets waiting to be sent"); #if 0 SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_pidx", CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, 0, "#tunneled packets queue producer index"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "queue_cidx", CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, 0, "#tunneled packets queue consumer index"); #endif SYSCTL_ADD_INT(ctx, txqpoidlist, OID_AUTO, "processed", CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, 0, "#tunneled packets processed by the card"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "cleaned", CTLFLAG_RD, &txq->cleaned, 0, "#tunneled packets cleaned"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "in_use", CTLFLAG_RD, &txq->in_use, 0, "#tunneled packet slots in use"); SYSCTL_ADD_ULONG(ctx, txqpoidlist, OID_AUTO, "frees", CTLFLAG_RD, &txq->txq_frees, "#tunneled packets freed"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "skipped", CTLFLAG_RD, &txq->txq_skipped, 0, "#tunneled packet descriptors skipped"); SYSCTL_ADD_QUAD(ctx, txqpoidlist, OID_AUTO, "coalesced", CTLFLAG_RD, &txq->txq_coalesced, "#tunneled packets coalesced"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "enqueued", CTLFLAG_RD, &txq->txq_enqueued, 0, "#tunneled packets enqueued to hardware"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "stopped_flags", CTLFLAG_RD, &qs->txq_stopped, 0, "tx queues stopped"); SYSCTL_ADD_XLONG(ctx, txqpoidlist, OID_AUTO, "phys_addr", CTLFLAG_RD, &txq->phys_addr, "physical_address_of the queue"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "qgen", CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, 0, "txq generation"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_cidx", CTLFLAG_RD, &txq->cidx, 0, "hardware queue cidx"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "hw_pidx", CTLFLAG_RD, &txq->pidx, 0, "hardware queue pidx"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_start", CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, 0, "txq start idx for dump"); SYSCTL_ADD_UINT(ctx, txqpoidlist, OID_AUTO, "dump_count", CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, 0, "txq #entries to dump"); SYSCTL_ADD_PROC(ctx, txqpoidlist, OID_AUTO, "qdump", CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH], 0, t3_dump_txq_eth, "A", "dump of the transmit queue"); SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_start", CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, 0, "ctrlq start idx for dump"); SYSCTL_ADD_UINT(ctx, ctrlqpoidlist, OID_AUTO, "dump_count", CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, 0, "ctrl #entries to dump"); SYSCTL_ADD_PROC(ctx, ctrlqpoidlist, OID_AUTO, "qdump", CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_CTRL], 0, t3_dump_txq_ctrl, "A", "dump of the transmit queue"); #ifdef LRO_SUPPORTED SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_queued", CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL); SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_flushed", CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL); SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_bad_csum", CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL); SYSCTL_ADD_INT(ctx, lropoidlist, OID_AUTO, "lro_cnt", CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL); #endif } /* Now add a node for mac stats. */ poid = SYSCTL_ADD_NODE(ctx, poidlist, OID_AUTO, "mac_stats", CTLFLAG_RD, NULL, "MAC statistics"); poidlist = SYSCTL_CHILDREN(poid); /* * We (ab)use the length argument (arg2) to pass on the offset * of the data that we are interested in. This is only required * for the quad counters that are updated from the hardware (we * make sure that we return the latest value). * sysctl_handle_macstat first updates *all* the counters from * the hardware, and then returns the latest value of the * requested counter. Best would be to update only the * requested counter from hardware, but t3_mac_update_stats() * hides all the register details and we don't want to dive into * all that here. */ #define CXGB_SYSCTL_ADD_QUAD(a) SYSCTL_ADD_OID(ctx, poidlist, OID_AUTO, #a, \ (CTLTYPE_QUAD | CTLFLAG_RD), pi, offsetof(struct mac_stats, a), \ sysctl_handle_macstat, "QU", 0) CXGB_SYSCTL_ADD_QUAD(tx_octets); CXGB_SYSCTL_ADD_QUAD(tx_octets_bad); CXGB_SYSCTL_ADD_QUAD(tx_frames); CXGB_SYSCTL_ADD_QUAD(tx_mcast_frames); CXGB_SYSCTL_ADD_QUAD(tx_bcast_frames); CXGB_SYSCTL_ADD_QUAD(tx_pause); CXGB_SYSCTL_ADD_QUAD(tx_deferred); CXGB_SYSCTL_ADD_QUAD(tx_late_collisions); CXGB_SYSCTL_ADD_QUAD(tx_total_collisions); CXGB_SYSCTL_ADD_QUAD(tx_excess_collisions); CXGB_SYSCTL_ADD_QUAD(tx_underrun); CXGB_SYSCTL_ADD_QUAD(tx_len_errs); CXGB_SYSCTL_ADD_QUAD(tx_mac_internal_errs); CXGB_SYSCTL_ADD_QUAD(tx_excess_deferral); CXGB_SYSCTL_ADD_QUAD(tx_fcs_errs); CXGB_SYSCTL_ADD_QUAD(tx_frames_64); CXGB_SYSCTL_ADD_QUAD(tx_frames_65_127); CXGB_SYSCTL_ADD_QUAD(tx_frames_128_255); CXGB_SYSCTL_ADD_QUAD(tx_frames_256_511); CXGB_SYSCTL_ADD_QUAD(tx_frames_512_1023); CXGB_SYSCTL_ADD_QUAD(tx_frames_1024_1518); CXGB_SYSCTL_ADD_QUAD(tx_frames_1519_max); CXGB_SYSCTL_ADD_QUAD(rx_octets); CXGB_SYSCTL_ADD_QUAD(rx_octets_bad); CXGB_SYSCTL_ADD_QUAD(rx_frames); CXGB_SYSCTL_ADD_QUAD(rx_mcast_frames); CXGB_SYSCTL_ADD_QUAD(rx_bcast_frames); CXGB_SYSCTL_ADD_QUAD(rx_pause); CXGB_SYSCTL_ADD_QUAD(rx_fcs_errs); CXGB_SYSCTL_ADD_QUAD(rx_align_errs); CXGB_SYSCTL_ADD_QUAD(rx_symbol_errs); CXGB_SYSCTL_ADD_QUAD(rx_data_errs); CXGB_SYSCTL_ADD_QUAD(rx_sequence_errs); CXGB_SYSCTL_ADD_QUAD(rx_runt); CXGB_SYSCTL_ADD_QUAD(rx_jabber); CXGB_SYSCTL_ADD_QUAD(rx_short); CXGB_SYSCTL_ADD_QUAD(rx_too_long); CXGB_SYSCTL_ADD_QUAD(rx_mac_internal_errs); CXGB_SYSCTL_ADD_QUAD(rx_cong_drops); CXGB_SYSCTL_ADD_QUAD(rx_frames_64); CXGB_SYSCTL_ADD_QUAD(rx_frames_65_127); CXGB_SYSCTL_ADD_QUAD(rx_frames_128_255); CXGB_SYSCTL_ADD_QUAD(rx_frames_256_511); CXGB_SYSCTL_ADD_QUAD(rx_frames_512_1023); CXGB_SYSCTL_ADD_QUAD(rx_frames_1024_1518); CXGB_SYSCTL_ADD_QUAD(rx_frames_1519_max); #undef CXGB_SYSCTL_ADD_QUAD #define CXGB_SYSCTL_ADD_ULONG(a) SYSCTL_ADD_ULONG(ctx, poidlist, OID_AUTO, #a, \ CTLFLAG_RD, &mstats->a, 0) CXGB_SYSCTL_ADD_ULONG(tx_fifo_parity_err); CXGB_SYSCTL_ADD_ULONG(rx_fifo_parity_err); CXGB_SYSCTL_ADD_ULONG(tx_fifo_urun); CXGB_SYSCTL_ADD_ULONG(rx_fifo_ovfl); CXGB_SYSCTL_ADD_ULONG(serdes_signal_loss); CXGB_SYSCTL_ADD_ULONG(xaui_pcs_ctc_err); CXGB_SYSCTL_ADD_ULONG(xaui_pcs_align_change); CXGB_SYSCTL_ADD_ULONG(num_toggled); CXGB_SYSCTL_ADD_ULONG(num_resets); CXGB_SYSCTL_ADD_ULONG(link_faults); #undef CXGB_SYSCTL_ADD_ULONG } } /** * t3_get_desc - dump an SGE descriptor for debugging purposes * @qs: the queue set * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) * @idx: the descriptor index in the queue * @data: where to dump the descriptor contents * * Dumps the contents of a HW descriptor of an SGE queue. Returns the * size of the descriptor. */ int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, unsigned char *data) { if (qnum >= 6) return (EINVAL); if (qnum < 3) { if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) return -EINVAL; memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); return sizeof(struct tx_desc); } if (qnum == 3) { if (!qs->rspq.desc || idx >= qs->rspq.size) return (EINVAL); memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); return sizeof(struct rsp_desc); } qnum -= 4; if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) return (EINVAL); memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); return sizeof(struct rx_desc); } Index: user/imp/tbemd/sys/dev/e1000/if_em.c =================================================================== --- user/imp/tbemd/sys/dev/e1000/if_em.c (revision 203849) +++ user/imp/tbemd/sys/dev/e1000/if_em.c (revision 203850) @@ -1,5517 +1,5517 @@ /****************************************************************************** Copyright (c) 2001-2010, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ /*$FreeBSD$*/ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #include "opt_inet.h" #include "opt_altq.h" #endif #include #include #if __FreeBSD_version >= 800000 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #if __FreeBSD_version >= 700029 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "e1000_api.h" #include "e1000_82571.h" #include "if_em.h" /********************************************************************* * Set this to one to display debug statistics *********************************************************************/ int em_display_debug_stats = 0; /********************************************************************* * Driver version: *********************************************************************/ char em_driver_version[] = "6.9.25"; /********************************************************************* * PCI Device ID Table * * Used by probe to select devices to load on * Last field stores an index into e1000_strings * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ static em_vendor_info_t em_vendor_info_array[] = { /* Intel(R) PRO/1000 Network Connection */ { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, /* required last entry */ { 0, 0, 0, 0, 0} }; /********************************************************************* * Table of branding strings for all supported NICs. *********************************************************************/ static char *em_strings[] = { "Intel(R) PRO/1000 Network Connection" }; /********************************************************************* * Function prototypes *********************************************************************/ static int em_probe(device_t); static int em_attach(device_t); static int em_detach(device_t); static int em_shutdown(device_t); static int em_suspend(device_t); static int em_resume(device_t); static void em_start(struct ifnet *); static void em_start_locked(struct ifnet *ifp); #if __FreeBSD_version >= 800000 static int em_mq_start(struct ifnet *, struct mbuf *); static int em_mq_start_locked(struct ifnet *, struct mbuf *); static void em_qflush(struct ifnet *); #endif static int em_ioctl(struct ifnet *, u_long, caddr_t); static void em_init(void *); static void em_init_locked(struct adapter *); static void em_stop(void *); static void em_media_status(struct ifnet *, struct ifmediareq *); static int em_media_change(struct ifnet *); static void em_identify_hardware(struct adapter *); static int em_allocate_pci_resources(struct adapter *); static int em_allocate_legacy(struct adapter *adapter); static int em_allocate_msix(struct adapter *adapter); static int em_setup_msix(struct adapter *); static void em_free_pci_resources(struct adapter *); static void em_local_timer(void *); static int em_hardware_init(struct adapter *); static void em_setup_interface(device_t, struct adapter *); static void em_setup_transmit_structures(struct adapter *); static void em_initialize_transmit_unit(struct adapter *); static int em_setup_receive_structures(struct adapter *); static void em_initialize_receive_unit(struct adapter *); static void em_enable_intr(struct adapter *); static void em_disable_intr(struct adapter *); static void em_free_transmit_structures(struct adapter *); static void em_free_receive_structures(struct adapter *); static void em_update_stats_counters(struct adapter *); static void em_txeof(struct adapter *); static void em_tx_purge(struct adapter *); static int em_allocate_receive_structures(struct adapter *); static int em_allocate_transmit_structures(struct adapter *); static int em_rxeof(struct adapter *, int); #ifndef __NO_STRICT_ALIGNMENT static int em_fixup_rx(struct adapter *); #endif static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, struct mbuf *); static void em_transmit_checksum_setup(struct adapter *, struct mbuf *, u32 *, u32 *); #if __FreeBSD_version >= 700000 static bool em_tso_setup(struct adapter *, struct mbuf *, u32 *, u32 *); #endif /* FreeBSD_version >= 700000 */ static void em_set_promisc(struct adapter *); static void em_disable_promisc(struct adapter *); static void em_set_multi(struct adapter *); static void em_print_hw_stats(struct adapter *); static void em_update_link_status(struct adapter *); static int em_get_buf(struct adapter *, int); #if __FreeBSD_version >= 700029 static void em_register_vlan(void *, struct ifnet *, u16); static void em_unregister_vlan(void *, struct ifnet *, u16); static void em_setup_vlan_hw_support(struct adapter *); #endif static int em_xmit(struct adapter *, struct mbuf **); static void em_smartspeed(struct adapter *); static int em_82547_fifo_workaround(struct adapter *, int); static void em_82547_update_fifo_head(struct adapter *, int); static int em_82547_tx_fifo_reset(struct adapter *); static void em_82547_move_tail(void *); static int em_dma_malloc(struct adapter *, bus_size_t, struct em_dma_alloc *, int); static void em_dma_free(struct adapter *, struct em_dma_alloc *); static void em_print_debug_info(struct adapter *); static void em_print_nvm_info(struct adapter *); static int em_is_valid_ether_addr(u8 *); static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); static u32 em_fill_descriptors (bus_addr_t address, u32 length, PDESC_ARRAY desc_array); static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); static void em_add_int_delay_sysctl(struct adapter *, const char *, const char *, struct em_int_delay_info *, int, int); /* Management and WOL Support */ static void em_init_manageability(struct adapter *); static void em_release_manageability(struct adapter *); static void em_get_hw_control(struct adapter *); static void em_release_hw_control(struct adapter *); static void em_get_wakeup(device_t); static void em_enable_wakeup(device_t); static int em_enable_phy_wakeup(struct adapter *); #ifdef EM_LEGACY_IRQ static void em_intr(void *); #else /* FAST IRQ */ #if __FreeBSD_version < 700000 static void em_irq_fast(void *); #else static int em_irq_fast(void *); #endif /* MSIX handlers */ static void em_msix_tx(void *); static void em_msix_rx(void *); static void em_msix_link(void *); static void em_handle_rx(void *context, int pending); static void em_handle_tx(void *context, int pending); static void em_handle_rxtx(void *context, int pending); static void em_handle_link(void *context, int pending); static void em_add_rx_process_limit(struct adapter *, const char *, const char *, int *, int); #endif /* ~EM_LEGACY_IRQ */ #ifdef DEVICE_POLLING static poll_handler_t em_poll; #endif /* POLLING */ /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ static device_method_t em_methods[] = { /* Device interface */ DEVMETHOD(device_probe, em_probe), DEVMETHOD(device_attach, em_attach), DEVMETHOD(device_detach, em_detach), DEVMETHOD(device_shutdown, em_shutdown), DEVMETHOD(device_suspend, em_suspend), DEVMETHOD(device_resume, em_resume), {0, 0} }; static driver_t em_driver = { "em", em_methods, sizeof(struct adapter), }; static devclass_t em_devclass; DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); MODULE_DEPEND(em, pci, 1, 1, 1); MODULE_DEPEND(em, ether, 1, 1, 1); /********************************************************************* * Tunable default values. *********************************************************************/ #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) #define M_TSO_LEN 66 /* Allow common code without TSO */ #ifndef CSUM_TSO #define CSUM_TSO 0 #endif static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); static int em_rxd = EM_DEFAULT_RXD; static int em_txd = EM_DEFAULT_TXD; static int em_smart_pwr_down = FALSE; /* Controls whether promiscuous also shows bad packets */ static int em_debug_sbp = FALSE; /* Local switch for MSI/MSIX */ static int em_enable_msi = TRUE; TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); TUNABLE_INT("hw.em.rxd", &em_rxd); TUNABLE_INT("hw.em.txd", &em_txd); TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); TUNABLE_INT("hw.em.sbp", &em_debug_sbp); TUNABLE_INT("hw.em.enable_msi", &em_enable_msi); #ifndef EM_LEGACY_IRQ /* How many packets rxeof tries to clean at a time */ static int em_rx_process_limit = 100; TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); #endif /* Flow control setting - default to FULL */ static int em_fc_setting = e1000_fc_full; TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); /* ** Shadow VFTA table, this is needed because ** the real vlan filter table gets cleared during ** a soft reset and the driver needs to be able ** to repopulate it. */ static u32 em_shadow_vfta[EM_VFTA_SIZE]; /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; /********************************************************************* * Device identification routine * * em_probe determines if the driver should be loaded on * adapter based on PCI vendor/device id of the adapter. * * return BUS_PROBE_DEFAULT on success, positive on failure *********************************************************************/ static int em_probe(device_t dev) { char adapter_name[60]; u16 pci_vendor_id = 0; u16 pci_device_id = 0; u16 pci_subvendor_id = 0; u16 pci_subdevice_id = 0; em_vendor_info_t *ent; INIT_DEBUGOUT("em_probe: begin"); pci_vendor_id = pci_get_vendor(dev); if (pci_vendor_id != EM_VENDOR_ID) return (ENXIO); pci_device_id = pci_get_device(dev); pci_subvendor_id = pci_get_subvendor(dev); pci_subdevice_id = pci_get_subdevice(dev); ent = em_vendor_info_array; while (ent->vendor_id != 0) { if ((pci_vendor_id == ent->vendor_id) && (pci_device_id == ent->device_id) && ((pci_subvendor_id == ent->subvendor_id) || (ent->subvendor_id == PCI_ANY_ID)) && ((pci_subdevice_id == ent->subdevice_id) || (ent->subdevice_id == PCI_ANY_ID))) { sprintf(adapter_name, "%s %s", em_strings[ent->index], em_driver_version); device_set_desc_copy(dev, adapter_name); return (BUS_PROBE_DEFAULT); } ent++; } return (ENXIO); } /********************************************************************* * Device initialization routine * * The attach entry point is called when the driver is being loaded. * This routine identifies the type of hardware, allocates all resources * and initializes the hardware. * * return 0 on success, positive on failure *********************************************************************/ static int em_attach(device_t dev) { struct adapter *adapter; int tsize, rsize; int error = 0; INIT_DEBUGOUT("em_attach: begin"); adapter = device_get_softc(dev); adapter->dev = adapter->osdep.dev = dev; EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); /* SYSCTL stuff */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, em_sysctl_debug_info, "I", "Debug Information"); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, em_sysctl_stats, "I", "Statistics"); callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); /* Determine hardware and mac info */ em_identify_hardware(adapter); /* Setup PCI resources */ if (em_allocate_pci_resources(adapter)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_pci; } /* ** For ICH8 and family we need to ** map the flash memory, and this ** must happen after the MAC is ** identified */ if ((adapter->hw.mac.type == e1000_ich8lan) || (adapter->hw.mac.type == e1000_pchlan) || (adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_ich10lan)) { int rid = EM_BAR_TYPE_FLASH; adapter->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (adapter->flash == NULL) { device_printf(dev, "Mapping of Flash failed\n"); error = ENXIO; goto err_pci; } /* This is used in the shared code */ adapter->hw.flash_address = (u8 *)adapter->flash; adapter->osdep.flash_bus_space_tag = rman_get_bustag(adapter->flash); adapter->osdep.flash_bus_space_handle = rman_get_bushandle(adapter->flash); } /* Do Shared Code initialization */ if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { device_printf(dev, "Setup of Shared code failed\n"); error = ENXIO; goto err_pci; } e1000_get_bus_info(&adapter->hw); /* Set up some sysctls for the tunable interrupt delays */ em_add_int_delay_sysctl(adapter, "rx_int_delay", "receive interrupt delay in usecs", &adapter->rx_int_delay, E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); em_add_int_delay_sysctl(adapter, "tx_int_delay", "transmit interrupt delay in usecs", &adapter->tx_int_delay, E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); if (adapter->hw.mac.type >= e1000_82540) { em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", "receive interrupt delay limit in usecs", &adapter->rx_abs_int_delay, E1000_REGISTER(&adapter->hw, E1000_RADV), em_rx_abs_int_delay_dflt); em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", "transmit interrupt delay limit in usecs", &adapter->tx_abs_int_delay, E1000_REGISTER(&adapter->hw, E1000_TADV), em_tx_abs_int_delay_dflt); } #ifndef EM_LEGACY_IRQ /* Sysctls for limiting the amount of work done in the taskqueue */ em_add_rx_process_limit(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, em_rx_process_limit); #endif /* * Validate number of transmit and receive descriptors. It * must not exceed hardware maximum, and must be multiple * of E1000_DBA_ALIGN. */ if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || (em_txd < EM_MIN_TXD)) { device_printf(dev, "Using %d TX descriptors instead of %d!\n", EM_DEFAULT_TXD, em_txd); adapter->num_tx_desc = EM_DEFAULT_TXD; } else adapter->num_tx_desc = em_txd; if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || (em_rxd < EM_MIN_RXD)) { device_printf(dev, "Using %d RX descriptors instead of %d!\n", EM_DEFAULT_RXD, em_rxd); adapter->num_rx_desc = EM_DEFAULT_RXD; } else adapter->num_rx_desc = em_rxd; adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_wait_to_complete = FALSE; adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; adapter->rx_buffer_len = 2048; e1000_init_script_state_82541(&adapter->hw, TRUE); e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); /* Copper options */ if (adapter->hw.phy.media_type == e1000_media_type_copper) { adapter->hw.phy.mdix = AUTO_ALL_MODES; adapter->hw.phy.disable_polarity_correction = FALSE; adapter->hw.phy.ms_type = EM_MASTER_SLAVE; } /* * Set the frame limits assuming * standard ethernet sized frames. */ adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; /* * This controls when hardware reports transmit completion * status. */ adapter->hw.mac.report_tx_early = 1; tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); /* Allocate Transmit Descriptor ring */ if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate tx_desc memory\n"); error = ENOMEM; goto err_tx_desc; } adapter->tx_desc_base = (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); /* Allocate Receive Descriptor ring */ if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate rx_desc memory\n"); error = ENOMEM; goto err_rx_desc; } adapter->rx_desc_base = (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; /* ** Start from a known state, this is ** important in reading the nvm and ** mac from that. */ e1000_reset_hw(&adapter->hw); /* Make sure we have a good EEPROM before we read from it */ if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { /* ** Some PCI-E parts fail the first check due to ** the link being in sleep state, call it again, ** if it fails a second time its a real issue. */ if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { device_printf(dev, "The EEPROM Checksum Is Not Valid\n"); error = EIO; goto err_hw_init; } } /* Copy the permanent MAC address out of the EEPROM */ if (e1000_read_mac_addr(&adapter->hw) < 0) { device_printf(dev, "EEPROM read error while reading MAC" " address\n"); error = EIO; goto err_hw_init; } if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { device_printf(dev, "Invalid MAC address\n"); error = EIO; goto err_hw_init; } /* Initialize the hardware */ if (em_hardware_init(adapter)) { device_printf(dev, "Unable to initialize the hardware\n"); error = EIO; goto err_hw_init; } /* Allocate transmit descriptors and buffers */ if (em_allocate_transmit_structures(adapter)) { device_printf(dev, "Could not setup transmit structures\n"); error = ENOMEM; goto err_tx_struct; } /* Allocate receive descriptors and buffers */ if (em_allocate_receive_structures(adapter)) { device_printf(dev, "Could not setup receive structures\n"); error = ENOMEM; goto err_rx_struct; } /* ** Do interrupt configuration */ if (adapter->msi > 1) /* Do MSI/X */ error = em_allocate_msix(adapter); else /* MSI or Legacy */ error = em_allocate_legacy(adapter); if (error) goto err_rx_struct; /* * Get Wake-on-Lan and Management info for later use */ em_get_wakeup(dev); /* Setup OS specific network interface */ em_setup_interface(dev, adapter); /* Initialize statistics */ em_update_stats_counters(adapter); adapter->hw.mac.get_link_status = 1; em_update_link_status(adapter); /* Indicate SOL/IDER usage */ if (e1000_check_reset_block(&adapter->hw)) device_printf(dev, "PHY reset is blocked due to SOL/IDER session.\n"); /* Do we need workaround for 82544 PCI-X adapter? */ if (adapter->hw.bus.type == e1000_bus_type_pcix && adapter->hw.mac.type == e1000_82544) adapter->pcix_82544 = TRUE; else adapter->pcix_82544 = FALSE; #if __FreeBSD_version >= 700029 /* Register for VLAN events */ adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); #endif /* Non-AMT based hardware can now take control from firmware */ if (adapter->has_manage && !adapter->has_amt) em_get_hw_control(adapter); /* Tell the stack that the interface is not active */ adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); INIT_DEBUGOUT("em_attach: end"); return (0); err_rx_struct: em_free_transmit_structures(adapter); err_tx_struct: err_hw_init: em_release_hw_control(adapter); em_dma_free(adapter, &adapter->rxdma); err_rx_desc: em_dma_free(adapter, &adapter->txdma); err_tx_desc: err_pci: em_free_pci_resources(adapter); EM_TX_LOCK_DESTROY(adapter); EM_RX_LOCK_DESTROY(adapter); EM_CORE_LOCK_DESTROY(adapter); return (error); } /********************************************************************* * Device removal routine * * The detach entry point is called when the driver is being removed. * This routine stops the adapter and deallocates all the resources * that were allocated for driver operation. * * return 0 on success, positive on failure *********************************************************************/ static int em_detach(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct ifnet *ifp = adapter->ifp; INIT_DEBUGOUT("em_detach: begin"); /* Make sure VLANS are not using driver */ #if __FreeBSD_version >= 700000 if (adapter->ifp->if_vlantrunk != NULL) { #else if (adapter->ifp->if_nvlans != 0) { #endif device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } #ifdef DEVICE_POLLING if (ifp->if_capenable & IFCAP_POLLING) ether_poll_deregister(ifp); #endif EM_CORE_LOCK(adapter); EM_TX_LOCK(adapter); adapter->in_detach = 1; em_stop(adapter); e1000_phy_hw_reset(&adapter->hw); em_release_manageability(adapter); EM_TX_UNLOCK(adapter); EM_CORE_UNLOCK(adapter); #if __FreeBSD_version >= 700029 /* Unregister VLAN events */ if (adapter->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); if (adapter->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); #endif ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); callout_drain(&adapter->tx_fifo_timer); em_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); em_free_transmit_structures(adapter); em_free_receive_structures(adapter); /* Free Transmit Descriptor ring */ if (adapter->tx_desc_base) { em_dma_free(adapter, &adapter->txdma); adapter->tx_desc_base = NULL; } /* Free Receive Descriptor ring */ if (adapter->rx_desc_base) { em_dma_free(adapter, &adapter->rxdma); adapter->rx_desc_base = NULL; } em_release_hw_control(adapter); EM_TX_LOCK_DESTROY(adapter); EM_RX_LOCK_DESTROY(adapter); EM_CORE_LOCK_DESTROY(adapter); return (0); } /********************************************************************* * * Shutdown entry point * **********************************************************************/ static int em_shutdown(device_t dev) { return em_suspend(dev); } /* * Suspend/resume device methods. */ static int em_suspend(device_t dev) { struct adapter *adapter = device_get_softc(dev); EM_CORE_LOCK(adapter); em_release_manageability(adapter); em_release_hw_control(adapter); em_enable_wakeup(dev); EM_CORE_UNLOCK(adapter); return bus_generic_suspend(dev); } static int em_resume(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct ifnet *ifp = adapter->ifp; EM_CORE_LOCK(adapter); em_init_locked(adapter); em_init_manageability(adapter); EM_CORE_UNLOCK(adapter); em_start(ifp); return bus_generic_resume(dev); } /********************************************************************* * Transmit entry point * * em_start is called by the stack to initiate a transmit. * The driver will remain in this routine as long as there are * packets to transmit and transmit resources are available. * In case resources are not available stack is notified and * the packet is requeued. **********************************************************************/ #if __FreeBSD_version >= 800000 static int em_mq_start_locked(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; struct mbuf *next; int error = E1000_SUCCESS; EM_TX_LOCK_ASSERT(adapter); /* To allow being called from a tasklet */ if (m == NULL) goto process; if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) || (!adapter->link_active)) { error = drbr_enqueue(ifp, adapter->br, m); return (error); - } else if (drbr_empty(ifp, adapter->br) && + } else if (!drbr_needs_enqueue(ifp, adapter->br) && (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { if ((error = em_xmit(adapter, &m)) != 0) { if (m) error = drbr_enqueue(ifp, adapter->br, m); return (error); } else { /* * We've bypassed the buf ring so we need to update * ifp directly */ drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags); /* ** Send a copy of the frame to the BPF ** listener and set the watchdog on. */ ETHER_BPF_MTAP(ifp, m); adapter->watchdog_check = TRUE; } } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) return (error); process: if (drbr_empty(ifp, adapter->br)) return(error); /* Process the queue */ while (TRUE) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; next = drbr_dequeue(ifp, adapter->br); if (next == NULL) break; if ((error = em_xmit(adapter, &next)) != 0) { if (next != NULL) error = drbr_enqueue(ifp, adapter->br, next); break; } drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); ETHER_BPF_MTAP(ifp, next); /* Set the watchdog */ adapter->watchdog_check = TRUE; } if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) ifp->if_drv_flags |= IFF_DRV_OACTIVE; return (error); } /* ** Multiqueue capable stack interface, this is not ** yet truely multiqueue, but that is coming... */ static int em_mq_start(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; int error = 0; if (EM_TX_TRYLOCK(adapter)) { if (ifp->if_drv_flags & IFF_DRV_RUNNING) error = em_mq_start_locked(ifp, m); EM_TX_UNLOCK(adapter); } else error = drbr_enqueue(ifp, adapter->br, m); return (error); } static void em_qflush(struct ifnet *ifp) { struct mbuf *m; struct adapter *adapter = (struct adapter *)ifp->if_softc; EM_TX_LOCK(adapter); while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL) m_freem(m); if_qflush(ifp); EM_TX_UNLOCK(adapter); } #endif /* FreeBSD_version */ static void em_start_locked(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct mbuf *m_head; EM_TX_LOCK_ASSERT(adapter); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!adapter->link_active) return; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; /* * Encapsulation can modify our pointer, and or make it * NULL on failure. In that event, we can't requeue. */ if (em_xmit(adapter, &m_head)) { if (m_head == NULL) break; ifp->if_drv_flags |= IFF_DRV_OACTIVE; IFQ_DRV_PREPEND(&ifp->if_snd, m_head); break; } /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); /* Set timeout in case hardware has problems transmitting. */ adapter->watchdog_check = TRUE; } if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) ifp->if_drv_flags |= IFF_DRV_OACTIVE; return; } static void em_start(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; EM_TX_LOCK(adapter); if (ifp->if_drv_flags & IFF_DRV_RUNNING) em_start_locked(ifp); EM_TX_UNLOCK(adapter); } /********************************************************************* * Ioctl entry point * * em_ioctl is called when the user wants to configure the * interface. * * return 0 on success, positive on failure **********************************************************************/ static int em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct adapter *adapter = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; #ifdef INET struct ifaddr *ifa = (struct ifaddr *)data; #endif int error = 0; if (adapter->in_detach) return (error); switch (command) { case SIOCSIFADDR: #ifdef INET if (ifa->ifa_addr->sa_family == AF_INET) { /* * XXX * Since resetting hardware takes a very long time * and results in link renegotiation we only * initialize the hardware only when it is absolutely * required. */ ifp->if_flags |= IFF_UP; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { EM_CORE_LOCK(adapter); em_init_locked(adapter); EM_CORE_UNLOCK(adapter); } arp_ifinit(ifp, ifa); } else #endif error = ether_ioctl(ifp, command, data); break; case SIOCSIFMTU: { int max_frame_size; u16 eeprom_data = 0; IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); EM_CORE_LOCK(adapter); switch (adapter->hw.mac.type) { case e1000_82573: /* * 82573 only supports jumbo frames * if ASPM is disabled. */ e1000_read_nvm(&adapter->hw, NVM_INIT_3GIO_3, 1, &eeprom_data); if (eeprom_data & NVM_WORD1A_ASPM_MASK) { max_frame_size = ETHER_MAX_LEN; break; } /* Allow Jumbo frames - fall thru */ case e1000_82571: case e1000_82572: case e1000_ich9lan: case e1000_ich10lan: case e1000_82574: case e1000_80003es2lan: /* Limit Jumbo Frame size */ max_frame_size = 9234; break; case e1000_pchlan: max_frame_size = 4096; break; /* Adapters that do not support jumbo frames */ case e1000_82542: case e1000_82583: case e1000_ich8lan: max_frame_size = ETHER_MAX_LEN; break; default: max_frame_size = MAX_JUMBO_FRAME_SIZE; } if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { EM_CORE_UNLOCK(adapter); error = EINVAL; break; } ifp->if_mtu = ifr->ifr_mtu; adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; em_init_locked(adapter); EM_CORE_UNLOCK(adapter); break; } case SIOCSIFFLAGS: IOCTL_DEBUGOUT("ioctl rcv'd:\ SIOCSIFFLAGS (Set Interface Flags)"); EM_CORE_LOCK(adapter); if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { if ((ifp->if_flags ^ adapter->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { em_disable_promisc(adapter); em_set_promisc(adapter); } } else em_init_locked(adapter); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { EM_TX_LOCK(adapter); em_stop(adapter); EM_TX_UNLOCK(adapter); } adapter->if_flags = ifp->if_flags; EM_CORE_UNLOCK(adapter); break; case SIOCADDMULTI: case SIOCDELMULTI: IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { EM_CORE_LOCK(adapter); em_disable_intr(adapter); em_set_multi(adapter); if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { em_initialize_receive_unit(adapter); } #ifdef DEVICE_POLLING if (!(ifp->if_capenable & IFCAP_POLLING)) #endif em_enable_intr(adapter); EM_CORE_UNLOCK(adapter); } break; case SIOCSIFMEDIA: /* Check SOL/IDER usage */ EM_CORE_LOCK(adapter); if (e1000_check_reset_block(&adapter->hw)) { EM_CORE_UNLOCK(adapter); device_printf(adapter->dev, "Media change is" " blocked due to SOL/IDER session.\n"); break; } EM_CORE_UNLOCK(adapter); case SIOCGIFMEDIA: IOCTL_DEBUGOUT("ioctl rcv'd: \ SIOCxIFMEDIA (Get/Set Interface Media)"); error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); break; case SIOCSIFCAP: { int mask, reinit; IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); reinit = 0; mask = ifr->ifr_reqcap ^ ifp->if_capenable; #ifdef DEVICE_POLLING if (mask & IFCAP_POLLING) { if (ifr->ifr_reqcap & IFCAP_POLLING) { error = ether_poll_register(em_poll, ifp); if (error) return (error); EM_CORE_LOCK(adapter); em_disable_intr(adapter); ifp->if_capenable |= IFCAP_POLLING; EM_CORE_UNLOCK(adapter); } else { error = ether_poll_deregister(ifp); /* Enable interrupt even in error case */ EM_CORE_LOCK(adapter); em_enable_intr(adapter); ifp->if_capenable &= ~IFCAP_POLLING; EM_CORE_UNLOCK(adapter); } } #endif if (mask & IFCAP_HWCSUM) { ifp->if_capenable ^= IFCAP_HWCSUM; reinit = 1; } #if __FreeBSD_version >= 700000 if (mask & IFCAP_TSO4) { ifp->if_capenable ^= IFCAP_TSO4; reinit = 1; } #endif if (mask & IFCAP_VLAN_HWTAGGING) { ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; reinit = 1; } if (mask & IFCAP_VLAN_HWFILTER) { ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; reinit = 1; } if ((mask & IFCAP_WOL) && (ifp->if_capabilities & IFCAP_WOL) != 0) { if (mask & IFCAP_WOL_MCAST) ifp->if_capenable ^= IFCAP_WOL_MCAST; if (mask & IFCAP_WOL_MAGIC) ifp->if_capenable ^= IFCAP_WOL_MAGIC; } if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) em_init(adapter); #if __FreeBSD_version >= 700000 VLAN_CAPABILITIES(ifp); #endif break; } default: error = ether_ioctl(ifp, command, data); break; } return (error); } /********************************************************************* * Init entry point * * This routine is used in two ways. It is used by the stack as * init entry point in network interface structure. It is also used * by the driver as a hw/sw initialization routine to get to a * consistent state. * * return 0 on success, positive on failure **********************************************************************/ static void em_init_locked(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; u32 pba; INIT_DEBUGOUT("em_init: begin"); EM_CORE_LOCK_ASSERT(adapter); EM_TX_LOCK(adapter); em_stop(adapter); EM_TX_UNLOCK(adapter); /* * Packet Buffer Allocation (PBA) * Writing PBA sets the receive portion of the buffer * the remainder is used for the transmit buffer. * * Devices before the 82547 had a Packet Buffer of 64K. * Default allocation: PBA=48K for Rx, leaving 16K for Tx. * After the 82547 the buffer was reduced to 40K. * Default allocation: PBA=30K for Rx, leaving 10K for Tx. * Note: default does not leave enough room for Jumbo Frame >10k. */ switch (adapter->hw.mac.type) { case e1000_82547: case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ if (adapter->max_frame_size > 8192) pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ else pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ adapter->tx_fifo_head = 0; adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; adapter->tx_fifo_size = (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; break; /* Total Packet Buffer on these is 48K */ case e1000_82571: case e1000_82572: case e1000_80003es2lan: pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ break; case e1000_82573: /* 82573: Total Packet Buffer is 32K */ pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ break; case e1000_82574: case e1000_82583: pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ break; case e1000_ich9lan: case e1000_ich10lan: case e1000_pchlan: pba = E1000_PBA_10K; break; case e1000_ich8lan: pba = E1000_PBA_8K; break; default: /* Devices before 82547 had a Packet Buffer of 64K. */ if (adapter->max_frame_size > 8192) pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ else pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ } INIT_DEBUGOUT1("em_init: pba=%dK",pba); E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); /* Get the latest mac address, User can use a LAA */ bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); /* * With the 82571 adapter, RAR[0] may be overwritten * when the other port is reset, we make a duplicate * in RAR[14] for that eventuality, this assures * the interface continues to function. */ if (adapter->hw.mac.type == e1000_82571) { e1000_set_laa_state_82571(&adapter->hw, TRUE); e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, E1000_RAR_ENTRIES - 1); } /* Initialize the hardware */ if (em_hardware_init(adapter)) { device_printf(dev, "Unable to initialize the hardware\n"); return; } em_update_link_status(adapter); /* Setup VLAN support, basic and offload if available */ E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) /* Use real VLAN Filter support */ em_setup_vlan_hw_support(adapter); else { u32 ctrl; ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); ctrl |= E1000_CTRL_VME; E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); } } /* Set hardware offload abilities */ ifp->if_hwassist = 0; if (adapter->hw.mac.type >= e1000_82543) { if (ifp->if_capenable & IFCAP_TXCSUM) ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); #if __FreeBSD_version >= 700000 if (ifp->if_capenable & IFCAP_TSO4) ifp->if_hwassist |= CSUM_TSO; #endif } /* Configure for OS presence */ em_init_manageability(adapter); /* Prepare transmit descriptors and buffers */ em_setup_transmit_structures(adapter); em_initialize_transmit_unit(adapter); /* Setup Multicast table */ em_set_multi(adapter); /* Prepare receive descriptors and buffers */ if (em_setup_receive_structures(adapter)) { device_printf(dev, "Could not setup receive structures\n"); EM_TX_LOCK(adapter); em_stop(adapter); EM_TX_UNLOCK(adapter); return; } em_initialize_receive_unit(adapter); /* Don't lose promiscuous settings */ em_set_promisc(adapter); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; callout_reset(&adapter->timer, hz, em_local_timer, adapter); e1000_clear_hw_cntrs_base_generic(&adapter->hw); /* MSI/X configuration for 82574 */ if (adapter->hw.mac.type == e1000_82574) { int tmp; tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); tmp |= E1000_CTRL_EXT_PBA_CLR; E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); /* ** Set the IVAR - interrupt vector routing. ** Each nibble represents a vector, high bit ** is enable, other 3 bits are the MSIX table ** entry, we map RXQ0 to 0, TXQ0 to 1, and ** Link (other) to 2, hence the magic number. */ E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); } #ifdef DEVICE_POLLING /* * Only enable interrupts if we are not polling, make sure * they are off otherwise. */ if (ifp->if_capenable & IFCAP_POLLING) em_disable_intr(adapter); else #endif /* DEVICE_POLLING */ em_enable_intr(adapter); /* AMT based hardware can now take control from firmware */ if (adapter->has_manage && adapter->has_amt) em_get_hw_control(adapter); /* Don't reset the phy next time init gets called */ adapter->hw.phy.reset_disable = TRUE; } static void em_init(void *arg) { struct adapter *adapter = arg; EM_CORE_LOCK(adapter); em_init_locked(adapter); EM_CORE_UNLOCK(adapter); } #ifdef DEVICE_POLLING /********************************************************************* * * Legacy polling routine * *********************************************************************/ static int em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { struct adapter *adapter = ifp->if_softc; u32 reg_icr, rx_done = 0; EM_CORE_LOCK(adapter); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { EM_CORE_UNLOCK(adapter); return (rx_done); } if (cmd == POLL_AND_CHECK_STATUS) { reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; em_update_link_status(adapter); } if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; } EM_CORE_UNLOCK(adapter); rx_done = em_rxeof(adapter, count); EM_TX_LOCK(adapter); em_txeof(adapter); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, adapter->br)) em_mq_start_locked(ifp, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) em_start_locked(ifp); #endif EM_TX_UNLOCK(adapter); return (rx_done); } #endif /* DEVICE_POLLING */ #ifdef EM_LEGACY_IRQ /********************************************************************* * * Legacy Interrupt Service routine * *********************************************************************/ static void em_intr(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; u32 reg_icr; if (ifp->if_capenable & IFCAP_POLLING) return; EM_CORE_LOCK(adapter); reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; if ((reg_icr == 0xffffffff) || (reg_icr == 0)|| (adapter->hw.mac.type >= e1000_82571 && (reg_icr & E1000_ICR_INT_ASSERTED) == 0)) goto out; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) goto out; if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { callout_stop(&adapter->timer); adapter->hw.mac.get_link_status = 1; em_update_link_status(adapter); /* Deal with TX cruft when link lost */ em_tx_purge(adapter); callout_reset(&adapter->timer, hz, em_local_timer, adapter); goto out; } EM_TX_LOCK(adapter); em_txeof(adapter); em_rxeof(adapter, -1); em_txeof(adapter); if (ifp->if_drv_flags & IFF_DRV_RUNNING && !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) em_start_locked(ifp); EM_TX_UNLOCK(adapter); out: EM_CORE_UNLOCK(adapter); return; } #else /* EM_FAST_IRQ, then fast interrupt routines only */ static void em_handle_link(void *context, int pending) { struct adapter *adapter = context; struct ifnet *ifp = adapter->ifp; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) return; EM_CORE_LOCK(adapter); callout_stop(&adapter->timer); em_update_link_status(adapter); /* Deal with TX cruft when link lost */ em_tx_purge(adapter); callout_reset(&adapter->timer, hz, em_local_timer, adapter); EM_CORE_UNLOCK(adapter); } /* Combined RX/TX handler, used by Legacy and MSI */ static void em_handle_rxtx(void *context, int pending) { struct adapter *adapter = context; struct ifnet *ifp = adapter->ifp; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (em_rxeof(adapter, adapter->rx_process_limit) != 0) taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); EM_TX_LOCK(adapter); em_txeof(adapter); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, adapter->br)) em_mq_start_locked(ifp, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) em_start_locked(ifp); #endif EM_TX_UNLOCK(adapter); } em_enable_intr(adapter); } /********************************************************************* * * Fast Legacy/MSI Combined Interrupt Service routine * *********************************************************************/ #if __FreeBSD_version < 700000 #define FILTER_STRAY #define FILTER_HANDLED static void #else static int #endif em_irq_fast(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp; u32 reg_icr; ifp = adapter->ifp; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); /* Hot eject? */ if (reg_icr == 0xffffffff) return FILTER_STRAY; /* Definitely not our interrupt. */ if (reg_icr == 0x0) return FILTER_STRAY; /* * Starting with the 82571 chip, bit 31 should be used to * determine whether the interrupt belongs to us. */ if (adapter->hw.mac.type >= e1000_82571 && (reg_icr & E1000_ICR_INT_ASSERTED) == 0) return FILTER_STRAY; /* * Mask interrupts until the taskqueue is finished running. This is * cheap, just assume that it is needed. This also works around the * MSI message reordering errata on certain systems. */ em_disable_intr(adapter); taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; taskqueue_enqueue(taskqueue_fast, &adapter->link_task); } if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; return FILTER_HANDLED; } /********************************************************************* * * MSIX Interrupt Service Routines * **********************************************************************/ #define EM_MSIX_TX 0x00040000 #define EM_MSIX_RX 0x00010000 #define EM_MSIX_LINK 0x00100000 static void em_msix_tx(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; ++adapter->tx_irq; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { EM_TX_LOCK(adapter); em_txeof(adapter); EM_TX_UNLOCK(adapter); taskqueue_enqueue(adapter->tq, &adapter->tx_task); } /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX); return; } /********************************************************************* * * MSIX RX Interrupt Service routine * **********************************************************************/ static void em_msix_rx(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; ++adapter->rx_irq; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && (em_rxeof(adapter, adapter->rx_process_limit) != 0)) taskqueue_enqueue(adapter->tq, &adapter->rx_task); /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX); return; } /********************************************************************* * * MSIX Link Fast Interrupt Service routine * **********************************************************************/ static void em_msix_link(void *arg) { struct adapter *adapter = arg; u32 reg_icr; ++adapter->link_irq; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; taskqueue_enqueue(taskqueue_fast, &adapter->link_task); } E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); return; } static void em_handle_rx(void *context, int pending) { struct adapter *adapter = context; struct ifnet *ifp = adapter->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && (em_rxeof(adapter, adapter->rx_process_limit) != 0)) taskqueue_enqueue(adapter->tq, &adapter->rx_task); } static void em_handle_tx(void *context, int pending) { struct adapter *adapter = context; struct ifnet *ifp = adapter->ifp; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (!EM_TX_TRYLOCK(adapter)) return; em_txeof(adapter); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, adapter->br)) em_mq_start_locked(ifp, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) em_start_locked(ifp); #endif EM_TX_UNLOCK(adapter); } } #endif /* EM_FAST_IRQ */ /********************************************************************* * * Media Ioctl callback * * This routine is called whenever the user queries the status of * the interface using ifconfig. * **********************************************************************/ static void em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct adapter *adapter = ifp->if_softc; u_char fiber_type = IFM_1000_SX; INIT_DEBUGOUT("em_media_status: begin"); EM_CORE_LOCK(adapter); em_update_link_status(adapter); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { EM_CORE_UNLOCK(adapter); return; } ifmr->ifm_status |= IFM_ACTIVE; if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { if (adapter->hw.mac.type == e1000_82545) fiber_type = IFM_1000_LX; ifmr->ifm_active |= fiber_type | IFM_FDX; } else { switch (adapter->link_speed) { case 10: ifmr->ifm_active |= IFM_10_T; break; case 100: ifmr->ifm_active |= IFM_100_TX; break; case 1000: ifmr->ifm_active |= IFM_1000_T; break; } if (adapter->link_duplex == FULL_DUPLEX) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; } EM_CORE_UNLOCK(adapter); } /********************************************************************* * * Media Ioctl callback * * This routine is called when the user changes speed/duplex using * media/mediopt option with ifconfig. * **********************************************************************/ static int em_media_change(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct ifmedia *ifm = &adapter->media; INIT_DEBUGOUT("em_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); EM_CORE_LOCK(adapter); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; break; case IFM_1000_LX: case IFM_1000_SX: case IFM_1000_T: adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; break; case IFM_100_TX: adapter->hw.mac.autoneg = FALSE; adapter->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; else adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; break; case IFM_10_T: adapter->hw.mac.autoneg = FALSE; adapter->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; else adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; break; default: device_printf(adapter->dev, "Unsupported media type\n"); } /* As the speed/duplex settings my have changed we need to * reset the PHY. */ adapter->hw.phy.reset_disable = FALSE; em_init_locked(adapter); EM_CORE_UNLOCK(adapter); return (0); } /********************************************************************* * * This routine maps the mbufs to tx descriptors. * * return 0 on success, positive on failure **********************************************************************/ static int em_xmit(struct adapter *adapter, struct mbuf **m_headp) { bus_dma_segment_t segs[EM_MAX_SCATTER]; bus_dmamap_t map; struct em_buffer *tx_buffer, *tx_buffer_mapped; struct e1000_tx_desc *ctxd = NULL; struct mbuf *m_head; u32 txd_upper, txd_lower, txd_used, txd_saved; int nsegs, i, j, first, last = 0; int error, do_tso, tso_desc = 0; #if __FreeBSD_version < 700000 struct m_tag *mtag; #endif m_head = *m_headp; txd_upper = txd_lower = txd_used = txd_saved = 0; #if __FreeBSD_version >= 700000 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); #else do_tso = 0; #endif /* * Force a cleanup if number of TX descriptors * available hits the threshold */ if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { em_txeof(adapter); /* Now do we at least have a minimal? */ if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { adapter->no_tx_desc_avail1++; return (ENOBUFS); } } /* * TSO workaround: * If an mbuf is only header we need * to pull 4 bytes of data into it. */ if (do_tso && (m_head->m_len <= M_TSO_LEN)) { m_head = m_pullup(m_head, M_TSO_LEN + 4); *m_headp = m_head; if (m_head == NULL) return (ENOBUFS); } /* * Map the packet for DMA * * Capture the first descriptor index, * this descriptor will have the index * of the EOP which is the only one that * now gets a DONE bit writeback. */ first = adapter->next_avail_tx_desc; tx_buffer = &adapter->tx_buffer_area[first]; tx_buffer_mapped = tx_buffer; map = tx_buffer->map; error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); /* * There are two types of errors we can (try) to handle: * - EFBIG means the mbuf chain was too long and bus_dma ran * out of segments. Defragment the mbuf chain and try again. * - ENOMEM means bus_dma could not obtain enough bounce buffers * at this point in time. Defer sending and try again later. * All other errors, in particular EINVAL, are fatal and prevent the * mbuf chain from ever going through. Drop it and report error. */ if (error == EFBIG) { struct mbuf *m; m = m_defrag(*m_headp, M_DONTWAIT); if (m == NULL) { adapter->mbuf_alloc_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); } *m_headp = m; /* Try it again */ error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } } else if (error != 0) { adapter->no_tx_dma_setup++; return (error); } /* * TSO Hardware workaround, if this packet is not * TSO, and is only a single descriptor long, and * it follows a TSO burst, then we need to add a * sentinel descriptor to prevent premature writeback. */ if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { if (nsegs == 1) tso_desc = TRUE; adapter->tx_tso = FALSE; } if (nsegs > (adapter->num_tx_desc_avail - 2)) { adapter->no_tx_desc_avail2++; bus_dmamap_unload(adapter->txtag, map); return (ENOBUFS); } m_head = *m_headp; /* Do hardware assists */ #if __FreeBSD_version >= 700000 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); if (error != TRUE) return (ENXIO); /* something foobar */ /* we need to make a final sentinel transmit desc */ tso_desc = TRUE; } else #endif if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) em_transmit_checksum_setup(adapter, m_head, &txd_upper, &txd_lower); i = adapter->next_avail_tx_desc; if (adapter->pcix_82544) txd_saved = i; /* Set up our transmit descriptors */ for (j = 0; j < nsegs; j++) { bus_size_t seg_len; bus_addr_t seg_addr; /* If adapter is 82544 and on PCIX bus */ if(adapter->pcix_82544) { DESC_ARRAY desc_array; u32 array_elements, counter; /* * Check the Address and Length combination and * split the data accordingly */ array_elements = em_fill_descriptors(segs[j].ds_addr, segs[j].ds_len, &desc_array); for (counter = 0; counter < array_elements; counter++) { if (txd_used == adapter->num_tx_desc_avail) { adapter->next_avail_tx_desc = txd_saved; adapter->no_tx_desc_avail2++; bus_dmamap_unload(adapter->txtag, map); return (ENOBUFS); } tx_buffer = &adapter->tx_buffer_area[i]; ctxd = &adapter->tx_desc_base[i]; ctxd->buffer_addr = htole64( desc_array.descriptor[counter].address); ctxd->lower.data = htole32( (adapter->txd_cmd | txd_lower | (u16) desc_array.descriptor[counter].length)); ctxd->upper.data = htole32((txd_upper)); last = i; if (++i == adapter->num_tx_desc) i = 0; tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; txd_used++; } } else { tx_buffer = &adapter->tx_buffer_area[i]; ctxd = &adapter->tx_desc_base[i]; seg_addr = segs[j].ds_addr; seg_len = segs[j].ds_len; /* ** TSO Workaround: ** If this is the last descriptor, we want to ** split it so we have a small final sentinel */ if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { seg_len -= 4; ctxd->buffer_addr = htole64(seg_addr); ctxd->lower.data = htole32( adapter->txd_cmd | txd_lower | seg_len); ctxd->upper.data = htole32(txd_upper); if (++i == adapter->num_tx_desc) i = 0; /* Now make the sentinel */ ++txd_used; /* using an extra txd */ ctxd = &adapter->tx_desc_base[i]; tx_buffer = &adapter->tx_buffer_area[i]; ctxd->buffer_addr = htole64(seg_addr + seg_len); ctxd->lower.data = htole32( adapter->txd_cmd | txd_lower | 4); ctxd->upper.data = htole32(txd_upper); last = i; if (++i == adapter->num_tx_desc) i = 0; } else { ctxd->buffer_addr = htole64(seg_addr); ctxd->lower.data = htole32( adapter->txd_cmd | txd_lower | seg_len); ctxd->upper.data = htole32(txd_upper); last = i; if (++i == adapter->num_tx_desc) i = 0; } tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; } } adapter->next_avail_tx_desc = i; if (adapter->pcix_82544) adapter->num_tx_desc_avail -= txd_used; else { adapter->num_tx_desc_avail -= nsegs; if (tso_desc) /* TSO used an extra for sentinel */ adapter->num_tx_desc_avail -= txd_used; } /* ** Handle VLAN tag, this is the ** biggest difference between ** 6.x and 7 */ #if __FreeBSD_version < 700000 /* Find out if we are in vlan mode. */ mtag = VLAN_OUTPUT_TAG(ifp, m_head); if (mtag != NULL) { ctxd->upper.fields.special = htole16(VLAN_TAG_VALUE(mtag)); #else /* FreeBSD 7 */ if (m_head->m_flags & M_VLANTAG) { /* Set the vlan id. */ ctxd->upper.fields.special = htole16(m_head->m_pkthdr.ether_vtag); #endif /* Tell hardware to add tag */ ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); } tx_buffer->m_head = m_head; tx_buffer_mapped->map = tx_buffer->map; tx_buffer->map = map; bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); /* * Last Descriptor of Packet * needs End Of Packet (EOP) * and Report Status (RS) */ ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); /* * Keep track in the first buffer which * descriptor will be written back */ tx_buffer = &adapter->tx_buffer_area[first]; tx_buffer->next_eop = last; adapter->watchdog_time = ticks; /* * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 * that this frame is available to transmit. */ bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); if (adapter->hw.mac.type == e1000_82547 && adapter->link_duplex == HALF_DUPLEX) em_82547_move_tail(adapter); else { E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); if (adapter->hw.mac.type == e1000_82547) em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len); } return (0); } /********************************************************************* * * 82547 workaround to avoid controller hang in half-duplex environment. * The workaround is to avoid queuing a large packet that would span * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers * in this case. We do that only when FIFO is quiescent. * **********************************************************************/ static void em_82547_move_tail(void *arg) { struct adapter *adapter = arg; struct e1000_tx_desc *tx_desc; u16 hw_tdt, sw_tdt, length = 0; bool eop = 0; EM_TX_LOCK_ASSERT(adapter); hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); sw_tdt = adapter->next_avail_tx_desc; while (hw_tdt != sw_tdt) { tx_desc = &adapter->tx_desc_base[hw_tdt]; length += tx_desc->lower.flags.length; eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; if (++hw_tdt == adapter->num_tx_desc) hw_tdt = 0; if (eop) { if (em_82547_fifo_workaround(adapter, length)) { adapter->tx_fifo_wrk_cnt++; callout_reset(&adapter->tx_fifo_timer, 1, em_82547_move_tail, adapter); break; } E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); em_82547_update_fifo_head(adapter, length); length = 0; } } } static int em_82547_fifo_workaround(struct adapter *adapter, int len) { int fifo_space, fifo_pkt_len; fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); if (adapter->link_duplex == HALF_DUPLEX) { fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { if (em_82547_tx_fifo_reset(adapter)) return (0); else return (1); } } return (0); } static void em_82547_update_fifo_head(struct adapter *adapter, int len) { int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); /* tx_fifo_head is always 16 byte aligned */ adapter->tx_fifo_head += fifo_pkt_len; if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { adapter->tx_fifo_head -= adapter->tx_fifo_size; } } static int em_82547_tx_fifo_reset(struct adapter *adapter) { u32 tctl; if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && (E1000_READ_REG(&adapter->hw, E1000_TDFT) == E1000_READ_REG(&adapter->hw, E1000_TDFH)) && (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { /* Disable TX unit */ tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl & ~E1000_TCTL_EN); /* Reset FIFO pointers */ E1000_WRITE_REG(&adapter->hw, E1000_TDFT, adapter->tx_head_addr); E1000_WRITE_REG(&adapter->hw, E1000_TDFH, adapter->tx_head_addr); E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, adapter->tx_head_addr); E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, adapter->tx_head_addr); /* Re-enable TX unit */ E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); E1000_WRITE_FLUSH(&adapter->hw); adapter->tx_fifo_head = 0; adapter->tx_fifo_reset_cnt++; return (TRUE); } else { return (FALSE); } } static void em_set_promisc(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; u32 reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); if (ifp->if_flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } else if (ifp->if_flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } } static void em_disable_promisc(struct adapter *adapter) { u32 reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= (~E1000_RCTL_UPE); reg_rctl &= (~E1000_RCTL_MPE); reg_rctl &= (~E1000_RCTL_SBP); E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } /********************************************************************* * Multicast Update * * This routine is called whenever multicast address list is updated. * **********************************************************************/ static void em_set_multi(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; struct ifmultiaddr *ifma; u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; IOCTL_DEBUGOUT("em_set_multi: begin"); if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) e1000_pci_clear_mwi(&adapter->hw); reg_rctl |= E1000_RCTL_RST; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); msec_delay(5); } /* Allocate temporary memory to setup array */ mta = malloc(sizeof(u8) * (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), M_DEVBUF, M_NOWAIT | M_ZERO); if (mta == NULL) panic("em_set_multi memory failure\n"); #if __FreeBSD_version < 800000 IF_ADDR_LOCK(ifp); #else if_maddr_rlock(ifp); #endif TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) break; bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); mcnt++; } #if __FreeBSD_version < 800000 IF_ADDR_UNLOCK(ifp); #else if_maddr_runlock(ifp); #endif if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl |= E1000_RCTL_MPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } else e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); if (adapter->hw.mac.type == e1000_82542 && adapter->hw.revision_id == E1000_REVISION_2) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= ~E1000_RCTL_RST; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); msec_delay(5); if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) e1000_pci_set_mwi(&adapter->hw); } free(mta, M_DEVBUF); } /********************************************************************* * Timer routine * * This routine checks for link status and updates statistics. * **********************************************************************/ static void em_local_timer(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; EM_CORE_LOCK_ASSERT(adapter); #ifndef DEVICE_POLLING taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); #endif em_update_link_status(adapter); em_update_stats_counters(adapter); /* Reset LAA into RAR[0] on 82571 */ if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) em_print_hw_stats(adapter); em_smartspeed(adapter); /* * We check the watchdog: the time since * the last TX descriptor was cleaned. * This implies a functional TX engine. */ if ((adapter->watchdog_check == TRUE) && (ticks - adapter->watchdog_time > EM_WATCHDOG)) goto hung; callout_reset(&adapter->timer, hz, em_local_timer, adapter); return; hung: device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; adapter->watchdog_events++; em_init_locked(adapter); } static void em_update_link_status(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; u32 link_check = 0; /* Get the cached link value or read phy for real */ switch (hw->phy.media_type) { case e1000_media_type_copper: if (hw->mac.get_link_status) { /* Do the work to read phy */ e1000_check_for_link(hw); link_check = !hw->mac.get_link_status; if (link_check) /* ESB2 fix */ e1000_cfg_on_link_up(hw); } else link_check = TRUE; break; case e1000_media_type_fiber: e1000_check_for_link(hw); link_check = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); break; case e1000_media_type_internal_serdes: e1000_check_for_link(hw); link_check = adapter->hw.mac.serdes_has_link; break; default: case e1000_media_type_unknown: break; } /* Now check for a transition */ if (link_check && (adapter->link_active == 0)) { e1000_get_speed_and_duplex(hw, &adapter->link_speed, &adapter->link_duplex); /* Check if we must disable SPEED_MODE bit on PCI-E */ if ((adapter->link_speed != SPEED_1000) && ((hw->mac.type == e1000_82571) || (hw->mac.type == e1000_82572))) { int tarc0; tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); tarc0 &= ~SPEED_MODE_BIT; E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); } if (bootverbose) device_printf(dev, "Link is up %d Mbps %s\n", adapter->link_speed, ((adapter->link_duplex == FULL_DUPLEX) ? "Full Duplex" : "Half Duplex")); adapter->link_active = 1; adapter->smartspeed = 0; ifp->if_baudrate = adapter->link_speed * 1000000; if_link_state_change(ifp, LINK_STATE_UP); } else if (!link_check && (adapter->link_active == 1)) { ifp->if_baudrate = adapter->link_speed = 0; adapter->link_duplex = 0; if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; /* Link down, disable watchdog */ adapter->watchdog_check = FALSE; if_link_state_change(ifp, LINK_STATE_DOWN); } } /********************************************************************* * * This routine disables all traffic on the adapter by issuing a * global reset on the MAC and deallocates TX/RX buffers. * * This routine should always be called with BOTH the CORE * and TX locks. **********************************************************************/ static void em_stop(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; EM_CORE_LOCK_ASSERT(adapter); EM_TX_LOCK_ASSERT(adapter); INIT_DEBUGOUT("em_stop: begin"); em_disable_intr(adapter); callout_stop(&adapter->timer); callout_stop(&adapter->tx_fifo_timer); /* Tell the stack that the interface is no longer active */ ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); e1000_reset_hw(&adapter->hw); if (adapter->hw.mac.type >= e1000_82544) E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); } /********************************************************************* * * Determine hardware revision. * **********************************************************************/ static void em_identify_hardware(struct adapter *adapter) { device_t dev = adapter->dev; /* Make sure our PCI config space has the necessary stuff set */ adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { device_printf(dev, "Memory Access and/or Bus Master bits " "were not set!\n"); adapter->hw.bus.pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); pci_write_config(dev, PCIR_COMMAND, adapter->hw.bus.pci_cmd_word, 2); } /* Save off the information about this board */ adapter->hw.vendor_id = pci_get_vendor(dev); adapter->hw.device_id = pci_get_device(dev); adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); adapter->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); adapter->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); /* Do Shared Code Init and Setup */ if (e1000_set_mac_type(&adapter->hw)) { device_printf(dev, "Setup init failure\n"); return; } } static int em_allocate_pci_resources(struct adapter *adapter) { device_t dev = adapter->dev; int val, rid, error = E1000_SUCCESS; rid = PCIR_BAR(0); adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (adapter->memory == NULL) { device_printf(dev, "Unable to allocate bus resource: memory\n"); return (ENXIO); } adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->memory); adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->memory); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; /* Only older adapters use IO mapping */ if ((adapter->hw.mac.type > e1000_82543) && (adapter->hw.mac.type < e1000_82571)) { /* Figure our where our IO BAR is ? */ for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { val = pci_read_config(dev, rid, 4); if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { adapter->io_rid = rid; break; } rid += 4; /* check for 64bit BAR */ if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) rid += 4; } if (rid >= PCIR_CIS) { device_printf(dev, "Unable to locate IO BAR\n"); return (ENXIO); } adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); if (adapter->ioport == NULL) { device_printf(dev, "Unable to allocate bus resource: " "ioport\n"); return (ENXIO); } adapter->hw.io_base = 0; adapter->osdep.io_bus_space_tag = rman_get_bustag(adapter->ioport); adapter->osdep.io_bus_space_handle = rman_get_bushandle(adapter->ioport); } /* ** Init the resource arrays ** used by MSIX setup */ for (int i = 0; i < 3; i++) { adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ adapter->tag[i] = NULL; adapter->res[i] = NULL; } /* * Setup MSI/X or MSI if PCI Express */ if (em_enable_msi) adapter->msi = em_setup_msix(adapter); adapter->hw.back = &adapter->osdep; return (error); } /********************************************************************* * * Setup the Legacy or MSI Interrupt handler * **********************************************************************/ int em_allocate_legacy(struct adapter *adapter) { device_t dev = adapter->dev; int error; /* Manually turn off all interrupts */ E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); /* Legacy RID is 0 */ if (adapter->msi == 0) adapter->rid[0] = 0; /* We allocate a single interrupt resource */ adapter->res[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); if (adapter->res[0] == NULL) { device_printf(dev, "Unable to allocate bus resource: " "interrupt\n"); return (ENXIO); } #ifdef EM_LEGACY_IRQ /* We do Legacy setup */ if ((error = bus_setup_intr(dev, adapter->res[0], #if __FreeBSD_version > 700000 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, #else /* 6.X */ INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, #endif &adapter->tag[0])) != 0) { device_printf(dev, "Failed to register interrupt handler"); return (error); } #else /* FAST_IRQ */ /* * Try allocating a fast interrupt and the associated deferred * processing contexts. */ TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", device_get_nameunit(adapter->dev)); #if __FreeBSD_version < 700000 if ((error = bus_setup_intr(dev, adapter->res[0], INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, #else if ((error = bus_setup_intr(dev, adapter->res[0], INTR_TYPE_NET, em_irq_fast, NULL, adapter, #endif &adapter->tag[0])) != 0) { device_printf(dev, "Failed to register fast interrupt " "handler: %d\n", error); taskqueue_free(adapter->tq); adapter->tq = NULL; return (error); } #endif /* EM_LEGACY_IRQ */ return (0); } /********************************************************************* * * Setup the MSIX Interrupt handlers * This is not really Multiqueue, rather * its just multiple interrupt vectors. * **********************************************************************/ int em_allocate_msix(struct adapter *adapter) { device_t dev = adapter->dev; int error; /* Make sure all interrupts are disabled */ E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); /* First get the resources */ for (int i = 0; i < adapter->msi; i++) { adapter->res[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); if (adapter->res[i] == NULL) { device_printf(dev, "Unable to allocate bus resource: " "MSIX Interrupt\n"); return (ENXIO); } } /* * Now allocate deferred processing contexts. */ TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); /* * Handle compatibility for msi case for deferral due to * trylock failure */ TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter); TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", device_get_nameunit(adapter->dev)); /* * And setup the interrupt handlers */ /* First slot to RX */ if ((error = bus_setup_intr(dev, adapter->res[0], #if __FreeBSD_version > 700000 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter, #else /* 6.X */ INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter, #endif &adapter->tag[0])) != 0) { device_printf(dev, "Failed to register RX handler"); return (error); } /* Next TX */ if ((error = bus_setup_intr(dev, adapter->res[1], #if __FreeBSD_version > 700000 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter, #else /* 6.X */ INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter, #endif &adapter->tag[1])) != 0) { device_printf(dev, "Failed to register TX handler"); return (error); } /* And Link */ if ((error = bus_setup_intr(dev, adapter->res[2], #if __FreeBSD_version > 700000 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter, #else /* 6.X */ INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter, #endif &adapter->tag[2])) != 0) { device_printf(dev, "Failed to register TX handler"); return (error); } return (0); } static void em_free_pci_resources(struct adapter *adapter) { device_t dev = adapter->dev; /* Make sure the for loop below runs once */ if (adapter->msi == 0) adapter->msi = 1; /* * First release all the interrupt resources: * notice that since these are just kept * in an array we can do the same logic * whether its MSIX or just legacy. */ for (int i = 0; i < adapter->msi; i++) { if (adapter->tag[i] != NULL) { bus_teardown_intr(dev, adapter->res[i], adapter->tag[i]); adapter->tag[i] = NULL; } if (adapter->res[i] != NULL) { bus_release_resource(dev, SYS_RES_IRQ, adapter->rid[i], adapter->res[i]); } } if (adapter->msi) pci_release_msi(dev); if (adapter->msix != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(EM_MSIX_BAR), adapter->msix); if (adapter->memory != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), adapter->memory); if (adapter->flash != NULL) bus_release_resource(dev, SYS_RES_MEMORY, EM_FLASH, adapter->flash); if (adapter->ioport != NULL) bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid, adapter->ioport); } /* * Setup MSI or MSI/X */ static int em_setup_msix(struct adapter *adapter) { device_t dev = adapter->dev; int val = 0; if (adapter->hw.mac.type < e1000_82571) return (0); /* Setup MSI/X for Hartwell */ if (adapter->hw.mac.type == e1000_82574) { /* Map the MSIX BAR */ int rid = PCIR_BAR(EM_MSIX_BAR); adapter->msix = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!adapter->msix) { /* May not be enabled */ device_printf(adapter->dev, "Unable to map MSIX table \n"); goto msi; } val = pci_msix_count(dev); /* ** 82574 can be configured for 5 but ** we limit use to 3. */ if (val > 3) val = 3; if ((val) && pci_alloc_msix(dev, &val) == 0) { device_printf(adapter->dev,"Using MSIX interrupts\n"); return (val); } } msi: val = pci_msi_count(dev); if (val == 1 && pci_alloc_msi(dev, &val) == 0) { adapter->msi = 1; device_printf(adapter->dev,"Using MSI interrupt\n"); return (val); } return (0); } /********************************************************************* * * Initialize the hardware to a configuration * as specified by the adapter structure. * **********************************************************************/ static int em_hardware_init(struct adapter *adapter) { device_t dev = adapter->dev; u16 rx_buffer_size; INIT_DEBUGOUT("em_hardware_init: begin"); /* Issue a global reset */ e1000_reset_hw(&adapter->hw); /* When hardware is reset, fifo_head is also reset */ adapter->tx_fifo_head = 0; /* Set up smart power down as default off on newer adapters. */ if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || adapter->hw.mac.type == e1000_82572)) { u16 phy_tmp = 0; /* Speed up time to link by disabling smart power down. */ e1000_read_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); phy_tmp &= ~IGP02E1000_PM_SPD; e1000_write_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); } /* * These parameters control the automatic generation (Tx) and * response (Rx) to Ethernet PAUSE frames. * - High water mark should allow for at least two frames to be * received after sending an XOFF. * - Low water mark works best when it is very near the high water mark. * This allows the receiver to restart by sending XON when it has * drained a bit. Here we use an arbitary value of 1500 which will * restart after one full frame is pulled from the buffer. There * could be several smaller frames in the buffer and if so they will * not trigger the XON until their total number reduces the buffer * by 1500. * - The pause time is fairly large at 1000 x 512ns = 512 usec. */ rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) << 10 ); adapter->hw.fc.high_water = rx_buffer_size - roundup2(adapter->max_frame_size, 1024); adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; if (adapter->hw.mac.type == e1000_80003es2lan) adapter->hw.fc.pause_time = 0xFFFF; else adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; adapter->hw.fc.send_xon = TRUE; /* Set Flow control, use the tunable location if sane */ if ((em_fc_setting >= 0) || (em_fc_setting < 4)) adapter->hw.fc.requested_mode = em_fc_setting; else adapter->hw.fc.requested_mode = e1000_fc_none; /* Override - workaround for PCHLAN issue */ if (adapter->hw.mac.type == e1000_pchlan) adapter->hw.fc.requested_mode = e1000_fc_rx_pause; if (e1000_init_hw(&adapter->hw) < 0) { device_printf(dev, "Hardware Initialization Failed\n"); return (EIO); } e1000_check_for_link(&adapter->hw); return (0); } /********************************************************************* * * Setup networking device structure and register an interface. * **********************************************************************/ static void em_setup_interface(device_t dev, struct adapter *adapter) { struct ifnet *ifp; INIT_DEBUGOUT("em_setup_interface: begin"); ifp = adapter->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) panic("%s: can not if_alloc()", device_get_nameunit(dev)); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_mtu = ETHERMTU; ifp->if_init = em_init; ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = em_ioctl; ifp->if_start = em_start; IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; IFQ_SET_READY(&ifp->if_snd); ether_ifattach(ifp, adapter->hw.mac.addr); ifp->if_capabilities = ifp->if_capenable = 0; #if __FreeBSD_version >= 800000 /* Multiqueue tx functions */ ifp->if_transmit = em_mq_start; ifp->if_qflush = em_qflush; adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); #endif if (adapter->hw.mac.type >= e1000_82543) { int version_cap; #if __FreeBSD_version < 700000 version_cap = IFCAP_HWCSUM; #else version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; #endif ifp->if_capabilities |= version_cap; ifp->if_capenable |= version_cap; } #if __FreeBSD_version >= 700000 /* Identify TSO capable adapters */ if ((adapter->hw.mac.type > e1000_82544) && (adapter->hw.mac.type != e1000_82547)) ifp->if_capabilities |= IFCAP_TSO4; /* * By default only enable on PCI-E, this * can be overriden by ifconfig. */ if (adapter->hw.mac.type >= e1000_82571) ifp->if_capenable |= IFCAP_TSO4; #endif /* * Tell the upper layer(s) we * support full VLAN capability */ ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; ifp->if_capenable |= (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING); /* ** Dont turn this on by default, if vlans are ** created on another pseudo device (eg. lagg) ** then vlan events are not passed thru, breaking ** operation, but with HW FILTER off it works. If ** using vlans directly on the em driver you can ** enable this and get full hardware tag filtering. */ ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; #ifdef DEVICE_POLLING ifp->if_capabilities |= IFCAP_POLLING; #endif /* Limit WOL to MAGIC, not clear others are used */ if (adapter->wol) { ifp->if_capabilities |= IFCAP_WOL_MAGIC; ifp->if_capenable |= IFCAP_WOL_MAGIC; } /* * Specify the media types supported by this adapter and register * callbacks to update media and link information */ ifmedia_init(&adapter->media, IFM_IMASK, em_media_change, em_media_status); if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { u_char fiber_type = IFM_1000_SX; /* default type */ if (adapter->hw.mac.type == e1000_82545) fiber_type = IFM_1000_LX; ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); } else { ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); if (adapter->hw.phy.type != e1000_phy_ife) { ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } } ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); } /********************************************************************* * * Workaround for SmartSpeed on 82541 and 82547 controllers * **********************************************************************/ static void em_smartspeed(struct adapter *adapter) { u16 phy_tmp; if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || adapter->hw.mac.autoneg == 0 || (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) return; if (adapter->smartspeed == 0) { /* If Master/Slave config fault is asserted twice, * we assume back-to-back */ e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) return; e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); if(phy_tmp & CR_1000T_MS_ENABLE) { phy_tmp &= ~CR_1000T_MS_ENABLE; e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); adapter->smartspeed++; if(adapter->hw.mac.autoneg && !e1000_copper_link_autoneg(&adapter->hw) && !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { phy_tmp |= (MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG); e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); } } } return; } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { /* If still no link, perhaps using 2/3 pair cable */ e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); phy_tmp |= CR_1000T_MS_ENABLE; e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); if(adapter->hw.mac.autoneg && !e1000_copper_link_autoneg(&adapter->hw) && !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { phy_tmp |= (MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG); e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); } } /* Restart process after EM_SMARTSPEED_MAX iterations */ if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) adapter->smartspeed = 0; } /* * Manage DMA'able memory. */ static void em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { if (error) return; *(bus_addr_t *) arg = segs[0].ds_addr; } static int em_dma_malloc(struct adapter *adapter, bus_size_t size, struct em_dma_alloc *dma, int mapflags) { int error; #if __FreeBSD_version >= 700000 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ #else error = bus_dma_tag_create(NULL, /* parent */ #endif EM_DBA_ALIGN, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ 1, /* nsegments */ size, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &dma->dma_tag); if (error) { device_printf(adapter->dev, "%s: bus_dma_tag_create failed: %d\n", __func__, error); goto fail_0; } error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); if (error) { device_printf(adapter->dev, "%s: bus_dmamem_alloc(%ju) failed: %d\n", __func__, (uintmax_t)size, error); goto fail_2; } dma->dma_paddr = 0; error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); if (error || dma->dma_paddr == 0) { device_printf(adapter->dev, "%s: bus_dmamap_load failed: %d\n", __func__, error); goto fail_3; } return (0); fail_3: bus_dmamap_unload(dma->dma_tag, dma->dma_map); fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); fail_0: dma->dma_map = NULL; dma->dma_tag = NULL; return (error); } static void em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) { if (dma->dma_tag == NULL) return; if (dma->dma_map != NULL) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); dma->dma_map = NULL; } bus_dma_tag_destroy(dma->dma_tag); dma->dma_tag = NULL; } /********************************************************************* * * Allocate memory for tx_buffer structures. The tx_buffer stores all * the information needed to transmit a packet on the wire. * **********************************************************************/ static int em_allocate_transmit_structures(struct adapter *adapter) { device_t dev = adapter->dev; struct em_buffer *tx_buffer; int error; /* * Create DMA tags for tx descriptors */ #if __FreeBSD_version >= 700000 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ #else if ((error = bus_dma_tag_create(NULL, /* parent */ #endif 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ EM_TSO_SIZE, /* maxsize */ EM_MAX_SCATTER, /* nsegments */ EM_TSO_SEG_SIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &adapter->txtag)) != 0) { device_printf(dev, "Unable to allocate TX DMA tag\n"); goto fail; } adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); if (adapter->tx_buffer_area == NULL) { device_printf(dev, "Unable to allocate tx_buffer memory\n"); error = ENOMEM; goto fail; } /* Create the descriptor buffer dma maps */ for (int i = 0; i < adapter->num_tx_desc; i++) { tx_buffer = &adapter->tx_buffer_area[i]; error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); if (error != 0) { device_printf(dev, "Unable to create TX DMA map\n"); goto fail; } tx_buffer->next_eop = -1; } return (0); fail: em_free_transmit_structures(adapter); return (error); } /********************************************************************* * * (Re)Initialize transmit structures. * **********************************************************************/ static void em_setup_transmit_structures(struct adapter *adapter) { struct em_buffer *tx_buffer; /* Clear the old ring contents */ bzero(adapter->tx_desc_base, (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); /* Free any existing TX buffers */ for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { tx_buffer = &adapter->tx_buffer_area[i]; bus_dmamap_sync(adapter->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(adapter->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; } /* Reset state */ adapter->next_avail_tx_desc = 0; adapter->next_tx_to_clean = 0; adapter->num_tx_desc_avail = adapter->num_tx_desc; bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return; } /********************************************************************* * * Enable transmit unit. * **********************************************************************/ static void em_initialize_transmit_unit(struct adapter *adapter) { u32 tctl, tarc, tipg = 0; u64 bus_addr; INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); /* Setup the Base and Length of the Tx Descriptor Ring */ bus_addr = adapter->txdma.dma_paddr; E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), (u32)(bus_addr >> 32)); E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), (u32)bus_addr); /* Setup the HW Tx Head and Tail descriptor pointers */ E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); HW_DEBUGOUT2("Base = %x, Length = %x\n", E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); /* Set the default values for the Tx Inter Packet Gap timer */ switch (adapter->hw.mac.type) { case e1000_82542: tipg = DEFAULT_82542_TIPG_IPGT; tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; break; case e1000_80003es2lan: tipg = DEFAULT_82543_TIPG_IPGR1; tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; break; default: if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) tipg = DEFAULT_82543_TIPG_IPGT_FIBER; else tipg = DEFAULT_82543_TIPG_IPGT_COPPER; tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; } E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); if(adapter->hw.mac.type >= e1000_82540) E1000_WRITE_REG(&adapter->hw, E1000_TADV, adapter->tx_abs_int_delay.value); if ((adapter->hw.mac.type == e1000_82571) || (adapter->hw.mac.type == e1000_82572)) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= SPEED_MODE_BIT; E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } else if (adapter->hw.mac.type == e1000_80003es2lan) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= 1; E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); tarc |= 1; E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); } /* Program the Transmit Control Register */ tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); tctl &= ~E1000_TCTL_CT; tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); if (adapter->hw.mac.type >= e1000_82571) tctl |= E1000_TCTL_MULR; /* This write will effectively turn on the transmit unit. */ E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); /* Setup Transmit Descriptor Base Settings */ adapter->txd_cmd = E1000_TXD_CMD_IFCS; if (adapter->tx_int_delay.value > 0) adapter->txd_cmd |= E1000_TXD_CMD_IDE; } /********************************************************************* * * Free all transmit related data structures. * **********************************************************************/ static void em_free_transmit_structures(struct adapter *adapter) { struct em_buffer *tx_buffer; INIT_DEBUGOUT("free_transmit_structures: begin"); if (adapter->tx_buffer_area != NULL) { for (int i = 0; i < adapter->num_tx_desc; i++) { tx_buffer = &adapter->tx_buffer_area[i]; if (tx_buffer->m_head != NULL) { bus_dmamap_sync(adapter->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(adapter->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; } else if (tx_buffer->map != NULL) bus_dmamap_unload(adapter->txtag, tx_buffer->map); if (tx_buffer->map != NULL) { bus_dmamap_destroy(adapter->txtag, tx_buffer->map); tx_buffer->map = NULL; } } } if (adapter->tx_buffer_area != NULL) { free(adapter->tx_buffer_area, M_DEVBUF); adapter->tx_buffer_area = NULL; } if (adapter->txtag != NULL) { bus_dma_tag_destroy(adapter->txtag); adapter->txtag = NULL; } #if __FreeBSD_version >= 800000 if (adapter->br != NULL) buf_ring_free(adapter->br, M_DEVBUF); #endif } /********************************************************************* * * The offload context needs to be set when we transfer the first * packet of a particular protocol (TCP/UDP). This routine has been * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) * * Added back the old method of keeping the current context type * and not setting if unnecessary, as this is reported to be a * big performance win. -jfv **********************************************************************/ static void em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, u32 *txd_lower) { struct e1000_context_desc *TXD = NULL; struct em_buffer *tx_buffer; struct ether_vlan_header *eh; struct ip *ip = NULL; struct ip6_hdr *ip6; int curr_txd, ehdrlen; u32 cmd, hdr_len, ip_hlen; u16 etype; u8 ipproto; cmd = hdr_len = ipproto = 0; curr_txd = adapter->next_avail_tx_desc; /* * Determine where frame payload starts. * Jump over vlan headers if already present, * helpful for QinQ too. */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } /* * We only support TCP/UDP for IPv4 and IPv6 for the moment. * TODO: Support SCTP too when it hits the tree. */ switch (etype) { case ETHERTYPE_IP: ip = (struct ip *)(mp->m_data + ehdrlen); ip_hlen = ip->ip_hl << 2; /* Setup of IP header checksum. */ if (mp->m_pkthdr.csum_flags & CSUM_IP) { /* * Start offset for header checksum calculation. * End offset for header checksum calculation. * Offset of place to put the checksum. */ TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; TXD->lower_setup.ip_fields.ipcss = ehdrlen; TXD->lower_setup.ip_fields.ipcse = htole16(ehdrlen + ip_hlen); TXD->lower_setup.ip_fields.ipcso = ehdrlen + offsetof(struct ip, ip_sum); cmd |= E1000_TXD_CMD_IP; *txd_upper |= E1000_TXD_POPTS_IXSM << 8; } if (mp->m_len < ehdrlen + ip_hlen) return; /* failure */ hdr_len = ehdrlen + ip_hlen; ipproto = ip->ip_p; break; case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ if (mp->m_len < ehdrlen + ip_hlen) return; /* failure */ /* IPv6 doesn't have a header checksum. */ hdr_len = ehdrlen + ip_hlen; ipproto = ip6->ip6_nxt; break; default: *txd_upper = 0; *txd_lower = 0; return; } switch (ipproto) { case IPPROTO_TCP: if (mp->m_pkthdr.csum_flags & CSUM_TCP) { *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; *txd_upper |= E1000_TXD_POPTS_TXSM << 8; /* no need for context if already set */ if (adapter->last_hw_offload == CSUM_TCP) return; adapter->last_hw_offload = CSUM_TCP; /* * Start offset for payload checksum calculation. * End offset for payload checksum calculation. * Offset of place to put the checksum. */ TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; TXD->upper_setup.tcp_fields.tucss = hdr_len; TXD->upper_setup.tcp_fields.tucse = htole16(0); TXD->upper_setup.tcp_fields.tucso = hdr_len + offsetof(struct tcphdr, th_sum); cmd |= E1000_TXD_CMD_TCP; } break; case IPPROTO_UDP: { if (mp->m_pkthdr.csum_flags & CSUM_UDP) { *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; *txd_upper |= E1000_TXD_POPTS_TXSM << 8; /* no need for context if already set */ if (adapter->last_hw_offload == CSUM_UDP) return; adapter->last_hw_offload = CSUM_UDP; /* * Start offset for header checksum calculation. * End offset for header checksum calculation. * Offset of place to put the checksum. */ TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; TXD->upper_setup.tcp_fields.tucss = hdr_len; TXD->upper_setup.tcp_fields.tucse = htole16(0); TXD->upper_setup.tcp_fields.tucso = hdr_len + offsetof(struct udphdr, uh_sum); } /* Fall Thru */ } default: break; } TXD->tcp_seg_setup.data = htole32(0); TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); tx_buffer = &adapter->tx_buffer_area[curr_txd]; tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; if (++curr_txd == adapter->num_tx_desc) curr_txd = 0; adapter->num_tx_desc_avail--; adapter->next_avail_tx_desc = curr_txd; } #if __FreeBSD_version >= 700000 /********************************************************************** * * Setup work for hardware segmentation offload (TSO) * **********************************************************************/ static bool em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, u32 *txd_lower) { struct e1000_context_desc *TXD; struct em_buffer *tx_buffer; struct ether_vlan_header *eh; struct ip *ip; struct ip6_hdr *ip6; struct tcphdr *th; int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; u16 etype; /* * This function could/should be extended to support IP/IPv6 * fragmentation as well. But as they say, one step at a time. */ /* * Determine where frame payload starts. * Jump over vlan headers if already present, * helpful for QinQ too. */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } /* Ensure we have at least the IP+TCP header in the first mbuf. */ if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) return FALSE; /* -1 */ /* * We only support TCP for IPv4 and IPv6 (notyet) for the moment. * TODO: Support SCTP too when it hits the tree. */ switch (etype) { case ETHERTYPE_IP: isip6 = 0; ip = (struct ip *)(mp->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) return FALSE; /* 0 */ ip->ip_len = 0; ip->ip_sum = 0; ip_hlen = ip->ip_hl << 2; if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) return FALSE; /* -1 */ th = (struct tcphdr *)((caddr_t)ip + ip_hlen); #if 1 th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); #else th->th_sum = mp->m_pkthdr.csum_data; #endif break; case ETHERTYPE_IPV6: isip6 = 1; return FALSE; /* Not supported yet. */ ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); if (ip6->ip6_nxt != IPPROTO_TCP) return FALSE; /* 0 */ ip6->ip6_plen = 0; ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) return FALSE; /* -1 */ th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); #if 0 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, htons(IPPROTO_TCP)); /* XXX: function notyet. */ #else th->th_sum = mp->m_pkthdr.csum_data; #endif break; default: return FALSE; } hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ E1000_TXD_DTYP_D | /* Data descr type */ E1000_TXD_CMD_TSE); /* Do TSE on this packet */ /* IP and/or TCP header checksum calculation and insertion. */ *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | E1000_TXD_POPTS_TXSM) << 8; curr_txd = adapter->next_avail_tx_desc; tx_buffer = &adapter->tx_buffer_area[curr_txd]; TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; /* IPv6 doesn't have a header checksum. */ if (!isip6) { /* * Start offset for header checksum calculation. * End offset for header checksum calculation. * Offset of place put the checksum. */ TXD->lower_setup.ip_fields.ipcss = ehdrlen; TXD->lower_setup.ip_fields.ipcse = htole16(ehdrlen + ip_hlen - 1); TXD->lower_setup.ip_fields.ipcso = ehdrlen + offsetof(struct ip, ip_sum); } /* * Start offset for payload checksum calculation. * End offset for payload checksum calculation. * Offset of place to put the checksum. */ TXD->upper_setup.tcp_fields.tucss = ehdrlen + ip_hlen; TXD->upper_setup.tcp_fields.tucse = 0; TXD->upper_setup.tcp_fields.tucso = ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); /* * Payload size per packet w/o any headers. * Length of all headers up to payload. */ TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); TXD->tcp_seg_setup.fields.hdr_len = hdr_len; TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | /* Extended descr */ E1000_TXD_CMD_TSE | /* TSE context */ (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ E1000_TXD_CMD_TCP | /* Do TCP checksum */ (mp->m_pkthdr.len - (hdr_len))); /* Total len */ tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; if (++curr_txd == adapter->num_tx_desc) curr_txd = 0; adapter->num_tx_desc_avail--; adapter->next_avail_tx_desc = curr_txd; adapter->tx_tso = TRUE; return TRUE; } #endif /* __FreeBSD_version >= 700000 */ /********************************************************************** * * Examine each tx_buffer in the used queue. If the hardware is done * processing the packet then free associated resources. The * tx_buffer is put back on the free queue. * **********************************************************************/ static void em_txeof(struct adapter *adapter) { int first, last, done, num_avail; struct em_buffer *tx_buffer; struct e1000_tx_desc *tx_desc, *eop_desc; struct ifnet *ifp = adapter->ifp; EM_TX_LOCK_ASSERT(adapter); if (adapter->num_tx_desc_avail == adapter->num_tx_desc) return; num_avail = adapter->num_tx_desc_avail; first = adapter->next_tx_to_clean; tx_desc = &adapter->tx_desc_base[first]; tx_buffer = &adapter->tx_buffer_area[first]; last = tx_buffer->next_eop; eop_desc = &adapter->tx_desc_base[last]; /* * What this does is get the index of the * first descriptor AFTER the EOP of the * first packet, that way we can do the * simple comparison on the inner while loop. */ if (++last == adapter->num_tx_desc) last = 0; done = last; bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_POSTREAD); while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { /* We clean the range of the packet */ while (first != done) { tx_desc->upper.data = 0; tx_desc->lower.data = 0; tx_desc->buffer_addr = 0; ++num_avail; if (tx_buffer->m_head) { ifp->if_opackets++; bus_dmamap_sync(adapter->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(adapter->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; } tx_buffer->next_eop = -1; adapter->watchdog_time = ticks; if (++first == adapter->num_tx_desc) first = 0; tx_buffer = &adapter->tx_buffer_area[first]; tx_desc = &adapter->tx_desc_base[first]; } /* See if we can continue to the next packet */ last = tx_buffer->next_eop; if (last != -1) { eop_desc = &adapter->tx_desc_base[last]; /* Get new done point */ if (++last == adapter->num_tx_desc) last = 0; done = last; } else break; } bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); adapter->next_tx_to_clean = first; /* * If we have enough room, clear IFF_DRV_OACTIVE to * tell the stack that it is OK to send packets. * If there are no pending descriptors, clear the watchdog. */ if (num_avail > EM_TX_CLEANUP_THRESHOLD) { ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; if (num_avail == adapter->num_tx_desc) { adapter->watchdog_check = FALSE; adapter->num_tx_desc_avail = num_avail; return; } } adapter->num_tx_desc_avail = num_avail; return; } /********************************************************************* * * When Link is lost sometimes there is work still in the TX ring * which may result in a watchdog, rather than allow that we do an * attempted cleanup and then reinit here. Note that this has been * seens mostly with fiber adapters. * **********************************************************************/ static void em_tx_purge(struct adapter *adapter) { if ((!adapter->link_active) && (adapter->watchdog_check)) { EM_TX_LOCK(adapter); em_txeof(adapter); EM_TX_UNLOCK(adapter); if (adapter->watchdog_check) /* Still outstanding? */ em_init_locked(adapter); } } /********************************************************************* * * Get a buffer from system mbuf buffer pool. * **********************************************************************/ static int em_get_buf(struct adapter *adapter, int i) { struct mbuf *m; bus_dma_segment_t segs[1]; bus_dmamap_t map; struct em_buffer *rx_buffer; int error, nsegs; m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); if (m == NULL) { adapter->mbuf_cluster_failed++; return (ENOBUFS); } m->m_len = m->m_pkthdr.len = MCLBYTES; if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) m_adj(m, ETHER_ALIGN); /* * Using memory from the mbuf cluster pool, invoke the * bus_dma machinery to arrange the memory mapping. */ error = bus_dmamap_load_mbuf_sg(adapter->rxtag, adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { m_free(m); return (error); } /* If nsegs is wrong then the stack is corrupt. */ KASSERT(nsegs == 1, ("Too many segments returned!")); rx_buffer = &adapter->rx_buffer_area[i]; if (rx_buffer->m_head != NULL) bus_dmamap_unload(adapter->rxtag, rx_buffer->map); map = rx_buffer->map; rx_buffer->map = adapter->rx_sparemap; adapter->rx_sparemap = map; bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); rx_buffer->m_head = m; adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); return (0); } /********************************************************************* * * Allocate memory for rx_buffer structures. Since we use one * rx_buffer per received packet, the maximum number of rx_buffer's * that we'll need is equal to the number of receive descriptors * that we've allocated. * **********************************************************************/ static int em_allocate_receive_structures(struct adapter *adapter) { device_t dev = adapter->dev; struct em_buffer *rx_buffer; int i, error; adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); if (adapter->rx_buffer_area == NULL) { device_printf(dev, "Unable to allocate rx_buffer memory\n"); return (ENOMEM); } #if __FreeBSD_version >= 700000 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ #else error = bus_dma_tag_create(NULL, /* parent */ #endif 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MCLBYTES, /* maxsize */ 1, /* nsegments */ MCLBYTES, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &adapter->rxtag); if (error) { device_printf(dev, "%s: bus_dma_tag_create failed %d\n", __func__, error); goto fail; } /* Create the spare map (used by getbuf) */ error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, &adapter->rx_sparemap); if (error) { device_printf(dev, "%s: bus_dmamap_create failed: %d\n", __func__, error); goto fail; } rx_buffer = adapter->rx_buffer_area; for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, &rx_buffer->map); if (error) { device_printf(dev, "%s: bus_dmamap_create failed: %d\n", __func__, error); goto fail; } } return (0); fail: em_free_receive_structures(adapter); return (error); } /********************************************************************* * * (Re)initialize receive structures. * **********************************************************************/ static int em_setup_receive_structures(struct adapter *adapter) { struct em_buffer *rx_buffer; int i, error; /* Reset descriptor ring */ bzero(adapter->rx_desc_base, (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); /* Free current RX buffers. */ rx_buffer = adapter->rx_buffer_area; for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { if (rx_buffer->m_head != NULL) { bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(adapter->rxtag, rx_buffer->map); m_freem(rx_buffer->m_head); rx_buffer->m_head = NULL; } } /* Allocate new ones. */ for (i = 0; i < adapter->num_rx_desc; i++) { error = em_get_buf(adapter, i); if (error) return (error); } /* Setup our descriptor pointers */ adapter->next_rx_desc_to_check = 0; bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return (0); } /********************************************************************* * * Enable receive unit. * **********************************************************************/ #define MAX_INTS_PER_SEC 8000 #define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) static void em_initialize_receive_unit(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; u64 bus_addr; u32 rctl, rxcsum; INIT_DEBUGOUT("em_initialize_receive_unit: begin"); /* * Make sure receives are disabled while setting * up the descriptor ring */ rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); if (adapter->hw.mac.type >= e1000_82540) { E1000_WRITE_REG(&adapter->hw, E1000_RADV, adapter->rx_abs_int_delay.value); /* * Set the interrupt throttling rate. Value is calculated * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) */ E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); } /* ** When using MSIX interrupts we need to throttle ** using the EITR register (82574 only) */ if (adapter->msix) for (int i = 0; i < 4; i++) E1000_WRITE_REG(&adapter->hw, E1000_EITR_82574(i), DEFAULT_ITR); /* Disable accelerated ackknowledge */ if (adapter->hw.mac.type == e1000_82574) E1000_WRITE_REG(&adapter->hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); /* Setup the Base and Length of the Rx Descriptor Ring */ bus_addr = adapter->rxdma.dma_paddr; E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), (u32)(bus_addr >> 32)); E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), (u32)bus_addr); /* Setup the Receive Control Register */ rctl &= ~(3 << E1000_RCTL_MO_SHIFT); rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); /* Make sure VLAN Filters are off */ rctl &= ~E1000_RCTL_VFE; if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) rctl |= E1000_RCTL_SBP; else rctl &= ~E1000_RCTL_SBP; switch (adapter->rx_buffer_len) { default: case 2048: rctl |= E1000_RCTL_SZ_2048; break; case 4096: rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE; break; case 8192: rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE; break; case 16384: rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE; break; } if (ifp->if_mtu > ETHERMTU) rctl |= E1000_RCTL_LPE; else rctl &= ~E1000_RCTL_LPE; /* Enable 82543 Receive Checksum Offload for TCP and UDP */ if ((adapter->hw.mac.type >= e1000_82543) && (ifp->if_capenable & IFCAP_RXCSUM)) { rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); } /* ** XXX TEMPORARY WORKAROUND: on some systems with 82573 ** long latencies are observed, like Lenovo X60. This ** change eliminates the problem, but since having positive ** values in RDTR is a known source of problems on other ** platforms another solution is being sought. */ if (adapter->hw.mac.type == e1000_82573) E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); /* Enable Receives */ E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); /* * Setup the HW Rx Head and * Tail Descriptor Pointers */ E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); return; } /********************************************************************* * * Free receive related data structures. * **********************************************************************/ static void em_free_receive_structures(struct adapter *adapter) { struct em_buffer *rx_buffer; int i; INIT_DEBUGOUT("free_receive_structures: begin"); if (adapter->rx_sparemap) { bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); adapter->rx_sparemap = NULL; } /* Cleanup any existing buffers */ if (adapter->rx_buffer_area != NULL) { rx_buffer = adapter->rx_buffer_area; for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { if (rx_buffer->m_head != NULL) { bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(adapter->rxtag, rx_buffer->map); m_freem(rx_buffer->m_head); rx_buffer->m_head = NULL; } else if (rx_buffer->map != NULL) bus_dmamap_unload(adapter->rxtag, rx_buffer->map); if (rx_buffer->map != NULL) { bus_dmamap_destroy(adapter->rxtag, rx_buffer->map); rx_buffer->map = NULL; } } } if (adapter->rx_buffer_area != NULL) { free(adapter->rx_buffer_area, M_DEVBUF); adapter->rx_buffer_area = NULL; } if (adapter->rxtag != NULL) { bus_dma_tag_destroy(adapter->rxtag); adapter->rxtag = NULL; } } /********************************************************************* * * This routine executes in interrupt context. It replenishes * the mbufs in the descriptor and sends data which has been * dma'ed into host memory to upper layer. * * We loop at most count times if count is > 0, or until done if * count < 0. * * For polling we also now return the number of cleaned packets *********************************************************************/ static int em_rxeof(struct adapter *adapter, int count) { struct ifnet *ifp = adapter->ifp;; struct mbuf *mp; u8 status, accept_frame = 0, eop = 0; u16 len, desc_len, prev_len_adj; int i, rx_sent = 0; struct e1000_rx_desc *current_desc; EM_RX_LOCK(adapter); i = adapter->next_rx_desc_to_check; current_desc = &adapter->rx_desc_base[i]; bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_POSTREAD); if (!((current_desc->status) & E1000_RXD_STAT_DD)) { EM_RX_UNLOCK(adapter); return (rx_sent); } while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0) && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { struct mbuf *m = NULL; mp = adapter->rx_buffer_area[i].m_head; /* * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT * needs to access the last received byte in the mbuf. */ bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, BUS_DMASYNC_POSTREAD); accept_frame = 1; prev_len_adj = 0; desc_len = le16toh(current_desc->length); status = current_desc->status; if (status & E1000_RXD_STAT_EOP) { count--; eop = 1; if (desc_len < ETHER_CRC_LEN) { len = 0; prev_len_adj = ETHER_CRC_LEN - desc_len; } else len = desc_len - ETHER_CRC_LEN; } else { eop = 0; len = desc_len; } if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { u8 last_byte; u32 pkt_len = desc_len; if (adapter->fmp != NULL) pkt_len += adapter->fmp->m_pkthdr.len; last_byte = *(mtod(mp, caddr_t) + desc_len - 1); if (TBI_ACCEPT(&adapter->hw, status, current_desc->errors, pkt_len, last_byte, adapter->min_frame_size, adapter->max_frame_size)) { e1000_tbi_adjust_stats_82543(&adapter->hw, &adapter->stats, pkt_len, adapter->hw.mac.addr, adapter->max_frame_size); if (len > 0) len--; } else accept_frame = 0; } if (accept_frame) { if (em_get_buf(adapter, i) != 0) { ifp->if_iqdrops++; goto discard; } /* Assign correct length to the current fragment */ mp->m_len = len; if (adapter->fmp == NULL) { mp->m_pkthdr.len = len; adapter->fmp = mp; /* Store the first mbuf */ adapter->lmp = mp; } else { /* Chain mbuf's together */ mp->m_flags &= ~M_PKTHDR; /* * Adjust length of previous mbuf in chain if * we received less than 4 bytes in the last * descriptor. */ if (prev_len_adj > 0) { adapter->lmp->m_len -= prev_len_adj; adapter->fmp->m_pkthdr.len -= prev_len_adj; } adapter->lmp->m_next = mp; adapter->lmp = adapter->lmp->m_next; adapter->fmp->m_pkthdr.len += len; } if (eop) { adapter->fmp->m_pkthdr.rcvif = ifp; ifp->if_ipackets++; em_receive_checksum(adapter, current_desc, adapter->fmp); #ifndef __NO_STRICT_ALIGNMENT if (adapter->max_frame_size > (MCLBYTES - ETHER_ALIGN) && em_fixup_rx(adapter) != 0) goto skip; #endif if (status & E1000_RXD_STAT_VP) { #if __FreeBSD_version < 700000 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, (le16toh(current_desc->special) & E1000_RXD_SPC_VLAN_MASK)); #else adapter->fmp->m_pkthdr.ether_vtag = (le16toh(current_desc->special) & E1000_RXD_SPC_VLAN_MASK); adapter->fmp->m_flags |= M_VLANTAG; #endif } #ifndef __NO_STRICT_ALIGNMENT skip: #endif m = adapter->fmp; adapter->fmp = NULL; adapter->lmp = NULL; } } else { ifp->if_ierrors++; discard: /* Reuse loaded DMA map and just update mbuf chain */ mp = adapter->rx_buffer_area[i].m_head; mp->m_len = mp->m_pkthdr.len = MCLBYTES; mp->m_data = mp->m_ext.ext_buf; mp->m_next = NULL; if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) m_adj(mp, ETHER_ALIGN); if (adapter->fmp != NULL) { m_freem(adapter->fmp); adapter->fmp = NULL; adapter->lmp = NULL; } m = NULL; } /* Zero out the receive descriptors status. */ current_desc->status = 0; bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* Advance our pointers to the next descriptor. */ if (++i == adapter->num_rx_desc) i = 0; /* Call into the stack */ if (m != NULL) { adapter->next_rx_desc_to_check = i; EM_RX_UNLOCK(adapter); (*ifp->if_input)(ifp, m); EM_RX_LOCK(adapter); rx_sent++; i = adapter->next_rx_desc_to_check; } current_desc = &adapter->rx_desc_base[i]; } adapter->next_rx_desc_to_check = i; /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ if (--i < 0) i = adapter->num_rx_desc - 1; E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); EM_RX_UNLOCK(adapter); return (rx_sent); } #ifndef __NO_STRICT_ALIGNMENT /* * When jumbo frames are enabled we should realign entire payload on * architecures with strict alignment. This is serious design mistake of 8254x * as it nullifies DMA operations. 8254x just allows RX buffer size to be * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its * payload. On architecures without strict alignment restrictions 8254x still * performs unaligned memory access which would reduce the performance too. * To avoid copying over an entire frame to align, we allocate a new mbuf and * copy ethernet header to the new mbuf. The new mbuf is prepended into the * existing mbuf chain. * * Be aware, best performance of the 8254x is achived only when jumbo frame is * not used at all on architectures with strict alignment. */ static int em_fixup_rx(struct adapter *adapter) { struct mbuf *m, *n; int error; error = 0; m = adapter->fmp; if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); m->m_data += ETHER_HDR_LEN; } else { MGETHDR(n, M_DONTWAIT, MT_DATA); if (n != NULL) { bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); m->m_data += ETHER_HDR_LEN; m->m_len -= ETHER_HDR_LEN; n->m_len = ETHER_HDR_LEN; M_MOVE_PKTHDR(n, m); n->m_next = m; adapter->fmp = n; } else { adapter->dropped_pkts++; m_freem(adapter->fmp); adapter->fmp = NULL; error = ENOMEM; } } return (error); } #endif /********************************************************************* * * Verify that the hardware indicated that the checksum is valid. * Inform the stack about the status of checksum so that stack * doesn't spend time verifying the checksum. * *********************************************************************/ static void em_receive_checksum(struct adapter *adapter, struct e1000_rx_desc *rx_desc, struct mbuf *mp) { /* 82543 or newer only */ if ((adapter->hw.mac.type < e1000_82543) || /* Ignore Checksum bit is set */ (rx_desc->status & E1000_RXD_STAT_IXSM)) { mp->m_pkthdr.csum_flags = 0; return; } if (rx_desc->status & E1000_RXD_STAT_IPCS) { /* Did it pass? */ if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { /* IP Checksum Good */ mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; } else { mp->m_pkthdr.csum_flags = 0; } } if (rx_desc->status & E1000_RXD_STAT_TCPCS) { /* Did it pass? */ if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); mp->m_pkthdr.csum_data = htons(0xffff); } } } #if __FreeBSD_version >= 700029 /* * This routine is run via an vlan * config EVENT */ static void em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u32 index, bit; if (ifp->if_softc != arg) /* Not our event */ return; if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; em_shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; /* Re-init to load the changes */ em_init(adapter); } /* * This routine is run via an vlan * unconfig EVENT */ static void em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u32 index, bit; if (ifp->if_softc != arg) return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; em_shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; /* Re-init to load the changes */ em_init(adapter); } static void em_setup_vlan_hw_support(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; u32 reg; /* ** We get here thru init_locked, meaning ** a soft reset, this has already cleared ** the VFTA and other state, so if there ** have been no vlan's registered do nothing. */ if (adapter->num_vlans == 0) return; /* ** A soft reset zero's out the VFTA, so ** we need to repopulate it now. */ for (int i = 0; i < EM_VFTA_SIZE; i++) if (em_shadow_vfta[i] != 0) E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, em_shadow_vfta[i]); reg = E1000_READ_REG(hw, E1000_CTRL); reg |= E1000_CTRL_VME; E1000_WRITE_REG(hw, E1000_CTRL, reg); /* Enable the Filter Table */ reg = E1000_READ_REG(hw, E1000_RCTL); reg &= ~E1000_RCTL_CFIEN; reg |= E1000_RCTL_VFE; E1000_WRITE_REG(hw, E1000_RCTL, reg); /* Update the frame size */ E1000_WRITE_REG(&adapter->hw, E1000_RLPML, adapter->max_frame_size + VLAN_TAG_SIZE); } #endif static void em_enable_intr(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; u32 ims_mask = IMS_ENABLE_MASK; if (adapter->msix) { E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); ims_mask |= EM_MSIX_MASK; } E1000_WRITE_REG(hw, E1000_IMS, ims_mask); } static void em_disable_intr(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; if (adapter->msix) E1000_WRITE_REG(hw, EM_EIAC, 0); E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); } /* * Bit of a misnomer, what this really means is * to enable OS management of the system... aka * to disable special hardware management features */ static void em_init_manageability(struct adapter *adapter) { /* A shared code workaround */ #define E1000_82542_MANC2H E1000_MANC2H if (adapter->has_manage) { int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); /* disable hardware interception of ARP */ manc &= ~(E1000_MANC_ARP_EN); /* enable receiving management packets to the host */ if (adapter->hw.mac.type >= e1000_82571) { manc |= E1000_MANC_EN_MNG2HOST; #define E1000_MNG2HOST_PORT_623 (1 << 5) #define E1000_MNG2HOST_PORT_664 (1 << 6) manc2h |= E1000_MNG2HOST_PORT_623; manc2h |= E1000_MNG2HOST_PORT_664; E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); } E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); } } /* * Give control back to hardware management * controller if there is one. */ static void em_release_manageability(struct adapter *adapter) { if (adapter->has_manage) { int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); /* re-enable hardware interception of ARP */ manc |= E1000_MANC_ARP_EN; if (adapter->hw.mac.type >= e1000_82571) manc &= ~E1000_MANC_EN_MNG2HOST; E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); } } /* * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means * that the driver is loaded. For AMT version type f/w * this means that the network i/f is open. */ static void em_get_hw_control(struct adapter *adapter) { u32 ctrl_ext, swsm; if (adapter->hw.mac.type == e1000_82573) { swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); E1000_WRITE_REG(&adapter->hw, E1000_SWSM, swsm | E1000_SWSM_DRV_LOAD); return; } /* else */ ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); return; } /* * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means that * the driver is no longer loaded. For AMT versions of the * f/w this means that the network i/f is closed. */ static void em_release_hw_control(struct adapter *adapter) { u32 ctrl_ext, swsm; if (!adapter->has_manage) return; if (adapter->hw.mac.type == e1000_82573) { swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); E1000_WRITE_REG(&adapter->hw, E1000_SWSM, swsm & ~E1000_SWSM_DRV_LOAD); return; } /* else */ ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); return; } static int em_is_valid_ether_addr(u8 *addr) { char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { return (FALSE); } return (TRUE); } /* ** Parse the interface capabilities with regard ** to both system management and wake-on-lan for ** later use. */ static void em_get_wakeup(device_t dev) { struct adapter *adapter = device_get_softc(dev); u16 eeprom_data = 0, device_id, apme_mask; adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); apme_mask = EM_EEPROM_APME; switch (adapter->hw.mac.type) { case e1000_82542: case e1000_82543: break; case e1000_82544: e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL2_REG, 1, &eeprom_data); apme_mask = EM_82544_APME; break; case e1000_82573: case e1000_82583: adapter->has_amt = TRUE; /* Falls thru */ case e1000_82546: case e1000_82546_rev_3: case e1000_82571: case e1000_82572: case e1000_80003es2lan: if (adapter->hw.bus.func == 1) { e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); break; } else e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); break; case e1000_ich8lan: case e1000_ich9lan: case e1000_ich10lan: case e1000_pchlan: apme_mask = E1000_WUC_APME; adapter->has_amt = TRUE; eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); break; default: e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); break; } if (eeprom_data & apme_mask) adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); /* * We have the eeprom settings, now apply the special cases * where the eeprom may be wrong or the board won't support * wake on lan on a particular port */ device_id = pci_get_device(dev); switch (device_id) { case E1000_DEV_ID_82546GB_PCIE: adapter->wol = 0; break; case E1000_DEV_ID_82546EB_FIBER: case E1000_DEV_ID_82546GB_FIBER: case E1000_DEV_ID_82571EB_FIBER: /* Wake events only supported on port A for dual fiber * regardless of eeprom setting */ if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_FUNC_1) adapter->wol = 0; break; case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: case E1000_DEV_ID_82571EB_QUAD_COPPER: case E1000_DEV_ID_82571EB_QUAD_FIBER: case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: /* if quad port adapter, disable WoL on all but port A */ if (global_quad_port_a != 0) adapter->wol = 0; /* Reset for multiple quad port adapters */ if (++global_quad_port_a == 4) global_quad_port_a = 0; break; } return; } /* * Enable PCI Wake On Lan capability */ void em_enable_wakeup(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct ifnet *ifp = adapter->ifp; u32 pmc, ctrl, ctrl_ext, rctl; u16 status; if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) return; /* Advertise the wakeup capability */ ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); /* ICH workaround code */ if ((adapter->hw.mac.type == e1000_ich8lan) || (adapter->hw.mac.type == e1000_pchlan) || (adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_ich10lan)) { e1000_disable_gig_wol_ich8lan(&adapter->hw); e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); } /* Keep the laser running on Fiber adapters */ if (adapter->hw.phy.media_type == e1000_media_type_fiber || adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); } /* ** Determine type of Wakeup: note that wol ** is set with all bits on by default. */ if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) adapter->wol &= ~E1000_WUFC_MAG; if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) adapter->wol &= ~E1000_WUFC_MC; else { rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); rctl |= E1000_RCTL_MPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); } if (adapter->hw.mac.type == e1000_pchlan) { if (em_enable_phy_wakeup(adapter)) return; } else { E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); } if (adapter->hw.phy.type == e1000_phy_igp_3) e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); /* Request PME */ status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); if (ifp->if_capenable & IFCAP_WOL) status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); return; } /* ** WOL in the newer chipset interfaces (pchlan) ** require thing to be copied into the phy */ static int em_enable_phy_wakeup(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; u32 mreg, ret = 0; u16 preg; /* copy MAC RARs to PHY RARs */ for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { mreg = E1000_READ_REG(hw, E1000_RAL(i)); e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); e1000_write_phy_reg(hw, BM_RAR_M(i), (u16)((mreg >> 16) & 0xFFFF)); mreg = E1000_READ_REG(hw, E1000_RAH(i)); e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); e1000_write_phy_reg(hw, BM_RAR_CTRL(i), (u16)((mreg >> 16) & 0xFFFF)); } /* copy MAC MTA to PHY MTA */ for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); e1000_write_phy_reg(hw, BM_MTA(i) + 1, (u16)((mreg >> 16) & 0xFFFF)); } /* configure PHY Rx Control register */ e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); mreg = E1000_READ_REG(hw, E1000_RCTL); if (mreg & E1000_RCTL_UPE) preg |= BM_RCTL_UPE; if (mreg & E1000_RCTL_MPE) preg |= BM_RCTL_MPE; preg &= ~(BM_RCTL_MO_MASK); if (mreg & E1000_RCTL_MO_3) preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) << BM_RCTL_MO_SHIFT); if (mreg & E1000_RCTL_BAM) preg |= BM_RCTL_BAM; if (mreg & E1000_RCTL_PMCF) preg |= BM_RCTL_PMCF; mreg = E1000_READ_REG(hw, E1000_CTRL); if (mreg & E1000_CTRL_RFCE) preg |= BM_RCTL_RFCE; e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); /* enable PHY wakeup in MAC register */ E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); /* configure and enable PHY wakeup in PHY registers */ e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); /* activate PHY wakeup */ ret = hw->phy.ops.acquire(hw); if (ret) { printf("Could not acquire PHY\n"); return ret; } e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); if (ret) { printf("Could not read PHY page 769\n"); goto out; } preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); if (ret) printf("Could not set PHY Host Wakeup bit\n"); out: hw->phy.ops.release(hw); return ret; } /********************************************************************* * 82544 Coexistence issue workaround. * There are 2 issues. * 1. Transmit Hang issue. * To detect this issue, following equation can be used... * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. * If SUM[3:0] is in between 1 to 4, we will have this issue. * * 2. DAC issue. * To detect this issue, following equation can be used... * SIZE[3:0] + ADDR[2:0] = SUM[3:0]. * If SUM[3:0] is in between 9 to c, we will have this issue. * * * WORKAROUND: * Make sure we do not have ending address * as 1,2,3,4(Hang) or 9,a,b,c (DAC) * *************************************************************************/ static u32 em_fill_descriptors (bus_addr_t address, u32 length, PDESC_ARRAY desc_array) { u32 safe_terminator; /* Since issue is sensitive to length and address.*/ /* Let us first check the address...*/ if (length <= 4) { desc_array->descriptor[0].address = address; desc_array->descriptor[0].length = length; desc_array->elements = 1; return (desc_array->elements); } safe_terminator = (u32)((((u32)address & 0x7) + (length & 0xF)) & 0xF); /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ if (safe_terminator == 0 || (safe_terminator > 4 && safe_terminator < 9) || (safe_terminator > 0xC && safe_terminator <= 0xF)) { desc_array->descriptor[0].address = address; desc_array->descriptor[0].length = length; desc_array->elements = 1; return (desc_array->elements); } desc_array->descriptor[0].address = address; desc_array->descriptor[0].length = length - 4; desc_array->descriptor[1].address = address + (length - 4); desc_array->descriptor[1].length = 4; desc_array->elements = 2; return (desc_array->elements); } /********************************************************************** * * Update the board statistics counters. * **********************************************************************/ static void em_update_stats_counters(struct adapter *adapter) { struct ifnet *ifp; if(adapter->hw.phy.media_type == e1000_media_type_copper || (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); } adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); /* For the 64-bit byte counters the low dword must be read first. */ /* Both registers clear on the read of the high dword */ adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); if (adapter->hw.mac.type >= e1000_82543) { adapter->stats.algnerrc += E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); adapter->stats.rxerrc += E1000_READ_REG(&adapter->hw, E1000_RXERRC); adapter->stats.tncrs += E1000_READ_REG(&adapter->hw, E1000_TNCRS); adapter->stats.cexterr += E1000_READ_REG(&adapter->hw, E1000_CEXTERR); adapter->stats.tsctc += E1000_READ_REG(&adapter->hw, E1000_TSCTC); adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); } ifp = adapter->ifp; ifp->if_collisions = adapter->stats.colc; /* Rx Errors */ ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + adapter->stats.crcerrs + adapter->stats.algnerrc + adapter->stats.ruc + adapter->stats.roc + adapter->stats.mpc + adapter->stats.cexterr; /* Tx Errors */ ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol + adapter->watchdog_events; } /********************************************************************** * * This routine is called only when em_display_debug_stats is enabled. * This routine provides a way to take a look at important statistics * maintained by the driver and hardware. * **********************************************************************/ static void em_print_debug_info(struct adapter *adapter) { device_t dev = adapter->dev; u8 *hw_addr = adapter->hw.hw_addr; device_printf(dev, "Adapter hardware address = %p \n", hw_addr); device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", E1000_READ_REG(&adapter->hw, E1000_CTRL), E1000_READ_REG(&adapter->hw, E1000_RCTL)); device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); device_printf(dev, "Flow control watermarks high = %d low = %d\n", adapter->hw.fc.high_water, adapter->hw.fc.low_water); device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", E1000_READ_REG(&adapter->hw, E1000_TIDV), E1000_READ_REG(&adapter->hw, E1000_TADV)); device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", E1000_READ_REG(&adapter->hw, E1000_RDTR), E1000_READ_REG(&adapter->hw, E1000_RADV)); device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", (long long)adapter->tx_fifo_wrk_cnt, (long long)adapter->tx_fifo_reset_cnt); device_printf(dev, "hw tdh = %d, hw tdt = %d\n", E1000_READ_REG(&adapter->hw, E1000_TDH(0)), E1000_READ_REG(&adapter->hw, E1000_TDT(0))); device_printf(dev, "hw rdh = %d, hw rdt = %d\n", E1000_READ_REG(&adapter->hw, E1000_RDH(0)), E1000_READ_REG(&adapter->hw, E1000_RDT(0))); device_printf(dev, "Num Tx descriptors avail = %d\n", adapter->num_tx_desc_avail); device_printf(dev, "Tx Descriptors not avail1 = %ld\n", adapter->no_tx_desc_avail1); device_printf(dev, "Tx Descriptors not avail2 = %ld\n", adapter->no_tx_desc_avail2); device_printf(dev, "Std mbuf failed = %ld\n", adapter->mbuf_alloc_failed); device_printf(dev, "Std mbuf cluster failed = %ld\n", adapter->mbuf_cluster_failed); device_printf(dev, "Driver dropped packets = %ld\n", adapter->dropped_pkts); device_printf(dev, "Driver tx dma failure in encap = %ld\n", adapter->no_tx_dma_setup); } static void em_print_hw_stats(struct adapter *adapter) { device_t dev = adapter->dev; device_printf(dev, "Excessive collisions = %lld\n", (long long)adapter->stats.ecol); #if (DEBUG_HW > 0) /* Dont output these errors normally */ device_printf(dev, "Symbol errors = %lld\n", (long long)adapter->stats.symerrs); #endif device_printf(dev, "Sequence errors = %lld\n", (long long)adapter->stats.sec); device_printf(dev, "Defer count = %lld\n", (long long)adapter->stats.dc); device_printf(dev, "Missed Packets = %lld\n", (long long)adapter->stats.mpc); device_printf(dev, "Receive No Buffers = %lld\n", (long long)adapter->stats.rnbc); /* RLEC is inaccurate on some hardware, calculate our own. */ device_printf(dev, "Receive Length Errors = %lld\n", ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); device_printf(dev, "Receive errors = %lld\n", (long long)adapter->stats.rxerrc); device_printf(dev, "Crc errors = %lld\n", (long long)adapter->stats.crcerrs); device_printf(dev, "Alignment errors = %lld\n", (long long)adapter->stats.algnerrc); device_printf(dev, "Collision/Carrier extension errors = %lld\n", (long long)adapter->stats.cexterr); device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); device_printf(dev, "watchdog timeouts = %ld\n", adapter->watchdog_events); device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld" " LINK MSIX IRQ = %ld\n", adapter->rx_irq, adapter->tx_irq , adapter->link_irq); device_printf(dev, "XON Rcvd = %lld\n", (long long)adapter->stats.xonrxc); device_printf(dev, "XON Xmtd = %lld\n", (long long)adapter->stats.xontxc); device_printf(dev, "XOFF Rcvd = %lld\n", (long long)adapter->stats.xoffrxc); device_printf(dev, "XOFF Xmtd = %lld\n", (long long)adapter->stats.xofftxc); device_printf(dev, "Good Packets Rcvd = %lld\n", (long long)adapter->stats.gprc); device_printf(dev, "Good Packets Xmtd = %lld\n", (long long)adapter->stats.gptc); device_printf(dev, "TSO Contexts Xmtd = %lld\n", (long long)adapter->stats.tsctc); device_printf(dev, "TSO Contexts Failed = %lld\n", (long long)adapter->stats.tsctfc); } /********************************************************************** * * This routine provides a way to dump out the adapter eeprom, * often a useful debug/service tool. This only dumps the first * 32 words, stuff that matters is in that extent. * **********************************************************************/ static void em_print_nvm_info(struct adapter *adapter) { u16 eeprom_data; int i, j, row = 0; /* Its a bit crude, but it gets the job done */ printf("\nInterface EEPROM Dump:\n"); printf("Offset\n0x0000 "); for (i = 0, j = 0; i < 32; i++, j++) { if (j == 8) { /* Make the offset block */ j = 0; ++row; printf("\n0x00%x0 ",row); } e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); printf("%04x ", eeprom_data); } printf("\n"); } static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) { struct adapter *adapter; int error; int result; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *)arg1; em_print_debug_info(adapter); } /* * This value will cause a hex dump of the * first 32 16-bit words of the EEPROM to * the screen. */ if (result == 2) { adapter = (struct adapter *)arg1; em_print_nvm_info(adapter); } return (error); } static int em_sysctl_stats(SYSCTL_HANDLER_ARGS) { struct adapter *adapter; int error; int result; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *)arg1; em_print_hw_stats(adapter); } return (error); } static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) { struct em_int_delay_info *info; struct adapter *adapter; u32 regval; int error; int usecs; int ticks; info = (struct em_int_delay_info *)arg1; usecs = info->value; error = sysctl_handle_int(oidp, &usecs, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) return (EINVAL); info->value = usecs; ticks = EM_USECS_TO_TICKS(usecs); adapter = info->adapter; EM_CORE_LOCK(adapter); regval = E1000_READ_OFFSET(&adapter->hw, info->offset); regval = (regval & ~0xffff) | (ticks & 0xffff); /* Handle a few special cases. */ switch (info->offset) { case E1000_RDTR: break; case E1000_TIDV: if (ticks == 0) { adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; /* Don't write 0 into the TIDV register. */ regval++; } else adapter->txd_cmd |= E1000_TXD_CMD_IDE; break; } E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); EM_CORE_UNLOCK(adapter); return (0); } static void em_add_int_delay_sysctl(struct adapter *adapter, const char *name, const char *description, struct em_int_delay_info *info, int offset, int value) { info->adapter = adapter; info->offset = offset; info->value = value; SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, info, 0, em_sysctl_int_delay, "I", description); } #ifndef EM_LEGACY_IRQ static void em_add_rx_process_limit(struct adapter *adapter, const char *name, const char *description, int *limit, int value) { *limit = value; SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); } #endif Index: user/imp/tbemd/sys/dev/e1000/if_igb.c =================================================================== --- user/imp/tbemd/sys/dev/e1000/if_igb.c (revision 203849) +++ user/imp/tbemd/sys/dev/e1000/if_igb.c (revision 203850) @@ -1,4967 +1,4971 @@ /****************************************************************************** Copyright (c) 2001-2010, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ /*$FreeBSD$*/ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #include "opt_inet.h" #include "opt_altq.h" #endif #include #include #if __FreeBSD_version >= 800000 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef IGB_IEEE1588 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "e1000_api.h" #include "e1000_82575.h" #include "if_igb.h" /********************************************************************* * Set this to one to display debug statistics *********************************************************************/ int igb_display_debug_stats = 0; /********************************************************************* * Driver version: *********************************************************************/ char igb_driver_version[] = "version - 1.9.1"; /********************************************************************* * PCI Device ID Table * * Used by probe to select devices to load on * Last field stores an index into e1000_strings * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ static igb_vendor_info_t igb_vendor_info_array[] = { { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL, PCI_ANY_ID, PCI_ANY_ID, 0}, /* required last entry */ { 0, 0, 0, 0, 0} }; /********************************************************************* * Table of branding strings for all supported NICs. *********************************************************************/ static char *igb_strings[] = { "Intel(R) PRO/1000 Network Connection" }; /********************************************************************* * Function prototypes *********************************************************************/ static int igb_probe(device_t); static int igb_attach(device_t); static int igb_detach(device_t); static int igb_shutdown(device_t); static int igb_suspend(device_t); static int igb_resume(device_t); static void igb_start(struct ifnet *); static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); #if __FreeBSD_version >= 800000 static int igb_mq_start(struct ifnet *, struct mbuf *); static int igb_mq_start_locked(struct ifnet *, struct tx_ring *, struct mbuf *); static void igb_qflush(struct ifnet *); #endif static int igb_ioctl(struct ifnet *, u_long, caddr_t); static void igb_init(void *); static void igb_init_locked(struct adapter *); static void igb_stop(void *); static void igb_media_status(struct ifnet *, struct ifmediareq *); static int igb_media_change(struct ifnet *); static void igb_identify_hardware(struct adapter *); static int igb_allocate_pci_resources(struct adapter *); static int igb_allocate_msix(struct adapter *); static int igb_allocate_legacy(struct adapter *); static int igb_setup_msix(struct adapter *); static void igb_free_pci_resources(struct adapter *); static void igb_local_timer(void *); static void igb_reset(struct adapter *); static void igb_setup_interface(device_t, struct adapter *); static int igb_allocate_queues(struct adapter *); static void igb_configure_queues(struct adapter *); static int igb_allocate_transmit_buffers(struct tx_ring *); static void igb_setup_transmit_structures(struct adapter *); static void igb_setup_transmit_ring(struct tx_ring *); static void igb_initialize_transmit_units(struct adapter *); static void igb_free_transmit_structures(struct adapter *); static void igb_free_transmit_buffers(struct tx_ring *); static int igb_allocate_receive_buffers(struct rx_ring *); static int igb_setup_receive_structures(struct adapter *); static int igb_setup_receive_ring(struct rx_ring *); static void igb_initialize_receive_units(struct adapter *); static void igb_free_receive_structures(struct adapter *); static void igb_free_receive_buffers(struct rx_ring *); static void igb_free_receive_ring(struct rx_ring *); static void igb_enable_intr(struct adapter *); static void igb_disable_intr(struct adapter *); static void igb_update_stats_counters(struct adapter *); static bool igb_txeof(struct tx_ring *); static __inline void igb_rx_discard(struct rx_ring *, union e1000_adv_rx_desc *, int); static __inline void igb_rx_input(struct rx_ring *, struct ifnet *, struct mbuf *, u32); static bool igb_rxeof(struct rx_ring *, int); static void igb_rx_checksum(u32, struct mbuf *, u32); static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); static void igb_set_promisc(struct adapter *); static void igb_disable_promisc(struct adapter *); static void igb_set_multi(struct adapter *); static void igb_print_hw_stats(struct adapter *); static void igb_update_link_status(struct adapter *); static int igb_get_buf(struct rx_ring *, int, u8); static void igb_register_vlan(void *, struct ifnet *, u16); static void igb_unregister_vlan(void *, struct ifnet *, u16); static void igb_setup_vlan_hw_support(struct adapter *); static int igb_xmit(struct tx_ring *, struct mbuf **); static int igb_dma_malloc(struct adapter *, bus_size_t, struct igb_dma_alloc *, int); static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); static void igb_print_debug_info(struct adapter *); static void igb_print_nvm_info(struct adapter *); static int igb_is_valid_ether_addr(u8 *); static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS); static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS); /* Management and WOL Support */ static void igb_init_manageability(struct adapter *); static void igb_release_manageability(struct adapter *); static void igb_get_hw_control(struct adapter *); static void igb_release_hw_control(struct adapter *); static void igb_enable_wakeup(device_t); static int igb_irq_fast(void *); static void igb_add_rx_process_limit(struct adapter *, const char *, const char *, int *, int); static void igb_handle_rxtx(void *context, int pending); static void igb_handle_que(void *context, int pending); static void igb_handle_link(void *context, int pending); /* These are MSIX only irq handlers */ static void igb_msix_que(void *); static void igb_msix_link(void *); #ifdef DEVICE_POLLING static poll_handler_t igb_poll; #endif /* POLLING */ /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ static device_method_t igb_methods[] = { /* Device interface */ DEVMETHOD(device_probe, igb_probe), DEVMETHOD(device_attach, igb_attach), DEVMETHOD(device_detach, igb_detach), DEVMETHOD(device_shutdown, igb_shutdown), DEVMETHOD(device_suspend, igb_suspend), DEVMETHOD(device_resume, igb_resume), {0, 0} }; static driver_t igb_driver = { "igb", igb_methods, sizeof(struct adapter), }; static devclass_t igb_devclass; DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); MODULE_DEPEND(igb, pci, 1, 1, 1); MODULE_DEPEND(igb, ether, 1, 1, 1); /********************************************************************* * Tunable default values. *********************************************************************/ /* Descriptor defaults */ static int igb_rxd = IGB_DEFAULT_RXD; static int igb_txd = IGB_DEFAULT_TXD; TUNABLE_INT("hw.igb.rxd", &igb_rxd); TUNABLE_INT("hw.igb.txd", &igb_txd); /* ** AIM: Adaptive Interrupt Moderation ** which means that the interrupt rate ** is varied over time based on the ** traffic for that interrupt vector */ static int igb_enable_aim = TRUE; TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); /* * MSIX should be the default for best performance, * but this allows it to be forced off for testing. */ static int igb_enable_msix = 1; TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); /* * Header split has seemed to be beneficial in * many circumstances tested, however there have * been some stability issues, so the default is * off. */ static bool igb_header_split = FALSE; TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); /* ** This will autoconfigure based on ** the number of CPUs if left at 0. */ static int igb_num_queues = 0; TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); /* How many packets rxeof tries to clean at a time */ static int igb_rx_process_limit = 100; TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); /* Flow control setting - default to FULL */ static int igb_fc_setting = e1000_fc_full; TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); /* ** Shadow VFTA table, this is needed because ** the real filter table gets cleared during ** a soft reset and the driver needs to be able ** to repopulate it. */ static u32 igb_shadow_vfta[IGB_VFTA_SIZE]; /********************************************************************* * Device identification routine * * igb_probe determines if the driver should be loaded on * adapter based on PCI vendor/device id of the adapter. * * return BUS_PROBE_DEFAULT on success, positive on failure *********************************************************************/ static int igb_probe(device_t dev) { char adapter_name[60]; uint16_t pci_vendor_id = 0; uint16_t pci_device_id = 0; uint16_t pci_subvendor_id = 0; uint16_t pci_subdevice_id = 0; igb_vendor_info_t *ent; INIT_DEBUGOUT("igb_probe: begin"); pci_vendor_id = pci_get_vendor(dev); if (pci_vendor_id != IGB_VENDOR_ID) return (ENXIO); pci_device_id = pci_get_device(dev); pci_subvendor_id = pci_get_subvendor(dev); pci_subdevice_id = pci_get_subdevice(dev); ent = igb_vendor_info_array; while (ent->vendor_id != 0) { if ((pci_vendor_id == ent->vendor_id) && (pci_device_id == ent->device_id) && ((pci_subvendor_id == ent->subvendor_id) || (ent->subvendor_id == PCI_ANY_ID)) && ((pci_subdevice_id == ent->subdevice_id) || (ent->subdevice_id == PCI_ANY_ID))) { sprintf(adapter_name, "%s %s", igb_strings[ent->index], igb_driver_version); device_set_desc_copy(dev, adapter_name); return (BUS_PROBE_DEFAULT); } ent++; } return (ENXIO); } /********************************************************************* * Device initialization routine * * The attach entry point is called when the driver is being loaded. * This routine identifies the type of hardware, allocates all resources * and initializes the hardware. * * return 0 on success, positive on failure *********************************************************************/ static int igb_attach(device_t dev) { struct adapter *adapter; int error = 0; u16 eeprom_data; INIT_DEBUGOUT("igb_attach: begin"); adapter = device_get_softc(dev); adapter->dev = adapter->osdep.dev = dev; IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); /* SYSCTL stuff */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, igb_sysctl_debug_info, "I", "Debug Information"); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, igb_sysctl_stats, "I", "Statistics"); SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, &igb_fc_setting, 0, "Flow Control"); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, &igb_enable_aim, 1, "Interrupt Moderation"); callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); /* Determine hardware and mac info */ igb_identify_hardware(adapter); /* Setup PCI resources */ if (igb_allocate_pci_resources(adapter)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_pci; } /* Do Shared Code initialization */ if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { device_printf(dev, "Setup of Shared code failed\n"); error = ENXIO; goto err_pci; } e1000_get_bus_info(&adapter->hw); /* Sysctls for limiting the amount of work done in the taskqueue */ igb_add_rx_process_limit(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, igb_rx_process_limit); /* * Validate number of transmit and receive descriptors. It * must not exceed hardware maximum, and must be multiple * of E1000_DBA_ALIGN. */ if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { device_printf(dev, "Using %d TX descriptors instead of %d!\n", IGB_DEFAULT_TXD, igb_txd); adapter->num_tx_desc = IGB_DEFAULT_TXD; } else adapter->num_tx_desc = igb_txd; if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { device_printf(dev, "Using %d RX descriptors instead of %d!\n", IGB_DEFAULT_RXD, igb_rxd); adapter->num_rx_desc = IGB_DEFAULT_RXD; } else adapter->num_rx_desc = igb_rxd; adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_wait_to_complete = FALSE; adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; /* Copper options */ if (adapter->hw.phy.media_type == e1000_media_type_copper) { adapter->hw.phy.mdix = AUTO_ALL_MODES; adapter->hw.phy.disable_polarity_correction = FALSE; adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; } /* * Set the frame limits assuming * standard ethernet sized frames. */ adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; /* ** Allocate and Setup Queues */ if (igb_allocate_queues(adapter)) { error = ENOMEM; goto err_pci; } /* ** Start from a known state, this is ** important in reading the nvm and ** mac from that. */ e1000_reset_hw(&adapter->hw); /* Make sure we have a good EEPROM before we read from it */ if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { /* ** Some PCI-E parts fail the first check due to ** the link being in sleep state, call it again, ** if it fails a second time its a real issue. */ if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { device_printf(dev, "The EEPROM Checksum Is Not Valid\n"); error = EIO; goto err_late; } } /* ** Copy the permanent MAC address out of the EEPROM */ if (e1000_read_mac_addr(&adapter->hw) < 0) { device_printf(dev, "EEPROM read error while reading MAC" " address\n"); error = EIO; goto err_late; } /* Check its sanity */ if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { device_printf(dev, "Invalid MAC address\n"); error = EIO; goto err_late; } /* ** Configure Interrupts */ if ((adapter->msix > 1) && (igb_enable_msix)) error = igb_allocate_msix(adapter); else /* MSI or Legacy */ error = igb_allocate_legacy(adapter); if (error) goto err_late; /* Setup OS specific network interface */ igb_setup_interface(dev, adapter); /* Now get a good starting state */ igb_reset(adapter); /* Initialize statistics */ igb_update_stats_counters(adapter); adapter->hw.mac.get_link_status = 1; igb_update_link_status(adapter); /* Indicate SOL/IDER usage */ if (e1000_check_reset_block(&adapter->hw)) device_printf(dev, "PHY reset is blocked due to SOL/IDER session.\n"); /* Determine if we have to control management hardware */ adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); /* * Setup Wake-on-Lan */ /* APME bit in EEPROM is mapped to WUC.APME */ eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; if (eeprom_data) adapter->wol = E1000_WUFC_MAG; /* Register for VLAN events */ adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); /* Tell the stack that the interface is not active */ adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); INIT_DEBUGOUT("igb_attach: end"); return (0); err_late: igb_free_transmit_structures(adapter); igb_free_receive_structures(adapter); igb_release_hw_control(adapter); err_pci: igb_free_pci_resources(adapter); IGB_CORE_LOCK_DESTROY(adapter); return (error); } /********************************************************************* * Device removal routine * * The detach entry point is called when the driver is being removed. * This routine stops the adapter and deallocates all the resources * that were allocated for driver operation. * * return 0 on success, positive on failure *********************************************************************/ static int igb_detach(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct ifnet *ifp = adapter->ifp; INIT_DEBUGOUT("igb_detach: begin"); /* Make sure VLANS are not using driver */ if (adapter->ifp->if_vlantrunk != NULL) { device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } #ifdef DEVICE_POLLING if (ifp->if_capenable & IFCAP_POLLING) ether_poll_deregister(ifp); #endif IGB_CORE_LOCK(adapter); adapter->in_detach = 1; igb_stop(adapter); IGB_CORE_UNLOCK(adapter); e1000_phy_hw_reset(&adapter->hw); /* Give control back to firmware */ igb_release_manageability(adapter); igb_release_hw_control(adapter); if (adapter->wol) { E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); igb_enable_wakeup(dev); } /* Unregister VLAN events */ if (adapter->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); if (adapter->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); igb_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); igb_free_transmit_structures(adapter); igb_free_receive_structures(adapter); IGB_CORE_LOCK_DESTROY(adapter); return (0); } /********************************************************************* * * Shutdown entry point * **********************************************************************/ static int igb_shutdown(device_t dev) { return igb_suspend(dev); } /* * Suspend/resume device methods. */ static int igb_suspend(device_t dev) { struct adapter *adapter = device_get_softc(dev); IGB_CORE_LOCK(adapter); igb_stop(adapter); igb_release_manageability(adapter); igb_release_hw_control(adapter); if (adapter->wol) { E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); igb_enable_wakeup(dev); } IGB_CORE_UNLOCK(adapter); return bus_generic_suspend(dev); } static int igb_resume(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct ifnet *ifp = adapter->ifp; IGB_CORE_LOCK(adapter); igb_init_locked(adapter); igb_init_manageability(adapter); if ((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING)) igb_start(ifp); IGB_CORE_UNLOCK(adapter); return bus_generic_resume(dev); } /********************************************************************* * Transmit entry point * * igb_start is called by the stack to initiate a transmit. * The driver will remain in this routine as long as there are * packets to transmit and transmit resources are available. * In case resources are not available stack is notified and * the packet is requeued. **********************************************************************/ static void igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct mbuf *m_head; IGB_TX_LOCK_ASSERT(txr); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!adapter->link_active) return; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; /* * Encapsulation can modify our pointer, and or make it * NULL on failure. In that event, we can't requeue. */ if (igb_xmit(txr, &m_head)) { if (m_head == NULL) break; ifp->if_drv_flags |= IFF_DRV_OACTIVE; IFQ_DRV_PREPEND(&ifp->if_snd, m_head); break; } /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); /* Set watchdog on */ txr->watchdog_check = TRUE; } } /* * Legacy TX driver routine, called from the * stack, always uses tx[0], and spins for it. * Should not be used with multiqueue tx */ static void igb_start(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { IGB_TX_LOCK(txr); igb_start_locked(txr, ifp); IGB_TX_UNLOCK(txr); } return; } #if __FreeBSD_version >= 800000 /* ** Multiqueue Transmit driver ** */ static int igb_mq_start(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr; int i = 0, err = 0; /* Which queue to use */ if ((m->m_flags & M_FLOWID) != 0) i = m->m_pkthdr.flowid % adapter->num_queues; txr = &adapter->tx_rings[i]; if (IGB_TX_TRYLOCK(txr)) { err = igb_mq_start_locked(ifp, txr, m); IGB_TX_UNLOCK(txr); } else err = drbr_enqueue(ifp, txr->br, m); return (err); } static int igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) { struct adapter *adapter = txr->adapter; struct mbuf *next; int err = 0, enq; IGB_TX_LOCK_ASSERT(txr); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING || adapter->link_active == 0) { if (m != NULL) err = drbr_enqueue(ifp, txr->br, m); return (err); } enq = 0; - if (m == NULL) + if (m == NULL) { next = drbr_dequeue(ifp, txr->br); - else + } else if (drbr_needs_enqueue(ifp, txr->br)) { + if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) + return (err); + next = drbr_dequeue(ifp, txr->br); + } else next = m; /* Process the queue */ while (next != NULL) { if ((err = igb_xmit(txr, &next)) != 0) { if (next != NULL) err = drbr_enqueue(ifp, txr->br, next); break; } enq++; drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); ETHER_BPF_MTAP(ifp, next); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } next = drbr_dequeue(ifp, txr->br); } if (enq > 0) { /* Set the watchdog */ txr->watchdog_check = TRUE; } return (err); } /* ** Flush all ring buffers */ static void igb_qflush(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; struct mbuf *m; for (int i = 0; i < adapter->num_queues; i++, txr++) { IGB_TX_LOCK(txr); while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) m_freem(m); IGB_TX_UNLOCK(txr); } if_qflush(ifp); } #endif /* __FreeBSD_version >= 800000 */ /********************************************************************* * Ioctl entry point * * igb_ioctl is called when the user wants to configure the * interface. * * return 0 on success, positive on failure **********************************************************************/ static int igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct adapter *adapter = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; #ifdef INET struct ifaddr *ifa = (struct ifaddr *)data; #endif int error = 0; if (adapter->in_detach) return (error); switch (command) { case SIOCSIFADDR: #ifdef INET if (ifa->ifa_addr->sa_family == AF_INET) { /* * XXX * Since resetting hardware takes a very long time * and results in link renegotiation we only * initialize the hardware only when it is absolutely * required. */ ifp->if_flags |= IFF_UP; if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { IGB_CORE_LOCK(adapter); igb_init_locked(adapter); IGB_CORE_UNLOCK(adapter); } if (!(ifp->if_flags & IFF_NOARP)) arp_ifinit(ifp, ifa); } else #endif error = ether_ioctl(ifp, command, data); break; case SIOCSIFMTU: { int max_frame_size; IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); IGB_CORE_LOCK(adapter); max_frame_size = 9234; if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { IGB_CORE_UNLOCK(adapter); error = EINVAL; break; } ifp->if_mtu = ifr->ifr_mtu; adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; igb_init_locked(adapter); IGB_CORE_UNLOCK(adapter); break; } case SIOCSIFFLAGS: IOCTL_DEBUGOUT("ioctl rcv'd:\ SIOCSIFFLAGS (Set Interface Flags)"); IGB_CORE_LOCK(adapter); if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { if ((ifp->if_flags ^ adapter->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { igb_disable_promisc(adapter); igb_set_promisc(adapter); } } else igb_init_locked(adapter); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) igb_stop(adapter); adapter->if_flags = ifp->if_flags; IGB_CORE_UNLOCK(adapter); break; case SIOCADDMULTI: case SIOCDELMULTI: IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { IGB_CORE_LOCK(adapter); igb_disable_intr(adapter); igb_set_multi(adapter); #ifdef DEVICE_POLLING if (!(ifp->if_capenable & IFCAP_POLLING)) #endif igb_enable_intr(adapter); IGB_CORE_UNLOCK(adapter); } break; case SIOCSIFMEDIA: /* Check SOL/IDER usage */ IGB_CORE_LOCK(adapter); if (e1000_check_reset_block(&adapter->hw)) { IGB_CORE_UNLOCK(adapter); device_printf(adapter->dev, "Media change is" " blocked due to SOL/IDER session.\n"); break; } IGB_CORE_UNLOCK(adapter); case SIOCGIFMEDIA: IOCTL_DEBUGOUT("ioctl rcv'd: \ SIOCxIFMEDIA (Get/Set Interface Media)"); error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); break; case SIOCSIFCAP: { int mask, reinit; IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); reinit = 0; mask = ifr->ifr_reqcap ^ ifp->if_capenable; #ifdef DEVICE_POLLING if (mask & IFCAP_POLLING) { if (ifr->ifr_reqcap & IFCAP_POLLING) { error = ether_poll_register(igb_poll, ifp); if (error) return (error); IGB_CORE_LOCK(adapter); igb_disable_intr(adapter); ifp->if_capenable |= IFCAP_POLLING; IGB_CORE_UNLOCK(adapter); } else { error = ether_poll_deregister(ifp); /* Enable interrupt even in error case */ IGB_CORE_LOCK(adapter); igb_enable_intr(adapter); ifp->if_capenable &= ~IFCAP_POLLING; IGB_CORE_UNLOCK(adapter); } } #endif if (mask & IFCAP_HWCSUM) { ifp->if_capenable ^= IFCAP_HWCSUM; reinit = 1; } if (mask & IFCAP_TSO4) { ifp->if_capenable ^= IFCAP_TSO4; reinit = 1; } if (mask & IFCAP_VLAN_HWTAGGING) { ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; reinit = 1; } if (mask & IFCAP_LRO) { ifp->if_capenable ^= IFCAP_LRO; reinit = 1; } if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) igb_init(adapter); VLAN_CAPABILITIES(ifp); break; } default: error = ether_ioctl(ifp, command, data); break; } return (error); } /********************************************************************* * Init entry point * * This routine is used in two ways. It is used by the stack as * init entry point in network interface structure. It is also used * by the driver as a hw/sw initialization routine to get to a * consistent state. * * return 0 on success, positive on failure **********************************************************************/ static void igb_init_locked(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; INIT_DEBUGOUT("igb_init: begin"); IGB_CORE_LOCK_ASSERT(adapter); igb_disable_intr(adapter); callout_stop(&adapter->timer); /* Get the latest mac address, User can use a LAA */ bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); igb_reset(adapter); igb_update_link_status(adapter); E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); /* Set hardware offload abilities */ ifp->if_hwassist = 0; if (ifp->if_capenable & IFCAP_TXCSUM) { ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); #if __FreeBSD_version >= 800000 if (adapter->hw.mac.type == e1000_82576) ifp->if_hwassist |= CSUM_SCTP; #endif } if (ifp->if_capenable & IFCAP_TSO4) ifp->if_hwassist |= CSUM_TSO; /* Configure for OS presence */ igb_init_manageability(adapter); /* Prepare transmit descriptors and buffers */ igb_setup_transmit_structures(adapter); igb_initialize_transmit_units(adapter); /* Setup Multicast table */ igb_set_multi(adapter); /* ** Figure out the desired mbuf pool ** for doing jumbo/packetsplit */ if (ifp->if_mtu > ETHERMTU) adapter->rx_mbuf_sz = MJUMPAGESIZE; else adapter->rx_mbuf_sz = MCLBYTES; /* Prepare receive descriptors and buffers */ if (igb_setup_receive_structures(adapter)) { device_printf(dev, "Could not setup receive structures\n"); return; } igb_initialize_receive_units(adapter); /* Don't lose promiscuous settings */ igb_set_promisc(adapter); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; callout_reset(&adapter->timer, hz, igb_local_timer, adapter); e1000_clear_hw_cntrs_base_generic(&adapter->hw); if (adapter->msix > 1) /* Set up queue routing */ igb_configure_queues(adapter); /* Set up VLAN tag offload and filter */ igb_setup_vlan_hw_support(adapter); /* this clears any pending interrupts */ E1000_READ_REG(&adapter->hw, E1000_ICR); #ifdef DEVICE_POLLING /* * Only enable interrupts if we are not polling, make sure * they are off otherwise. */ if (ifp->if_capenable & IFCAP_POLLING) igb_disable_intr(adapter); else #endif /* DEVICE_POLLING */ { igb_enable_intr(adapter); E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); } /* Don't reset the phy next time init gets called */ adapter->hw.phy.reset_disable = TRUE; } static void igb_init(void *arg) { struct adapter *adapter = arg; IGB_CORE_LOCK(adapter); igb_init_locked(adapter); IGB_CORE_UNLOCK(adapter); } static void igb_handle_rxtx(void *context, int pending) { struct adapter *adapter = context; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; struct ifnet *ifp; ifp = adapter->ifp; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (igb_rxeof(rxr, adapter->rx_process_limit)) taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); IGB_TX_LOCK(txr); igb_txeof(txr); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, txr->br)) igb_mq_start_locked(ifp, txr, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) igb_start_locked(txr, ifp); #endif IGB_TX_UNLOCK(txr); } igb_enable_intr(adapter); } static void igb_handle_que(void *context, int pending) { struct igb_queue *que = context; struct adapter *adapter = que->adapter; struct tx_ring *txr = que->txr; struct rx_ring *rxr = que->rxr; struct ifnet *ifp = adapter->ifp; u32 loop = IGB_MAX_LOOP; bool more; /* RX first */ do { more = igb_rxeof(rxr, -1); } while (loop-- && more); if (IGB_TX_TRYLOCK(txr)) { loop = IGB_MAX_LOOP; do { more = igb_txeof(txr); } while (loop-- && more); #if __FreeBSD_version >= 800000 igb_mq_start_locked(ifp, txr, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) igb_start_locked(txr, ifp); #endif IGB_TX_UNLOCK(txr); } /* Reenable this interrupt */ #ifdef DEVICE_POLLING if (!(ifp->if_capenable & IFCAP_POLLING)) #endif E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); } /* Deal with link in a sleepable context */ static void igb_handle_link(void *context, int pending) { struct adapter *adapter = context; adapter->hw.mac.get_link_status = 1; igb_update_link_status(adapter); } /********************************************************************* * * MSI/Legacy Deferred * Interrupt Service routine * *********************************************************************/ static int igb_irq_fast(void *arg) { struct adapter *adapter = arg; uint32_t reg_icr; reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); /* Hot eject? */ if (reg_icr == 0xffffffff) return FILTER_STRAY; /* Definitely not our interrupt. */ if (reg_icr == 0x0) return FILTER_STRAY; if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) return FILTER_STRAY; /* * Mask interrupts until the taskqueue is finished running. This is * cheap, just assume that it is needed. This also works around the * MSI message reordering errata on certain systems. */ igb_disable_intr(adapter); taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) taskqueue_enqueue(adapter->tq, &adapter->link_task); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; return FILTER_HANDLED; } #ifdef DEVICE_POLLING /********************************************************************* * * Legacy polling routine * *********************************************************************/ #if __FreeBSD_version >= 800000 #define POLL_RETURN_COUNT(a) (a) static int #else #define POLL_RETURN_COUNT(a) static void #endif igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { struct adapter *adapter = ifp->if_softc; struct rx_ring *rxr = adapter->rx_rings; struct tx_ring *txr = adapter->tx_rings; u32 reg_icr, rx_done = 0; u32 loop = IGB_MAX_LOOP; bool more; IGB_CORE_LOCK(adapter); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { IGB_CORE_UNLOCK(adapter); return POLL_RETURN_COUNT(rx_done); } if (cmd == POLL_AND_CHECK_STATUS) { reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) taskqueue_enqueue(adapter->tq, &adapter->link_task); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; } IGB_CORE_UNLOCK(adapter); /* TODO: rx_count */ rx_done = igb_rxeof(rxr, count) ? 1 : 0; IGB_TX_LOCK(txr); do { more = igb_txeof(txr); } while (loop-- && more); #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, txr->br)) igb_mq_start_locked(ifp, txr, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) igb_start_locked(txr, ifp); #endif IGB_TX_UNLOCK(txr); return POLL_RETURN_COUNT(rx_done); } #endif /* DEVICE_POLLING */ /********************************************************************* * * MSIX TX Interrupt Service routine * **********************************************************************/ static void igb_msix_que(void *arg) { struct igb_queue *que = arg; struct adapter *adapter = que->adapter; struct tx_ring *txr = que->txr; struct rx_ring *rxr = que->rxr; u32 newitr = 0; bool more_tx, more_rx; E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); ++que->irqs; IGB_TX_LOCK(txr); more_tx = igb_txeof(txr); IGB_TX_UNLOCK(txr); more_rx = igb_rxeof(rxr, adapter->rx_process_limit); if (igb_enable_aim == FALSE) goto no_calc; /* ** Do Adaptive Interrupt Moderation: ** - Write out last calculated setting ** - Calculate based on average size over ** the last interval. */ if (que->eitr_setting) E1000_WRITE_REG(&adapter->hw, E1000_EITR(que->msix), que->eitr_setting); que->eitr_setting = 0; /* Idle, do nothing */ if ((txr->bytes == 0) && (rxr->bytes == 0)) goto no_calc; /* Used half Default if sub-gig */ if (adapter->link_speed != 1000) newitr = IGB_DEFAULT_ITR / 2; else { if ((txr->bytes) && (txr->packets)) newitr = txr->bytes/txr->packets; if ((rxr->bytes) && (rxr->packets)) newitr = max(newitr, (rxr->bytes / rxr->packets)); newitr += 24; /* account for hardware frame, crc */ /* set an upper boundary */ newitr = min(newitr, 3000); /* Be nice to the mid range */ if ((newitr > 300) && (newitr < 1200)) newitr = (newitr / 3); else newitr = (newitr / 2); } newitr &= 0x7FFC; /* Mask invalid bits */ if (adapter->hw.mac.type == e1000_82575) newitr |= newitr << 16; else newitr |= 0x8000000; /* save for next interrupt */ que->eitr_setting = newitr; /* Reset state */ txr->bytes = 0; txr->packets = 0; rxr->bytes = 0; rxr->packets = 0; no_calc: /* Schedule a clean task if needed*/ if (more_tx || more_rx) taskqueue_enqueue(que->tq, &que->que_task); else /* Reenable this interrupt */ E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); return; } /********************************************************************* * * MSIX Link Interrupt Service routine * **********************************************************************/ static void igb_msix_link(void *arg) { struct adapter *adapter = arg; u32 icr; ++adapter->link_irq; icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (!(icr & E1000_ICR_LSC)) goto spurious; taskqueue_enqueue(adapter->tq, &adapter->link_task); spurious: /* Rearm */ E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); return; } /********************************************************************* * * Media Ioctl callback * * This routine is called whenever the user queries the status of * the interface using ifconfig. * **********************************************************************/ static void igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { struct adapter *adapter = ifp->if_softc; u_char fiber_type = IFM_1000_SX; INIT_DEBUGOUT("igb_media_status: begin"); IGB_CORE_LOCK(adapter); igb_update_link_status(adapter); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { IGB_CORE_UNLOCK(adapter); return; } ifmr->ifm_status |= IFM_ACTIVE; if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) ifmr->ifm_active |= fiber_type | IFM_FDX; else { switch (adapter->link_speed) { case 10: ifmr->ifm_active |= IFM_10_T; break; case 100: ifmr->ifm_active |= IFM_100_TX; break; case 1000: ifmr->ifm_active |= IFM_1000_T; break; } if (adapter->link_duplex == FULL_DUPLEX) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; } IGB_CORE_UNLOCK(adapter); } /********************************************************************* * * Media Ioctl callback * * This routine is called when the user changes speed/duplex using * media/mediopt option with ifconfig. * **********************************************************************/ static int igb_media_change(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct ifmedia *ifm = &adapter->media; INIT_DEBUGOUT("igb_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); IGB_CORE_LOCK(adapter); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; break; case IFM_1000_LX: case IFM_1000_SX: case IFM_1000_T: adapter->hw.mac.autoneg = DO_AUTO_NEG; adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; break; case IFM_100_TX: adapter->hw.mac.autoneg = FALSE; adapter->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; else adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; break; case IFM_10_T: adapter->hw.mac.autoneg = FALSE; adapter->hw.phy.autoneg_advertised = 0; if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; else adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; break; default: device_printf(adapter->dev, "Unsupported media type\n"); } /* As the speed/duplex settings my have changed we need to * reset the PHY. */ adapter->hw.phy.reset_disable = FALSE; igb_init_locked(adapter); IGB_CORE_UNLOCK(adapter); return (0); } /********************************************************************* * * This routine maps the mbufs to Advanced TX descriptors. * used by the 82575 adapter. * **********************************************************************/ static int igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) { struct adapter *adapter = txr->adapter; bus_dma_segment_t segs[IGB_MAX_SCATTER]; bus_dmamap_t map; struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped; union e1000_adv_tx_desc *txd = NULL; struct mbuf *m_head; u32 olinfo_status = 0, cmd_type_len = 0; int nsegs, i, j, error, first, last = 0; u32 hdrlen = 0; m_head = *m_headp; /* Set basic descriptor constants */ cmd_type_len |= E1000_ADVTXD_DTYP_DATA; cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; if (m_head->m_flags & M_VLANTAG) cmd_type_len |= E1000_ADVTXD_DCMD_VLE; /* * Force a cleanup if number of TX descriptors * available hits the threshold */ if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { igb_txeof(txr); /* Now do we at least have a minimal? */ if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { txr->no_desc_avail++; return (ENOBUFS); } } /* * Map the packet for DMA. * * Capture the first descriptor index, * this descriptor will have the index * of the EOP which is the only one that * now gets a DONE bit writeback. */ first = txr->next_avail_desc; tx_buffer = &txr->tx_buffers[first]; tx_buffer_mapped = tx_buffer; map = tx_buffer->map; error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == EFBIG) { struct mbuf *m; m = m_defrag(*m_headp, M_DONTWAIT); if (m == NULL) { adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); } *m_headp = m; /* Try it again */ error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == ENOMEM) { adapter->no_tx_dma_setup++; return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } } else if (error == ENOMEM) { adapter->no_tx_dma_setup++; return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } /* Check again to be sure we have enough descriptors */ if (nsegs > (txr->tx_avail - 2)) { txr->no_desc_avail++; bus_dmamap_unload(txr->txtag, map); return (ENOBUFS); } m_head = *m_headp; /* * Set up the context descriptor: * used when any hardware offload is done. * This includes CSUM, VLAN, and TSO. It * will use the first descriptor. */ if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { if (igb_tso_setup(txr, m_head, &hdrlen)) { cmd_type_len |= E1000_ADVTXD_DCMD_TSE; olinfo_status |= E1000_TXD_POPTS_IXSM << 8; olinfo_status |= E1000_TXD_POPTS_TXSM << 8; } else return (ENXIO); } else if (igb_tx_ctx_setup(txr, m_head)) olinfo_status |= E1000_TXD_POPTS_TXSM << 8; /* Calculate payload length */ olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) << E1000_ADVTXD_PAYLEN_SHIFT); /* 82575 needs the queue index added */ if (adapter->hw.mac.type == e1000_82575) olinfo_status |= txr->me << 4; /* Set up our transmit descriptors */ i = txr->next_avail_desc; for (j = 0; j < nsegs; j++) { bus_size_t seg_len; bus_addr_t seg_addr; tx_buffer = &txr->tx_buffers[i]; txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; seg_addr = segs[j].ds_addr; seg_len = segs[j].ds_len; txd->read.buffer_addr = htole64(seg_addr); txd->read.cmd_type_len = htole32(cmd_type_len | seg_len); txd->read.olinfo_status = htole32(olinfo_status); last = i; if (++i == adapter->num_tx_desc) i = 0; tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; } txr->next_avail_desc = i; txr->tx_avail -= nsegs; tx_buffer->m_head = m_head; tx_buffer_mapped->map = tx_buffer->map; tx_buffer->map = map; bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); /* * Last Descriptor of Packet * needs End Of Packet (EOP) * and Report Status (RS) */ txd->read.cmd_type_len |= htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS); /* * Keep track in the first buffer which * descriptor will be written back */ tx_buffer = &txr->tx_buffers[first]; tx_buffer->next_eop = last; txr->watchdog_time = ticks; /* * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 * that this frame is available to transmit. */ bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); ++txr->tx_packets; return (0); } static void igb_set_promisc(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; uint32_t reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); if (ifp->if_flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } else if (ifp->if_flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } } static void igb_disable_promisc(struct adapter *adapter) { uint32_t reg_rctl; reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl &= (~E1000_RCTL_UPE); reg_rctl &= (~E1000_RCTL_MPE); E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } /********************************************************************* * Multicast Update * * This routine is called whenever multicast address list is updated. * **********************************************************************/ static void igb_set_multi(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; struct ifmultiaddr *ifma; u32 reg_rctl = 0; u8 mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN]; int mcnt = 0; IOCTL_DEBUGOUT("igb_set_multi: begin"); #if __FreeBSD_version < 800000 IF_ADDR_LOCK(ifp); #else if_maddr_rlock(ifp); #endif TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) break; bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); mcnt++; } #if __FreeBSD_version < 800000 IF_ADDR_UNLOCK(ifp); #else if_maddr_runlock(ifp); #endif if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); reg_rctl |= E1000_RCTL_MPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } else e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); } /********************************************************************* * Timer routine: * This routine checks for link status, * updates statistics, and does the watchdog. * **********************************************************************/ static void igb_local_timer(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; IGB_CORE_LOCK_ASSERT(adapter); igb_update_link_status(adapter); igb_update_stats_counters(adapter); if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) igb_print_hw_stats(adapter); /* ** Watchdog: check for time since any descriptor was cleaned */ for (int i = 0; i < adapter->num_queues; i++, txr++) { if (txr->watchdog_check == FALSE) continue; if ((ticks - txr->watchdog_time) > IGB_WATCHDOG) goto timeout; } /* Trigger an RX interrupt on all queues */ #ifdef DEVICE_POLLING if (!(ifp->if_capenable & IFCAP_POLLING)) #endif E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask); callout_reset(&adapter->timer, hz, igb_local_timer, adapter); return; timeout: device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); device_printf(dev,"TX(%d) desc avail = %d," "Next TX to Clean = %d\n", txr->me, txr->tx_avail, txr->next_to_clean); adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; adapter->watchdog_events++; igb_init_locked(adapter); } static void igb_update_link_status(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; u32 link_check = 0; /* Get the cached link value or read for real */ switch (hw->phy.media_type) { case e1000_media_type_copper: if (hw->mac.get_link_status) { /* Do the work to read phy */ e1000_check_for_link(hw); link_check = !hw->mac.get_link_status; } else link_check = TRUE; break; case e1000_media_type_fiber: e1000_check_for_link(hw); link_check = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); break; case e1000_media_type_internal_serdes: e1000_check_for_link(hw); link_check = adapter->hw.mac.serdes_has_link; break; default: case e1000_media_type_unknown: break; } /* Now we check if a transition has happened */ if (link_check && (adapter->link_active == 0)) { e1000_get_speed_and_duplex(&adapter->hw, &adapter->link_speed, &adapter->link_duplex); if (bootverbose) device_printf(dev, "Link is up %d Mbps %s\n", adapter->link_speed, ((adapter->link_duplex == FULL_DUPLEX) ? "Full Duplex" : "Half Duplex")); adapter->link_active = 1; ifp->if_baudrate = adapter->link_speed * 1000000; if_link_state_change(ifp, LINK_STATE_UP); } else if (!link_check && (adapter->link_active == 1)) { ifp->if_baudrate = adapter->link_speed = 0; adapter->link_duplex = 0; if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; if_link_state_change(ifp, LINK_STATE_DOWN); /* Turn off watchdogs */ for (int i = 0; i < adapter->num_queues; i++, txr++) txr->watchdog_check = FALSE; } } /********************************************************************* * * This routine disables all traffic on the adapter by issuing a * global reset on the MAC and deallocates TX/RX buffers. * **********************************************************************/ static void igb_stop(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; IGB_CORE_LOCK_ASSERT(adapter); INIT_DEBUGOUT("igb_stop: begin"); igb_disable_intr(adapter); callout_stop(&adapter->timer); /* Tell the stack that the interface is no longer active */ ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); /* Unarm watchdog timer. */ for (int i = 0; i < adapter->num_queues; i++, txr++) { IGB_TX_LOCK(txr); txr->watchdog_check = FALSE; IGB_TX_UNLOCK(txr); } e1000_reset_hw(&adapter->hw); E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); } /********************************************************************* * * Determine hardware revision. * **********************************************************************/ static void igb_identify_hardware(struct adapter *adapter) { device_t dev = adapter->dev; /* Make sure our PCI config space has the necessary stuff set */ adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { device_printf(dev, "Memory Access and/or Bus Master bits " "were not set!\n"); adapter->hw.bus.pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); pci_write_config(dev, PCIR_COMMAND, adapter->hw.bus.pci_cmd_word, 2); } /* Save off the information about this board */ adapter->hw.vendor_id = pci_get_vendor(dev); adapter->hw.device_id = pci_get_device(dev); adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); adapter->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); adapter->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); /* Do Shared Code Init and Setup */ if (e1000_set_mac_type(&adapter->hw)) { device_printf(dev, "Setup init failure\n"); return; } } static int igb_allocate_pci_resources(struct adapter *adapter) { device_t dev = adapter->dev; int rid; rid = PCIR_BAR(0); adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (adapter->pci_mem == NULL) { device_printf(dev, "Unable to allocate bus resource: memory\n"); return (ENXIO); } adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem); adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->pci_mem); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; adapter->num_queues = 1; /* Defaults for Legacy or MSI */ /* This will setup either MSI/X or MSI */ adapter->msix = igb_setup_msix(adapter); adapter->hw.back = &adapter->osdep; return (0); } /********************************************************************* * * Setup the Legacy or MSI Interrupt handler * **********************************************************************/ static int igb_allocate_legacy(struct adapter *adapter) { device_t dev = adapter->dev; int error, rid = 0; /* Turn off all interrupts */ E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); /* MSI RID is 1 */ if (adapter->msix == 1) rid = 1; /* We allocate a single interrupt resource */ adapter->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (adapter->res == NULL) { device_printf(dev, "Unable to allocate bus resource: " "interrupt\n"); return (ENXIO); } /* * Try allocating a fast interrupt and the associated deferred * processing contexts. */ TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter); /* Make tasklet for deferred link handling */ TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", device_get_nameunit(adapter->dev)); if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, adapter, &adapter->tag)) != 0) { device_printf(dev, "Failed to register fast interrupt " "handler: %d\n", error); taskqueue_free(adapter->tq); adapter->tq = NULL; return (error); } return (0); } /********************************************************************* * * Setup the MSIX Queue Interrupt handlers: * **********************************************************************/ static int igb_allocate_msix(struct adapter *adapter) { device_t dev = adapter->dev; struct igb_queue *que = adapter->queues; int error, rid, vector = 0; for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { rid = vector +1; que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (que->res == NULL) { device_printf(dev, "Unable to allocate bus resource: " "MSIX Queue Interrupt\n"); return (ENXIO); } error = bus_setup_intr(dev, que->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_que, que, &que->tag); if (error) { que->res = NULL; device_printf(dev, "Failed to register Queue handler"); return (error); } que->msix = vector; if (adapter->hw.mac.type == e1000_82575) que->eims = E1000_EICR_TX_QUEUE0 << i; else que->eims = 1 << vector; /* ** Bind the msix vector, and thus the ** rings to the corresponding cpu. */ if (adapter->num_queues > 1) bus_bind_intr(dev, que->res, i); /* Make tasklet for deferred handling */ TASK_INIT(&que->que_task, 0, igb_handle_que, que); que->tq = taskqueue_create_fast("igb_que", M_NOWAIT, taskqueue_thread_enqueue, &que->tq); taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", device_get_nameunit(adapter->dev)); } /* And Link */ rid = vector + 1; adapter->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (adapter->res == NULL) { device_printf(dev, "Unable to allocate bus resource: " "MSIX Link Interrupt\n"); return (ENXIO); } if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_link, adapter, &adapter->tag)) != 0) { device_printf(dev, "Failed to register Link handler"); return (error); } adapter->linkvec = vector; /* Make tasklet for deferred handling */ TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); adapter->tq = taskqueue_create_fast("igb_link", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s link", device_get_nameunit(adapter->dev)); return (0); } static void igb_configure_queues(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; struct igb_queue *que; u32 tmp, ivar = 0; u32 newitr = IGB_DEFAULT_ITR; /* First turn on RSS capability */ if (adapter->hw.mac.type > e1000_82575) E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | E1000_GPIE_PBA | E1000_GPIE_NSICR); /* Turn on MSIX */ switch (adapter->hw.mac.type) { case e1000_82580: /* RX entries */ for (int i = 0; i < adapter->num_queues; i++) { u32 index = i >> 1; ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); que = &adapter->queues[i]; if (i & 1) { ivar &= 0xFF00FFFF; ivar |= (que->msix | E1000_IVAR_VALID) << 16; } else { ivar &= 0xFFFFFF00; ivar |= que->msix | E1000_IVAR_VALID; } E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); } /* TX entries */ for (int i = 0; i < adapter->num_queues; i++) { u32 index = i >> 1; ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); que = &adapter->queues[i]; if (i & 1) { ivar &= 0x00FFFFFF; ivar |= (que->msix | E1000_IVAR_VALID) << 24; } else { ivar &= 0xFFFF00FF; ivar |= (que->msix | E1000_IVAR_VALID) << 8; } E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); adapter->eims_mask |= que->eims; } /* And for the link interrupt */ ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; adapter->link_mask = 1 << adapter->linkvec; adapter->eims_mask |= adapter->link_mask; E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); break; case e1000_82576: /* RX entries */ for (int i = 0; i < adapter->num_queues; i++) { u32 index = i & 0x7; /* Each IVAR has two entries */ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); que = &adapter->queues[i]; if (i < 8) { ivar &= 0xFFFFFF00; ivar |= que->msix | E1000_IVAR_VALID; } else { ivar &= 0xFF00FFFF; ivar |= (que->msix | E1000_IVAR_VALID) << 16; } E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); adapter->eims_mask |= que->eims; } /* TX entries */ for (int i = 0; i < adapter->num_queues; i++) { u32 index = i & 0x7; /* Each IVAR has two entries */ ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); que = &adapter->queues[i]; if (i < 8) { ivar &= 0xFFFF00FF; ivar |= (que->msix | E1000_IVAR_VALID) << 8; } else { ivar &= 0x00FFFFFF; ivar |= (que->msix | E1000_IVAR_VALID) << 24; } E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); adapter->eims_mask |= que->eims; } /* And for the link interrupt */ ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; adapter->link_mask = 1 << adapter->linkvec; adapter->eims_mask |= adapter->link_mask; E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); break; case e1000_82575: /* enable MSI-X support*/ tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); tmp |= E1000_CTRL_EXT_PBA_CLR; /* Auto-Mask interrupts upon ICR read. */ tmp |= E1000_CTRL_EXT_EIAME; tmp |= E1000_CTRL_EXT_IRCA; E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); /* Queues */ for (int i = 0; i < adapter->num_queues; i++) { que = &adapter->queues[i]; tmp = E1000_EICR_RX_QUEUE0 << i; tmp |= E1000_EICR_TX_QUEUE0 << i; que->eims = tmp; E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), i, que->eims); adapter->eims_mask |= que->eims; } /* Link */ E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), E1000_EIMS_OTHER); adapter->link_mask |= E1000_EIMS_OTHER; adapter->eims_mask |= adapter->link_mask; default: break; } /* Set the starting interrupt rate */ if (hw->mac.type == e1000_82575) newitr |= newitr << 16; else newitr |= 0x8000000; for (int i = 0; i < adapter->num_queues; i++) { que = &adapter->queues[i]; E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); } return; } static void igb_free_pci_resources(struct adapter *adapter) { struct igb_queue *que = adapter->queues; device_t dev = adapter->dev; int rid; /* ** There is a slight possibility of a failure mode ** in attach that will result in entering this function ** before interrupt resources have been initialized, and ** in that case we do not want to execute the loops below ** We can detect this reliably by the state of the adapter ** res pointer. */ if (adapter->res == NULL) goto mem; /* * First release all the interrupt resources: */ for (int i = 0; i < adapter->num_queues; i++, que++) { rid = que->msix + 1; if (que->tag != NULL) { bus_teardown_intr(dev, que->res, que->tag); que->tag = NULL; } if (que->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); } /* Clean the Legacy or Link interrupt last */ if (adapter->linkvec) /* we are doing MSIX */ rid = adapter->linkvec + 1; else (adapter->msix != 0) ? (rid = 1):(rid = 0); if (adapter->tag != NULL) { bus_teardown_intr(dev, adapter->res, adapter->tag); adapter->tag = NULL; } if (adapter->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); mem: if (adapter->msix) pci_release_msi(dev); if (adapter->msix_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), adapter->pci_mem); } /* * Setup Either MSI/X or MSI */ static int igb_setup_msix(struct adapter *adapter) { device_t dev = adapter->dev; int rid, want, queues, msgs; /* tuneable override */ if (igb_enable_msix == 0) goto msi; /* First try MSI/X */ rid = PCIR_BAR(IGB_MSIX_BAR); adapter->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!adapter->msix_mem) { /* May not be enabled */ device_printf(adapter->dev, "Unable to map MSIX table \n"); goto msi; } msgs = pci_msix_count(dev); if (msgs == 0) { /* system has msix disabled */ bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); adapter->msix_mem = NULL; goto msi; } /* Figure out a reasonable auto config value */ queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; /* Can have max of 4 queues on 82575 */ if (adapter->hw.mac.type == e1000_82575) { if (queues > 4) queues = 4; if (igb_num_queues > 4) igb_num_queues = 4; } if (igb_num_queues == 0) igb_num_queues = queues; /* ** One vector (RX/TX pair) per queue ** plus an additional for Link interrupt */ want = igb_num_queues + 1; if (msgs >= want) msgs = want; else { device_printf(adapter->dev, "MSIX Configuration Problem, " "%d vectors configured, but %d queues wanted!\n", msgs, want); return (ENXIO); } if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { device_printf(adapter->dev, "Using MSIX interrupts with %d vectors\n", msgs); adapter->num_queues = igb_num_queues; return (msgs); } msi: msgs = pci_msi_count(dev); if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) device_printf(adapter->dev,"Using MSI interrupt\n"); return (msgs); } /********************************************************************* * * Set up an fresh starting state * **********************************************************************/ static void igb_reset(struct adapter *adapter) { device_t dev = adapter->dev; struct e1000_hw *hw = &adapter->hw; struct e1000_fc_info *fc = &hw->fc; struct ifnet *ifp = adapter->ifp; u32 pba = 0; u16 hwm; INIT_DEBUGOUT("igb_reset: begin"); /* Let the firmware know the OS is in control */ igb_get_hw_control(adapter); /* * Packet Buffer Allocation (PBA) * Writing PBA sets the receive portion of the buffer * the remainder is used for the transmit buffer. */ switch (hw->mac.type) { case e1000_82575: pba = E1000_PBA_32K; break; case e1000_82576: pba = E1000_PBA_64K; break; case e1000_82580: pba = E1000_PBA_35K; default: break; } /* Special needs in case of Jumbo frames */ if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { u32 tx_space, min_tx, min_rx; pba = E1000_READ_REG(hw, E1000_PBA); tx_space = pba >> 16; pba &= 0xffff; min_tx = (adapter->max_frame_size + sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; min_tx = roundup2(min_tx, 1024); min_tx >>= 10; min_rx = adapter->max_frame_size; min_rx = roundup2(min_rx, 1024); min_rx >>= 10; if (tx_space < min_tx && ((min_tx - tx_space) < pba)) { pba = pba - (min_tx - tx_space); /* * if short on rx space, rx wins * and must trump tx adjustment */ if (pba < min_rx) pba = min_rx; } E1000_WRITE_REG(hw, E1000_PBA, pba); } INIT_DEBUGOUT1("igb_init: pba=%dK",pba); /* * These parameters control the automatic generation (Tx) and * response (Rx) to Ethernet PAUSE frames. * - High water mark should allow for at least two frames to be * received after sending an XOFF. * - Low water mark works best when it is very near the high water mark. * This allows the receiver to restart by sending XON when it has * drained a bit. */ hwm = min(((pba << 10) * 9 / 10), ((pba << 10) - 2 * adapter->max_frame_size)); if (hw->mac.type < e1000_82576) { fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ fc->low_water = fc->high_water - 8; } else { fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ fc->low_water = fc->high_water - 16; } fc->pause_time = IGB_FC_PAUSE_TIME; fc->send_xon = TRUE; /* Set Flow control, use the tunable location if sane */ if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) fc->requested_mode = igb_fc_setting; else fc->requested_mode = e1000_fc_none; fc->current_mode = fc->requested_mode; /* Issue a global reset */ e1000_reset_hw(hw); E1000_WRITE_REG(hw, E1000_WUC, 0); if (e1000_init_hw(hw) < 0) device_printf(dev, "Hardware Initialization Failed\n"); if (hw->mac.type == e1000_82580) { u32 reg; hwm = (pba << 10) - (2 * adapter->max_frame_size); /* * 0x80000000 - enable DMA COAL * 0x10000000 - use L0s as low power * 0x20000000 - use L1 as low power * X << 16 - exit dma coal when rx data exceeds X kB * Y - upper limit to stay in dma coal in units of 32usecs */ E1000_WRITE_REG(hw, E1000_DMACR, 0xA0000006 | ((hwm << 6) & 0x00FF0000)); /* set hwm to PBA - 2 * max frame size */ E1000_WRITE_REG(hw, E1000_FCRTC, hwm); /* * This sets the time to wait before requesting transition to * low power state to number of usecs needed to receive 1 512 * byte frame at gigabit line rate */ E1000_WRITE_REG(hw, E1000_DMCTLX, 4); /* free space in tx packet buffer to wake from DMA coal */ E1000_WRITE_REG(hw, E1000_DMCTXTH, (20480 - (2 * adapter->max_frame_size)) >> 6); /* make low power state decision controlled by DMA coal */ reg = E1000_READ_REG(hw, E1000_PCIEMISC); E1000_WRITE_REG(hw, E1000_PCIEMISC, reg | E1000_PCIEMISC_LX_DECISION); } E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); e1000_get_phy_info(hw); e1000_check_for_link(hw); return; } /********************************************************************* * * Setup networking device structure and register an interface. * **********************************************************************/ static void igb_setup_interface(device_t dev, struct adapter *adapter) { struct ifnet *ifp; INIT_DEBUGOUT("igb_setup_interface: begin"); ifp = adapter->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) panic("%s: can not if_alloc()", device_get_nameunit(dev)); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_mtu = ETHERMTU; ifp->if_init = igb_init; ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = igb_ioctl; ifp->if_start = igb_start; #if __FreeBSD_version >= 800000 ifp->if_transmit = igb_mq_start; ifp->if_qflush = igb_qflush; #endif IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; IFQ_SET_READY(&ifp->if_snd); ether_ifattach(ifp, adapter->hw.mac.addr); ifp->if_capabilities = ifp->if_capenable = 0; ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU; ifp->if_capabilities |= IFCAP_TSO4; ifp->if_capabilities |= IFCAP_JUMBO_MTU; if (igb_header_split) ifp->if_capabilities |= IFCAP_LRO; ifp->if_capenable = ifp->if_capabilities; #ifdef DEVICE_POLLING ifp->if_capabilities |= IFCAP_POLLING; #endif /* * Tell the upper layer(s) we support long frames. */ ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; /* * Specify the media types supported by this adapter and register * callbacks to update media and link information */ ifmedia_init(&adapter->media, IFM_IMASK, igb_media_change, igb_media_status); if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); } else { ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); if (adapter->hw.phy.type != e1000_phy_ife) { ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } } ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); } /* * Manage DMA'able memory. */ static void igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { if (error) return; *(bus_addr_t *) arg = segs[0].ds_addr; } static int igb_dma_malloc(struct adapter *adapter, bus_size_t size, struct igb_dma_alloc *dma, int mapflags) { int error; error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ IGB_DBA_ALIGN, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ 1, /* nsegments */ size, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &dma->dma_tag); if (error) { device_printf(adapter->dev, "%s: bus_dma_tag_create failed: %d\n", __func__, error); goto fail_0; } error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, BUS_DMA_NOWAIT, &dma->dma_map); if (error) { device_printf(adapter->dev, "%s: bus_dmamem_alloc(%ju) failed: %d\n", __func__, (uintmax_t)size, error); goto fail_2; } dma->dma_paddr = 0; error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); if (error || dma->dma_paddr == 0) { device_printf(adapter->dev, "%s: bus_dmamap_load failed: %d\n", __func__, error); goto fail_3; } return (0); fail_3: bus_dmamap_unload(dma->dma_tag, dma->dma_map); fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); fail_0: dma->dma_map = NULL; dma->dma_tag = NULL; return (error); } static void igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) { if (dma->dma_tag == NULL) return; if (dma->dma_map != NULL) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); dma->dma_map = NULL; } bus_dma_tag_destroy(dma->dma_tag); dma->dma_tag = NULL; } /********************************************************************* * * Allocate memory for the transmit and receive rings, and then * the descriptors associated with each, called only once at attach. * **********************************************************************/ static int igb_allocate_queues(struct adapter *adapter) { device_t dev = adapter->dev; struct igb_queue *que = NULL; struct tx_ring *txr = NULL; struct rx_ring *rxr = NULL; int rsize, tsize, error = E1000_SUCCESS; int txconf = 0, rxconf = 0; /* First allocate the top level queue structs */ if (!(adapter->queues = (struct igb_queue *) malloc(sizeof(struct igb_queue) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate queue memory\n"); error = ENOMEM; goto fail; } /* Next allocate the TX ring struct memory */ if (!(adapter->tx_rings = (struct tx_ring *) malloc(sizeof(struct tx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate TX ring memory\n"); error = ENOMEM; goto tx_fail; } /* Now allocate the RX */ if (!(adapter->rx_rings = (struct rx_ring *) malloc(sizeof(struct rx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate RX ring memory\n"); error = ENOMEM; goto rx_fail; } tsize = roundup2(adapter->num_tx_desc * sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); /* * Now set up the TX queues, txconf is needed to handle the * possibility that things fail midcourse and we need to * undo memory gracefully */ for (int i = 0; i < adapter->num_queues; i++, txconf++) { /* Set up some basics */ txr = &adapter->tx_rings[i]; txr->adapter = adapter; txr->me = i; /* Initialize the TX lock */ snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", device_get_nameunit(dev), txr->me); mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); if (igb_dma_malloc(adapter, tsize, &txr->txdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate TX Descriptor memory\n"); error = ENOMEM; goto err_tx_desc; } txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; bzero((void *)txr->tx_base, tsize); /* Now allocate transmit buffers for the ring */ if (igb_allocate_transmit_buffers(txr)) { device_printf(dev, "Critical Failure setting up transmit buffers\n"); error = ENOMEM; goto err_tx_desc; } #if __FreeBSD_version >= 800000 /* Allocate a buf ring */ txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF, M_WAITOK, &txr->tx_mtx); #endif } /* * Next the RX queues... */ rsize = roundup2(adapter->num_rx_desc * sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); for (int i = 0; i < adapter->num_queues; i++, rxconf++) { rxr = &adapter->rx_rings[i]; rxr->adapter = adapter; rxr->me = i; /* Initialize the RX lock */ snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", device_get_nameunit(dev), txr->me); mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); if (igb_dma_malloc(adapter, rsize, &rxr->rxdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate RxDescriptor memory\n"); error = ENOMEM; goto err_rx_desc; } rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; bzero((void *)rxr->rx_base, rsize); /* Allocate receive buffers for the ring*/ if (igb_allocate_receive_buffers(rxr)) { device_printf(dev, "Critical Failure setting up receive buffers\n"); error = ENOMEM; goto err_rx_desc; } } /* ** Finally set up the queue holding structs */ for (int i = 0; i < adapter->num_queues; i++) { que = &adapter->queues[i]; que->adapter = adapter; que->txr = &adapter->tx_rings[i]; que->rxr = &adapter->rx_rings[i]; } return (0); err_rx_desc: for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) igb_dma_free(adapter, &rxr->rxdma); err_tx_desc: for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) igb_dma_free(adapter, &txr->txdma); free(adapter->rx_rings, M_DEVBUF); rx_fail: #if __FreeBSD_version >= 800000 buf_ring_free(txr->br, M_DEVBUF); #endif free(adapter->tx_rings, M_DEVBUF); tx_fail: free(adapter->queues, M_DEVBUF); fail: return (error); } /********************************************************************* * * Allocate memory for tx_buffer structures. The tx_buffer stores all * the information needed to transmit a packet on the wire. This is * called only once at attach, setup is done every reset. * **********************************************************************/ static int igb_allocate_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; device_t dev = adapter->dev; struct igb_tx_buffer *txbuf; int error, i; /* * Setup DMA descriptor areas. */ if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ IGB_TSO_SIZE, /* maxsize */ IGB_MAX_SCATTER, /* nsegments */ PAGE_SIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &txr->txtag))) { device_printf(dev,"Unable to allocate TX DMA tag\n"); goto fail; } if (!(txr->tx_buffers = (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) * adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate tx_buffer memory\n"); error = ENOMEM; goto fail; } /* Create the descriptor buffer dma maps */ txbuf = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); if (error != 0) { device_printf(dev, "Unable to create TX DMA map\n"); goto fail; } } return 0; fail: /* We free all, it handles case where we are in the middle */ igb_free_transmit_structures(adapter); return (error); } /********************************************************************* * * Initialize a transmit ring. * **********************************************************************/ static void igb_setup_transmit_ring(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct igb_tx_buffer *txbuf; int i; /* Clear the old descriptor contents */ IGB_TX_LOCK(txr); bzero((void *)txr->tx_base, (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); /* Reset indices */ txr->next_avail_desc = 0; txr->next_to_clean = 0; /* Free any existing tx buffers. */ txbuf = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { if (txbuf->m_head != NULL) { bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, txbuf->map); m_freem(txbuf->m_head); txbuf->m_head = NULL; } /* clear the watch index */ txbuf->next_eop = -1; } /* Set number of descriptors available */ txr->tx_avail = adapter->num_tx_desc; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); IGB_TX_UNLOCK(txr); } /********************************************************************* * * Initialize all transmit rings. * **********************************************************************/ static void igb_setup_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) igb_setup_transmit_ring(txr); return; } /********************************************************************* * * Enable transmit unit. * **********************************************************************/ static void igb_initialize_transmit_units(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; struct e1000_hw *hw = &adapter->hw; u32 tctl, txdctl; INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); /* Setup the Tx Descriptor Rings */ for (int i = 0; i < adapter->num_queues; i++, txr++) { u64 bus_addr = txr->txdma.dma_paddr; E1000_WRITE_REG(hw, E1000_TDLEN(i), adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); E1000_WRITE_REG(hw, E1000_TDBAH(i), (uint32_t)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_TDBAL(i), (uint32_t)bus_addr); /* Setup the HW Tx Head and Tail descriptor pointers */ E1000_WRITE_REG(hw, E1000_TDT(i), 0); E1000_WRITE_REG(hw, E1000_TDH(i), 0); HW_DEBUGOUT2("Base = %x, Length = %x\n", E1000_READ_REG(hw, E1000_TDBAL(i)), E1000_READ_REG(hw, E1000_TDLEN(i))); txr->watchdog_check = FALSE; txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); txdctl |= IGB_TX_PTHRESH; txdctl |= IGB_TX_HTHRESH << 8; txdctl |= IGB_TX_WTHRESH << 16; txdctl |= E1000_TXDCTL_QUEUE_ENABLE; E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); } /* Program the Transmit Control Register */ tctl = E1000_READ_REG(hw, E1000_TCTL); tctl &= ~E1000_TCTL_CT; tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); e1000_config_collision_dist(hw); /* This write will effectively turn on the transmit unit. */ E1000_WRITE_REG(hw, E1000_TCTL, tctl); } /********************************************************************* * * Free all transmit rings. * **********************************************************************/ static void igb_free_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) { IGB_TX_LOCK(txr); igb_free_transmit_buffers(txr); igb_dma_free(adapter, &txr->txdma); IGB_TX_UNLOCK(txr); IGB_TX_LOCK_DESTROY(txr); } free(adapter->tx_rings, M_DEVBUF); } /********************************************************************* * * Free transmit ring related data structures. * **********************************************************************/ static void igb_free_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct igb_tx_buffer *tx_buffer; int i; INIT_DEBUGOUT("free_transmit_ring: begin"); if (txr->tx_buffers == NULL) return; tx_buffer = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { if (tx_buffer->m_head != NULL) { bus_dmamap_sync(txr->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; if (tx_buffer->map != NULL) { bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } else if (tx_buffer->map != NULL) { bus_dmamap_unload(txr->txtag, tx_buffer->map); bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } #if __FreeBSD_version >= 800000 if (txr->br != NULL) buf_ring_free(txr->br, M_DEVBUF); #endif if (txr->tx_buffers != NULL) { free(txr->tx_buffers, M_DEVBUF); txr->tx_buffers = NULL; } if (txr->txtag != NULL) { bus_dma_tag_destroy(txr->txtag); txr->txtag = NULL; } return; } /********************************************************************** * * Setup work for hardware segmentation offload (TSO) * **********************************************************************/ static boolean_t igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) { struct adapter *adapter = txr->adapter; struct e1000_adv_tx_context_desc *TXD; struct igb_tx_buffer *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; u32 mss_l4len_idx = 0; u16 vtag = 0; int ctxd, ehdrlen, ip_hlen, tcp_hlen; struct ether_vlan_header *eh; struct ip *ip; struct tcphdr *th; /* * Determine where frame payload starts. * Jump over vlan headers if already present */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; else ehdrlen = ETHER_HDR_LEN; /* Ensure we have at least the IP+TCP header in the first mbuf. */ if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) return FALSE; /* Only supports IPV4 for now */ ctxd = txr->next_avail_desc; tx_buffer = &txr->tx_buffers[ctxd]; TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; ip = (struct ip *)(mp->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) return FALSE; /* 0 */ ip->ip_sum = 0; ip_hlen = ip->ip_hl << 2; th = (struct tcphdr *)((caddr_t)ip + ip_hlen); th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); tcp_hlen = th->th_off << 2; /* * Calculate header length, this is used * in the transmit desc in igb_xmit */ *hdrlen = ehdrlen + ip_hlen + tcp_hlen; /* VLAN MACLEN IPLEN */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); } vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); vlan_macip_lens |= ip_hlen; TXD->vlan_macip_lens |= htole32(vlan_macip_lens); /* ADV DTYPE TUCMD */ type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); /* MSS L4LEN IDX */ mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); /* 82575 needs the queue index added */ if (adapter->hw.mac.type == e1000_82575) mss_l4len_idx |= txr->me << 4; TXD->mss_l4len_idx = htole32(mss_l4len_idx); TXD->seqnum_seed = htole32(0); tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; if (++ctxd == adapter->num_tx_desc) ctxd = 0; txr->tx_avail--; txr->next_avail_desc = ctxd; return TRUE; } /********************************************************************* * * Context Descriptor setup for VLAN or CSUM * **********************************************************************/ static bool igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) { struct adapter *adapter = txr->adapter; struct e1000_adv_tx_context_desc *TXD; struct igb_tx_buffer *tx_buffer; u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx; struct ether_vlan_header *eh; struct ip *ip = NULL; struct ip6_hdr *ip6; int ehdrlen, ctxd, ip_hlen = 0; u16 etype, vtag = 0; u8 ipproto = 0; bool offload = TRUE; if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) offload = FALSE; vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0; ctxd = txr->next_avail_desc; tx_buffer = &txr->tx_buffers[ctxd]; TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; /* ** In advanced descriptors the vlan tag must ** be placed into the context descriptor, thus ** we need to be here just for that setup. */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); } else if (offload == FALSE) return FALSE; /* * Determine where frame payload starts. * Jump over vlan headers if already present, * helpful for QinQ too. */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } /* Set the ether header length */ vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; switch (etype) { case ETHERTYPE_IP: ip = (struct ip *)(mp->m_data + ehdrlen); ip_hlen = ip->ip_hl << 2; if (mp->m_len < ehdrlen + ip_hlen) { offload = FALSE; break; } ipproto = ip->ip_p; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; break; case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); ip_hlen = sizeof(struct ip6_hdr); if (mp->m_len < ehdrlen + ip_hlen) return (FALSE); ipproto = ip6->ip6_nxt; type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; break; default: offload = FALSE; break; } vlan_macip_lens |= ip_hlen; type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; switch (ipproto) { case IPPROTO_TCP: if (mp->m_pkthdr.csum_flags & CSUM_TCP) type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; break; case IPPROTO_UDP: if (mp->m_pkthdr.csum_flags & CSUM_UDP) type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; break; #if __FreeBSD_version >= 800000 case IPPROTO_SCTP: if (mp->m_pkthdr.csum_flags & CSUM_SCTP) type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; break; #endif default: offload = FALSE; break; } /* 82575 needs the queue index added */ if (adapter->hw.mac.type == e1000_82575) mss_l4len_idx = txr->me << 4; /* Now copy bits into descriptor */ TXD->vlan_macip_lens |= htole32(vlan_macip_lens); TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); TXD->seqnum_seed = htole32(0); TXD->mss_l4len_idx = htole32(mss_l4len_idx); tx_buffer->m_head = NULL; tx_buffer->next_eop = -1; /* We've consumed the first desc, adjust counters */ if (++ctxd == adapter->num_tx_desc) ctxd = 0; txr->next_avail_desc = ctxd; --txr->tx_avail; return (offload); } /********************************************************************** * * Examine each tx_buffer in the used queue. If the hardware is done * processing the packet then free associated resources. The * tx_buffer is put back on the free queue. * * TRUE return means there's work in the ring to clean, FALSE its empty. **********************************************************************/ static bool igb_txeof(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; int first, last, done; struct igb_tx_buffer *tx_buffer; struct e1000_tx_desc *tx_desc, *eop_desc; struct ifnet *ifp = adapter->ifp; IGB_TX_LOCK_ASSERT(txr); if (txr->tx_avail == adapter->num_tx_desc) return FALSE; first = txr->next_to_clean; tx_desc = &txr->tx_base[first]; tx_buffer = &txr->tx_buffers[first]; last = tx_buffer->next_eop; eop_desc = &txr->tx_base[last]; /* * What this does is get the index of the * first descriptor AFTER the EOP of the * first packet, that way we can do the * simple comparison on the inner while loop. */ if (++last == adapter->num_tx_desc) last = 0; done = last; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { /* We clean the range of the packet */ while (first != done) { tx_desc->upper.data = 0; tx_desc->lower.data = 0; tx_desc->buffer_addr = 0; ++txr->tx_avail; if (tx_buffer->m_head) { txr->bytes += tx_buffer->m_head->m_pkthdr.len; bus_dmamap_sync(txr->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; } tx_buffer->next_eop = -1; txr->watchdog_time = ticks; if (++first == adapter->num_tx_desc) first = 0; tx_buffer = &txr->tx_buffers[first]; tx_desc = &txr->tx_base[first]; } ++txr->packets; ++ifp->if_opackets; /* See if we can continue to the next packet */ last = tx_buffer->next_eop; if (last != -1) { eop_desc = &txr->tx_base[last]; /* Get new done point */ if (++last == adapter->num_tx_desc) last = 0; done = last; } else break; } bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); txr->next_to_clean = first; /* * If we have enough room, clear IFF_DRV_OACTIVE * to tell the stack that it is OK to send packets. */ if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) { ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; /* All clean, turn off the watchdog */ if (txr->tx_avail == adapter->num_tx_desc) { txr->watchdog_check = FALSE; return FALSE; } } return (TRUE); } /********************************************************************* * * Setup descriptor buffer(s) from system mbuf buffer pools. * i - designates the ring index * clean - tells the function whether to update * the header, the packet buffer, or both. * **********************************************************************/ static int igb_get_buf(struct rx_ring *rxr, int i, u8 clean) { struct adapter *adapter = rxr->adapter; struct igb_rx_buf *rxbuf; struct mbuf *mh, *mp; bus_dma_segment_t hseg[1]; bus_dma_segment_t pseg[1]; bus_dmamap_t map; int nsegs, error; rxbuf = &rxr->rx_buffers[i]; mh = mp = NULL; if ((clean & IGB_CLEAN_HEADER) != 0) { mh = m_gethdr(M_DONTWAIT, MT_DATA); if (mh == NULL) { adapter->mbuf_header_failed++; return (ENOBUFS); } mh->m_pkthdr.len = mh->m_len = MHLEN; /* * Because IGB_HDR_BUF size is less than MHLEN * and we configure controller to split headers * we can align mbuf on ETHER_ALIGN boundary. */ m_adj(mh, ETHER_ALIGN); error = bus_dmamap_load_mbuf_sg(rxr->rx_htag, rxr->rx_hspare_map, mh, hseg, &nsegs, 0); if (error != 0) { m_freem(mh); return (error); } mh->m_flags &= ~M_PKTHDR; } if ((clean & IGB_CLEAN_PAYLOAD) != 0) { mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); if (mp == NULL) { if (mh != NULL) { adapter->mbuf_packet_failed++; bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map); mh->m_flags |= M_PKTHDR; m_freem(mh); } return (ENOBUFS); } mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; error = bus_dmamap_load_mbuf_sg(rxr->rx_ptag, rxr->rx_pspare_map, mp, pseg, &nsegs, 0); if (error != 0) { if (mh != NULL) { bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map); mh->m_flags |= M_PKTHDR; m_freem(mh); } m_freem(mp); return (error); } mp->m_flags &= ~M_PKTHDR; } /* Loading new DMA maps complete, unload maps for received buffers. */ if ((clean & IGB_CLEAN_HEADER) != 0 && rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map); } if ((clean & IGB_CLEAN_PAYLOAD) != 0 && rxbuf->m_pack != NULL) { bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map); } /* Reflect loaded dmamaps. */ if ((clean & IGB_CLEAN_HEADER) != 0) { map = rxbuf->head_map; rxbuf->head_map = rxr->rx_hspare_map; rxr->rx_hspare_map = map; rxbuf->m_head = mh; bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map, BUS_DMASYNC_PREREAD); rxr->rx_base[i].read.hdr_addr = htole64(hseg[0].ds_addr); } if ((clean & IGB_CLEAN_PAYLOAD) != 0) { map = rxbuf->pack_map; rxbuf->pack_map = rxr->rx_pspare_map; rxr->rx_pspare_map = map; rxbuf->m_pack = mp; bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map, BUS_DMASYNC_PREREAD); rxr->rx_base[i].read.pkt_addr = htole64(pseg[0].ds_addr); } return (0); } /********************************************************************* * * Allocate memory for rx_buffer structures. Since we use one * rx_buffer per received packet, the maximum number of rx_buffer's * that we'll need is equal to the number of receive descriptors * that we've allocated. * **********************************************************************/ static int igb_allocate_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; device_t dev = adapter->dev; struct igb_rx_buf *rxbuf; int i, bsize, error; bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; if (!(rxr->rx_buffers = (struct igb_rx_buf *) malloc(bsize, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate rx_buffer memory\n"); error = ENOMEM; goto fail; } if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MSIZE, /* maxsize */ 1, /* nsegments */ MSIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &rxr->rx_htag))) { device_printf(dev, "Unable to create RX DMA tag\n"); goto fail; } if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUMPAGESIZE, /* maxsize */ 1, /* nsegments */ MJUMPAGESIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &rxr->rx_ptag))) { device_printf(dev, "Unable to create RX payload DMA tag\n"); goto fail; } /* Create the spare maps (used by getbuf) */ error = bus_dmamap_create(rxr->rx_htag, BUS_DMA_NOWAIT, &rxr->rx_hspare_map); if (error) { device_printf(dev, "%s: bus_dmamap_create header spare failed: %d\n", __func__, error); goto fail; } error = bus_dmamap_create(rxr->rx_ptag, BUS_DMA_NOWAIT, &rxr->rx_pspare_map); if (error) { device_printf(dev, "%s: bus_dmamap_create packet spare failed: %d\n", __func__, error); goto fail; } for (i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; error = bus_dmamap_create(rxr->rx_htag, BUS_DMA_NOWAIT, &rxbuf->head_map); if (error) { device_printf(dev, "Unable to create RX head DMA maps\n"); goto fail; } error = bus_dmamap_create(rxr->rx_ptag, BUS_DMA_NOWAIT, &rxbuf->pack_map); if (error) { device_printf(dev, "Unable to create RX packet DMA maps\n"); goto fail; } } return (0); fail: /* Frees all, but can handle partial completion */ igb_free_receive_structures(adapter); return (error); } static void igb_free_receive_ring(struct rx_ring *rxr) { struct adapter *adapter; struct igb_rx_buf *rxbuf; int i; adapter = rxr->adapter; for (i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map); rxbuf->m_head->m_flags |= M_PKTHDR; m_freem(rxbuf->m_head); } if (rxbuf->m_pack != NULL) { bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map); rxbuf->m_pack->m_flags |= M_PKTHDR; m_freem(rxbuf->m_pack); } rxbuf->m_head = NULL; rxbuf->m_pack = NULL; } } /********************************************************************* * * Initialize a receive ring and its buffers. * **********************************************************************/ static int igb_setup_receive_ring(struct rx_ring *rxr) { struct adapter *adapter; struct ifnet *ifp; device_t dev; struct lro_ctrl *lro = &rxr->lro; int j, rsize, error = 0; adapter = rxr->adapter; dev = adapter->dev; ifp = adapter->ifp; /* Clear the ring contents */ IGB_RX_LOCK(rxr); rsize = roundup2(adapter->num_rx_desc * sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); /* ** Free current RX buffer structures and their mbufs */ igb_free_receive_ring(rxr); /* Now replenish the ring mbufs */ for (j = 0; j < adapter->num_rx_desc; j++) { if ((error = igb_get_buf(rxr, j, IGB_CLEAN_BOTH)) != 0) goto fail; } /* Setup our descriptor indices */ rxr->next_to_check = 0; rxr->last_cleaned = 0; rxr->lro_enabled = FALSE; if (igb_header_split) rxr->hdr_split = TRUE; else ifp->if_capabilities &= ~IFCAP_LRO; rxr->fmp = NULL; rxr->lmp = NULL; rxr->discard = FALSE; bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* ** Now set up the LRO interface, we ** also only do head split when LRO ** is enabled, since so often they ** are undesireable in similar setups. */ if (ifp->if_capenable & IFCAP_LRO) { int err = tcp_lro_init(lro); if (err) { device_printf(dev, "LRO Initialization failed!\n"); goto fail; } INIT_DEBUGOUT("RX LRO Initialized\n"); rxr->lro_enabled = TRUE; lro->ifp = adapter->ifp; } IGB_RX_UNLOCK(rxr); return (0); fail: igb_free_receive_ring(rxr); IGB_RX_UNLOCK(rxr); return (error); } /********************************************************************* * * Initialize all receive rings. * **********************************************************************/ static int igb_setup_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; int i, j; for (i = 0; i < adapter->num_queues; i++, rxr++) if (igb_setup_receive_ring(rxr)) goto fail; return (0); fail: /* * Free RX buffers allocated so far, we will only handle * the rings that completed, the failing case will have * cleaned up for itself. The value of 'i' will be the * failed ring so we must pre-decrement it. */ rxr = adapter->rx_rings; for (--i; i > 0; i--, rxr++) { for (j = 0; j < adapter->num_rx_desc; j++) igb_free_receive_ring(rxr); } return (ENOBUFS); } /********************************************************************* * * Enable receive unit. * **********************************************************************/ static void igb_initialize_receive_units(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; struct ifnet *ifp = adapter->ifp; struct e1000_hw *hw = &adapter->hw; u32 rctl, rxcsum, psize, srrctl = 0; INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); /* * Make sure receives are disabled while setting * up the descriptor ring */ rctl = E1000_READ_REG(hw, E1000_RCTL); E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); /* ** Set up for header split */ if (rxr->hdr_split) { /* Use a standard mbuf for the header */ srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; } else srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; /* ** Set up for jumbo frames */ if (ifp->if_mtu > ETHERMTU) { rctl |= E1000_RCTL_LPE; srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; /* Set maximum packet len */ psize = adapter->max_frame_size; /* are we on a vlan? */ if (adapter->ifp->if_vlantrunk != NULL) psize += VLAN_TAG_SIZE; E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); } else { rctl &= ~E1000_RCTL_LPE; srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; rctl |= E1000_RCTL_SZ_2048; } /* Setup the Base and Length of the Rx Descriptor Rings */ for (int i = 0; i < adapter->num_queues; i++, rxr++) { u64 bus_addr = rxr->rxdma.dma_paddr; u32 rxdctl; E1000_WRITE_REG(hw, E1000_RDLEN(i), adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); E1000_WRITE_REG(hw, E1000_RDBAH(i), (uint32_t)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_RDBAL(i), (uint32_t)bus_addr); E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); /* Enable this Queue */ rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; rxdctl &= 0xFFF00000; rxdctl |= IGB_RX_PTHRESH; rxdctl |= IGB_RX_HTHRESH << 8; rxdctl |= IGB_RX_WTHRESH << 16; E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); } /* ** Setup for RX MultiQueue */ rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); if (adapter->num_queues >1) { u32 random[10], mrqc, shift = 0; union igb_reta { u32 dword; u8 bytes[4]; } reta; arc4rand(&random, sizeof(random), 0); if (adapter->hw.mac.type == e1000_82575) shift = 6; /* Warning FM follows */ for (int i = 0; i < 128; i++) { reta.bytes[i & 3] = (i % adapter->num_queues) << shift; if ((i & 3) == 3) E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta.dword); } /* Now fill in hash table */ mrqc = E1000_MRQC_ENABLE_RSS_4Q; for (int i = 0; i < 10; i++) E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), i, random[i]); mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | E1000_MRQC_RSS_FIELD_IPV4_TCP); mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | E1000_MRQC_RSS_FIELD_IPV6_TCP); mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | E1000_MRQC_RSS_FIELD_IPV6_UDP); mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); E1000_WRITE_REG(hw, E1000_MRQC, mrqc); /* ** NOTE: Receive Full-Packet Checksum Offload ** is mutually exclusive with Multiqueue. However ** this is not the same as TCP/IP checksums which ** still work. */ rxcsum |= E1000_RXCSUM_PCSD; #if __FreeBSD_version >= 800000 /* For SCTP Offload */ if ((hw->mac.type == e1000_82576) && (ifp->if_capenable & IFCAP_RXCSUM)) rxcsum |= E1000_RXCSUM_CRCOFL; #endif } else { /* Non RSS setup */ if (ifp->if_capenable & IFCAP_RXCSUM) { rxcsum |= E1000_RXCSUM_IPPCSE; #if __FreeBSD_version >= 800000 if (adapter->hw.mac.type == e1000_82576) rxcsum |= E1000_RXCSUM_CRCOFL; #endif } else rxcsum &= ~E1000_RXCSUM_TUOFL; } E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); /* Setup the Receive Control Register */ rctl &= ~(3 << E1000_RCTL_MO_SHIFT); rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); /* Strip CRC bytes. */ rctl |= E1000_RCTL_SECRC; /* Make sure VLAN Filters are off */ rctl &= ~E1000_RCTL_VFE; /* Don't store bad packets */ rctl &= ~E1000_RCTL_SBP; /* Enable Receives */ E1000_WRITE_REG(hw, E1000_RCTL, rctl); /* * Setup the HW Rx Head and Tail Descriptor Pointers * - needs to be after enable */ for (int i = 0; i < adapter->num_queues; i++) { E1000_WRITE_REG(hw, E1000_RDH(i), 0); E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1); } return; } /********************************************************************* * * Free receive rings. * **********************************************************************/ static void igb_free_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; for (int i = 0; i < adapter->num_queues; i++, rxr++) { struct lro_ctrl *lro = &rxr->lro; igb_free_receive_buffers(rxr); tcp_lro_free(lro); igb_dma_free(adapter, &rxr->rxdma); } free(adapter->rx_rings, M_DEVBUF); } /********************************************************************* * * Free receive ring data structures. * **********************************************************************/ static void igb_free_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; struct igb_rx_buf *rxbuf; int i; INIT_DEBUGOUT("free_receive_structures: begin"); if (rxr->rx_hspare_map != NULL) { bus_dmamap_destroy(rxr->rx_htag, rxr->rx_hspare_map); rxr->rx_hspare_map = NULL; } if (rxr->rx_hspare_map != NULL) { bus_dmamap_destroy(rxr->rx_ptag, rxr->rx_pspare_map); rxr->rx_pspare_map = NULL; } /* Cleanup any existing buffers */ if (rxr->rx_buffers != NULL) { for (i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map); rxbuf->m_head->m_flags |= M_PKTHDR; m_freem(rxbuf->m_head); } if (rxbuf->m_pack != NULL) { bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map); rxbuf->m_pack->m_flags |= M_PKTHDR; m_freem(rxbuf->m_pack); } rxbuf->m_head = NULL; rxbuf->m_pack = NULL; if (rxbuf->head_map != NULL) { bus_dmamap_destroy(rxr->rx_htag, rxbuf->head_map); rxbuf->head_map = NULL; } if (rxbuf->pack_map != NULL) { bus_dmamap_destroy(rxr->rx_ptag, rxbuf->pack_map); rxbuf->pack_map = NULL; } } if (rxr->rx_buffers != NULL) { free(rxr->rx_buffers, M_DEVBUF); rxr->rx_buffers = NULL; } } if (rxr->rx_htag != NULL) { bus_dma_tag_destroy(rxr->rx_htag); rxr->rx_htag = NULL; } if (rxr->rx_ptag != NULL) { bus_dma_tag_destroy(rxr->rx_ptag); rxr->rx_ptag = NULL; } } static __inline void igb_rx_discard(struct rx_ring *rxr, union e1000_adv_rx_desc *cur, int i) { if (rxr->fmp != NULL) { rxr->fmp->m_flags |= M_PKTHDR; m_freem(rxr->fmp); rxr->fmp = NULL; rxr->lmp = NULL; } } static __inline void igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) { /* * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet * should be computed by hardware. Also it should not have VLAN tag in * ethernet header. */ if (rxr->lro_enabled && (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { /* * Send to the stack if: ** - LRO not enabled, or ** - no LRO resources, or ** - lro enqueue fails */ if (rxr->lro.lro_cnt != 0) if (tcp_lro_rx(&rxr->lro, m, 0) == 0) return; } (*ifp->if_input)(ifp, m); } /********************************************************************* * * This routine executes in interrupt context. It replenishes * the mbufs in the descriptor and sends data which has been * dma'ed into host memory to upper layer. * * We loop at most count times if count is > 0, or until done if * count < 0. * * Return TRUE if more to clean, FALSE otherwise *********************************************************************/ static bool igb_rxeof(struct rx_ring *rxr, int count) { struct adapter *adapter = rxr->adapter; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; struct lro_entry *queued; int i, prog = 0; u32 ptype, staterr = 0; union e1000_adv_rx_desc *cur; IGB_RX_LOCK(rxr); /* Main clean loop */ for (i = rxr->next_to_check; count > 0; prog++) { struct mbuf *sendmp, *mh, *mp; u16 hlen, plen, hdr, vtag; bool eop = FALSE; u8 dopayload; /* Sync the ring. */ bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cur = &rxr->rx_base[i]; staterr = le32toh(cur->wb.upper.status_error); if ((staterr & E1000_RXD_STAT_DD) == 0) break; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; count--; sendmp = mh = mp = NULL; cur->wb.upper.status_error = 0; plen = le16toh(cur->wb.upper.length); ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); /* Make sure all segments of a bad packet are discarded */ if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) || (rxr->discard)) { ifp->if_ierrors++; ++rxr->rx_discarded; if (!eop) /* Catch subsequent segs */ rxr->discard = TRUE; else rxr->discard = FALSE; igb_rx_discard(rxr, cur, i); goto next_desc; } /* ** The way the hardware is configured to ** split, it will ONLY use the header buffer ** when header split is enabled, otherwise we ** get normal behavior, ie, both header and ** payload are DMA'd into the payload buffer. ** ** The fmp test is to catch the case where a ** packet spans multiple descriptors, in that ** case only the first header is valid. */ if (rxr->hdr_split && rxr->fmp == NULL) { hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT; if (hlen > IGB_HDR_BUF) hlen = IGB_HDR_BUF; /* Handle the header mbuf */ mh = rxr->rx_buffers[i].m_head; mh->m_len = hlen; dopayload = IGB_CLEAN_HEADER; /* ** Get the payload length, this ** could be zero if its a small ** packet. */ if (plen > 0) { mp = rxr->rx_buffers[i].m_pack; mp->m_len = plen; mh->m_next = mp; dopayload = IGB_CLEAN_BOTH; rxr->rx_split_packets++; } } else { /* ** Either no header split, or a ** secondary piece of a fragmented ** split packet. */ mh = rxr->rx_buffers[i].m_pack; mh->m_len = plen; dopayload = IGB_CLEAN_PAYLOAD; } /* ** get_buf will overwrite the writeback ** descriptor so save the VLAN tag now. */ vtag = le16toh(cur->wb.upper.vlan); if (igb_get_buf(rxr, i, dopayload) != 0) { ifp->if_iqdrops++; /* * We've dropped a frame due to lack of resources * so we should drop entire multi-segmented * frames until we encounter EOP. */ if ((staterr & E1000_RXD_STAT_EOP) != 0) rxr->discard = TRUE; igb_rx_discard(rxr, cur, i); goto next_desc; } /* Initial frame - setup */ if (rxr->fmp == NULL) { mh->m_pkthdr.len = mh->m_len; /* Store the first mbuf */ rxr->fmp = mh; rxr->lmp = mh; if (mp != NULL) { /* Add payload if split */ mh->m_pkthdr.len += mp->m_len; rxr->lmp = mh->m_next; } } else { /* Chain mbuf's together */ rxr->lmp->m_next = mh; rxr->lmp = rxr->lmp->m_next; rxr->fmp->m_pkthdr.len += mh->m_len; } if (eop) { rxr->fmp->m_pkthdr.rcvif = ifp; ifp->if_ipackets++; rxr->rx_packets++; /* capture data for AIM */ rxr->packets++; rxr->bytes += rxr->fmp->m_pkthdr.len; rxr->rx_bytes += rxr->fmp->m_pkthdr.len; if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) igb_rx_checksum(staterr, rxr->fmp, ptype); /* XXX igb(4) always strips VLAN. */ if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && (staterr & E1000_RXD_STAT_VP) != 0) { rxr->fmp->m_pkthdr.ether_vtag = vtag; rxr->fmp->m_flags |= M_VLANTAG; } #if __FreeBSD_version >= 800000 rxr->fmp->m_pkthdr.flowid = curcpu; rxr->fmp->m_flags |= M_FLOWID; #endif sendmp = rxr->fmp; /* Make sure to set M_PKTHDR. */ sendmp->m_flags |= M_PKTHDR; rxr->fmp = NULL; rxr->lmp = NULL; } next_desc: bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); rxr->last_cleaned = i; /* For updating tail */ /* Advance our pointers to the next descriptor. */ if (++i == adapter->num_rx_desc) i = 0; /* ** Note that we hold the RX lock thru ** the following call so this ring's ** next_to_check is not gonna change. */ if (sendmp != NULL) igb_rx_input(rxr, ifp, sendmp, ptype); } if (prog == 0) { IGB_RX_UNLOCK(rxr); return (FALSE); } rxr->next_to_check = i; /* Advance the E1000's Receive Queue "Tail Pointer". */ E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned); /* * Flush any outstanding LRO work */ while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { SLIST_REMOVE_HEAD(&lro->lro_active, next); tcp_lro_flush(lro, queued); } IGB_RX_UNLOCK(rxr); /* ** We still have cleaning to do? ** Schedule another interrupt if so. */ if ((staterr & E1000_RXD_STAT_DD) != 0) return (TRUE); return (FALSE); } /********************************************************************* * * Verify that the hardware indicated that the checksum is valid. * Inform the stack about the status of checksum so that stack * doesn't spend time verifying the checksum. * *********************************************************************/ static void igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) { u16 status = (u16)staterr; u8 errors = (u8) (staterr >> 24); int sctp; /* Ignore Checksum bit is set */ if (status & E1000_RXD_STAT_IXSM) { mp->m_pkthdr.csum_flags = 0; return; } if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) sctp = 1; else sctp = 0; if (status & E1000_RXD_STAT_IPCS) { /* Did it pass? */ if (!(errors & E1000_RXD_ERR_IPE)) { /* IP Checksum Good */ mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; } else mp->m_pkthdr.csum_flags = 0; } if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); #if __FreeBSD_version >= 800000 if (sctp) /* reassign */ type = CSUM_SCTP_VALID; #endif /* Did it pass? */ if (!(errors & E1000_RXD_ERR_TCPE)) { mp->m_pkthdr.csum_flags |= type; if (sctp == 0) mp->m_pkthdr.csum_data = htons(0xffff); } } return; } /* * This routine is run via an vlan * config EVENT */ static void igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u32 index, bit; if (ifp->if_softc != arg) /* Not our event */ return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; igb_shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; /* Re-init to load the changes */ igb_init(adapter); } /* * This routine is run via an vlan * unconfig EVENT */ static void igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u32 index, bit; if (ifp->if_softc != arg) return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; igb_shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; /* Re-init to load the changes */ igb_init(adapter); } static void igb_setup_vlan_hw_support(struct adapter *adapter) { struct e1000_hw *hw = &adapter->hw; u32 reg; /* ** We get here thru init_locked, meaning ** a soft reset, this has already cleared ** the VFTA and other state, so if there ** have been no vlan's registered do nothing. */ if (adapter->num_vlans == 0) return; /* ** A soft reset zero's out the VFTA, so ** we need to repopulate it now. */ for (int i = 0; i < IGB_VFTA_SIZE; i++) if (igb_shadow_vfta[i] != 0) E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, igb_shadow_vfta[i]); reg = E1000_READ_REG(hw, E1000_CTRL); reg |= E1000_CTRL_VME; E1000_WRITE_REG(hw, E1000_CTRL, reg); /* Enable the Filter Table */ reg = E1000_READ_REG(hw, E1000_RCTL); reg &= ~E1000_RCTL_CFIEN; reg |= E1000_RCTL_VFE; E1000_WRITE_REG(hw, E1000_RCTL, reg); /* Update the frame size */ E1000_WRITE_REG(&adapter->hw, E1000_RLPML, adapter->max_frame_size + VLAN_TAG_SIZE); } static void igb_enable_intr(struct adapter *adapter) { /* With RSS set up what to auto clear */ if (adapter->msix_mem) { E1000_WRITE_REG(&adapter->hw, E1000_EIAC, adapter->eims_mask); E1000_WRITE_REG(&adapter->hw, E1000_EIAM, adapter->eims_mask); E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->eims_mask); E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); } else { E1000_WRITE_REG(&adapter->hw, E1000_IMS, IMS_ENABLE_MASK); } E1000_WRITE_FLUSH(&adapter->hw); return; } static void igb_disable_intr(struct adapter *adapter) { if (adapter->msix_mem) { E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); } E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); E1000_WRITE_FLUSH(&adapter->hw); return; } /* * Bit of a misnomer, what this really means is * to enable OS management of the system... aka * to disable special hardware management features */ static void igb_init_manageability(struct adapter *adapter) { if (adapter->has_manage) { int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); /* disable hardware interception of ARP */ manc &= ~(E1000_MANC_ARP_EN); /* enable receiving management packets to the host */ manc |= E1000_MANC_EN_MNG2HOST; manc2h |= 1 << 5; /* Mng Port 623 */ manc2h |= 1 << 6; /* Mng Port 664 */ E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); } } /* * Give control back to hardware management * controller if there is one. */ static void igb_release_manageability(struct adapter *adapter) { if (adapter->has_manage) { int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); /* re-enable hardware interception of ARP */ manc |= E1000_MANC_ARP_EN; manc &= ~E1000_MANC_EN_MNG2HOST; E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); } } /* * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means that * the driver is loaded. * */ static void igb_get_hw_control(struct adapter *adapter) { u32 ctrl_ext; /* Let firmware know the driver has taken over */ ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); } /* * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. * For ASF and Pass Through versions of f/w this means that the * driver is no longer loaded. * */ static void igb_release_hw_control(struct adapter *adapter) { u32 ctrl_ext; /* Let firmware taken over control of h/w */ ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); } static int igb_is_valid_ether_addr(uint8_t *addr) { char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { return (FALSE); } return (TRUE); } /* * Enable PCI Wake On Lan capability */ void igb_enable_wakeup(device_t dev) { u16 cap, status; u8 id; /* First find the capabilities pointer*/ cap = pci_read_config(dev, PCIR_CAP_PTR, 2); /* Read the PM Capabilities */ id = pci_read_config(dev, cap, 1); if (id != PCIY_PMG) /* Something wrong */ return; /* OK, we have the power capabilities, so now get the status register */ cap += PCIR_POWER_STATUS; status = pci_read_config(dev, cap, 2); status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; pci_write_config(dev, cap, status, 2); return; } /********************************************************************** * * Update the board statistics counters. * **********************************************************************/ static void igb_update_stats_counters(struct adapter *adapter) { struct ifnet *ifp; if(adapter->hw.phy.media_type == e1000_media_type_copper || (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); } adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); /* For the 64-bit byte counters the low dword must be read first. */ /* Both registers clear on the read of the high dword */ adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); adapter->stats.algnerrc += E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); adapter->stats.rxerrc += E1000_READ_REG(&adapter->hw, E1000_RXERRC); adapter->stats.tncrs += E1000_READ_REG(&adapter->hw, E1000_TNCRS); adapter->stats.cexterr += E1000_READ_REG(&adapter->hw, E1000_CEXTERR); adapter->stats.tsctc += E1000_READ_REG(&adapter->hw, E1000_TSCTC); adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, E1000_TSCTFC); ifp = adapter->ifp; ifp->if_collisions = adapter->stats.colc; /* Rx Errors */ ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + adapter->stats.crcerrs + adapter->stats.algnerrc + adapter->stats.ruc + adapter->stats.roc + adapter->stats.mpc + adapter->stats.cexterr; /* Tx Errors */ ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol + adapter->watchdog_events; } /********************************************************************** * * This routine is called only when igb_display_debug_stats is enabled. * This routine provides a way to take a look at important statistics * maintained by the driver and hardware. * **********************************************************************/ static void igb_print_debug_info(struct adapter *adapter) { device_t dev = adapter->dev; struct igb_queue *que = adapter->queues; struct rx_ring *rxr = adapter->rx_rings; struct tx_ring *txr = adapter->tx_rings; uint8_t *hw_addr = adapter->hw.hw_addr; device_printf(dev, "Adapter hardware address = %p \n", hw_addr); device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", E1000_READ_REG(&adapter->hw, E1000_CTRL), E1000_READ_REG(&adapter->hw, E1000_RCTL)); #if (DEBUG_HW > 0) /* Dont output these errors normally */ device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n", E1000_READ_REG(&adapter->hw, E1000_IMS), E1000_READ_REG(&adapter->hw, E1000_EIMS)); #endif device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); device_printf(dev, "Flow control watermarks high = %d low = %d\n", adapter->hw.fc.high_water, adapter->hw.fc.low_water); for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { device_printf(dev, "Queue(%d) tdh = %d, tdt = %d ", i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)), E1000_READ_REG(&adapter->hw, E1000_TDT(i))); device_printf(dev, "rdh = %d, rdt = %d\n", E1000_READ_REG(&adapter->hw, E1000_RDH(i)), E1000_READ_REG(&adapter->hw, E1000_RDT(i))); device_printf(dev, "TX(%d) no descriptors avail event = %lld\n", txr->me, (long long)txr->no_desc_avail); device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me, (long long)txr->tx_packets); device_printf(dev, "RX(%d) Packets received = %lld ", rxr->me, (long long)rxr->rx_packets); } for (int i = 0; i < adapter->num_queues; i++, rxr++) { struct lro_ctrl *lro = &rxr->lro; device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i, E1000_READ_REG(&adapter->hw, E1000_RDH(i)), E1000_READ_REG(&adapter->hw, E1000_RDT(i))); device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me, (long long)rxr->rx_packets); device_printf(dev, " Split Packets = %lld ", (long long)rxr->rx_split_packets); device_printf(dev, " Byte count = %lld\n", (long long)rxr->rx_bytes); device_printf(dev,"RX(%d) LRO Queued= %d ", i, lro->lro_queued); device_printf(dev,"LRO Flushed= %d\n",lro->lro_flushed); } for (int i = 0; i < adapter->num_queues; i++, que++) device_printf(dev,"QUE(%d) IRQs = %llx\n", i, (long long)que->irqs); device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq); device_printf(dev, "Mbuf defrag failed = %ld\n", adapter->mbuf_defrag_failed); device_printf(dev, "Std mbuf header failed = %ld\n", adapter->mbuf_header_failed); device_printf(dev, "Std mbuf packet failed = %ld\n", adapter->mbuf_packet_failed); device_printf(dev, "Driver dropped packets = %ld\n", adapter->dropped_pkts); device_printf(dev, "Driver tx dma failure in xmit = %ld\n", adapter->no_tx_dma_setup); } static void igb_print_hw_stats(struct adapter *adapter) { device_t dev = adapter->dev; device_printf(dev, "Excessive collisions = %lld\n", (long long)adapter->stats.ecol); #if (DEBUG_HW > 0) /* Dont output these errors normally */ device_printf(dev, "Symbol errors = %lld\n", (long long)adapter->stats.symerrs); #endif device_printf(dev, "Sequence errors = %lld\n", (long long)adapter->stats.sec); device_printf(dev, "Defer count = %lld\n", (long long)adapter->stats.dc); device_printf(dev, "Missed Packets = %lld\n", (long long)adapter->stats.mpc); device_printf(dev, "Receive No Buffers = %lld\n", (long long)adapter->stats.rnbc); /* RLEC is inaccurate on some hardware, calculate our own. */ device_printf(dev, "Receive Length Errors = %lld\n", ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); device_printf(dev, "Receive errors = %lld\n", (long long)adapter->stats.rxerrc); device_printf(dev, "Crc errors = %lld\n", (long long)adapter->stats.crcerrs); device_printf(dev, "Alignment errors = %lld\n", (long long)adapter->stats.algnerrc); /* On 82575 these are collision counts */ device_printf(dev, "Collision/Carrier extension errors = %lld\n", (long long)adapter->stats.cexterr); device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); device_printf(dev, "watchdog timeouts = %ld\n", adapter->watchdog_events); device_printf(dev, "XON Rcvd = %lld\n", (long long)adapter->stats.xonrxc); device_printf(dev, "XON Xmtd = %lld\n", (long long)adapter->stats.xontxc); device_printf(dev, "XOFF Rcvd = %lld\n", (long long)adapter->stats.xoffrxc); device_printf(dev, "XOFF Xmtd = %lld\n", (long long)adapter->stats.xofftxc); device_printf(dev, "Good Packets Rcvd = %lld\n", (long long)adapter->stats.gprc); device_printf(dev, "Good Packets Xmtd = %lld\n", (long long)adapter->stats.gptc); device_printf(dev, "TSO Contexts Xmtd = %lld\n", (long long)adapter->stats.tsctc); device_printf(dev, "TSO Contexts Failed = %lld\n", (long long)adapter->stats.tsctfc); } /********************************************************************** * * This routine provides a way to dump out the adapter eeprom, * often a useful debug/service tool. This only dumps the first * 32 words, stuff that matters is in that extent. * **********************************************************************/ static void igb_print_nvm_info(struct adapter *adapter) { u16 eeprom_data; int i, j, row = 0; /* Its a bit crude, but it gets the job done */ printf("\nInterface EEPROM Dump:\n"); printf("Offset\n0x0000 "); for (i = 0, j = 0; i < 32; i++, j++) { if (j == 8) { /* Make the offset block */ j = 0; ++row; printf("\n0x00%x0 ",row); } e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); printf("%04x ", eeprom_data); } printf("\n"); } static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS) { struct adapter *adapter; int error; int result; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *)arg1; igb_print_debug_info(adapter); } /* * This value will cause a hex dump of the * first 32 16-bit words of the EEPROM to * the screen. */ if (result == 2) { adapter = (struct adapter *)arg1; igb_print_nvm_info(adapter); } return (error); } static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS) { struct adapter *adapter; int error; int result; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *)arg1; igb_print_hw_stats(adapter); } return (error); } static void igb_add_rx_process_limit(struct adapter *adapter, const char *name, const char *description, int *limit, int value) { *limit = value; SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); } Index: user/imp/tbemd/sys/dev/ixgbe/ixgbe.c =================================================================== --- user/imp/tbemd/sys/dev/ixgbe/ixgbe.c (revision 203849) +++ user/imp/tbemd/sys/dev/ixgbe/ixgbe.c (revision 203850) @@ -1,5223 +1,5223 @@ /****************************************************************************** Copyright (c) 2001-2009, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the Intel Corporation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ******************************************************************************/ /*$FreeBSD$*/ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #endif #include "ixgbe.h" /********************************************************************* * Set this to one to display debug statistics *********************************************************************/ int ixgbe_display_debug_stats = 0; /********************************************************************* * Driver version *********************************************************************/ char ixgbe_driver_version[] = "2.0.7"; /********************************************************************* * PCI Device ID Table * * Used by probe to select devices to load on * Last field stores an index into ixgbe_strings * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = { {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} }; /********************************************************************* * Table of branding strings *********************************************************************/ static char *ixgbe_strings[] = { "Intel(R) PRO/10GbE PCI-Express Network Driver" }; /********************************************************************* * Function prototypes *********************************************************************/ static int ixgbe_probe(device_t); static int ixgbe_attach(device_t); static int ixgbe_detach(device_t); static int ixgbe_shutdown(device_t); static void ixgbe_start(struct ifnet *); static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); #if __FreeBSD_version >= 800000 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *, struct mbuf *); static void ixgbe_qflush(struct ifnet *); #endif static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); static void ixgbe_init(void *); static void ixgbe_init_locked(struct adapter *); static void ixgbe_stop(void *); static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); static int ixgbe_media_change(struct ifnet *); static void ixgbe_identify_hardware(struct adapter *); static int ixgbe_allocate_pci_resources(struct adapter *); static int ixgbe_allocate_msix(struct adapter *); static int ixgbe_allocate_legacy(struct adapter *); static int ixgbe_allocate_queues(struct adapter *); static int ixgbe_setup_msix(struct adapter *); static void ixgbe_free_pci_resources(struct adapter *); static void ixgbe_local_timer(void *); static void ixgbe_setup_interface(device_t, struct adapter *); static void ixgbe_config_link(struct adapter *); static int ixgbe_allocate_transmit_buffers(struct tx_ring *); static int ixgbe_setup_transmit_structures(struct adapter *); static void ixgbe_setup_transmit_ring(struct tx_ring *); static void ixgbe_initialize_transmit_units(struct adapter *); static void ixgbe_free_transmit_structures(struct adapter *); static void ixgbe_free_transmit_buffers(struct tx_ring *); static int ixgbe_allocate_receive_buffers(struct rx_ring *); static int ixgbe_setup_receive_structures(struct adapter *); static int ixgbe_setup_receive_ring(struct rx_ring *); static void ixgbe_initialize_receive_units(struct adapter *); static void ixgbe_free_receive_structures(struct adapter *); static void ixgbe_free_receive_buffers(struct rx_ring *); static void ixgbe_setup_hw_rsc(struct rx_ring *); static void ixgbe_init_moderation(struct adapter *); static void ixgbe_enable_intr(struct adapter *); static void ixgbe_disable_intr(struct adapter *); static void ixgbe_update_stats_counters(struct adapter *); static bool ixgbe_txeof(struct tx_ring *); static bool ixgbe_rxeof(struct rx_ring *, int); static void ixgbe_rx_checksum(u32, struct mbuf *); static void ixgbe_set_promisc(struct adapter *); static void ixgbe_disable_promisc(struct adapter *); static void ixgbe_set_multi(struct adapter *); static void ixgbe_print_hw_stats(struct adapter *); static void ixgbe_print_debug_info(struct adapter *); static void ixgbe_update_link_status(struct adapter *); static int ixgbe_get_buf(struct rx_ring *, int, int); static int ixgbe_xmit(struct tx_ring *, struct mbuf **); static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS); static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS); static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS); static int ixgbe_dma_malloc(struct adapter *, bus_size_t, struct ixgbe_dma_alloc *, int); static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); static void ixgbe_add_rx_process_limit(struct adapter *, const char *, const char *, int *, int); static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); static void ixgbe_configure_ivars(struct adapter *); static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); static void ixgbe_setup_vlan_hw_support(struct adapter *); static void ixgbe_register_vlan(void *, struct ifnet *, u16); static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); static void ixgbe_update_aim(struct rx_ring *); /* Support for pluggable optic modules */ static bool ixgbe_sfp_probe(struct adapter *); /* Legacy (single vector interrupt handler */ static void ixgbe_legacy_irq(void *); /* The MSI/X Interrupt handlers */ static void ixgbe_msix_tx(void *); static void ixgbe_msix_rx(void *); static void ixgbe_msix_link(void *); /* Deferred interrupt tasklets */ static void ixgbe_handle_tx(void *, int); static void ixgbe_handle_rx(void *, int); static void ixgbe_handle_link(void *, int); static void ixgbe_handle_msf(void *, int); static void ixgbe_handle_mod(void *, int); #ifdef IXGBE_FDIR static void ixgbe_atr(struct tx_ring *, struct mbuf *); static void ixgbe_reinit_fdir(void *, int); #endif /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ static device_method_t ixgbe_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ixgbe_probe), DEVMETHOD(device_attach, ixgbe_attach), DEVMETHOD(device_detach, ixgbe_detach), DEVMETHOD(device_shutdown, ixgbe_shutdown), {0, 0} }; static driver_t ixgbe_driver = { "ix", ixgbe_methods, sizeof(struct adapter), }; static devclass_t ixgbe_devclass; DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); MODULE_DEPEND(ixgbe, pci, 1, 1, 1); MODULE_DEPEND(ixgbe, ether, 1, 1, 1); /* ** TUNEABLE PARAMETERS: */ /* ** These parameters are used in Adaptive ** Interrupt Moderation. The value is set ** into EITR and controls the interrupt ** frequency. They can be modified but ** be careful in tuning them. */ static int ixgbe_enable_aim = TRUE; TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); static int ixgbe_low_latency = IXGBE_LOW_LATENCY; TUNABLE_INT("hw.ixgbe.low_latency", &ixgbe_low_latency); static int ixgbe_ave_latency = IXGBE_AVE_LATENCY; TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_ave_latency); static int ixgbe_bulk_latency = IXGBE_BULK_LATENCY; TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency); /* How many packets rxeof tries to clean at a time */ static int ixgbe_rx_process_limit = 100; TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); /* Flow control setting, default to full */ static int ixgbe_flow_control = ixgbe_fc_full; TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); /* ** Smart speed setting, default to on ** this only works as a compile option ** right now as its during attach, set ** this to 'ixgbe_smart_speed_off' to ** disable. */ static int ixgbe_smart_speed = ixgbe_smart_speed_on; /* * MSIX should be the default for best performance, * but this allows it to be forced off for testing. */ static int ixgbe_enable_msix = 1; TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); /* * Header split has seemed to be beneficial in * all circumstances tested, so its on by default * however this variable will allow it to be disabled * for some debug purposes. */ static bool ixgbe_header_split = TRUE; TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); /* * Number of Queues, should normally * be left at 0, it then autoconfigures to * the number of cpus. Each queue is a pair * of RX and TX rings with a dedicated interrupt */ static int ixgbe_num_queues = 0; TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); /* Number of TX descriptors per ring */ static int ixgbe_txd = DEFAULT_TXD; TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); /* Number of RX descriptors per ring */ static int ixgbe_rxd = DEFAULT_RXD; TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); /* Keep running tab on them for sanity check */ static int ixgbe_total_ports; /* ** Shadow VFTA table, this is needed because ** the real filter table gets cleared during ** a soft reset and we need to repopulate it. */ static u32 ixgbe_shadow_vfta[IXGBE_VFTA_SIZE]; /* ** The number of scatter-gather segments ** differs for 82598 and 82599, default to ** the former. */ static int ixgbe_num_segs = IXGBE_82598_SCATTER; #ifdef IXGBE_FDIR /* ** For Flow Director: this is the ** number of TX packets we sample ** for the filter pool, this means ** every 20th packet will be probed. ** ** This feature can be disabled by ** setting this to 0. */ static int atr_sample_rate = 20; /* ** Flow Director actually 'steals' ** part of the packet buffer as its ** filter pool, this variable controls ** how much it uses: ** 0 = 64K, 1 = 128K, 2 = 256K */ static int fdir_pballoc = 1; #endif /********************************************************************* * Device identification routine * * ixgbe_probe determines if the driver should be loaded on * adapter based on PCI vendor/device id of the adapter. * * return 0 on success, positive on failure *********************************************************************/ static int ixgbe_probe(device_t dev) { ixgbe_vendor_info_t *ent; u16 pci_vendor_id = 0; u16 pci_device_id = 0; u16 pci_subvendor_id = 0; u16 pci_subdevice_id = 0; char adapter_name[256]; INIT_DEBUGOUT("ixgbe_probe: begin"); pci_vendor_id = pci_get_vendor(dev); if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) return (ENXIO); pci_device_id = pci_get_device(dev); pci_subvendor_id = pci_get_subvendor(dev); pci_subdevice_id = pci_get_subdevice(dev); ent = ixgbe_vendor_info_array; while (ent->vendor_id != 0) { if ((pci_vendor_id == ent->vendor_id) && (pci_device_id == ent->device_id) && ((pci_subvendor_id == ent->subvendor_id) || (ent->subvendor_id == 0)) && ((pci_subdevice_id == ent->subdevice_id) || (ent->subdevice_id == 0))) { sprintf(adapter_name, "%s, Version - %s", ixgbe_strings[ent->index], ixgbe_driver_version); device_set_desc_copy(dev, adapter_name); ++ixgbe_total_ports; return (0); } ent++; } return (ENXIO); } /********************************************************************* * Device initialization routine * * The attach entry point is called when the driver is being loaded. * This routine identifies the type of hardware, allocates all resources * and initializes the hardware. * * return 0 on success, positive on failure *********************************************************************/ static int ixgbe_attach(device_t dev) { struct adapter *adapter; struct ixgbe_hw *hw; int error = 0; u16 pci_device_id, csum; u32 ctrl_ext; INIT_DEBUGOUT("ixgbe_attach: begin"); /* Allocate, clear, and link in our adapter structure */ adapter = device_get_softc(dev); adapter->dev = adapter->osdep.dev = dev; hw = &adapter->hw; /* Core Lock Init*/ IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); /* Keep track of optics */ pci_device_id = pci_get_device(dev); switch (pci_device_id) { case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : case IXGBE_DEV_ID_82598EB_CX4 : adapter->optics = IFM_10G_CX4; break; case IXGBE_DEV_ID_82598 : case IXGBE_DEV_ID_82598AF_DUAL_PORT : case IXGBE_DEV_ID_82598_DA_DUAL_PORT : case IXGBE_DEV_ID_82598AF_SINGLE_PORT : case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM : case IXGBE_DEV_ID_82598EB_SFP_LOM : case IXGBE_DEV_ID_82598AT : case IXGBE_DEV_ID_82598AT2 : adapter->optics = IFM_10G_SR; break; case IXGBE_DEV_ID_82598EB_XF_LR : adapter->optics = IFM_10G_LR; break; case IXGBE_DEV_ID_82599_SFP : adapter->optics = IFM_10G_SR; ixgbe_num_segs = IXGBE_82599_SCATTER; break; case IXGBE_DEV_ID_82599_KX4 : case IXGBE_DEV_ID_82599_KX4_MEZZ: case IXGBE_DEV_ID_82599_CX4 : adapter->optics = IFM_10G_CX4; ixgbe_num_segs = IXGBE_82599_SCATTER; break; case IXGBE_DEV_ID_82599_XAUI_LOM : case IXGBE_DEV_ID_82599_COMBO_BACKPLANE : ixgbe_num_segs = IXGBE_82599_SCATTER; default: break; } /* SYSCTL APIs */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_stats, "I", "Statistics"); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_sysctl_debug, "I", "Debug Info"); SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control"); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_enable_aim, 1, "Interrupt Moderation"); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_low_latency, 1, "Low Latency"); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_ave_latency, 1, "Average Latency"); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_bulk_latency, 1, "Bulk Latency"); /* Set up the timer callout */ callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); /* Determine hardware revision */ ixgbe_identify_hardware(adapter); /* Do base PCI setup - map BAR0 */ if (ixgbe_allocate_pci_resources(adapter)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_out; } /* Do descriptor calc and sanity checks */ if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { device_printf(dev, "TXD config issue, using default!\n"); adapter->num_tx_desc = DEFAULT_TXD; } else adapter->num_tx_desc = ixgbe_txd; /* ** With many RX rings it is easy to exceed the ** system mbuf allocation. Tuning nmbclusters ** can alleviate this. */ if (nmbclusters > 0 ) { int s; s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; if (s > nmbclusters) { device_printf(dev, "RX Descriptors exceed " "system mbuf max, using default instead!\n"); ixgbe_rxd = DEFAULT_RXD; } } if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { device_printf(dev, "RXD config issue, using default!\n"); adapter->num_rx_desc = DEFAULT_RXD; } else adapter->num_rx_desc = ixgbe_rxd; /* Allocate our TX/RX Queues */ if (ixgbe_allocate_queues(adapter)) { error = ENOMEM; goto err_out; } /* Initialize the shared code */ error = ixgbe_init_shared_code(hw); if (error == IXGBE_ERR_SFP_NOT_PRESENT) { /* ** No optics in this port, set up ** so the timer routine will probe ** for later insertion. */ adapter->sfp_probe = TRUE; error = 0; } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { device_printf(dev,"Unsupported SFP+ module detected!\n"); error = EIO; goto err_late; } else if (error) { device_printf(dev,"Unable to initialize the shared code\n"); error = EIO; goto err_late; } /* Make sure we have a good EEPROM before we read from it */ if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { device_printf(dev,"The EEPROM Checksum Is Not Valid\n"); error = EIO; goto err_late; } /* Pick up the smart speed setting */ if (hw->mac.type == ixgbe_mac_82599EB) hw->phy.smart_speed = ixgbe_smart_speed; /* Get Hardware Flow Control setting */ hw->fc.requested_mode = ixgbe_fc_full; hw->fc.pause_time = IXGBE_FC_PAUSE; hw->fc.low_water = IXGBE_FC_LO; hw->fc.high_water = IXGBE_FC_HI; hw->fc.send_xon = TRUE; error = ixgbe_init_hw(hw); if (error == IXGBE_ERR_EEPROM_VERSION) { device_printf(dev, "This device is a pre-production adapter/" "LOM. Please be aware there may be issues associated " "with your hardware.\n If you are experiencing problems " "please contact your Intel or hardware representative " "who provided you with this hardware.\n"); } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) device_printf(dev,"Unsupported SFP+ Module\n"); if (error) { error = EIO; device_printf(dev,"Hardware Initialization Failure\n"); goto err_late; } if ((adapter->msix > 1) && (ixgbe_enable_msix)) error = ixgbe_allocate_msix(adapter); else error = ixgbe_allocate_legacy(adapter); if (error) goto err_late; /* Setup OS specific network interface */ ixgbe_setup_interface(dev, adapter); #ifdef IXGBE_IEEE1588 /* ** Setup the timer: IEEE 1588 support */ adapter->cycles.read = ixgbe_read_clock; adapter->cycles.mask = (u64)-1; adapter->cycles.mult = 1; adapter->cycles.shift = IXGBE_TSYNC_SHIFT; IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, (1<<24) | IXGBE_TSYNC_CYCLE_TIME * IXGBE_TSYNC_SHIFT); IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x00000000); IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0xFF800000); // JFV - this is not complete yet #endif /* Sysctl for limiting the amount of work done in the taskqueue */ ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, ixgbe_rx_process_limit); /* Initialize statistics */ ixgbe_update_stats_counters(adapter); /* Register for VLAN events */ adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); /* Print PCIE bus type/speed/width info */ ixgbe_get_bus_info(hw); device_printf(dev,"PCI Express Bus: Speed %s %s\n", ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : ("Unknown")); if (hw->bus.width <= ixgbe_bus_width_pcie_x4) { device_printf(dev, "PCI-Express bandwidth available" " for this card\n is not sufficient for" " optimal performance.\n"); device_printf(dev, "For optimal performance a x8 " "PCI-Express slot is required.\n"); } /* let hardware know driver is loaded */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); INIT_DEBUGOUT("ixgbe_attach: end"); return (0); err_late: ixgbe_free_transmit_structures(adapter); ixgbe_free_receive_structures(adapter); err_out: ixgbe_free_pci_resources(adapter); return (error); } /********************************************************************* * Device removal routine * * The detach entry point is called when the driver is being removed. * This routine stops the adapter and deallocates all the resources * that were allocated for driver operation. * * return 0 on success, positive on failure *********************************************************************/ static int ixgbe_detach(device_t dev) { struct adapter *adapter = device_get_softc(dev); struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; u32 ctrl_ext; INIT_DEBUGOUT("ixgbe_detach: begin"); /* Make sure VLANS are not using driver */ if (adapter->ifp->if_vlantrunk != NULL) { device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } IXGBE_CORE_LOCK(adapter); ixgbe_stop(adapter); IXGBE_CORE_UNLOCK(adapter); for (int i = 0; i < adapter->num_queues; i++, txr++) { if (txr->tq) { taskqueue_drain(txr->tq, &txr->tx_task); taskqueue_free(txr->tq); } } for (int i = 0; i < adapter->num_queues; i++, rxr++) { if (rxr->tq) { taskqueue_drain(rxr->tq, &rxr->rx_task); taskqueue_free(rxr->tq); } } /* Drain the Link queue */ if (adapter->tq) { taskqueue_drain(adapter->tq, &adapter->link_task); taskqueue_drain(adapter->tq, &adapter->mod_task); taskqueue_drain(adapter->tq, &adapter->msf_task); #ifdef IXGBE_FDIR taskqueue_drain(adapter->tq, &adapter->fdir_task); #endif taskqueue_free(adapter->tq); } /* let hardware know driver is unloading */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); /* Unregister VLAN events */ if (adapter->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); if (adapter->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); ixgbe_free_pci_resources(adapter); bus_generic_detach(dev); if_free(adapter->ifp); ixgbe_free_transmit_structures(adapter); ixgbe_free_receive_structures(adapter); IXGBE_CORE_LOCK_DESTROY(adapter); return (0); } /********************************************************************* * * Shutdown entry point * **********************************************************************/ static int ixgbe_shutdown(device_t dev) { struct adapter *adapter = device_get_softc(dev); IXGBE_CORE_LOCK(adapter); ixgbe_stop(adapter); IXGBE_CORE_UNLOCK(adapter); return (0); } /********************************************************************* * Transmit entry point * * ixgbe_start is called by the stack to initiate a transmit. * The driver will remain in this routine as long as there are * packets to transmit and transmit resources are available. * In case resources are not available stack is notified and * the packet is requeued. **********************************************************************/ static void ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) { struct mbuf *m_head; struct adapter *adapter = txr->adapter; IXGBE_TX_LOCK_ASSERT(txr); if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; if (!adapter->link_active) return; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; if (ixgbe_xmit(txr, &m_head)) { if (m_head == NULL) break; ifp->if_drv_flags |= IFF_DRV_OACTIVE; IFQ_DRV_PREPEND(&ifp->if_snd, m_head); break; } /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m_head); /* Set watchdog on */ txr->watchdog_check = TRUE; } return; } /* * Legacy TX start - called by the stack, this * always uses the first tx ring, and should * not be used with multiqueue tx enabled. */ static void ixgbe_start(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { IXGBE_TX_LOCK(txr); ixgbe_start_locked(txr, ifp); IXGBE_TX_UNLOCK(txr); } return; } #if __FreeBSD_version >= 800000 /* ** Multiqueue Transmit driver ** */ static int ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr; int i = 0, err = 0; /* Which queue to use */ if ((m->m_flags & M_FLOWID) != 0) i = m->m_pkthdr.flowid % adapter->num_queues; txr = &adapter->tx_rings[i]; if (IXGBE_TX_TRYLOCK(txr)) { err = ixgbe_mq_start_locked(ifp, txr, m); IXGBE_TX_UNLOCK(txr); } else err = drbr_enqueue(ifp, txr->br, m); return (err); } static int ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) { struct adapter *adapter = txr->adapter; struct mbuf *next; int err = 0; if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || (!adapter->link_active)) { err = drbr_enqueue(ifp, txr->br, m); return (err); } if (m == NULL) /* Called by tasklet */ goto process; /* If nothing queued go right to xmit */ - if (drbr_empty(ifp, txr->br)) { + if (!drbr_needs_enqueue(ifp, txr->br)) { if ((err = ixgbe_xmit(txr, &m)) != 0) { if (m != NULL) err = drbr_enqueue(ifp, txr->br, m); return (err); } else { /* Success, update stats */ drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags); /* Send a copy of the frame to the BPF listener */ ETHER_BPF_MTAP(ifp, m); /* Set the watchdog */ txr->watchdog_check = TRUE; } } else if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) return (err); process: if (drbr_empty(ifp, txr->br)) return (err); /* Process the queue */ while (TRUE) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; next = drbr_dequeue(ifp, txr->br); if (next == NULL) break; if ((err = ixgbe_xmit(txr, &next)) != 0) { if (next != NULL) err = drbr_enqueue(ifp, txr->br, next); break; } drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); ETHER_BPF_MTAP(ifp, next); /* Set the watchdog */ txr->watchdog_check = TRUE; } if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) ifp->if_drv_flags |= IFF_DRV_OACTIVE; return (err); } /* ** Flush all ring buffers */ static void ixgbe_qflush(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; struct mbuf *m; for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_TX_LOCK(txr); while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) m_freem(m); IXGBE_TX_UNLOCK(txr); } if_qflush(ifp); } #endif /* __FreeBSD_version >= 800000 */ /********************************************************************* * Ioctl entry point * * ixgbe_ioctl is called when the user wants to configure the * interface. * * return 0 on success, positive on failure **********************************************************************/ static int ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) { struct adapter *adapter = ifp->if_softc; struct ifreq *ifr = (struct ifreq *) data; int error = 0; switch (command) { case SIOCSIFMTU: IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) { error = EINVAL; } else { IXGBE_CORE_LOCK(adapter); ifp->if_mtu = ifr->ifr_mtu; adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; ixgbe_init_locked(adapter); IXGBE_CORE_UNLOCK(adapter); } break; case SIOCSIFFLAGS: IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); IXGBE_CORE_LOCK(adapter); if (ifp->if_flags & IFF_UP) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { if ((ifp->if_flags ^ adapter->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { ixgbe_disable_promisc(adapter); ixgbe_set_promisc(adapter); } } else ixgbe_init_locked(adapter); } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) ixgbe_stop(adapter); adapter->if_flags = ifp->if_flags; IXGBE_CORE_UNLOCK(adapter); break; case SIOCADDMULTI: case SIOCDELMULTI: IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { IXGBE_CORE_LOCK(adapter); ixgbe_disable_intr(adapter); ixgbe_set_multi(adapter); ixgbe_enable_intr(adapter); IXGBE_CORE_UNLOCK(adapter); } break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); break; case SIOCSIFCAP: { int mask = ifr->ifr_reqcap ^ ifp->if_capenable; IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); if (mask & IFCAP_HWCSUM) ifp->if_capenable ^= IFCAP_HWCSUM; if (mask & IFCAP_TSO4) ifp->if_capenable ^= IFCAP_TSO4; /* Only allow changing when using header split */ if ((mask & IFCAP_LRO) && (ixgbe_header_split)) ifp->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_VLAN_HWTAGGING) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (ifp->if_drv_flags & IFF_DRV_RUNNING) ixgbe_init(adapter); VLAN_CAPABILITIES(ifp); break; } #ifdef IXGBE_IEEE1588 /* ** IOCTL support for Precision Time (IEEE 1588) Support */ case SIOCSHWTSTAMP: error = ixgbe_hwtstamp_ioctl(adapter, ifp); break; #endif default: IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); error = ether_ioctl(ifp, command, data); break; } return (error); } /********************************************************************* * Init entry point * * This routine is used in two ways. It is used by the stack as * init entry point in network interface structure. It is also used * by the driver as a hw/sw initialization routine to get to a * consistent state. * * return 0 on success, positive on failure **********************************************************************/ #define IXGBE_MHADD_MFS_SHIFT 16 static void ixgbe_init_locked(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; struct ixgbe_hw *hw; u32 k, txdctl, mhadd, gpie; u32 rxdctl, rxctrl; int err; INIT_DEBUGOUT("ixgbe_init: begin"); hw = &adapter->hw; mtx_assert(&adapter->core_mtx, MA_OWNED); /* Get the latest mac address, User can use a LAA */ bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, IXGBE_ETH_LENGTH_OF_ADDRESS); ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); hw->addr_ctrl.rar_used_count = 1; /* Do a warm reset */ ixgbe_reset_hw(hw); /* Prepare transmit descriptors and buffers */ if (ixgbe_setup_transmit_structures(adapter)) { device_printf(dev,"Could not setup transmit structures\n"); ixgbe_stop(adapter); return; } ixgbe_initialize_transmit_units(adapter); /* Setup Multicast table */ ixgbe_set_multi(adapter); /* ** Determine the correct mbuf pool ** for doing jumbo/headersplit */ if (ifp->if_mtu > ETHERMTU) adapter->rx_mbuf_sz = MJUMPAGESIZE; else adapter->rx_mbuf_sz = MCLBYTES; /* Prepare receive descriptors and buffers */ if (ixgbe_setup_receive_structures(adapter)) { device_printf(dev,"Could not setup receive structures\n"); ixgbe_stop(adapter); return; } /* Configure RX settings */ ixgbe_initialize_receive_units(adapter); /* Configure Interrupt Moderation */ ixgbe_init_moderation(adapter); gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); if (hw->mac.type == ixgbe_mac_82599EB) { gpie |= IXGBE_SDP1_GPIEN; gpie |= IXGBE_SDP2_GPIEN; } /* Enable Fan Failure Interrupt */ if (hw->device_id == IXGBE_DEV_ID_82598AT) gpie |= IXGBE_SDP1_GPIEN; if (adapter->msix > 1) { /* Enable Enhanced MSIX mode */ gpie |= IXGBE_GPIE_MSIX_MODE; gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; } IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); /* Set the various hardware offload abilities */ ifp->if_hwassist = 0; if (ifp->if_capenable & IFCAP_TSO4) ifp->if_hwassist |= CSUM_TSO; if (ifp->if_capenable & IFCAP_TXCSUM) ifp->if_hwassist = (CSUM_TCP | CSUM_UDP); /* Set MTU size */ if (ifp->if_mtu > ETHERMTU) { mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); mhadd &= ~IXGBE_MHADD_MFS_MASK; mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); } /* Now enable all the queues */ for (int i = 0; i < adapter->num_queues; i++) { txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); txdctl |= IXGBE_TXDCTL_ENABLE; /* Set WTHRESH to 8, burst writeback */ txdctl |= (8 << 16); IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); } for (int i = 0; i < adapter->num_queues; i++) { rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); if (hw->mac.type == ixgbe_mac_82598EB) { /* ** PTHRESH = 21 ** HTHRESH = 4 ** WTHRESH = 8 */ rxdctl &= ~0x3FFFFF; rxdctl |= 0x080420; } rxdctl |= IXGBE_RXDCTL_ENABLE; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); for (k = 0; k < 10; k++) { if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & IXGBE_RXDCTL_ENABLE) break; else msec_delay(1); } wmb(); IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); } /* Set up VLAN offloads and filter */ ixgbe_setup_vlan_hw_support(adapter); /* Enable Receive engine */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); if (hw->mac.type == ixgbe_mac_82598EB) rxctrl |= IXGBE_RXCTRL_DMBYPS; rxctrl |= IXGBE_RXCTRL_RXEN; IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl); callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); /* Set up MSI/X routing */ if (ixgbe_enable_msix) ixgbe_configure_ivars(adapter); else { /* Simple settings for Legacy/MSI */ ixgbe_set_ivar(adapter, 0, 0, 0); ixgbe_set_ivar(adapter, 0, 0, 1); } #ifdef IXGBE_FDIR /* Init Flow director */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); #endif /* ** Check on any SFP devices that ** need to be kick-started */ err = hw->phy.ops.identify(hw); if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { device_printf(dev, "Unsupported SFP+ module type was detected.\n"); ixgbe_detach(dev); return; } /* Config/Enable Link */ ixgbe_config_link(adapter); /* And now turn on interrupts */ ixgbe_enable_intr(adapter); /* Now inform the stack we're ready */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; return; } static void ixgbe_init(void *arg) { struct adapter *adapter = arg; IXGBE_CORE_LOCK(adapter); ixgbe_init_locked(adapter); IXGBE_CORE_UNLOCK(adapter); return; } /* ** ** MSIX Interrupt Handlers and Tasklets ** */ static inline void ixgbe_enable_queue(struct adapter *adapter, u32 vector) { struct ixgbe_hw *hw = &adapter->hw; u64 queue = (u64)(1 << vector); u32 mask; if (hw->mac.type == ixgbe_mac_82598EB) { mask = (IXGBE_EIMS_RTX_QUEUE & queue); IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); } else { mask = (queue & 0xFFFFFFFF); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); mask = (queue >> 32); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); } } static inline void ixgbe_disable_queue(struct adapter *adapter, u32 vector) { struct ixgbe_hw *hw = &adapter->hw; u64 queue = (u64)(1 << vector); u32 mask; if (hw->mac.type == ixgbe_mac_82598EB) { mask = (IXGBE_EIMS_RTX_QUEUE & queue); IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); } else { mask = (queue & 0xFFFFFFFF); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); mask = (queue >> 32); if (mask) IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); } } static inline void ixgbe_rearm_queues(struct adapter *adapter, u64 queues) { u32 mask; if (adapter->hw.mac.type == ixgbe_mac_82598EB) { mask = (IXGBE_EIMS_RTX_QUEUE & queues); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); } else { mask = (queues & 0xFFFFFFFF); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); mask = (queues >> 32); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); } } static void ixgbe_handle_rx(void *context, int pending) { struct rx_ring *rxr = context; struct adapter *adapter = rxr->adapter; u32 loop = MAX_LOOP; bool more; do { more = ixgbe_rxeof(rxr, -1); } while (loop-- && more); /* Reenable this interrupt */ ixgbe_enable_queue(adapter, rxr->msix); } static void ixgbe_handle_tx(void *context, int pending) { struct tx_ring *txr = context; struct adapter *adapter = txr->adapter; struct ifnet *ifp = adapter->ifp; u32 loop = MAX_LOOP; bool more; IXGBE_TX_LOCK(txr); do { more = ixgbe_txeof(txr); } while (loop-- && more); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { #if __FreeBSD_version >= 800000 if (!drbr_empty(ifp, txr->br)) ixgbe_mq_start_locked(ifp, txr, NULL); #else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) ixgbe_start_locked(txr, ifp); #endif } IXGBE_TX_UNLOCK(txr); /* Reenable this interrupt */ ixgbe_enable_queue(adapter, txr->msix); } /********************************************************************* * * Legacy Interrupt Service routine * **********************************************************************/ static void ixgbe_legacy_irq(void *arg) { struct adapter *adapter = arg; struct ixgbe_hw *hw = &adapter->hw; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; bool more; u32 reg_eicr, loop = MAX_LOOP; reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); if (reg_eicr == 0) { ixgbe_enable_intr(adapter); return; } if (ixgbe_rxeof(rxr, adapter->rx_process_limit)) taskqueue_enqueue(rxr->tq, &rxr->rx_task); IXGBE_TX_LOCK(txr); ++txr->tx_irq; do { more = ixgbe_txeof(txr); } while (loop-- && more); IXGBE_TX_UNLOCK(txr); if (more) taskqueue_enqueue(txr->tq, &txr->tx_task); /* Check for fan failure */ if ((hw->phy.media_type == ixgbe_media_type_copper) && (reg_eicr & IXGBE_EICR_GPI_SDP1)) { device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " "REPLACE IMMEDIATELY!!\n"); IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); } /* Link status change */ if (reg_eicr & IXGBE_EICR_LSC) { ixgbe_check_link(&adapter->hw, &adapter->link_speed, &adapter->link_up, 0); ixgbe_update_link_status(adapter); } /* Update interrupt rate */ if (ixgbe_enable_aim == TRUE) ixgbe_update_aim(rxr); ixgbe_enable_intr(adapter); return; } /********************************************************************* * * MSI TX Interrupt Service routine * **********************************************************************/ void ixgbe_msix_tx(void *arg) { struct tx_ring *txr = arg; struct adapter *adapter = txr->adapter; bool more; ixgbe_disable_queue(adapter, txr->msix); IXGBE_TX_LOCK(txr); ++txr->tx_irq; more = ixgbe_txeof(txr); IXGBE_TX_UNLOCK(txr); if (more) taskqueue_enqueue(txr->tq, &txr->tx_task); else /* Reenable this interrupt */ ixgbe_enable_queue(adapter, txr->msix); return; } /********************************************************************* * * MSIX RX Interrupt Service routine * **********************************************************************/ static void ixgbe_msix_rx(void *arg) { struct rx_ring *rxr = arg; struct adapter *adapter = rxr->adapter; bool more; ixgbe_disable_queue(adapter, rxr->msix); ++rxr->rx_irq; more = ixgbe_rxeof(rxr, adapter->rx_process_limit); /* Update interrupt rate */ if (ixgbe_enable_aim == TRUE) ixgbe_update_aim(rxr); if (more) taskqueue_enqueue(rxr->tq, &rxr->rx_task); else ixgbe_enable_queue(adapter, rxr->msix); return; } static void ixgbe_msix_link(void *arg) { struct adapter *adapter = arg; struct ixgbe_hw *hw = &adapter->hw; u32 reg_eicr; ++adapter->link_irq; /* First get the cause */ reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); /* Clear interrupt with write */ IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); /* Link status change */ if (reg_eicr & IXGBE_EICR_LSC) taskqueue_enqueue(adapter->tq, &adapter->link_task); if (adapter->hw.mac.type == ixgbe_mac_82599EB) { #ifdef IXGBE_FDIR if (reg_eicr & IXGBE_EICR_FLOW_DIR) { /* This is probably overkill :) */ if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) return; /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); /* Turn off the interface */ adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; taskqueue_enqueue(adapter->tq, &adapter->fdir_task); } else #endif if (reg_eicr & IXGBE_EICR_ECC) { device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " "Please Reboot!!\n"); IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); } else if (reg_eicr & IXGBE_EICR_GPI_SDP1) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); taskqueue_enqueue(adapter->tq, &adapter->msf_task); } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); taskqueue_enqueue(adapter->tq, &adapter->mod_task); } } /* Check for fan failure */ if ((hw->device_id == IXGBE_DEV_ID_82598AT) && (reg_eicr & IXGBE_EICR_GPI_SDP1)) { device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " "REPLACE IMMEDIATELY!!\n"); IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); } IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); return; } /* ** Routine to do adjust the RX EITR value based on traffic, ** its a simple three state model, but seems to help. ** ** Note that the three EITR values are tuneable using ** sysctl in real time. The feature can be effectively ** nullified by setting them equal. */ #define BULK_THRESHOLD 10000 #define AVE_THRESHOLD 1600 static void ixgbe_update_aim(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; u32 olditr, newitr; /* Update interrupt moderation based on traffic */ olditr = rxr->eitr_setting; newitr = olditr; /* Idle, don't change setting */ if (rxr->bytes == 0) return; if (olditr == ixgbe_low_latency) { if (rxr->bytes > AVE_THRESHOLD) newitr = ixgbe_ave_latency; } else if (olditr == ixgbe_ave_latency) { if (rxr->bytes < AVE_THRESHOLD) newitr = ixgbe_low_latency; else if (rxr->bytes > BULK_THRESHOLD) newitr = ixgbe_bulk_latency; } else if (olditr == ixgbe_bulk_latency) { if (rxr->bytes < BULK_THRESHOLD) newitr = ixgbe_ave_latency; } if (olditr != newitr) { /* Change interrupt rate */ rxr->eitr_setting = newitr; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->me), newitr | (newitr << 16)); } rxr->bytes = 0; return; } static void ixgbe_init_moderation(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; struct tx_ring *txr = adapter->tx_rings; /* Single interrupt - MSI or Legacy? */ if (adapter->msix < 2) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(0), 100); return; } /* TX irq moderation rate is fixed */ for (int i = 0; i < adapter->num_queues; i++, txr++) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(txr->msix), ixgbe_ave_latency); /* RX moderation will be adapted over time, set default */ for (int i = 0; i < adapter->num_queues; i++, rxr++) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->msix), ixgbe_low_latency); } /* Set Link moderation */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); } /********************************************************************* * * Media Ioctl callback * * This routine is called whenever the user queries the status of * the interface using ifconfig. * **********************************************************************/ static void ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) { struct adapter *adapter = ifp->if_softc; INIT_DEBUGOUT("ixgbe_media_status: begin"); IXGBE_CORE_LOCK(adapter); ixgbe_update_link_status(adapter); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { IXGBE_CORE_UNLOCK(adapter); return; } ifmr->ifm_status |= IFM_ACTIVE; switch (adapter->link_speed) { case IXGBE_LINK_SPEED_1GB_FULL: ifmr->ifm_active |= IFM_1000_T | IFM_FDX; break; case IXGBE_LINK_SPEED_10GB_FULL: ifmr->ifm_active |= adapter->optics | IFM_FDX; break; } IXGBE_CORE_UNLOCK(adapter); return; } /********************************************************************* * * Media Ioctl callback * * This routine is called when the user changes speed/duplex using * media/mediopt option with ifconfig. * **********************************************************************/ static int ixgbe_media_change(struct ifnet * ifp) { struct adapter *adapter = ifp->if_softc; struct ifmedia *ifm = &adapter->media; INIT_DEBUGOUT("ixgbe_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.phy.autoneg_advertised = IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; break; default: device_printf(adapter->dev, "Only auto media type\n"); return (EINVAL); } return (0); } /********************************************************************* * * This routine maps the mbufs to tx descriptors. * WARNING: while this code is using an MQ style infrastructure, * it would NOT work as is with more than 1 queue. * * return 0 on success, positive on failure **********************************************************************/ static int ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) { struct adapter *adapter = txr->adapter; u32 olinfo_status = 0, cmd_type_len; u32 paylen = 0; int i, j, error, nsegs; int first, last = 0; struct mbuf *m_head; bus_dma_segment_t segs[ixgbe_num_segs]; bus_dmamap_t map; struct ixgbe_tx_buf *txbuf, *txbuf_mapped; union ixgbe_adv_tx_desc *txd = NULL; m_head = *m_headp; /* Basic descriptor defines */ cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); if (m_head->m_flags & M_VLANTAG) cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; /* Do a clean if descriptors are low */ if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { ixgbe_txeof(txr); /* Now do we at least have a minimal? */ if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) return (ENOBUFS); } /* * Important to capture the first descriptor * used because it will contain the index of * the one we tell the hardware to report back */ first = txr->next_avail_desc; txbuf = &txr->tx_buffers[first]; txbuf_mapped = txbuf; map = txbuf->map; /* * Map the packet for DMA. */ error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == EFBIG) { struct mbuf *m; m = m_defrag(*m_headp, M_DONTWAIT); if (m == NULL) { adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); } *m_headp = m; /* Try it again */ error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == ENOMEM) { adapter->no_tx_dma_setup++; return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } } else if (error == ENOMEM) { adapter->no_tx_dma_setup++; return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } /* Make certain there are enough descriptors */ if (nsegs > txr->tx_avail - 2) { txr->no_tx_desc_avail++; error = ENOBUFS; goto xmit_fail; } m_head = *m_headp; /* ** Set up the appropriate offload context ** this becomes the first descriptor of ** a packet. */ if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { if (ixgbe_tso_setup(txr, m_head, &paylen)) { cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; ++adapter->tso_tx; } else return (ENXIO); } else if (ixgbe_tx_ctx_setup(txr, m_head)) olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; #ifdef IXGBE_IEEE1588 /* This is changing soon to an mtag detection */ if (we detect this mbuf has a TSTAMP mtag) cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; #endif #ifdef IXGBE_FDIR /* Do the flow director magic */ if ((txr->atr_sample) && (!adapter->fdir_reinit)) { ++txr->atr_count; if (txr->atr_count >= atr_sample_rate) { ixgbe_atr(txr, m_head); txr->atr_count = 0; } } #endif /* Record payload length */ if (paylen == 0) olinfo_status |= m_head->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; i = txr->next_avail_desc; for (j = 0; j < nsegs; j++) { bus_size_t seglen; bus_addr_t segaddr; txbuf = &txr->tx_buffers[i]; txd = &txr->tx_base[i]; seglen = segs[j].ds_len; segaddr = htole64(segs[j].ds_addr); txd->read.buffer_addr = segaddr; txd->read.cmd_type_len = htole32(txr->txd_cmd | cmd_type_len |seglen); txd->read.olinfo_status = htole32(olinfo_status); last = i; /* Next descriptor that will get completed */ if (++i == adapter->num_tx_desc) i = 0; txbuf->m_head = NULL; txbuf->eop_index = -1; } txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); txr->tx_avail -= nsegs; txr->next_avail_desc = i; txbuf->m_head = m_head; txbuf->map = map; bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); /* Set the index of the descriptor that will be marked done */ txbuf = &txr->tx_buffers[first]; txbuf->eop_index = last; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* * Advance the Transmit Descriptor Tail (Tdt), this tells the * hardware that this frame is available to transmit. */ ++txr->total_packets; IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); return (0); xmit_fail: bus_dmamap_unload(txr->txtag, txbuf->map); return (error); } static void ixgbe_set_promisc(struct adapter *adapter) { u_int32_t reg_rctl; struct ifnet *ifp = adapter->ifp; reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); if (ifp->if_flags & IFF_PROMISC) { reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); } else if (ifp->if_flags & IFF_ALLMULTI) { reg_rctl |= IXGBE_FCTRL_MPE; reg_rctl &= ~IXGBE_FCTRL_UPE; IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); } return; } static void ixgbe_disable_promisc(struct adapter * adapter) { u_int32_t reg_rctl; reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); reg_rctl &= (~IXGBE_FCTRL_UPE); reg_rctl &= (~IXGBE_FCTRL_MPE); IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); return; } /********************************************************************* * Multicast Update * * This routine is called whenever multicast address list is updated. * **********************************************************************/ #define IXGBE_RAR_ENTRIES 16 static void ixgbe_set_multi(struct adapter *adapter) { u32 fctrl; u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; u8 *update_ptr; struct ifmultiaddr *ifma; int mcnt = 0; struct ifnet *ifp = adapter->ifp; IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); if (ifp->if_flags & IFF_PROMISC) fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); else if (ifp->if_flags & IFF_ALLMULTI) { fctrl |= IXGBE_FCTRL_MPE; fctrl &= ~IXGBE_FCTRL_UPE; } else fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); #if __FreeBSD_version < 800000 IF_ADDR_LOCK(ifp); #else if_maddr_rlock(ifp); #endif TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], IXGBE_ETH_LENGTH_OF_ADDRESS); mcnt++; } #if __FreeBSD_version < 800000 IF_ADDR_UNLOCK(ifp); #else if_maddr_runlock(ifp); #endif update_ptr = mta; ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, ixgbe_mc_array_itr); return; } /* * This is an iterator function now needed by the multicast * shared code. It simply feeds the shared code routine the * addresses in the array of ixgbe_set_multi() one by one. */ static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) { u8 *addr = *update_ptr; u8 *newptr; *vmdq = 0; newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; *update_ptr = newptr; return addr; } /********************************************************************* * Timer routine * * This routine checks for link status,updates statistics, * and runs the watchdog check. * **********************************************************************/ static void ixgbe_local_timer(void *arg) { struct adapter *adapter = arg; struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; bool tx_hung = FALSE; mtx_assert(&adapter->core_mtx, MA_OWNED); /* Check for pluggable optics */ if (adapter->sfp_probe) if (!ixgbe_sfp_probe(adapter)) goto out; /* Nothing to do */ ixgbe_update_link_status(adapter); ixgbe_update_stats_counters(adapter); if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) { ixgbe_print_hw_stats(adapter); } /* ** Check for time since any descriptor was cleaned */ for (int i = 0; i < adapter->num_queues; i++, txr++) { if (txr->watchdog_check == FALSE) continue; if ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG) { tx_hung = TRUE; goto hung; } } out: callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); return; hung: device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); device_printf(dev,"TX(%d) desc avail = %d," "Next TX to Clean = %d\n", txr->me, txr->tx_avail, txr->next_to_clean); adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; adapter->watchdog_events++; ixgbe_init_locked(adapter); } /* ** Note: this routine updates the OS on the link state ** the real check of the hardware only happens with ** a link interrupt. */ static void ixgbe_update_link_status(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; struct tx_ring *txr = adapter->tx_rings; device_t dev = adapter->dev; if (adapter->link_up){ if (adapter->link_active == FALSE) { if (bootverbose) device_printf(dev,"Link is up %d Gbps %s \n", ((adapter->link_speed == 128)? 10:1), "Full Duplex"); adapter->link_active = TRUE; if_link_state_change(ifp, LINK_STATE_UP); } } else { /* Link down */ if (adapter->link_active == TRUE) { if (bootverbose) device_printf(dev,"Link is Down\n"); if_link_state_change(ifp, LINK_STATE_DOWN); adapter->link_active = FALSE; for (int i = 0; i < adapter->num_queues; i++, txr++) txr->watchdog_check = FALSE; } } return; } /********************************************************************* * * This routine disables all traffic on the adapter by issuing a * global reset on the MAC and deallocates TX/RX buffers. * **********************************************************************/ static void ixgbe_stop(void *arg) { struct ifnet *ifp; struct adapter *adapter = arg; ifp = adapter->ifp; mtx_assert(&adapter->core_mtx, MA_OWNED); INIT_DEBUGOUT("ixgbe_stop: begin\n"); ixgbe_disable_intr(adapter); /* Tell the stack that the interface is no longer active */ ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); ixgbe_reset_hw(&adapter->hw); adapter->hw.adapter_stopped = FALSE; ixgbe_stop_adapter(&adapter->hw); callout_stop(&adapter->timer); /* reprogram the RAR[0] in case user changed it. */ ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); return; } /********************************************************************* * * Determine hardware revision. * **********************************************************************/ static void ixgbe_identify_hardware(struct adapter *adapter) { device_t dev = adapter->dev; /* Save off the information about this board */ adapter->hw.vendor_id = pci_get_vendor(dev); adapter->hw.device_id = pci_get_device(dev); adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); adapter->hw.subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); adapter->hw.subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); return; } /********************************************************************* * * Setup the Legacy or MSI Interrupt handler * **********************************************************************/ static int ixgbe_allocate_legacy(struct adapter *adapter) { device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; int error, rid = 0; /* MSI RID at 1 */ if (adapter->msix == 1) rid = 1; /* We allocate a single interrupt resource */ adapter->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (adapter->res == NULL) { device_printf(dev, "Unable to allocate bus resource: " "interrupt\n"); return (ENXIO); } /* * Try allocating a fast interrupt and the associated deferred * processing contexts. */ TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, taskqueue_thread_enqueue, &txr->tq); rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, taskqueue_thread_enqueue, &rxr->tq); taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", device_get_nameunit(adapter->dev)); taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", device_get_nameunit(adapter->dev)); /* Tasklets for Link, SFP and Multispeed Fiber */ TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter); TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter); TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter); #ifdef IXGBE_FDIR TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter); #endif adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", device_get_nameunit(adapter->dev)); if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq, adapter, &adapter->tag)) != 0) { device_printf(dev, "Failed to register fast interrupt " "handler: %d\n", error); taskqueue_free(txr->tq); taskqueue_free(rxr->tq); txr->tq = NULL; rxr->tq = NULL; return (error); } return (0); } /********************************************************************* * * Setup MSIX Interrupt resources and handlers * **********************************************************************/ static int ixgbe_allocate_msix(struct adapter *adapter) { device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; int error, rid, vector = 0; /* TX setup: the code is here for multi tx, there are other parts of the driver not ready for it */ for (int i = 0; i < adapter->num_queues; i++, vector++, txr++) { rid = vector + 1; txr->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!txr->res) { device_printf(dev,"Unable to allocate" " bus resource: tx interrupt [%d]\n", vector); return (ENXIO); } /* Set the handler function */ error = bus_setup_intr(dev, txr->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_tx, txr, &txr->tag); if (error) { txr->res = NULL; device_printf(dev, "Failed to register TX handler"); return (error); } txr->msix = vector; /* ** Bind the msix vector, and thus the ** ring to the corresponding cpu. */ if (adapter->num_queues > 1) bus_bind_intr(dev, txr->res, i); TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, taskqueue_thread_enqueue, &txr->tq); taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", device_get_nameunit(adapter->dev)); } /* RX setup */ for (int i = 0; i < adapter->num_queues; i++, vector++, rxr++) { rid = vector + 1; rxr->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!rxr->res) { device_printf(dev,"Unable to allocate" " bus resource: rx interrupt [%d]," "rid = %d\n", i, rid); return (ENXIO); } /* Set the handler function */ error = bus_setup_intr(dev, rxr->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_rx, rxr, &rxr->tag); if (error) { rxr->res = NULL; device_printf(dev, "Failed to register RX handler"); return (error); } rxr->msix = vector; /* used in local timer */ adapter->rx_mask |= (u64)(1 << vector); /* ** Bind the msix vector, and thus the ** ring to the corresponding cpu. */ if (adapter->num_queues > 1) bus_bind_intr(dev, rxr->res, i); TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, taskqueue_thread_enqueue, &rxr->tq); taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", device_get_nameunit(adapter->dev)); } /* Now for Link changes */ rid = vector + 1; adapter->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!adapter->res) { device_printf(dev,"Unable to allocate" " bus resource: Link interrupt [%d]\n", rid); return (ENXIO); } /* Set the link handler function */ error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_link, adapter, &adapter->tag); if (error) { adapter->res = NULL; device_printf(dev, "Failed to register LINK handler"); return (error); } adapter->linkvec = vector; /* Tasklets for Link, SFP and Multispeed Fiber */ TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter); TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter); TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter); #ifdef IXGBE_FDIR TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter); #endif adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT, taskqueue_thread_enqueue, &adapter->tq); taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", device_get_nameunit(adapter->dev)); return (0); } /* * Setup Either MSI/X or MSI */ static int ixgbe_setup_msix(struct adapter *adapter) { device_t dev = adapter->dev; int rid, want, queues, msgs; /* Override by tuneable */ if (ixgbe_enable_msix == 0) goto msi; /* First try MSI/X */ rid = PCIR_BAR(MSIX_82598_BAR); adapter->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!adapter->msix_mem) { rid += 4; /* 82599 maps in higher BAR */ adapter->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); } if (!adapter->msix_mem) { /* May not be enabled */ device_printf(adapter->dev, "Unable to map MSIX table \n"); goto msi; } msgs = pci_msix_count(dev); if (msgs == 0) { /* system has msix disabled */ bus_release_resource(dev, SYS_RES_MEMORY, rid, adapter->msix_mem); adapter->msix_mem = NULL; goto msi; } /* Figure out a reasonable auto config value */ queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; if (ixgbe_num_queues == 0) ixgbe_num_queues = queues; /* ** Want two vectors (RX/TX) per queue ** plus an additional for Link. */ want = (ixgbe_num_queues * 2) + 1; if (msgs >= want) msgs = want; else { device_printf(adapter->dev, "MSIX Configuration Problem, " "%d vectors but %d queues wanted!\n", msgs, want); return (0); /* Will go to Legacy setup */ } if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { device_printf(adapter->dev, "Using MSIX interrupts with %d vectors\n", msgs); adapter->num_queues = ixgbe_num_queues; return (msgs); } msi: msgs = pci_msi_count(dev); if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) device_printf(adapter->dev,"Using MSI interrupt\n"); return (msgs); } static int ixgbe_allocate_pci_resources(struct adapter *adapter) { int rid; device_t dev = adapter->dev; rid = PCIR_BAR(0); adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!(adapter->pci_mem)) { device_printf(dev,"Unable to allocate bus resource: memory\n"); return (ENXIO); } adapter->osdep.mem_bus_space_tag = rman_get_bustag(adapter->pci_mem); adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->pci_mem); adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; /* Legacy defaults */ adapter->num_queues = 1; adapter->hw.back = &adapter->osdep; /* ** Now setup MSI or MSI/X, should ** return us the number of supported ** vectors. (Will be 1 for MSI) */ adapter->msix = ixgbe_setup_msix(adapter); return (0); } static void ixgbe_free_pci_resources(struct adapter * adapter) { struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; device_t dev = adapter->dev; int rid, memrid; if (adapter->hw.mac.type == ixgbe_mac_82598EB) memrid = PCIR_BAR(MSIX_82598_BAR); else memrid = PCIR_BAR(MSIX_82599_BAR); /* ** There is a slight possibility of a failure mode ** in attach that will result in entering this function ** before interrupt resources have been initialized, and ** in that case we do not want to execute the loops below ** We can detect this reliably by the state of the adapter ** res pointer. */ if (adapter->res == NULL) goto mem; /* ** Release all the interrupt resources: ** notice this is harmless for Legacy or ** MSI since pointers will always be NULL */ for (int i = 0; i < adapter->num_queues; i++, txr++) { rid = txr->msix + 1; if (txr->tag != NULL) { bus_teardown_intr(dev, txr->res, txr->tag); txr->tag = NULL; } if (txr->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, txr->res); } for (int i = 0; i < adapter->num_queues; i++, rxr++) { rid = rxr->msix + 1; if (rxr->tag != NULL) { bus_teardown_intr(dev, rxr->res, rxr->tag); rxr->tag = NULL; } if (rxr->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, rxr->res); } /* Clean the Legacy or Link interrupt last */ if (adapter->linkvec) /* we are doing MSIX */ rid = adapter->linkvec + 1; else (adapter->msix != 0) ? (rid = 1):(rid = 0); if (adapter->tag != NULL) { bus_teardown_intr(dev, adapter->res, adapter->tag); adapter->tag = NULL; } if (adapter->res != NULL) bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); mem: if (adapter->msix) pci_release_msi(dev); if (adapter->msix_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, memrid, adapter->msix_mem); if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), adapter->pci_mem); return; } /********************************************************************* * * Setup networking device structure and register an interface. * **********************************************************************/ static void ixgbe_setup_interface(device_t dev, struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct ifnet *ifp; INIT_DEBUGOUT("ixgbe_setup_interface: begin"); ifp = adapter->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_mtu = ETHERMTU; ifp->if_baudrate = 1000000000; ifp->if_init = ixgbe_init; ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = ixgbe_ioctl; ifp->if_start = ixgbe_start; #if __FreeBSD_version >= 800000 ifp->if_transmit = ixgbe_mq_start; ifp->if_qflush = ixgbe_qflush; #endif ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; ether_ifattach(ifp, adapter->hw.mac.addr); adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; /* * Tell the upper layer(s) we support long frames. */ ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; ifp->if_capabilities |= IFCAP_JUMBO_MTU; if (ixgbe_header_split) ifp->if_capabilities |= IFCAP_LRO; ifp->if_capenable = ifp->if_capabilities; /* * Specify the media types supported by this adapter and register * callbacks to update media and link information */ ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, ixgbe_media_status); ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics | IFM_FDX, 0, NULL); if (hw->device_id == IXGBE_DEV_ID_82598AT) { ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); return; } static void ixgbe_config_link(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 autoneg, err = 0; bool sfp, negotiate; switch (hw->phy.type) { case ixgbe_phy_sfp_avago: case ixgbe_phy_sfp_ftl: case ixgbe_phy_sfp_intel: case ixgbe_phy_sfp_unknown: case ixgbe_phy_tw_tyco: case ixgbe_phy_tw_unknown: sfp = TRUE; default: sfp = FALSE; } if (sfp) { if (hw->phy.multispeed_fiber) { hw->mac.ops.setup_sfp(hw); taskqueue_enqueue(adapter->tq, &adapter->msf_task); } else taskqueue_enqueue(adapter->tq, &adapter->mod_task); } else { if (hw->mac.ops.check_link) err = ixgbe_check_link(hw, &autoneg, &adapter->link_up, FALSE); if (err) goto out; autoneg = hw->phy.autoneg_advertised; if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) err = hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); if (err) goto out; if (hw->mac.ops.setup_link) err = hw->mac.ops.setup_link(hw, autoneg, negotiate, adapter->link_up); } out: return; } /******************************************************************** * Manage DMA'able memory. *******************************************************************/ static void ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) { if (error) return; *(bus_addr_t *) arg = segs->ds_addr; return; } static int ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size, struct ixgbe_dma_alloc *dma, int mapflags) { device_t dev = adapter->dev; int r; r = bus_dma_tag_create(NULL, /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ size, /* maxsize */ 1, /* nsegments */ size, /* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &dma->dma_tag); if (r != 0) { device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; " "error %u\n", r); goto fail_0; } r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, BUS_DMA_NOWAIT, &dma->dma_map); if (r != 0) { device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; " "error %u\n", r); goto fail_1; } r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, ixgbe_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); if (r != 0) { device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; " "error %u\n", r); goto fail_2; } dma->dma_size = size; return (0); fail_2: bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); fail_1: bus_dma_tag_destroy(dma->dma_tag); fail_0: dma->dma_map = NULL; dma->dma_tag = NULL; return (r); } static void ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) { bus_dmamap_sync(dma->dma_tag, dma->dma_map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->dma_tag, dma->dma_map); bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); bus_dma_tag_destroy(dma->dma_tag); } /********************************************************************* * * Allocate memory for the transmit and receive rings, and then * the descriptors associated with each, called only once at attach. * **********************************************************************/ static int ixgbe_allocate_queues(struct adapter *adapter) { device_t dev = adapter->dev; struct tx_ring *txr; struct rx_ring *rxr; int rsize, tsize, error = IXGBE_SUCCESS; int txconf = 0, rxconf = 0; /* First allocate the TX ring struct memory */ if (!(adapter->tx_rings = (struct tx_ring *) malloc(sizeof(struct tx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate TX ring memory\n"); error = ENOMEM; goto fail; } txr = adapter->tx_rings; /* Next allocate the RX */ if (!(adapter->rx_rings = (struct rx_ring *) malloc(sizeof(struct rx_ring) * adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate RX ring memory\n"); error = ENOMEM; goto rx_fail; } rxr = adapter->rx_rings; /* For the ring itself */ tsize = roundup2(adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc), 4096); /* * Now set up the TX queues, txconf is needed to handle the * possibility that things fail midcourse and we need to * undo memory gracefully */ for (int i = 0; i < adapter->num_queues; i++, txconf++) { /* Set up some basics */ txr = &adapter->tx_rings[i]; txr->adapter = adapter; txr->me = i; /* Initialize the TX side lock */ snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", device_get_nameunit(dev), txr->me); mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate TX Descriptor memory\n"); error = ENOMEM; goto err_tx_desc; } txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; bzero((void *)txr->tx_base, tsize); /* Now allocate transmit buffers for the ring */ if (ixgbe_allocate_transmit_buffers(txr)) { device_printf(dev, "Critical Failure setting up transmit buffers\n"); error = ENOMEM; goto err_tx_desc; } #if __FreeBSD_version >= 800000 /* Allocate a buf ring */ txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, M_WAITOK, &txr->tx_mtx); #endif } /* * Next the RX queues... */ rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), 4096); for (int i = 0; i < adapter->num_queues; i++, rxconf++) { rxr = &adapter->rx_rings[i]; /* Set up some basics */ rxr->adapter = adapter; rxr->me = i; /* Initialize the RX side lock */ snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", device_get_nameunit(dev), rxr->me); mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma, BUS_DMA_NOWAIT)) { device_printf(dev, "Unable to allocate RxDescriptor memory\n"); error = ENOMEM; goto err_rx_desc; } rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; bzero((void *)rxr->rx_base, rsize); /* Allocate receive buffers for the ring*/ if (ixgbe_allocate_receive_buffers(rxr)) { device_printf(dev, "Critical Failure setting up receive buffers\n"); error = ENOMEM; goto err_rx_desc; } } return (0); err_rx_desc: for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) ixgbe_dma_free(adapter, &rxr->rxdma); err_tx_desc: for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) ixgbe_dma_free(adapter, &txr->txdma); free(adapter->rx_rings, M_DEVBUF); rx_fail: free(adapter->tx_rings, M_DEVBUF); fail: return (error); } /********************************************************************* * * Allocate memory for tx_buffer structures. The tx_buffer stores all * the information needed to transmit a packet on the wire. This is * called only once at attach, setup is done every reset. * **********************************************************************/ static int ixgbe_allocate_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; device_t dev = adapter->dev; struct ixgbe_tx_buf *txbuf; int error, i; /* * Setup DMA descriptor areas. */ if ((error = bus_dma_tag_create(NULL, /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ IXGBE_TSO_SIZE, /* maxsize */ ixgbe_num_segs, /* nsegments */ PAGE_SIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &txr->txtag))) { device_printf(dev,"Unable to allocate TX DMA tag\n"); goto fail; } if (!(txr->tx_buffers = (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate tx_buffer memory\n"); error = ENOMEM; goto fail; } /* Create the descriptor buffer dma maps */ txbuf = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); if (error != 0) { device_printf(dev, "Unable to create TX DMA map\n"); goto fail; } } return 0; fail: /* We free all, it handles case where we are in the middle */ ixgbe_free_transmit_structures(adapter); return (error); } /********************************************************************* * * Initialize a transmit ring. * **********************************************************************/ static void ixgbe_setup_transmit_ring(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *txbuf; int i; /* Clear the old ring contents */ bzero((void *)txr->tx_base, (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); /* Reset indices */ txr->next_avail_desc = 0; txr->next_to_clean = 0; /* Free any existing tx buffers. */ txbuf = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { if (txbuf->m_head != NULL) { bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, txbuf->map); m_freem(txbuf->m_head); txbuf->m_head = NULL; } /* Clear the EOP index */ txbuf->eop_index = -1; } #ifdef IXGBE_FDIR /* Set the rate at which we sample packets */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) txr->atr_sample = atr_sample_rate; #endif /* Set number of descriptors available */ txr->tx_avail = adapter->num_tx_desc; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } /********************************************************************* * * Initialize all transmit rings. * **********************************************************************/ static int ixgbe_setup_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) ixgbe_setup_transmit_ring(txr); return (0); } /********************************************************************* * * Enable transmit unit. * **********************************************************************/ static void ixgbe_initialize_transmit_units(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; struct ixgbe_hw *hw = &adapter->hw; /* Setup the Base and Length of the Tx Descriptor Ring */ for (int i = 0; i < adapter->num_queues; i++, txr++) { u64 tdba = txr->txdma.dma_paddr; u32 txctrl; IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), (tdba & 0x00000000ffffffffULL)); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); /* Setup the HW Tx Head and Tail descriptor pointers */ IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); /* Setup Transmit Descriptor Cmd Settings */ txr->txd_cmd = IXGBE_TXD_CMD_IFCS; txr->watchdog_check = FALSE; /* Disable Head Writeback */ switch (hw->mac.type) { case ixgbe_mac_82598EB: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); break; case ixgbe_mac_82599EB: default: txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); break; } txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; switch (hw->mac.type) { case ixgbe_mac_82598EB: IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); break; case ixgbe_mac_82599EB: default: IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); break; } } if (hw->mac.type == ixgbe_mac_82599EB) { u32 dmatxctl, rttdcs; dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); dmatxctl |= IXGBE_DMATXCTL_TE; IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); /* Disable arbiter to set MTQC */ rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); rttdcs |= IXGBE_RTTDCS_ARBDIS; IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); rttdcs &= ~IXGBE_RTTDCS_ARBDIS; IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); } return; } /********************************************************************* * * Free all transmit rings. * **********************************************************************/ static void ixgbe_free_transmit_structures(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_TX_LOCK(txr); ixgbe_free_transmit_buffers(txr); ixgbe_dma_free(adapter, &txr->txdma); IXGBE_TX_UNLOCK(txr); IXGBE_TX_LOCK_DESTROY(txr); } free(adapter->tx_rings, M_DEVBUF); } /********************************************************************* * * Free transmit ring related data structures. * **********************************************************************/ static void ixgbe_free_transmit_buffers(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ixgbe_tx_buf *tx_buffer; int i; INIT_DEBUGOUT("free_transmit_ring: begin"); if (txr->tx_buffers == NULL) return; tx_buffer = txr->tx_buffers; for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { if (tx_buffer->m_head != NULL) { bus_dmamap_sync(txr->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; if (tx_buffer->map != NULL) { bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } else if (tx_buffer->map != NULL) { bus_dmamap_unload(txr->txtag, tx_buffer->map); bus_dmamap_destroy(txr->txtag, tx_buffer->map); tx_buffer->map = NULL; } } #if __FreeBSD_version >= 800000 if (txr->br != NULL) buf_ring_free(txr->br, M_DEVBUF); #endif if (txr->tx_buffers != NULL) { free(txr->tx_buffers, M_DEVBUF); txr->tx_buffers = NULL; } if (txr->txtag != NULL) { bus_dma_tag_destroy(txr->txtag); txr->txtag = NULL; } return; } /********************************************************************* * * Advanced Context Descriptor setup for VLAN or CSUM * **********************************************************************/ static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) { struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; struct ixgbe_tx_buf *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; struct ether_vlan_header *eh; struct ip *ip; struct ip6_hdr *ip6; int ehdrlen, ip_hlen = 0; u16 etype; u8 ipproto = 0; bool offload = TRUE; int ctxd = txr->next_avail_desc; u16 vtag = 0; if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) offload = FALSE; tx_buffer = &txr->tx_buffers[ctxd]; TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; /* ** In advanced descriptors the vlan tag must ** be placed into the descriptor itself. */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); } else if (offload == FALSE) return FALSE; /* * Determine where frame payload starts. * Jump over vlan headers if already present, * helpful for QinQ too. */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { etype = ntohs(eh->evl_proto); ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { etype = ntohs(eh->evl_encap_proto); ehdrlen = ETHER_HDR_LEN; } /* Set the ether header length */ vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; switch (etype) { case ETHERTYPE_IP: ip = (struct ip *)(mp->m_data + ehdrlen); ip_hlen = ip->ip_hl << 2; if (mp->m_len < ehdrlen + ip_hlen) return (FALSE); ipproto = ip->ip_p; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; break; case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); ip_hlen = sizeof(struct ip6_hdr); if (mp->m_len < ehdrlen + ip_hlen) return (FALSE); ipproto = ip6->ip6_nxt; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; break; default: offload = FALSE; break; } vlan_macip_lens |= ip_hlen; type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; switch (ipproto) { case IPPROTO_TCP: if (mp->m_pkthdr.csum_flags & CSUM_TCP) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; break; case IPPROTO_UDP: if (mp->m_pkthdr.csum_flags & CSUM_UDP) type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; break; default: offload = FALSE; break; } /* Now copy bits into descriptor */ TXD->vlan_macip_lens |= htole32(vlan_macip_lens); TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); TXD->seqnum_seed = htole32(0); TXD->mss_l4len_idx = htole32(0); tx_buffer->m_head = NULL; tx_buffer->eop_index = -1; /* We've consumed the first desc, adjust counters */ if (++ctxd == adapter->num_tx_desc) ctxd = 0; txr->next_avail_desc = ctxd; --txr->tx_avail; return (offload); } /********************************************************************** * * Setup work for hardware segmentation offload (TSO) on * adapters using advanced tx descriptors * **********************************************************************/ static boolean_t ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) { struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; struct ixgbe_tx_buf *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; u32 mss_l4len_idx = 0; u16 vtag = 0; int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; struct ether_vlan_header *eh; struct ip *ip; struct tcphdr *th; /* * Determine where frame payload starts. * Jump over vlan headers if already present */ eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; else ehdrlen = ETHER_HDR_LEN; /* Ensure we have at least the IP+TCP header in the first mbuf. */ if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) return FALSE; ctxd = txr->next_avail_desc; tx_buffer = &txr->tx_buffers[ctxd]; TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; ip = (struct ip *)(mp->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) return FALSE; /* 0 */ ip->ip_sum = 0; ip_hlen = ip->ip_hl << 2; th = (struct tcphdr *)((caddr_t)ip + ip_hlen); th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); tcp_hlen = th->th_off << 2; hdrlen = ehdrlen + ip_hlen + tcp_hlen; /* This is used in the transmit desc in encap */ *paylen = mp->m_pkthdr.len - hdrlen; /* VLAN MACLEN IPLEN */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); } vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; vlan_macip_lens |= ip_hlen; TXD->vlan_macip_lens |= htole32(vlan_macip_lens); /* ADV DTYPE TUCMD */ type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); /* MSS L4LEN IDX */ mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); TXD->mss_l4len_idx = htole32(mss_l4len_idx); TXD->seqnum_seed = htole32(0); tx_buffer->m_head = NULL; tx_buffer->eop_index = -1; if (++ctxd == adapter->num_tx_desc) ctxd = 0; txr->tx_avail--; txr->next_avail_desc = ctxd; return TRUE; } #ifdef IXGBE_FDIR /* ** This routine parses packet headers so that Flow ** Director can make a hashed filter table entry ** allowing traffic flows to be identified and kept ** on the same cpu. This would be a performance ** hit, but we only do it at IXGBE_FDIR_RATE of ** packets. */ static void ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) { struct adapter *adapter = txr->adapter; struct ixgbe_atr_input atr_input; struct ip *ip; struct tcphdr *th; struct udphdr *uh; struct ether_vlan_header *eh; int ehdrlen, ip_hlen; u16 etype, vlan_id, src_port, dst_port, flex_bytes; u32 src_ipv4_addr, dst_ipv4_addr; u8 l4type = 0, ipproto = 0; eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; else ehdrlen = ETHER_HDR_LEN; etype = ntohs(eh->evl_proto); /* Only handling IPv4 */ if (etype != ETHERTYPE_IP) return; ip = (struct ip *)(mp->m_data + ehdrlen); ipproto = ip->ip_p; ip_hlen = ip->ip_hl << 2; src_port = dst_port = 0; /* check if we're UDP or TCP */ switch (ipproto) { case IPPROTO_TCP: th = (struct tcphdr *)((caddr_t)ip + ip_hlen); src_port = th->th_sport; dst_port = th->th_dport; l4type |= IXGBE_ATR_L4TYPE_TCP; break; case IPPROTO_UDP: uh = (struct udphdr *)((caddr_t)ip + ip_hlen); src_port = uh->uh_sport; dst_port = uh->uh_dport; l4type |= IXGBE_ATR_L4TYPE_UDP; break; default: return; } memset(&atr_input, 0, sizeof(struct ixgbe_atr_input)); vlan_id = htole16(mp->m_pkthdr.ether_vtag); src_ipv4_addr = ip->ip_src.s_addr; dst_ipv4_addr = ip->ip_dst.s_addr; flex_bytes = etype; ixgbe_atr_set_vlan_id_82599(&atr_input, vlan_id); ixgbe_atr_set_src_port_82599(&atr_input, dst_port); ixgbe_atr_set_dst_port_82599(&atr_input, src_port); ixgbe_atr_set_flex_byte_82599(&atr_input, flex_bytes); ixgbe_atr_set_l4type_82599(&atr_input, l4type); /* src and dst are inverted, think how the receiver sees them */ ixgbe_atr_set_src_ipv4_82599(&atr_input, dst_ipv4_addr); ixgbe_atr_set_dst_ipv4_82599(&atr_input, src_ipv4_addr); /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&adapter->hw, &atr_input, txr->msix); } #endif /********************************************************************** * * Examine each tx_buffer in the used queue. If the hardware is done * processing the packet then free associated resources. The * tx_buffer is put back on the free queue. * **********************************************************************/ static boolean_t ixgbe_txeof(struct tx_ring *txr) { struct adapter *adapter = txr->adapter; struct ifnet *ifp = adapter->ifp; u32 first, last, done; struct ixgbe_tx_buf *tx_buffer; struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; mtx_assert(&txr->tx_mtx, MA_OWNED); if (txr->tx_avail == adapter->num_tx_desc) return FALSE; first = txr->next_to_clean; tx_buffer = &txr->tx_buffers[first]; /* For cleanup we just use legacy struct */ tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; last = tx_buffer->eop_index; if (last == -1) return FALSE; eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; /* ** Get the index of the first descriptor ** BEYOND the EOP and call that 'done'. ** I do this so the comparison in the ** inner while loop below can be simple */ if (++last == adapter->num_tx_desc) last = 0; done = last; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD); /* ** Only the EOP descriptor of a packet now has the DD ** bit set, this is what we look for... */ while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { /* We clean the range of the packet */ while (first != done) { tx_desc->upper.data = 0; tx_desc->lower.data = 0; tx_desc->buffer_addr = 0; ++txr->tx_avail; if (tx_buffer->m_head) { ifp->if_opackets++; bus_dmamap_sync(txr->txtag, tx_buffer->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, tx_buffer->map); m_freem(tx_buffer->m_head); tx_buffer->m_head = NULL; tx_buffer->map = NULL; } tx_buffer->eop_index = -1; txr->watchdog_time = ticks; if (++first == adapter->num_tx_desc) first = 0; tx_buffer = &txr->tx_buffers[first]; tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; } /* See if there is more work now */ last = tx_buffer->eop_index; if (last != -1) { eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; /* Get next done point */ if (++last == adapter->num_tx_desc) last = 0; done = last; } else break; } bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); txr->next_to_clean = first; /* * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that * it is OK to send packets. If there are no pending descriptors, * clear the timeout. Otherwise, if some descriptors have been freed, * restart the timeout. */ if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; if (txr->tx_avail == adapter->num_tx_desc) { txr->watchdog_check = FALSE; return FALSE; } } return TRUE; } /********************************************************************* * * Refresh mbuf buffers for a range of descriptors * **********************************************************************/ static int ixgbe_get_buf(struct rx_ring *rxr, int first, int limit) { struct adapter *adapter = rxr->adapter; bus_dma_segment_t seg[2]; struct ixgbe_rx_buf *rxbuf; struct mbuf *mh, *mp; bus_dmamap_t map; int i, nsegs, error; i = first; while (i != limit) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->m_head == NULL) { mh = m_gethdr(M_DONTWAIT, MT_DATA); if (mh == NULL) goto failure; } else /* reuse */ mh = rxbuf->m_head; mh->m_len = MHLEN; mh->m_flags |= M_PKTHDR; if (rxbuf->m_pack == NULL) { mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); if (mp == NULL) goto failure; mp->m_len = adapter->rx_mbuf_sz; mp->m_flags &= ~M_PKTHDR; } else { /* reusing */ mp = rxbuf->m_pack; mp->m_len = adapter->rx_mbuf_sz; mp->m_flags &= ~M_PKTHDR; } /* ** Need to create a chain for the following ** dmamap call at this point. */ mh->m_next = mp; mh->m_pkthdr.len = mh->m_len + mp->m_len; /* Get the memory mapping */ error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxr->spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { printf("GET BUF: dmamap load failure - %d\n", error); m_free(mh); return (error); } /* Unload old mapping and update buffer struct */ if (rxbuf->m_head != NULL) bus_dmamap_unload(rxr->rxtag, rxbuf->map); map = rxbuf->map; rxbuf->map = rxr->spare_map; rxr->spare_map = map; rxbuf->m_head = mh; rxbuf->m_pack = mp; bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); /* Update descriptor */ rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); /* Calculate next index */ if (++i == adapter->num_rx_desc) i = 0; } return (0); failure: panic("GET BUF: ENOBUFS\n"); #if 0 /* ** If we get here, we have an mbuf resource ** issue, so we discard the incoming packet ** and attempt to reuse existing mbufs next ** pass thru the ring, but to do so we must ** fix up the descriptor which had the address ** clobbered with writeback info. */ remap: adapter->mbuf_header_failed++; merr = ENOBUFS; /* Is there a reusable buffer? */ mh = rxr->rx_buffers[i].m_head; if (mh == NULL) /* Nope, init error */ return (merr); mp = rxr->rx_buffers[i].m_pack; if (mp == NULL) /* Nope, init error */ return (merr); /* Get our old mapping */ rxbuf = &rxr->rx_buffers[i]; error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, mh, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { /* We really have a problem */ m_free(mh); return (error); } /* Now fix the descriptor as needed */ rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); return (merr); #endif } /********************************************************************* * * Allocate memory for rx_buffer structures. Since we use one * rx_buffer per received packet, the maximum number of rx_buffer's * that we'll need is equal to the number of receive descriptors * that we've allocated. * **********************************************************************/ static int ixgbe_allocate_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; device_t dev = adapter->dev; struct ixgbe_rx_buf *rxbuf; int i, bsize, error; bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; if (!(rxr->rx_buffers = (struct ixgbe_rx_buf *) malloc(bsize, M_DEVBUF, M_NOWAIT | M_ZERO))) { device_printf(dev, "Unable to allocate rx_buffer memory\n"); error = ENOMEM; goto fail; } /* ** The tag is made to accomodate the largest buffer size ** with packet split (hence the two segments, even though ** it may not always use this. */ if ((error = bus_dma_tag_create(NULL, /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ 2, /* nsegments */ MJUMPAGESIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &rxr->rxtag))) { device_printf(dev, "Unable to create RX DMA tag\n"); goto fail; } /* Create the spare map (used by getbuf) */ error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, &rxr->spare_map); if (error) { device_printf(dev, "%s: bus_dmamap_create failed: %d\n", __func__, error); goto fail; } for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { rxbuf = &rxr->rx_buffers[i]; error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, &rxbuf->map); if (error) { device_printf(dev, "Unable to create RX DMA map\n"); goto fail; } } return (0); fail: /* Frees all, but can handle partial completion */ ixgbe_free_receive_structures(adapter); return (error); } /* ** Used to detect a descriptor that has ** been merged by Hardware RSC. */ static inline u32 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) { return (le32toh(rx->wb.lower.lo_dword.data) & IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; } /********************************************************************* * * Initialize Hardware RSC (LRO) feature on 82599 * for an RX ring, this is toggled by the LRO capability * even though it is transparent to the stack. * **********************************************************************/ static void ixgbe_setup_hw_rsc(struct rx_ring *rxr) { struct adapter *adapter = rxr->adapter; struct ixgbe_hw *hw = &adapter->hw; u32 rscctrl, rdrxctl; rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); rscctrl |= IXGBE_RSCCTL_RSCEN; /* ** Limit the total number of descriptors that ** can be combined, so it does not exceed 64K */ if (adapter->rx_mbuf_sz == MCLBYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_16; else /* using 4K clusters */ rscctrl |= IXGBE_RSCCTL_MAXDESC_8; IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); /* Enable TCP header recognition */ IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR)); /* Disable RSC for ACK packets */ IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); rxr->hw_rsc = TRUE; } /********************************************************************* * * Initialize a receive ring and its buffers. * **********************************************************************/ static int ixgbe_setup_receive_ring(struct rx_ring *rxr) { struct adapter *adapter; struct ifnet *ifp; device_t dev; struct ixgbe_rx_buf *rxbuf; bus_dma_segment_t seg[2]; struct lro_ctrl *lro = &rxr->lro; int rsize, nsegs, error; adapter = rxr->adapter; ifp = adapter->ifp; dev = adapter->dev; /* Clear the ring contents */ rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); /* ** Free current RX buffer structs and their mbufs */ for (int i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rxtag, rxbuf->map); if (rxbuf->m_head) { rxbuf->m_head->m_next = rxbuf->m_pack; m_freem(rxbuf->m_head); } rxbuf->m_head = NULL; rxbuf->m_pack = NULL; } } /* Now replenish the mbufs */ for (int j = 0; j != adapter->num_rx_desc; ++j) { rxbuf = &rxr->rx_buffers[j]; rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); if (rxbuf->m_head == NULL) panic("RX ring hdr initialization failed!\n"); rxbuf->m_head->m_len = MHLEN; rxbuf->m_head->m_flags |= M_PKTHDR; rxbuf->m_head->m_pkthdr.len = rxbuf->m_head->m_len; rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); if (rxbuf->m_pack == NULL) panic("RX ring pkt initialization failed!\n"); rxbuf->m_pack->m_len = adapter->rx_mbuf_sz; rxbuf->m_head->m_next = rxbuf->m_pack; rxbuf->m_head->m_pkthdr.len += rxbuf->m_pack->m_len; /* Get the memory mapping */ error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, rxbuf->m_head, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) panic("RX ring dma initialization failed!\n"); bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_PREREAD); /* Update descriptor */ rxr->rx_base[j].read.hdr_addr = htole64(seg[0].ds_addr); rxr->rx_base[j].read.pkt_addr = htole64(seg[1].ds_addr); } /* Setup our descriptor indices */ rxr->next_to_check = 0; rxr->last_refreshed = 0; rxr->lro_enabled = FALSE; /* Use header split if configured */ if (ixgbe_header_split) rxr->hdr_split = TRUE; else ifp->if_capabilities &= ~IFCAP_LRO; bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* ** Now set up the LRO interface: ** 82598 uses software LRO, the ** 82599 on the other hand uses a ** hardware assist to do the same. ** We only do LRO when header split ** is enabled, its simpler that way. */ if ((ifp->if_capenable & IFCAP_LRO) && (rxr->hdr_split)) { int err = tcp_lro_init(lro); if (err) panic("LRO Initialization failed!\n"); INIT_DEBUGOUT("RX Soft LRO Initialized\n"); rxr->lro_enabled = TRUE; lro->ifp = adapter->ifp; ixgbe_setup_hw_rsc(rxr); } return (0); #if 0 fail: /* * We need to clean up any buffers allocated * so far, 'j' is the failing index. */ for (int i = 0; i < j; i++) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rxtag, rxbuf->map); m_freem(rxbuf->m_head); rxbuf->m_head = NULL; } } return (ENOBUFS); #endif } /********************************************************************* * * Initialize all receive rings. * **********************************************************************/ static int ixgbe_setup_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; int j; for (j = 0; j < adapter->num_queues; j++, rxr++) if (ixgbe_setup_receive_ring(rxr)) goto fail; return (0); fail: /* * Free RX buffers allocated so far, we will only handle * the rings that completed, the failing case will have * cleaned up for itself. 'j' failed, so its the terminus. */ for (int i = 0; i < j; ++i) { rxr = &adapter->rx_rings[i]; for (int n = 0; n < adapter->num_rx_desc; n++) { struct ixgbe_rx_buf *rxbuf; rxbuf = &rxr->rx_buffers[n]; if (rxbuf->m_head != NULL) { bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rxtag, rxbuf->map); m_freem(rxbuf->m_head); rxbuf->m_head = NULL; } } } return (ENOBUFS); } /********************************************************************* * * Setup receive registers and features. * **********************************************************************/ #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_initialize_receive_units(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; struct ixgbe_hw *hw = &adapter->hw; struct ifnet *ifp = adapter->ifp; u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; u32 reta, mrqc = 0, hlreg, random[10]; /* * Make sure receives are disabled while * setting up the descriptor ring */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); /* Enable broadcasts */ fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); fctrl |= IXGBE_FCTRL_BAM; fctrl |= IXGBE_FCTRL_DPF; fctrl |= IXGBE_FCTRL_PMCF; IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); /* Set for Jumbo Frames? */ hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); if (ifp->if_mtu > ETHERMTU) { hlreg |= IXGBE_HLREG0_JUMBOEN; bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } else { hlreg &= ~IXGBE_HLREG0_JUMBOEN; bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); for (int i = 0; i < adapter->num_queues; i++, rxr++) { u64 rdba = rxr->rxdma.dma_paddr; /* Setup the Base and Length of the Rx Descriptor Ring */ IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), (rdba & 0x00000000ffffffffULL)); IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); /* Set up the SRRCTL register */ srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; srrctl |= bufsz; if (rxr->hdr_split) { /* Use a standard mbuf for the header */ srrctl |= ((IXGBE_RX_HDR << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & IXGBE_SRRCTL_BSIZEHDR_MASK); srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; if (adapter->hw.mac.type == ixgbe_mac_82599EB) { /* PSRTYPE must be initialized in 82599 */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); } } else srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); /* Setup the HW Rx Head and Tail Descriptor Pointers */ IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); } rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); /* Setup RSS */ if (adapter->num_queues > 1) { int i, j; reta = 0; /* set up random bits */ arc4rand(&random, sizeof(random), 0); /* Set up the redirection table */ for (i = 0, j = 0; i < 128; i++, j++) { if (j == adapter->num_queues) j = 0; reta = (reta << 8) | (j * 0x11); if ((i & 3) == 3) IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); } /* Now fill our hash function seeds */ for (int i = 0; i < 10; i++) IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]); /* Perform hash on these packet types */ mrqc = IXGBE_MRQC_RSSEN | IXGBE_MRQC_RSS_FIELD_IPV4 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP | IXGBE_MRQC_RSS_FIELD_IPV4_UDP | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | IXGBE_MRQC_RSS_FIELD_IPV6_EX | IXGBE_MRQC_RSS_FIELD_IPV6 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP | IXGBE_MRQC_RSS_FIELD_IPV6_UDP | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); /* RSS and RX IPP Checksum are mutually exclusive */ rxcsum |= IXGBE_RXCSUM_PCSD; } if (ifp->if_capenable & IFCAP_RXCSUM) rxcsum |= IXGBE_RXCSUM_PCSD; if (!(rxcsum & IXGBE_RXCSUM_PCSD)) rxcsum |= IXGBE_RXCSUM_IPPCSE; IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); return; } /********************************************************************* * * Free all receive rings. * **********************************************************************/ static void ixgbe_free_receive_structures(struct adapter *adapter) { struct rx_ring *rxr = adapter->rx_rings; for (int i = 0; i < adapter->num_queues; i++, rxr++) { struct lro_ctrl *lro = &rxr->lro; ixgbe_free_receive_buffers(rxr); /* Free LRO memory */ tcp_lro_free(lro); /* Free the ring memory as well */ ixgbe_dma_free(adapter, &rxr->rxdma); } free(adapter->rx_rings, M_DEVBUF); } /********************************************************************* * * Free receive ring data structures * **********************************************************************/ void ixgbe_free_receive_buffers(struct rx_ring *rxr) { struct adapter *adapter = NULL; struct ixgbe_rx_buf *rxbuf = NULL; INIT_DEBUGOUT("free_receive_buffers: begin"); adapter = rxr->adapter; if (rxr->rx_buffers != NULL) { rxbuf = &rxr->rx_buffers[0]; for (int i = 0; i < adapter->num_rx_desc; i++) { if (rxbuf->map != NULL) { bus_dmamap_sync(rxr->rxtag, rxbuf->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rxr->rxtag, rxbuf->map); bus_dmamap_destroy(rxr->rxtag, rxbuf->map); } if (rxbuf->m_head != NULL) { m_freem(rxbuf->m_head); } rxbuf->m_head = NULL; ++rxbuf; } } if (rxr->rx_buffers != NULL) { free(rxr->rx_buffers, M_DEVBUF); rxr->rx_buffers = NULL; } if (rxr->rxtag != NULL) { bus_dma_tag_destroy(rxr->rxtag); rxr->rxtag = NULL; } return; } /********************************************************************* * * This routine executes in interrupt context. It replenishes * the mbufs in the descriptor and sends data which has been * dma'ed into host memory to upper layer. * * We loop at most count times if count is > 0, or until done if * count < 0. * * Return TRUE for more work, FALSE for all clean. *********************************************************************/ static bool ixgbe_rxeof(struct rx_ring *rxr, int count) { struct adapter *adapter = rxr->adapter; struct ifnet *ifp = adapter->ifp; struct lro_ctrl *lro = &rxr->lro; struct lro_entry *queued; int i, processed = 0; u32 staterr; union ixgbe_adv_rx_desc *cur; IXGBE_RX_LOCK(rxr); i = rxr->next_to_check; cur = &rxr->rx_base[i]; staterr = cur->wb.upper.status_error; if (!(staterr & IXGBE_RXD_STAT_DD)) { IXGBE_RX_UNLOCK(rxr); return FALSE; } /* Sync the ring */ bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_POSTREAD); while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) && (ifp->if_drv_flags & IFF_DRV_RUNNING)) { struct mbuf *sendmp, *mh, *mp, *nh, *np; struct ixgbe_rx_buf *nxtbuf; u32 rsc; u16 hlen, plen, hdr, nextp, vtag; bool accept_frame, eop; accept_frame = TRUE; hlen = plen = rsc = nextp = 0; sendmp = mh = mp = nh = np = NULL; /* Sync the buffers */ bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map, BUS_DMASYNC_POSTREAD); mh = rxr->rx_buffers[i].m_head; mp = rxr->rx_buffers[i].m_pack; vtag = le16toh(cur->wb.upper.vlan); eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); if (!eop) { /* ** On 82599 which supports a hardware ** LRO (called HW RSC), packets need ** not be fragmented across sequential ** descriptors, rather the next descriptor ** is indicated in bits of the current. ** This also means that we might proceses ** more than one packet at a time, something ** that has never been true before, it ** required eliminating global chain pointers ** in favor of what we are doing here. -jfv */ if (rxr->hw_rsc == TRUE) { rsc = ixgbe_rsc_count(cur); rxr->rsc_num += (rsc - 1); } if (rsc) { nextp = ((staterr & IXGBE_RXDADV_NEXTP_MASK) >> IXGBE_RXDADV_NEXTP_SHIFT); } else { nextp = i + 1; if (nextp == adapter->num_rx_desc) nextp = 0; } nxtbuf = &rxr->rx_buffers[nextp]; prefetch(nxtbuf); } /* ** The way the hardware is configured to ** split, it will ONLY use the header buffer ** when header split is enabled, otherwise we ** get legacy behavior, ie, both header and ** payload are DMA'd into JUST the payload buffer. ** ** Rather than using the fmp/lmp global pointers ** we now keep the head of a packet chain in the ** m_nextpkt pointer and pass this along from one ** descriptor to the next, until we get EOP. ** */ if ((rxr->hdr_split) && (mh->m_nextpkt == NULL)) { hdr = le16toh(cur-> wb.lower.lo_dword.hs_rss.hdr_info); hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> IXGBE_RXDADV_HDRBUFLEN_SHIFT; if (hlen > IXGBE_RX_HDR) hlen = IXGBE_RX_HDR; plen = le16toh(cur->wb.upper.length); mh->m_len = hlen; mh->m_flags |= M_PKTHDR; mh->m_next = NULL; mh->m_pkthdr.len = mh->m_len; /* Null this so getbuf replenishes */ rxr->rx_buffers[i].m_head = NULL; /* ** Get the payload length, this ** could be zero if its a small ** packet. */ if (plen) { mp->m_len = plen; mp->m_next = NULL; mp->m_flags &= ~M_PKTHDR; mh->m_next = mp; mh->m_pkthdr.len += mp->m_len; /* Null this so getbuf replenishes */ rxr->rx_buffers[i].m_pack = NULL; rxr->rx_split_packets++; } /* Setup the forward chain */ if (eop == 0) { nh = rxr->rx_buffers[nextp].m_head; np = rxr->rx_buffers[nextp].m_pack; nh->m_nextpkt = mh; if (plen) mp->m_next = np; else mh->m_next = np; } else { sendmp = mh; if (staterr & IXGBE_RXD_STAT_VP) { sendmp->m_pkthdr.ether_vtag = vtag; sendmp->m_flags |= M_VLANTAG; } } } else { /* ** Either no header split, or a ** secondary piece of a fragmented ** packet. */ mp->m_len = le16toh(cur->wb.upper.length); rxr->rx_buffers[i].m_pack = NULL; /* stored head pointer */ sendmp = mh->m_nextpkt; if (sendmp != NULL) { sendmp->m_pkthdr.len += mp->m_len; sendmp->m_nextpkt = NULL; } else { /* first desc of a non-ps chain */ sendmp = mp; sendmp->m_flags |= M_PKTHDR; sendmp->m_pkthdr.len = mp->m_len; if (staterr & IXGBE_RXD_STAT_VP) { sendmp->m_pkthdr.ether_vtag = vtag; sendmp->m_flags |= M_VLANTAG; } } /* Carry head forward */ if (eop == 0) { nh = rxr->rx_buffers[nextp].m_head; np = rxr->rx_buffers[nextp].m_pack; nh->m_nextpkt = sendmp; mp->m_next = np; sendmp = NULL; } mh->m_nextpkt = NULL; } if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) accept_frame = FALSE; if (accept_frame) { ++processed; if (eop) { --count; sendmp->m_pkthdr.rcvif = ifp; ifp->if_ipackets++; rxr->rx_packets++; /* capture data for AIM */ rxr->bytes += sendmp->m_pkthdr.len; rxr->rx_bytes += rxr->bytes; if (ifp->if_capenable & IFCAP_RXCSUM) ixgbe_rx_checksum(staterr, sendmp); else sendmp->m_pkthdr.csum_flags = 0; #if __FreeBSD_version >= 800000 /* Get the RSS Hash */ sendmp->m_pkthdr.flowid = le16toh(cur->wb.lower.hi_dword.rss); sendmp->m_flags |= M_FLOWID; #endif } } else { ifp->if_ierrors++; /* Reuse loaded DMA map and just update mbuf chain */ mh->m_len = MHLEN; mh->m_flags |= M_PKTHDR; mh->m_next = NULL; mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; mp->m_data = mp->m_ext.ext_buf; if (mp->m_next) { /* Free chain */ sendmp = mp->m_next; m_free(sendmp); } mp->m_next = NULL; if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) m_adj(mp, ETHER_ALIGN); sendmp = NULL; } bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); rxr->last_refreshed = i; /* for updating tail */ if (++i == adapter->num_rx_desc) i = 0; /* Prefetch next descriptor */ cur = &rxr->rx_base[i]; prefetch(cur); /* ** Now send up to the stack, ** note that the RX lock is ** held thru this call. */ if (sendmp != NULL) { /* ** Send to the stack if: ** - Soft LRO not enabled, or ** - no Soft LRO resources, or ** - soft lro enqueue fails */ if ((!rxr->lro_enabled) || ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0)))) (*ifp->if_input)(ifp, sendmp); } /* Replenish every 8 max */ if (processed == 8) { ixgbe_get_buf(rxr, rxr->next_to_check, i); processed = 0; IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_refreshed); rxr->next_to_check = i; } /* Next iteration */ staterr = cur->wb.upper.status_error; } /* Replenish remaining work */ if (processed != 0) { ixgbe_get_buf(rxr, rxr->next_to_check, i); processed = 0; IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_refreshed); } rxr->next_to_check = i; /* * Flush any outstanding LRO work */ while (!SLIST_EMPTY(&lro->lro_active)) { queued = SLIST_FIRST(&lro->lro_active); SLIST_REMOVE_HEAD(&lro->lro_active, next); tcp_lro_flush(lro, queued); } IXGBE_RX_UNLOCK(rxr); /* ** Leaving with more to clean? ** then schedule another interrupt. */ if (staterr & IXGBE_RXD_STAT_DD) { ixgbe_rearm_queues(adapter, (u64)(1 << rxr->msix)); return TRUE; } return FALSE; } /********************************************************************* * * Verify that the hardware indicated that the checksum is valid. * Inform the stack about the status of checksum so that stack * doesn't spend time verifying the checksum. * *********************************************************************/ static void ixgbe_rx_checksum(u32 staterr, struct mbuf * mp) { u16 status = (u16) staterr; u8 errors = (u8) (staterr >> 24); if (status & IXGBE_RXD_STAT_IPCS) { /* Did it pass? */ if (!(errors & IXGBE_RXD_ERR_IPE)) { /* IP Checksum Good */ mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; } else mp->m_pkthdr.csum_flags = 0; } if (status & IXGBE_RXD_STAT_L4CS) { /* Did it pass? */ if (!(errors & IXGBE_RXD_ERR_TCPE)) { mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); mp->m_pkthdr.csum_data = htons(0xffff); } } return; } /* ** This routine is run via an vlan config EVENT, ** it enables us to use the HW Filter table since ** we can get the vlan id. This just creates the ** entry in the soft version of the VFTA, init will ** repopulate the real table. */ static void ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u16 index, bit; if (ifp->if_softc != arg) /* Not our event */ return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; ixgbe_shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; /* Re-init to load the changes */ ixgbe_init(adapter); } /* ** This routine is run via an vlan ** unconfig EVENT, remove our entry ** in the soft vfta. */ static void ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) { struct adapter *adapter = ifp->if_softc; u16 index, bit; if (ifp->if_softc != arg) return; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; ixgbe_shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; /* Re-init to load the changes */ ixgbe_init(adapter); } static void ixgbe_setup_vlan_hw_support(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; u32 ctrl; /* ** We get here thru init_locked, meaning ** a soft reset, this has already cleared ** the VFTA and other state, so if there ** have been no vlan's registered do nothing. */ if (adapter->num_vlans == 0) return; /* ** A soft reset zero's out the VFTA, so ** we need to repopulate it now. */ for (int i = 0; i < IXGBE_VFTA_SIZE; i++) if (ixgbe_shadow_vfta[i] != 0) IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ixgbe_shadow_vfta[i]); /* Enable the Filter Table */ ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); ctrl &= ~IXGBE_VLNCTRL_CFIEN; ctrl |= IXGBE_VLNCTRL_VFE; if (hw->mac.type == ixgbe_mac_82598EB) ctrl |= IXGBE_VLNCTRL_VME; IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); /* On 82599 the VLAN enable is per/queue in RXDCTL */ if (hw->mac.type == ixgbe_mac_82599EB) for (int i = 0; i < adapter->num_queues; i++) { ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); ctrl |= IXGBE_RXDCTL_VME; IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); } } static void ixgbe_enable_intr(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); /* Enable Fan Failure detection */ if (hw->device_id == IXGBE_DEV_ID_82598AT) mask |= IXGBE_EIMS_GPI_SDP1; /* 82599 specific interrupts */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) { mask |= IXGBE_EIMS_ECC; mask |= IXGBE_EIMS_GPI_SDP1; mask |= IXGBE_EIMS_GPI_SDP2; #ifdef IXGBE_FDIR mask |= IXGBE_EIMS_FLOW_DIR; #endif } IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); /* With RSS we use auto clear */ if (adapter->msix_mem) { mask = IXGBE_EIMS_ENABLE_MASK; /* Dont autoclear Link */ mask &= ~IXGBE_EIMS_OTHER; mask &= ~IXGBE_EIMS_LSC; IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); } /* ** Now enable all queues, this is done seperately to ** allow for handling the extended (beyond 32) MSIX ** vectors that can be used by 82599 */ for (int i = 0; i < adapter->num_queues; i++, rxr++) ixgbe_enable_queue(adapter, rxr->msix); for (int i = 0; i < adapter->num_queues; i++, txr++) ixgbe_enable_queue(adapter, txr->msix); IXGBE_WRITE_FLUSH(hw); return; } static void ixgbe_disable_intr(struct adapter *adapter) { if (adapter->msix_mem) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); if (adapter->hw.mac.type == ixgbe_mac_82598EB) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); } else { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); } IXGBE_WRITE_FLUSH(&adapter->hw); return; } u16 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) { u16 value; value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, reg, 2); return (value); } void ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) { pci_write_config(((struct ixgbe_osdep *)hw->back)->dev, reg, value, 2); return; } /* ** Setup the correct IVAR register for a particular MSIX interrupt ** (yes this is all very magic and confusing :) ** - entry is the register array entry ** - vector is the MSIX vector for this queue ** - type is RX/TX/MISC */ static void ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) { struct ixgbe_hw *hw = &adapter->hw; u32 ivar, index; vector |= IXGBE_IVAR_ALLOC_VAL; switch (hw->mac.type) { case ixgbe_mac_82598EB: if (type == -1) entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; else entry += (type * 64); index = (entry >> 2) & 0x1F; ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); ivar &= ~(0xFF << (8 * (entry & 0x3))); ivar |= (vector << (8 * (entry & 0x3))); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); break; case ixgbe_mac_82599EB: if (type == -1) { /* MISC IVAR */ index = (entry & 1) * 8; ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); ivar &= ~(0xFF << index); ivar |= (vector << index); IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); } else { /* RX/TX IVARS */ index = (16 * (entry & 1)) + (8 * type); ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); ivar &= ~(0xFF << index); ivar |= (vector << index); IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); } default: break; } } static void ixgbe_configure_ivars(struct adapter *adapter) { struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; for (int i = 0; i < adapter->num_queues; i++, rxr++) ixgbe_set_ivar(adapter, i, rxr->msix, 0); for (int i = 0; i < adapter->num_queues; i++, txr++) ixgbe_set_ivar(adapter, i, txr->msix, 1); /* For the Link interrupt */ ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); } /* ** ixgbe_sfp_probe - called in the local timer to ** determine if a port had optics inserted. */ static bool ixgbe_sfp_probe(struct adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; device_t dev = adapter->dev; bool result = FALSE; if ((hw->phy.type == ixgbe_phy_nl) && (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { s32 ret = hw->phy.ops.identify_sfp(hw); if (ret) goto out; ret = hw->phy.ops.reset(hw); if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { device_printf(dev,"Unsupported SFP+ module detected!"); printf(" Reload driver with supported module.\n"); adapter->sfp_probe = FALSE; goto out; } else device_printf(dev,"SFP+ module detected!\n"); /* We now have supported optics */ adapter->sfp_probe = FALSE; result = TRUE; } out: return (result); } /* ** Tasklet handler for MSIX Link interrupts ** - do outside interrupt since it might sleep */ static void ixgbe_handle_link(void *context, int pending) { struct adapter *adapter = context; ixgbe_check_link(&adapter->hw, &adapter->link_speed, &adapter->link_up, 0); ixgbe_update_link_status(adapter); } /* ** Tasklet for handling SFP module interrupts */ static void ixgbe_handle_mod(void *context, int pending) { struct adapter *adapter = context; struct ixgbe_hw *hw = &adapter->hw; device_t dev = adapter->dev; u32 err; err = hw->phy.ops.identify_sfp(hw); if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { device_printf(dev, "Unsupported SFP+ module type was detected.\n"); return; } err = hw->mac.ops.setup_sfp(hw); if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { device_printf(dev, "Setup failure - unsupported SFP+ module type.\n"); return; } taskqueue_enqueue(adapter->tq, &adapter->msf_task); return; } /* ** Tasklet for handling MSF (multispeed fiber) interrupts */ static void ixgbe_handle_msf(void *context, int pending) { struct adapter *adapter = context; struct ixgbe_hw *hw = &adapter->hw; u32 autoneg; bool negotiate; autoneg = hw->phy.autoneg_advertised; if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); #if 0 ixgbe_check_link(&adapter->hw, &speed, &adapter->link_up, 0); ixgbe_update_link_status(adapter); #endif return; } #ifdef IXGBE_FDIR /* ** Tasklet for reinitializing the Flow Director filter table */ static void ixgbe_reinit_fdir(void *context, int pending) { struct adapter *adapter = context; struct ifnet *ifp = adapter->ifp; if (adapter->fdir_reinit != 1) /* Shouldn't happen */ return; ixgbe_reinit_fdir_tables_82599(&adapter->hw); adapter->fdir_reinit = 0; /* Restart the interface */ ifp->if_drv_flags |= IFF_DRV_RUNNING; return; } #endif /********************************************************************** * * Update the board statistics counters. * **********************************************************************/ static void ixgbe_update_stats_counters(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; struct ixgbe_hw *hw = &adapter->hw; u32 missed_rx = 0, bprc, lxon, lxoff, total; u64 total_missed_rx = 0; adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS); for (int i = 0; i < 8; i++) { u32 mp; mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); /* missed_rx tallies misses for the gprc workaround */ missed_rx += mp; /* global total per queue */ adapter->stats.mpc[i] += mp; /* Running comprehensive total for stats display */ total_missed_rx += adapter->stats.mpc[i]; if (hw->mac.type == ixgbe_mac_82598EB) adapter->stats.rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i)); } /* Hardware workaround, gprc counts missed packets */ adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC); adapter->stats.gprc -= missed_rx; if (hw->mac.type == ixgbe_mac_82599EB) { adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* clears register */ adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); IXGBE_READ_REG(hw, IXGBE_GOTCH); /* clears register */ adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL); IXGBE_READ_REG(hw, IXGBE_TORH); /* clears register */ adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); } else { adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC); adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); /* 82598 only has a counter in the high register */ adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH); adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GOTCH); adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH); } /* * Workaround: mprc hardware is incorrectly counting * broadcasts, so for now we subtract those. */ bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); adapter->stats.bprc += bprc; adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC); adapter->stats.mprc -= bprc; adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC); adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); adapter->stats.lxontxc += lxon; lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); adapter->stats.lxofftxc += lxoff; total = lxon + lxoff; adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64); adapter->stats.gptc -= total; adapter->stats.mptc -= total; adapter->stats.ptc64 -= total; adapter->stats.gotc -= total * ETHER_MIN_LEN; adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC); adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC); adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC); adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR); adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127); adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255); adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511); adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023); adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522); adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC); /* Fill out the OS statistics structure */ ifp->if_ipackets = adapter->stats.gprc; ifp->if_opackets = adapter->stats.gptc; ifp->if_ibytes = adapter->stats.gorc; ifp->if_obytes = adapter->stats.gotc; ifp->if_imcasts = adapter->stats.mprc; ifp->if_collisions = 0; /* Rx Errors */ ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs + adapter->stats.rlec; } /********************************************************************** * * This routine is called only when ixgbe_display_debug_stats is enabled. * This routine provides a way to take a look at important statistics * maintained by the driver and hardware. * **********************************************************************/ static void ixgbe_print_hw_stats(struct adapter * adapter) { device_t dev = adapter->dev; device_printf(dev,"Std Mbuf Failed = %lu\n", adapter->mbuf_defrag_failed); device_printf(dev,"Missed Packets = %llu\n", (long long)adapter->stats.mpc[0]); device_printf(dev,"Receive length errors = %llu\n", ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); device_printf(dev,"Crc errors = %llu\n", (long long)adapter->stats.crcerrs); device_printf(dev,"Driver dropped packets = %lu\n", adapter->dropped_pkts); device_printf(dev, "watchdog timeouts = %ld\n", adapter->watchdog_events); device_printf(dev,"XON Rcvd = %llu\n", (long long)adapter->stats.lxonrxc); device_printf(dev,"XON Xmtd = %llu\n", (long long)adapter->stats.lxontxc); device_printf(dev,"XOFF Rcvd = %llu\n", (long long)adapter->stats.lxoffrxc); device_printf(dev,"XOFF Xmtd = %llu\n", (long long)adapter->stats.lxofftxc); device_printf(dev,"Total Packets Rcvd = %llu\n", (long long)adapter->stats.tpr); device_printf(dev,"Good Packets Rcvd = %llu\n", (long long)adapter->stats.gprc); device_printf(dev,"Good Packets Xmtd = %llu\n", (long long)adapter->stats.gptc); device_printf(dev,"TSO Transmissions = %lu\n", adapter->tso_tx); return; } /********************************************************************** * * This routine is called only when em_display_debug_stats is enabled. * This routine provides a way to take a look at important statistics * maintained by the driver and hardware. * **********************************************************************/ static void ixgbe_print_debug_info(struct adapter *adapter) { device_t dev = adapter->dev; struct rx_ring *rxr = adapter->rx_rings; struct tx_ring *txr = adapter->tx_rings; struct ixgbe_hw *hw = &adapter->hw; device_printf(dev,"Error Byte Count = %u \n", IXGBE_READ_REG(hw, IXGBE_ERRBC)); for (int i = 0; i < adapter->num_queues; i++, rxr++) { struct lro_ctrl *lro = &rxr->lro; device_printf(dev,"Queue[%d]: rdh = %d, hw rdt = %d\n", i, IXGBE_READ_REG(hw, IXGBE_RDH(i)), IXGBE_READ_REG(hw, IXGBE_RDT(i))); device_printf(dev,"RX(%d) Packets Received: %lld\n", rxr->me, (long long)rxr->rx_packets); device_printf(dev,"RX(%d) Split RX Packets: %lld\n", rxr->me, (long long)rxr->rx_split_packets); device_printf(dev,"RX(%d) Bytes Received: %lu\n", rxr->me, (long)rxr->rx_bytes); device_printf(dev,"RX(%d) IRQ Handled: %lu\n", rxr->me, (long)rxr->rx_irq); device_printf(dev,"RX(%d) LRO Queued= %d\n", rxr->me, lro->lro_queued); device_printf(dev,"RX(%d) LRO Flushed= %d\n", rxr->me, lro->lro_flushed); device_printf(dev,"RX(%d) HW LRO Merges= %lu\n", rxr->me, (long)rxr->rsc_num); } for (int i = 0; i < adapter->num_queues; i++, txr++) { device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, IXGBE_READ_REG(hw, IXGBE_TDH(i)), IXGBE_READ_REG(hw, IXGBE_TDT(i))); device_printf(dev,"TX(%d) Packets Sent: %lu\n", txr->me, (long)txr->total_packets); device_printf(dev,"TX(%d) IRQ Handled: %lu\n", txr->me, (long)txr->tx_irq); device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", txr->me, (long)txr->no_tx_desc_avail); } device_printf(dev,"Link IRQ Handled: %lu\n", (long)adapter->link_irq); return; } static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS) { int error; int result; struct adapter *adapter; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *) arg1; ixgbe_print_hw_stats(adapter); } return error; } static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS) { int error, result; struct adapter *adapter; result = -1; error = sysctl_handle_int(oidp, &result, 0, req); if (error || !req->newptr) return (error); if (result == 1) { adapter = (struct adapter *) arg1; ixgbe_print_debug_info(adapter); } return error; } /* ** Set flow control using sysctl: ** Flow control values: ** 0 - off ** 1 - rx pause ** 2 - tx pause ** 3 - full */ static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) { int error; struct adapter *adapter; error = sysctl_handle_int(oidp, &ixgbe_flow_control, 0, req); if (error) return (error); adapter = (struct adapter *) arg1; switch (ixgbe_flow_control) { case ixgbe_fc_rx_pause: case ixgbe_fc_tx_pause: case ixgbe_fc_full: adapter->hw.fc.requested_mode = ixgbe_flow_control; break; case ixgbe_fc_none: default: adapter->hw.fc.requested_mode = ixgbe_fc_none; } ixgbe_fc_enable(&adapter->hw, 0); return error; } static void ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, const char *description, int *limit, int value) { *limit = value; SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); } #ifdef IXGBE_IEEE1588 /* ** ixgbe_hwtstamp_ioctl - control hardware time stamping ** ** Outgoing time stamping can be enabled and disabled. Play nice and ** disable it when requested, although it shouldn't case any overhead ** when no packet needs it. At most one packet in the queue may be ** marked for time stamping, otherwise it would be impossible to tell ** for sure to which packet the hardware time stamp belongs. ** ** Incoming time stamping has to be configured via the hardware ** filters. Not all combinations are supported, in particular event ** type has to be specified. Matching the kind of event packet is ** not supported, with the exception of "all V2 events regardless of ** level 2 or 4". ** */ static int ixgbe_hwtstamp_ioctl(struct adapter *adapter, struct ifreq *ifr) { struct ixgbe_hw *hw = &adapter->hw; struct hwtstamp_ctrl *config; u32 tsync_tx_ctl_bit = IXGBE_TSYNCTXCTL_ENABLED; u32 tsync_rx_ctl_bit = IXGBE_TSYNCRXCTL_ENABLED; u32 tsync_rx_ctl_type = 0; u32 tsync_rx_cfg = 0; int is_l4 = 0; int is_l2 = 0; u16 port = 319; /* PTP */ u32 regval; config = (struct hwtstamp_ctrl *) ifr->ifr_data; /* reserved for future extensions */ if (config->flags) return (EINVAL); switch (config->tx_type) { case HWTSTAMP_TX_OFF: tsync_tx_ctl_bit = 0; break; case HWTSTAMP_TX_ON: tsync_tx_ctl_bit = IXGBE_TSYNCTXCTL_ENABLED; break; default: return (ERANGE); } switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: tsync_rx_ctl_bit = 0; break; case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: case HWTSTAMP_FILTER_ALL: /* * register TSYNCRXCFG must be set, therefore it is not * possible to time stamp both Sync and Delay_Req messages * => fall back to time stamping all packets */ tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_ALL; config->rx_filter = HWTSTAMP_FILTER_ALL; break; case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_L4_V1; tsync_rx_cfg = IXGBE_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE; is_l4 = 1; break; case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_L4_V1; tsync_rx_cfg = IXGBE_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE; is_l4 = 1; break; case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_L2_L4_V2; tsync_rx_cfg = IXGBE_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE; is_l2 = 1; is_l4 = 1; config->rx_filter = HWTSTAMP_FILTER_SOME; break; case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_L2_L4_V2; tsync_rx_cfg = IXGBE_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE; is_l2 = 1; is_l4 = 1; config->rx_filter = HWTSTAMP_FILTER_SOME; break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_SYNC: case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: tsync_rx_ctl_type = IXGBE_TSYNCRXCTL_TYPE_EVENT_V2; config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; is_l2 = 1; break; default: return -ERANGE; } /* enable/disable TX */ regval = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL); regval = (regval & ~IXGBE_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit; IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, regval); /* enable/disable RX, define which PTP packets are time stamped */ regval = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL); regval = (regval & ~IXGBE_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit; regval = (regval & ~0xE) | tsync_rx_ctl_type; IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, regval); IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCFG, tsync_rx_cfg); /* * Ethertype Filter Queue Filter[0][15:0] = 0x88F7 * (Ethertype to filter on) * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter) * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping) */ IXGBE_WRITE_REG(hw, IXGBE_ETQF0, is_l2 ? 0x440088f7 : 0); /* L4 Queue Filter[0]: only filter by source and destination port */ IXGBE_WRITE_REG(hw, IXGBE_SPQF0, htons(port)); IXGBE_WRITE_REG(hw, IXGBE_IMIREXT(0), is_l4 ? ((1<<12) | (1<<19) /* bypass size and control flags */) : 0); IXGBE_WRITE_REG(hw, IXGBE_IMIR(0), is_l4 ? (htons(port) | (0<<16) /* immediate interrupt disabled */ | 0 /* (1<<17) bit cleared: do not bypass destination port check */) : 0); IXGBE_WRITE_REG(hw, IXGBE_FTQF0, is_l4 ? (0x11 /* UDP */ | (1<<15) /* VF not compared */ | (1<<27) /* Enable Timestamping */ | (7<<28) /* only source port filter enabled, source/target address and protocol masked */) : ((1<<15) | (15<<28) /* all mask bits set = filter not enabled */)); wrfl(); adapter->hwtstamp_ctrl = config; /* clear TX/RX time stamp registers, just to be sure */ regval = IXGBE_READ_REG(hw, IXGBE_TXSTMPH); regval = IXGBE_READ_REG(hw, IXGBE_RXSTMPH); return (error); } /* ** ixgbe_read_clock - read raw cycle counter (to be used by time counter) */ static cycle_t ixgbe_read_clock(const struct cyclecounter *tc) { struct adapter *adapter = container_of(tc, struct igb_adapter, cycles); struct ixgbe_hw *hw = &adapter->hw; u64 stamp; stamp = IXGBE_READ_REG(hw, IXGBE_SYSTIML); stamp |= (u64)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) << 32ULL; return (stamp); } #endif /* IXGBE_IEEE1588 */ Index: user/imp/tbemd/sys/dev/mxge/if_mxge.c =================================================================== --- user/imp/tbemd/sys/dev/mxge/if_mxge.c (revision 203849) +++ user/imp/tbemd/sys/dev/mxge/if_mxge.c (revision 203850) @@ -1,4857 +1,4857 @@ /****************************************************************************** Copyright (c) 2006-2009, Myricom Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Neither the name of the Myricom Inc, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ***************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* count xmits ourselves, rather than via drbr */ #define NO_SLOW_STATS #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* XXX for pci_cfg_restore */ #include /* for pmap_mapdev() */ #include #if defined(__i386) || defined(__amd64) #include #endif #include #include /*#define MXGE_FAKE_IFP*/ #include #ifdef IFNET_BUF_RING #include #endif #include "opt_inet.h" /* tunable params */ static int mxge_nvidia_ecrc_enable = 1; static int mxge_force_firmware = 0; static int mxge_intr_coal_delay = 30; static int mxge_deassert_wait = 1; static int mxge_flow_control = 1; static int mxge_verbose = 0; static int mxge_lro_cnt = 8; static int mxge_ticks; static int mxge_max_slices = 1; static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; static int mxge_always_promisc = 0; static int mxge_initial_mtu = ETHERMTU_JUMBO; static int mxge_throttle = 0; static char *mxge_fw_unaligned = "mxge_ethp_z8e"; static char *mxge_fw_aligned = "mxge_eth_z8e"; static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e"; static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e"; static int mxge_probe(device_t dev); static int mxge_attach(device_t dev); static int mxge_detach(device_t dev); static int mxge_shutdown(device_t dev); static void mxge_intr(void *arg); static device_method_t mxge_methods[] = { /* Device interface */ DEVMETHOD(device_probe, mxge_probe), DEVMETHOD(device_attach, mxge_attach), DEVMETHOD(device_detach, mxge_detach), DEVMETHOD(device_shutdown, mxge_shutdown), {0, 0} }; static driver_t mxge_driver = { "mxge", mxge_methods, sizeof(mxge_softc_t), }; static devclass_t mxge_devclass; /* Declare ourselves to be a child of the PCI bus.*/ DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, 0, 0); MODULE_DEPEND(mxge, firmware, 1, 1, 1); MODULE_DEPEND(mxge, zlib, 1, 1, 1); static int mxge_load_firmware(mxge_softc_t *sc, int adopt); static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data); static int mxge_close(mxge_softc_t *sc, int down); static int mxge_open(mxge_softc_t *sc); static void mxge_tick(void *arg); static int mxge_probe(device_t dev) { int rev; if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) && ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) || (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) { rev = pci_get_revid(dev); switch (rev) { case MXGE_PCI_REV_Z8E: device_set_desc(dev, "Myri10G-PCIE-8A"); break; case MXGE_PCI_REV_Z8ES: device_set_desc(dev, "Myri10G-PCIE-8B"); break; default: device_set_desc(dev, "Myri10G-PCIE-8??"); device_printf(dev, "Unrecognized rev %d NIC\n", rev); break; } return 0; } return ENXIO; } static void mxge_enable_wc(mxge_softc_t *sc) { #if defined(__i386) || defined(__amd64) vm_offset_t len; int err; sc->wc = 1; len = rman_get_size(sc->mem_res); err = pmap_change_attr((vm_offset_t) sc->sram, len, PAT_WRITE_COMBINING); if (err != 0) { device_printf(sc->dev, "pmap_change_attr failed, %d\n", err); sc->wc = 0; } #endif } /* callback to get our DMA address */ static void mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error == 0) { *(bus_addr_t *) arg = segs->ds_addr; } } static int mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes, bus_size_t alignment) { int err; device_t dev = sc->dev; bus_size_t boundary, maxsegsize; if (bytes > 4096 && alignment == 4096) { boundary = 0; maxsegsize = bytes; } else { boundary = 4096; maxsegsize = 4096; } /* allocate DMAable memory tags */ err = bus_dma_tag_create(sc->parent_dmat, /* parent */ alignment, /* alignment */ boundary, /* boundary */ BUS_SPACE_MAXADDR, /* low */ BUS_SPACE_MAXADDR, /* high */ NULL, NULL, /* filter */ bytes, /* maxsize */ 1, /* num segs */ maxsegsize, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lock */ &dma->dmat); /* tag */ if (err != 0) { device_printf(dev, "couldn't alloc tag (err = %d)\n", err); return err; } /* allocate DMAable memory & map */ err = bus_dmamem_alloc(dma->dmat, &dma->addr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO), &dma->map); if (err != 0) { device_printf(dev, "couldn't alloc mem (err = %d)\n", err); goto abort_with_dmat; } /* load the memory */ err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes, mxge_dmamap_callback, (void *)&dma->bus_addr, 0); if (err != 0) { device_printf(dev, "couldn't load map (err = %d)\n", err); goto abort_with_mem; } return 0; abort_with_mem: bus_dmamem_free(dma->dmat, dma->addr, dma->map); abort_with_dmat: (void)bus_dma_tag_destroy(dma->dmat); return err; } static void mxge_dma_free(mxge_dma_t *dma) { bus_dmamap_unload(dma->dmat, dma->map); bus_dmamem_free(dma->dmat, dma->addr, dma->map); (void)bus_dma_tag_destroy(dma->dmat); } /* * The eeprom strings on the lanaiX have the format * SN=x\0 * MAC=x:x:x:x:x:x\0 * PC=text\0 */ static int mxge_parse_strings(mxge_softc_t *sc) { #define MXGE_NEXT_STRING(p) while(ptr < limit && *ptr++) char *ptr, *limit; int i, found_mac; ptr = sc->eeprom_strings; limit = sc->eeprom_strings + MXGE_EEPROM_STRINGS_SIZE; found_mac = 0; while (ptr < limit && *ptr != '\0') { if (memcmp(ptr, "MAC=", 4) == 0) { ptr += 1; sc->mac_addr_string = ptr; for (i = 0; i < 6; i++) { ptr += 3; if ((ptr + 2) > limit) goto abort; sc->mac_addr[i] = strtoul(ptr, NULL, 16); found_mac = 1; } } else if (memcmp(ptr, "PC=", 3) == 0) { ptr += 3; strncpy(sc->product_code_string, ptr, sizeof (sc->product_code_string) - 1); } else if (memcmp(ptr, "SN=", 3) == 0) { ptr += 3; strncpy(sc->serial_number_string, ptr, sizeof (sc->serial_number_string) - 1); } MXGE_NEXT_STRING(ptr); } if (found_mac) return 0; abort: device_printf(sc->dev, "failed to parse eeprom_strings\n"); return ENXIO; } #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__ static void mxge_enable_nvidia_ecrc(mxge_softc_t *sc) { uint32_t val; unsigned long base, off; char *va, *cfgptr; device_t pdev, mcp55; uint16_t vendor_id, device_id, word; uintptr_t bus, slot, func, ivend, idev; uint32_t *ptr32; if (!mxge_nvidia_ecrc_enable) return; pdev = device_get_parent(device_get_parent(sc->dev)); if (pdev == NULL) { device_printf(sc->dev, "could not find parent?\n"); return; } vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2); device_id = pci_read_config(pdev, PCIR_DEVICE, 2); if (vendor_id != 0x10de) return; base = 0; if (device_id == 0x005d) { /* ck804, base address is magic */ base = 0xe0000000UL; } else if (device_id >= 0x0374 && device_id <= 0x378) { /* mcp55, base address stored in chipset */ mcp55 = pci_find_bsf(0, 0, 0); if (mcp55 && 0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) && 0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) { word = pci_read_config(mcp55, 0x90, 2); base = ((unsigned long)word & 0x7ffeU) << 25; } } if (!base) return; /* XXXX Test below is commented because it is believed that doing config read/write beyond 0xff will access the config space for the next larger function. Uncomment this and remove the hacky pmap_mapdev() way of accessing config space when FreeBSD grows support for extended pcie config space access */ #if 0 /* See if we can, by some miracle, access the extended config space */ val = pci_read_config(pdev, 0x178, 4); if (val != 0xffffffff) { val |= 0x40; pci_write_config(pdev, 0x178, val, 4); return; } #endif /* Rather than using normal pci config space writes, we must * map the Nvidia config space ourselves. This is because on * opteron/nvidia class machine the 0xe000000 mapping is * handled by the nvidia chipset, that means the internal PCI * device (the on-chip northbridge), or the amd-8131 bridge * and things behind them are not visible by this method. */ BUS_READ_IVAR(device_get_parent(pdev), pdev, PCI_IVAR_BUS, &bus); BUS_READ_IVAR(device_get_parent(pdev), pdev, PCI_IVAR_SLOT, &slot); BUS_READ_IVAR(device_get_parent(pdev), pdev, PCI_IVAR_FUNCTION, &func); BUS_READ_IVAR(device_get_parent(pdev), pdev, PCI_IVAR_VENDOR, &ivend); BUS_READ_IVAR(device_get_parent(pdev), pdev, PCI_IVAR_DEVICE, &idev); off = base + 0x00100000UL * (unsigned long)bus + 0x00001000UL * (unsigned long)(func + 8 * slot); /* map it into the kernel */ va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE); if (va == NULL) { device_printf(sc->dev, "pmap_kenter_temporary didn't\n"); return; } /* get a pointer to the config space mapped into the kernel */ cfgptr = va + (off & PAGE_MASK); /* make sure that we can really access it */ vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR); device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE); if (! (vendor_id == ivend && device_id == idev)) { device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n", vendor_id, device_id); pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); return; } ptr32 = (uint32_t*)(cfgptr + 0x178); val = *ptr32; if (val == 0xffffffff) { device_printf(sc->dev, "extended mapping failed\n"); pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); return; } *ptr32 = val | 0x40; pmap_unmapdev((vm_offset_t)va, PAGE_SIZE); if (mxge_verbose) device_printf(sc->dev, "Enabled ECRC on upstream Nvidia bridge " "at %d:%d:%d\n", (int)bus, (int)slot, (int)func); return; } #else static void mxge_enable_nvidia_ecrc(mxge_softc_t *sc) { device_printf(sc->dev, "Nforce 4 chipset on non-x86/amd64!?!?!\n"); return; } #endif static int mxge_dma_test(mxge_softc_t *sc, int test_type) { mxge_cmd_t cmd; bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr; int status; uint32_t len; char *test = " "; /* Run a small DMA test. * The magic multipliers to the length tell the firmware * to do DMA read, write, or read+write tests. The * results are returned in cmd.data0. The upper 16 * bits of the return is the number of transfers completed. * The lower 16 bits is the time in 0.5us ticks that the * transfers took to complete. */ len = sc->tx_boundary; cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); cmd.data2 = len * 0x10000; status = mxge_send_cmd(sc, test_type, &cmd); if (status != 0) { test = "read"; goto abort; } sc->read_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff); cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); cmd.data2 = len * 0x1; status = mxge_send_cmd(sc, test_type, &cmd); if (status != 0) { test = "write"; goto abort; } sc->write_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff); cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus); cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus); cmd.data2 = len * 0x10001; status = mxge_send_cmd(sc, test_type, &cmd); if (status != 0) { test = "read/write"; goto abort; } sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) / (cmd.data0 & 0xffff); abort: if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) device_printf(sc->dev, "DMA %s benchmark failed: %d\n", test, status); return status; } /* * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput * when the PCI-E Completion packets are aligned on an 8-byte * boundary. Some PCI-E chip sets always align Completion packets; on * the ones that do not, the alignment can be enforced by enabling * ECRC generation (if supported). * * When PCI-E Completion packets are not aligned, it is actually more * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. * * If the driver can neither enable ECRC nor verify that it has * already been enabled, then it must use a firmware image which works * around unaligned completion packets (ethp_z8e.dat), and it should * also ensure that it never gives the device a Read-DMA which is * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is * enabled, then the driver should use the aligned (eth_z8e.dat) * firmware image, and set tx_boundary to 4KB. */ static int mxge_firmware_probe(mxge_softc_t *sc) { device_t dev = sc->dev; int reg, status; uint16_t pectl; sc->tx_boundary = 4096; /* * Verify the max read request size was set to 4KB * before trying the test with 4KB. */ if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { pectl = pci_read_config(dev, reg + 0x8, 2); if ((pectl & (5 << 12)) != (5 << 12)) { device_printf(dev, "Max Read Req. size != 4k (0x%x\n", pectl); sc->tx_boundary = 2048; } } /* * load the optimized firmware (which assumes aligned PCIe * completions) in order to see if it works on this host. */ sc->fw_name = mxge_fw_aligned; status = mxge_load_firmware(sc, 1); if (status != 0) { return status; } /* * Enable ECRC if possible */ mxge_enable_nvidia_ecrc(sc); /* * Run a DMA test which watches for unaligned completions and * aborts on the first one seen. */ status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST); if (status == 0) return 0; /* keep the aligned firmware */ if (status != E2BIG) device_printf(dev, "DMA test failed: %d\n", status); if (status == ENOSYS) device_printf(dev, "Falling back to ethp! " "Please install up to date fw\n"); return status; } static int mxge_select_firmware(mxge_softc_t *sc) { int aligned = 0; int force_firmware = mxge_force_firmware; if (sc->throttle) force_firmware = sc->throttle; if (force_firmware != 0) { if (force_firmware == 1) aligned = 1; else aligned = 0; if (mxge_verbose) device_printf(sc->dev, "Assuming %s completions (forced)\n", aligned ? "aligned" : "unaligned"); goto abort; } /* if the PCIe link width is 4 or less, we can use the aligned firmware and skip any checks */ if (sc->link_width != 0 && sc->link_width <= 4) { device_printf(sc->dev, "PCIe x%d Link, expect reduced performance\n", sc->link_width); aligned = 1; goto abort; } if (0 == mxge_firmware_probe(sc)) return 0; abort: if (aligned) { sc->fw_name = mxge_fw_aligned; sc->tx_boundary = 4096; } else { sc->fw_name = mxge_fw_unaligned; sc->tx_boundary = 2048; } return (mxge_load_firmware(sc, 0)); } union qualhack { const char *ro_char; char *rw_char; }; static int mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr) { if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) { device_printf(sc->dev, "Bad firmware type: 0x%x\n", be32toh(hdr->mcp_type)); return EIO; } /* save firmware version for sysctl */ strncpy(sc->fw_version, hdr->version, sizeof (sc->fw_version)); if (mxge_verbose) device_printf(sc->dev, "firmware id: %s\n", hdr->version); sscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major, &sc->fw_ver_minor, &sc->fw_ver_tiny); if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) { device_printf(sc->dev, "Found firmware version %s\n", sc->fw_version); device_printf(sc->dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR); return EINVAL; } return 0; } static void * z_alloc(void *nil, u_int items, u_int size) { void *ptr; ptr = malloc(items * size, M_TEMP, M_NOWAIT); return ptr; } static void z_free(void *nil, void *ptr) { free(ptr, M_TEMP); } static int mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit) { z_stream zs; char *inflate_buffer; const struct firmware *fw; const mcp_gen_header_t *hdr; unsigned hdr_offset; int status; unsigned int i; char dummy; size_t fw_len; fw = firmware_get(sc->fw_name); if (fw == NULL) { device_printf(sc->dev, "Could not find firmware image %s\n", sc->fw_name); return ENOENT; } /* setup zlib and decompress f/w */ bzero(&zs, sizeof (zs)); zs.zalloc = z_alloc; zs.zfree = z_free; status = inflateInit(&zs); if (status != Z_OK) { status = EIO; goto abort_with_fw; } /* the uncompressed size is stored as the firmware version, which would otherwise go unused */ fw_len = (size_t) fw->version; inflate_buffer = malloc(fw_len, M_TEMP, M_NOWAIT); if (inflate_buffer == NULL) goto abort_with_zs; zs.avail_in = fw->datasize; zs.next_in = __DECONST(char *, fw->data); zs.avail_out = fw_len; zs.next_out = inflate_buffer; status = inflate(&zs, Z_FINISH); if (status != Z_STREAM_END) { device_printf(sc->dev, "zlib %d\n", status); status = EIO; goto abort_with_buffer; } /* check id */ hdr_offset = htobe32(*(const uint32_t *) (inflate_buffer + MCP_HEADER_PTR_OFFSET)); if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) { device_printf(sc->dev, "Bad firmware file"); status = EIO; goto abort_with_buffer; } hdr = (const void*)(inflate_buffer + hdr_offset); status = mxge_validate_firmware(sc, hdr); if (status != 0) goto abort_with_buffer; /* Copy the inflated firmware to NIC SRAM. */ for (i = 0; i < fw_len; i += 256) { mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, inflate_buffer + i, min(256U, (unsigned)(fw_len - i))); wmb(); dummy = *sc->sram; wmb(); } *limit = fw_len; status = 0; abort_with_buffer: free(inflate_buffer, M_TEMP); abort_with_zs: inflateEnd(&zs); abort_with_fw: firmware_put(fw, FIRMWARE_UNLOAD); return status; } /* * Enable or disable periodic RDMAs from the host to make certain * chipsets resend dropped PCIe messages */ static void mxge_dummy_rdma(mxge_softc_t *sc, int enable) { char buf_bytes[72]; volatile uint32_t *confirm; volatile char *submit; uint32_t *buf, dma_low, dma_high; int i; buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; wmb(); /* send an rdma command to the PCIe engine, and wait for the response in the confirmation address. The firmware should write a -1 there to indicate it is alive and well */ dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); buf[0] = htobe32(dma_high); /* confirm addr MSW */ buf[1] = htobe32(dma_low); /* confirm addr LSW */ buf[2] = htobe32(0xffffffff); /* confirm data */ dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr); dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr); buf[3] = htobe32(dma_high); /* dummy addr MSW */ buf[4] = htobe32(dma_low); /* dummy addr LSW */ buf[5] = htobe32(enable); /* enable? */ submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); mxge_pio_copy(submit, buf, 64); wmb(); DELAY(1000); wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000); i++; } if (*confirm != 0xffffffff) { device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)", (enable ? "enable" : "disable"), confirm, *confirm); } return; } static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data) { mcp_cmd_t *buf; char buf_bytes[sizeof(*buf) + 8]; volatile mcp_cmd_response_t *response = sc->cmd; volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD; uint32_t dma_low, dma_high; int err, sleep_total = 0; /* ensure buf is aligned to 8 bytes */ buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL); buf->data0 = htobe32(data->data0); buf->data1 = htobe32(data->data1); buf->data2 = htobe32(data->data2); buf->cmd = htobe32(cmd); dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); buf->response_addr.low = htobe32(dma_low); buf->response_addr.high = htobe32(dma_high); mtx_lock(&sc->cmd_mtx); response->result = 0xffffffff; wmb(); mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); /* wait up to 20ms */ err = EAGAIN; for (sleep_total = 0; sleep_total < 20; sleep_total++) { bus_dmamap_sync(sc->cmd_dma.dmat, sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); wmb(); switch (be32toh(response->result)) { case 0: data->data0 = be32toh(response->data); err = 0; break; case 0xffffffff: DELAY(1000); break; case MXGEFW_CMD_UNKNOWN: err = ENOSYS; break; case MXGEFW_CMD_ERROR_UNALIGNED: err = E2BIG; break; case MXGEFW_CMD_ERROR_BUSY: err = EBUSY; break; default: device_printf(sc->dev, "mxge: command %d " "failed, result = %d\n", cmd, be32toh(response->result)); err = ENXIO; break; } if (err != EAGAIN) break; } if (err == EAGAIN) device_printf(sc->dev, "mxge: command %d timed out" "result = %d\n", cmd, be32toh(response->result)); mtx_unlock(&sc->cmd_mtx); return err; } static int mxge_adopt_running_firmware(mxge_softc_t *sc) { struct mcp_gen_header *hdr; const size_t bytes = sizeof (struct mcp_gen_header); size_t hdr_offset; int status; /* find running firmware header */ hdr_offset = htobe32(*(volatile uint32_t *) (sc->sram + MCP_HEADER_PTR_OFFSET)); if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) { device_printf(sc->dev, "Running firmware has bad header offset (%d)\n", (int)hdr_offset); return EIO; } /* copy header of running firmware from SRAM to host memory to * validate firmware */ hdr = malloc(bytes, M_DEVBUF, M_NOWAIT); if (hdr == NULL) { device_printf(sc->dev, "could not malloc firmware hdr\n"); return ENOMEM; } bus_space_read_region_1(rman_get_bustag(sc->mem_res), rman_get_bushandle(sc->mem_res), hdr_offset, (char *)hdr, bytes); status = mxge_validate_firmware(sc, hdr); free(hdr, M_DEVBUF); /* * check to see if adopted firmware has bug where adopting * it will cause broadcasts to be filtered unless the NIC * is kept in ALLMULTI mode */ if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 && sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) { sc->adopted_rx_filter_bug = 1; device_printf(sc->dev, "Adopting fw %d.%d.%d: " "working around rx filter bug\n", sc->fw_ver_major, sc->fw_ver_minor, sc->fw_ver_tiny); } return status; } static int mxge_load_firmware(mxge_softc_t *sc, int adopt) { volatile uint32_t *confirm; volatile char *submit; char buf_bytes[72]; uint32_t *buf, size, dma_low, dma_high; int status, i; buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL); size = sc->sram_size; status = mxge_load_firmware_helper(sc, &size); if (status) { if (!adopt) return status; /* Try to use the currently running firmware, if it is new enough */ status = mxge_adopt_running_firmware(sc); if (status) { device_printf(sc->dev, "failed to adopt running firmware\n"); return status; } device_printf(sc->dev, "Successfully adopted running firmware\n"); if (sc->tx_boundary == 4096) { device_printf(sc->dev, "Using firmware currently running on NIC" ". For optimal\n"); device_printf(sc->dev, "performance consider loading optimized " "firmware\n"); } sc->fw_name = mxge_fw_unaligned; sc->tx_boundary = 2048; return 0; } /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; wmb(); /* send a reload command to the bootstrap MCP, and wait for the response in the confirmation address. The firmware should write a -1 there to indicate it is alive and well */ dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr); dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr); buf[0] = htobe32(dma_high); /* confirm addr MSW */ buf[1] = htobe32(dma_low); /* confirm addr LSW */ buf[2] = htobe32(0xffffffff); /* confirm data */ /* FIX: All newest firmware should un-protect the bottom of the sram before handoff. However, the very first interfaces do not. Therefore the handoff copy must skip the first 8 bytes */ /* where the code starts*/ buf[3] = htobe32(MXGE_FW_OFFSET + 8); buf[4] = htobe32(size - 8); /* length of code */ buf[5] = htobe32(8); /* where to copy to */ buf[6] = htobe32(0); /* where to jump to */ submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); mxge_pio_copy(submit, buf, 64); wmb(); DELAY(1000); wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000*10); i++; bus_dmamap_sync(sc->cmd_dma.dmat, sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); } if (*confirm != 0xffffffff) { device_printf(sc->dev,"handoff failed (%p = 0x%x)", confirm, *confirm); return ENXIO; } return 0; } static int mxge_update_mac_address(mxge_softc_t *sc) { mxge_cmd_t cmd; uint8_t *addr = sc->mac_addr; int status; cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3]); cmd.data1 = ((addr[4] << 8) | (addr[5])); status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd); return status; } static int mxge_change_pause(mxge_softc_t *sc, int pause) { mxge_cmd_t cmd; int status; if (pause) status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL, &cmd); else status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL, &cmd); if (status) { device_printf(sc->dev, "Failed to set flow control mode\n"); return ENXIO; } sc->pause = pause; return 0; } static void mxge_change_promisc(mxge_softc_t *sc, int promisc) { mxge_cmd_t cmd; int status; if (mxge_always_promisc) promisc = 1; if (promisc) status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC, &cmd); else status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC, &cmd); if (status) { device_printf(sc->dev, "Failed to set promisc mode\n"); } } static void mxge_set_multicast_list(mxge_softc_t *sc) { mxge_cmd_t cmd; struct ifmultiaddr *ifma; struct ifnet *ifp = sc->ifp; int err; /* This firmware is known to not support multicast */ if (!sc->fw_multicast_support) return; /* Disable multicast filtering while we play with the lists*/ err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd); if (err != 0) { device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI," " error status: %d\n", err); return; } if (sc->adopted_rx_filter_bug) return; if (ifp->if_flags & IFF_ALLMULTI) /* request to disable multicast filtering, so quit here */ return; /* Flush all the filters */ err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd); if (err != 0) { device_printf(sc->dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS" ", error status: %d\n", err); return; } /* Walk the multicast list, and add each address */ if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), &cmd.data0, 4); bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4, &cmd.data1, 2); cmd.data0 = htonl(cmd.data0); cmd.data1 = htonl(cmd.data1); err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd); if (err != 0) { device_printf(sc->dev, "Failed " "MXGEFW_JOIN_MULTICAST_GROUP, error status:" "%d\t", err); /* abort, leaving multicast filtering off */ if_maddr_runlock(ifp); return; } } if_maddr_runlock(ifp); /* Enable multicast filtering */ err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd); if (err != 0) { device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI" ", error status: %d\n", err); } } static int mxge_max_mtu(mxge_softc_t *sc) { mxge_cmd_t cmd; int status; if (MJUMPAGESIZE - MXGEFW_PAD > MXGEFW_MAX_MTU) return MXGEFW_MAX_MTU - MXGEFW_PAD; /* try to set nbufs to see if it we can use virtually contiguous jumbos */ cmd.data0 = 0; status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, &cmd); if (status == 0) return MXGEFW_MAX_MTU - MXGEFW_PAD; /* otherwise, we're limited to MJUMPAGESIZE */ return MJUMPAGESIZE - MXGEFW_PAD; } static int mxge_reset(mxge_softc_t *sc, int interrupts_setup) { struct mxge_slice_state *ss; mxge_rx_done_t *rx_done; volatile uint32_t *irq_claim; mxge_cmd_t cmd; int slice, status; /* try to send a reset command to the card to see if it is alive */ memset(&cmd, 0, sizeof (cmd)); status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); if (status != 0) { device_printf(sc->dev, "failed reset\n"); return ENXIO; } mxge_dummy_rdma(sc, 1); /* set the intrq size */ cmd.data0 = sc->rx_ring_size; status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); /* * Even though we already know how many slices are supported * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES * has magic side effects, and must be called after a reset. * It must be called prior to calling any RSS related cmds, * including assigning an interrupt queue for anything but * slice 0. It must also be called *after* * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by * the firmware to compute offsets. */ if (sc->num_slices > 1) { /* ask the maximum number of slices it supports */ status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd); if (status != 0) { device_printf(sc->dev, "failed to get number of slices\n"); return status; } /* * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior * to setting up the interrupt queue DMA */ cmd.data0 = sc->num_slices; cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; #ifdef IFNET_BUF_RING cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; #endif status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES, &cmd); if (status != 0) { device_printf(sc->dev, "failed to set number of slices\n"); return status; } } if (interrupts_setup) { /* Now exchange information about interrupts */ for (slice = 0; slice < sc->num_slices; slice++) { rx_done = &sc->ss[slice].rx_done; memset(rx_done->entry, 0, sc->rx_ring_size); cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr); cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr); cmd.data2 = slice; status |= mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_DMA, &cmd); } } status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd); sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0); status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd); irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0); status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, &cmd); sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0); if (status != 0) { device_printf(sc->dev, "failed set interrupt parameters\n"); return status; } *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay); /* run a DMA benchmark */ (void) mxge_dma_test(sc, MXGEFW_DMA_TEST); for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; ss->irq_claim = irq_claim + (2 * slice); /* reset mcp/driver shared state back to 0 */ ss->rx_done.idx = 0; ss->rx_done.cnt = 0; ss->tx.req = 0; ss->tx.done = 0; ss->tx.pkt_done = 0; ss->tx.queue_active = 0; ss->tx.activate = 0; ss->tx.deactivate = 0; ss->tx.wake = 0; ss->tx.defrag = 0; ss->tx.stall = 0; ss->rx_big.cnt = 0; ss->rx_small.cnt = 0; ss->lro_bad_csum = 0; ss->lro_queued = 0; ss->lro_flushed = 0; if (ss->fw_stats != NULL) { bzero(ss->fw_stats, sizeof *ss->fw_stats); } } sc->rdma_tags_available = 15; status = mxge_update_mac_address(sc); mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC); mxge_change_pause(sc, sc->pause); mxge_set_multicast_list(sc); if (sc->throttle) { cmd.data0 = sc->throttle; if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd)) { device_printf(sc->dev, "can't enable throttle\n"); } } return status; } static int mxge_change_throttle(SYSCTL_HANDLER_ARGS) { mxge_cmd_t cmd; mxge_softc_t *sc; int err; unsigned int throttle; sc = arg1; throttle = sc->throttle; err = sysctl_handle_int(oidp, &throttle, arg2, req); if (err != 0) { return err; } if (throttle == sc->throttle) return 0; if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE) return EINVAL; mtx_lock(&sc->driver_mtx); cmd.data0 = throttle; err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd); if (err == 0) sc->throttle = throttle; mtx_unlock(&sc->driver_mtx); return err; } static int mxge_change_intr_coal(SYSCTL_HANDLER_ARGS) { mxge_softc_t *sc; unsigned int intr_coal_delay; int err; sc = arg1; intr_coal_delay = sc->intr_coal_delay; err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req); if (err != 0) { return err; } if (intr_coal_delay == sc->intr_coal_delay) return 0; if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000) return EINVAL; mtx_lock(&sc->driver_mtx); *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay); sc->intr_coal_delay = intr_coal_delay; mtx_unlock(&sc->driver_mtx); return err; } static int mxge_change_flow_control(SYSCTL_HANDLER_ARGS) { mxge_softc_t *sc; unsigned int enabled; int err; sc = arg1; enabled = sc->pause; err = sysctl_handle_int(oidp, &enabled, arg2, req); if (err != 0) { return err; } if (enabled == sc->pause) return 0; mtx_lock(&sc->driver_mtx); err = mxge_change_pause(sc, enabled); mtx_unlock(&sc->driver_mtx); return err; } static int mxge_change_lro_locked(mxge_softc_t *sc, int lro_cnt) { struct ifnet *ifp; int err = 0; ifp = sc->ifp; if (lro_cnt == 0) ifp->if_capenable &= ~IFCAP_LRO; else ifp->if_capenable |= IFCAP_LRO; sc->lro_cnt = lro_cnt; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { mxge_close(sc, 0); err = mxge_open(sc); } return err; } static int mxge_change_lro(SYSCTL_HANDLER_ARGS) { mxge_softc_t *sc; unsigned int lro_cnt; int err; sc = arg1; lro_cnt = sc->lro_cnt; err = sysctl_handle_int(oidp, &lro_cnt, arg2, req); if (err != 0) return err; if (lro_cnt == sc->lro_cnt) return 0; if (lro_cnt > 128) return EINVAL; mtx_lock(&sc->driver_mtx); err = mxge_change_lro_locked(sc, lro_cnt); mtx_unlock(&sc->driver_mtx); return err; } static int mxge_handle_be32(SYSCTL_HANDLER_ARGS) { int err; if (arg1 == NULL) return EFAULT; arg2 = be32toh(*(int *)arg1); arg1 = NULL; err = sysctl_handle_int(oidp, arg1, arg2, req); return err; } static void mxge_rem_sysctls(mxge_softc_t *sc) { struct mxge_slice_state *ss; int slice; if (sc->slice_sysctl_tree == NULL) return; for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; if (ss == NULL || ss->sysctl_tree == NULL) continue; sysctl_ctx_free(&ss->sysctl_ctx); ss->sysctl_tree = NULL; } sysctl_ctx_free(&sc->slice_sysctl_ctx); sc->slice_sysctl_tree = NULL; } static void mxge_add_sysctls(mxge_softc_t *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; mcp_irq_data_t *fw; struct mxge_slice_state *ss; int slice; char slice_num[8]; ctx = device_get_sysctl_ctx(sc->dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); fw = sc->ss[0].fw_stats; /* random information */ SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "serial_number", CTLFLAG_RD, &sc->serial_number_string, 0, "serial number"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "product_code", CTLFLAG_RD, &sc->product_code_string, 0, "product_code"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcie_link_width", CTLFLAG_RD, &sc->link_width, 0, "tx_boundary"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_boundary", CTLFLAG_RD, &sc->tx_boundary, 0, "tx_boundary"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "write_combine", CTLFLAG_RD, &sc->wc, 0, "write combining PIO?"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "read_dma_MBs", CTLFLAG_RD, &sc->read_dma, 0, "DMA Read speed in MB/s"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "write_dma_MBs", CTLFLAG_RD, &sc->write_dma, 0, "DMA Write speed in MB/s"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "read_write_dma_MBs", CTLFLAG_RD, &sc->read_write_dma, 0, "DMA concurrent Read/Write speed in MB/s"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "watchdog_resets", CTLFLAG_RD, &sc->watchdog_resets, 0, "Number of times NIC was reset"); /* performance related tunables */ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal_delay", CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_intr_coal, "I", "interrupt coalescing delay in usecs"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "throttle", CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_throttle, "I", "transmit throttling"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "flow_control_enabled", CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_flow_control, "I", "interrupt coalescing delay in usecs"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "deassert_wait", CTLFLAG_RW, &mxge_deassert_wait, 0, "Wait for IRQ line to go low in ihandler"); /* stats block from firmware is in network byte order. Need to swap it */ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "link_up", CTLTYPE_INT|CTLFLAG_RD, &fw->link_up, 0, mxge_handle_be32, "I", "link up"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_tags_available", CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available, 0, mxge_handle_be32, "I", "rdma_tags_available"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_bad_crc32", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_bad_crc32, 0, mxge_handle_be32, "I", "dropped_bad_crc32"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_bad_phy", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_bad_phy, 0, mxge_handle_be32, "I", "dropped_bad_phy"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_link_error_or_filtered", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_error_or_filtered, 0, mxge_handle_be32, "I", "dropped_link_error_or_filtered"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_link_overflow", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow, 0, mxge_handle_be32, "I", "dropped_link_overflow"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_multicast_filtered", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_multicast_filtered, 0, mxge_handle_be32, "I", "dropped_multicast_filtered"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_no_big_buffer", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer, 0, mxge_handle_be32, "I", "dropped_no_big_buffer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_no_small_buffer", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_small_buffer, 0, mxge_handle_be32, "I", "dropped_no_small_buffer"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_overrun", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun, 0, mxge_handle_be32, "I", "dropped_overrun"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_pause", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_pause, 0, mxge_handle_be32, "I", "dropped_pause"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_runt", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt, 0, mxge_handle_be32, "I", "dropped_runt"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dropped_unicast_filtered", CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered, 0, mxge_handle_be32, "I", "dropped_unicast_filtered"); /* verbose printing? */ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "verbose", CTLFLAG_RW, &mxge_verbose, 0, "verbose printing"); /* lro */ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lro_cnt", CTLTYPE_INT|CTLFLAG_RW, sc, 0, mxge_change_lro, "I", "number of lro merge queues"); /* add counters exported for debugging from all slices */ sysctl_ctx_init(&sc->slice_sysctl_ctx); sc->slice_sysctl_tree = SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO, "slice", CTLFLAG_RD, 0, ""); for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; sysctl_ctx_init(&ss->sysctl_ctx); ctx = &ss->sysctl_ctx; children = SYSCTL_CHILDREN(sc->slice_sysctl_tree); sprintf(slice_num, "%d", slice); ss->sysctl_tree = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num, CTLFLAG_RD, 0, ""); children = SYSCTL_CHILDREN(ss->sysctl_tree); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_small_cnt", CTLFLAG_RD, &ss->rx_small.cnt, 0, "rx_small_cnt"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_big_cnt", CTLFLAG_RD, &ss->rx_big.cnt, 0, "rx_small_cnt"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, &ss->lro_flushed, 0, "number of lro merge queues flushed"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, &ss->lro_queued, 0, "number of frames appended to lro merge" "queues"); #ifndef IFNET_BUF_RING /* only transmit from slice 0 for now */ if (slice > 0) continue; #endif SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_req", CTLFLAG_RD, &ss->tx.req, 0, "tx_req"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_done", CTLFLAG_RD, &ss->tx.done, 0, "tx_done"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_pkt_done", CTLFLAG_RD, &ss->tx.pkt_done, 0, "tx_done"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_stall", CTLFLAG_RD, &ss->tx.stall, 0, "tx_stall"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_wake", CTLFLAG_RD, &ss->tx.wake, 0, "tx_wake"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_defrag", CTLFLAG_RD, &ss->tx.defrag, 0, "tx_defrag"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_queue_active", CTLFLAG_RD, &ss->tx.queue_active, 0, "tx_queue_active"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_activate", CTLFLAG_RD, &ss->tx.activate, 0, "tx_activate"); SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_deactivate", CTLFLAG_RD, &ss->tx.deactivate, 0, "tx_deactivate"); } } /* copy an array of mcp_kreq_ether_send_t's to the mcp. Copy backwards one at a time and handle ring wraps */ static inline void mxge_submit_req_backwards(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, int cnt) { int idx, starting_slot; starting_slot = tx->req; while (cnt > 1) { cnt--; idx = (starting_slot + cnt) & tx->mask; mxge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); wmb(); } } /* * copy an array of mcp_kreq_ether_send_t's to the mcp. Copy * at most 32 bytes at a time, so as to avoid involving the software * pio handler in the nic. We re-write the first segment's flags * to mark them valid only after writing the entire chain */ static inline void mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, int cnt) { int idx, i; uint32_t *src_ints; volatile uint32_t *dst_ints; mcp_kreq_ether_send_t *srcp; volatile mcp_kreq_ether_send_t *dstp, *dst; uint8_t last_flags; idx = tx->req & tx->mask; last_flags = src->flags; src->flags = 0; wmb(); dst = dstp = &tx->lanai[idx]; srcp = src; if ((idx + cnt) < tx->mask) { for (i = 0; i < (cnt - 1); i += 2) { mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); wmb(); /* force write every 32 bytes */ srcp += 2; dstp += 2; } } else { /* submit all but the first request, and ensure that it is submitted below */ mxge_submit_req_backwards(tx, src, cnt); i = 0; } if (i < cnt) { /* submit the first request */ mxge_pio_copy(dstp, srcp, sizeof(*src)); wmb(); /* barrier before setting valid flag */ } /* re-write the last 32-bits with the valid flags */ src->flags = last_flags; src_ints = (uint32_t *)src; src_ints+=3; dst_ints = (volatile uint32_t *)dst; dst_ints+=3; *dst_ints = *src_ints; tx->req += cnt; wmb(); } #if IFCAP_TSO4 static void mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m, int busdma_seg_cnt, int ip_off) { mxge_tx_ring_t *tx; mcp_kreq_ether_send_t *req; bus_dma_segment_t *seg; struct ip *ip; struct tcphdr *tcp; uint32_t low, high_swapped; int len, seglen, cum_len, cum_len_next; int next_is_first, chop, cnt, rdma_count, small; uint16_t pseudo_hdr_offset, cksum_offset, mss; uint8_t flags, flags_next; static int once; mss = m->m_pkthdr.tso_segsz; /* negative cum_len signifies to the * send loop that we are still in the * header portion of the TSO packet. */ /* ensure we have the ethernet, IP and TCP header together in the first mbuf, copy it to a scratch buffer if not */ if (__predict_false(m->m_len < ip_off + sizeof (*ip))) { m_copydata(m, 0, ip_off + sizeof (*ip), ss->scratch); ip = (struct ip *)(ss->scratch + ip_off); } else { ip = (struct ip *)(mtod(m, char *) + ip_off); } if (__predict_false(m->m_len < ip_off + (ip->ip_hl << 2) + sizeof (*tcp))) { m_copydata(m, 0, ip_off + (ip->ip_hl << 2) + sizeof (*tcp), ss->scratch); ip = (struct ip *)(mtod(m, char *) + ip_off); } tcp = (struct tcphdr *)((char *)ip + (ip->ip_hl << 2)); cum_len = -(ip_off + ((ip->ip_hl + tcp->th_off) << 2)); /* TSO implies checksum offload on this hardware */ cksum_offset = ip_off + (ip->ip_hl << 2); flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST; /* for TSO, pseudo_hdr_offset holds mss. * The firmware figures out where to put * the checksum by parsing the header. */ pseudo_hdr_offset = htobe16(mss); tx = &ss->tx; req = tx->req_list; seg = tx->seg_list; cnt = 0; rdma_count = 0; /* "rdma_count" is the number of RDMAs belonging to the * current packet BEFORE the current send request. For * non-TSO packets, this is equal to "count". * For TSO packets, rdma_count needs to be reset * to 0 after a segment cut. * * The rdma_count field of the send request is * the number of RDMAs of the packet starting at * that request. For TSO send requests with one ore more cuts * in the middle, this is the number of RDMAs starting * after the last cut in the request. All previous * segments before the last cut implicitly have 1 RDMA. * * Since the number of RDMAs is not known beforehand, * it must be filled-in retroactively - after each * segmentation cut or at the end of the entire packet. */ while (busdma_seg_cnt) { /* Break the busdma segment up into pieces*/ low = MXGE_LOWPART_TO_U32(seg->ds_addr); high_swapped = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); len = seg->ds_len; while (len) { flags_next = flags & ~MXGEFW_FLAGS_FIRST; seglen = len; cum_len_next = cum_len + seglen; (req-rdma_count)->rdma_count = rdma_count + 1; if (__predict_true(cum_len >= 0)) { /* payload */ chop = (cum_len_next > mss); cum_len_next = cum_len_next % mss; next_is_first = (cum_len_next == 0); flags |= chop * MXGEFW_FLAGS_TSO_CHOP; flags_next |= next_is_first * MXGEFW_FLAGS_FIRST; rdma_count |= -(chop | next_is_first); rdma_count += chop & !next_is_first; } else if (cum_len_next >= 0) { /* header ends */ rdma_count = -1; cum_len_next = 0; seglen = -cum_len; small = (mss <= MXGEFW_SEND_SMALL_SIZE); flags_next = MXGEFW_FLAGS_TSO_PLD | MXGEFW_FLAGS_FIRST | (small * MXGEFW_FLAGS_SMALL); } req->addr_high = high_swapped; req->addr_low = htobe32(low); req->pseudo_hdr_offset = pseudo_hdr_offset; req->pad = 0; req->rdma_count = 1; req->length = htobe16(seglen); req->cksum_offset = cksum_offset; req->flags = flags | ((cum_len & 1) * MXGEFW_FLAGS_ALIGN_ODD); low += seglen; len -= seglen; cum_len = cum_len_next; flags = flags_next; req++; cnt++; rdma_count++; if (__predict_false(cksum_offset > seglen)) cksum_offset -= seglen; else cksum_offset = 0; if (__predict_false(cnt > tx->max_desc)) goto drop; } busdma_seg_cnt--; seg++; } (req-rdma_count)->rdma_count = rdma_count; do { req--; req->flags |= MXGEFW_FLAGS_TSO_LAST; } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST))); tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; mxge_submit_req(tx, tx->req_list, cnt); #ifdef IFNET_BUF_RING if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { /* tell the NIC to start polling this slice */ *tx->send_go = 1; tx->queue_active = 1; tx->activate++; wmb(); } #endif return; drop: bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map); m_freem(m); ss->oerrors++; if (!once) { printf("tx->max_desc exceeded via TSO!\n"); printf("mss = %d, %ld, %d!\n", mss, (long)seg - (long)tx->seg_list, tx->max_desc); once = 1; } return; } #endif /* IFCAP_TSO4 */ #ifdef MXGE_NEW_VLAN_API /* * We reproduce the software vlan tag insertion from * net/if_vlan.c:vlan_start() here so that we can advertise "hardware" * vlan tag insertion. We need to advertise this in order to have the * vlan interface respect our csum offload flags. */ static struct mbuf * mxge_vlan_tag_insert(struct mbuf *m) { struct ether_vlan_header *evl; M_PREPEND(m, ETHER_VLAN_ENCAP_LEN, M_DONTWAIT); if (__predict_false(m == NULL)) return NULL; if (m->m_len < sizeof(*evl)) { m = m_pullup(m, sizeof(*evl)); if (__predict_false(m == NULL)) return NULL; } /* * Transform the Ethernet header into an Ethernet header * with 802.1Q encapsulation. */ evl = mtod(m, struct ether_vlan_header *); bcopy((char *)evl + ETHER_VLAN_ENCAP_LEN, (char *)evl, ETHER_HDR_LEN - ETHER_TYPE_LEN); evl->evl_encap_proto = htons(ETHERTYPE_VLAN); evl->evl_tag = htons(m->m_pkthdr.ether_vtag); m->m_flags &= ~M_VLANTAG; return m; } #endif /* MXGE_NEW_VLAN_API */ static void mxge_encap(struct mxge_slice_state *ss, struct mbuf *m) { mxge_softc_t *sc; mcp_kreq_ether_send_t *req; bus_dma_segment_t *seg; struct mbuf *m_tmp; struct ifnet *ifp; mxge_tx_ring_t *tx; struct ip *ip; int cnt, cum_len, err, i, idx, odd_flag, ip_off; uint16_t pseudo_hdr_offset; uint8_t flags, cksum_offset; sc = ss->sc; ifp = sc->ifp; tx = &ss->tx; ip_off = sizeof (struct ether_header); #ifdef MXGE_NEW_VLAN_API if (m->m_flags & M_VLANTAG) { m = mxge_vlan_tag_insert(m); if (__predict_false(m == NULL)) goto drop; ip_off += ETHER_VLAN_ENCAP_LEN; } #endif /* (try to) map the frame for DMA */ idx = tx->req & tx->mask; err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map, m, tx->seg_list, &cnt, BUS_DMA_NOWAIT); if (__predict_false(err == EFBIG)) { /* Too many segments in the chain. Try to defrag */ m_tmp = m_defrag(m, M_NOWAIT); if (m_tmp == NULL) { goto drop; } ss->tx.defrag++; m = m_tmp; err = bus_dmamap_load_mbuf_sg(tx->dmat, tx->info[idx].map, m, tx->seg_list, &cnt, BUS_DMA_NOWAIT); } if (__predict_false(err != 0)) { device_printf(sc->dev, "bus_dmamap_load_mbuf_sg returned %d" " packet len = %d\n", err, m->m_pkthdr.len); goto drop; } bus_dmamap_sync(tx->dmat, tx->info[idx].map, BUS_DMASYNC_PREWRITE); tx->info[idx].m = m; #if IFCAP_TSO4 /* TSO is different enough, we handle it in another routine */ if (m->m_pkthdr.csum_flags & (CSUM_TSO)) { mxge_encap_tso(ss, m, cnt, ip_off); return; } #endif req = tx->req_list; cksum_offset = 0; pseudo_hdr_offset = 0; flags = MXGEFW_FLAGS_NO_TSO; /* checksum offloading? */ if (m->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) { /* ensure ip header is in first mbuf, copy it to a scratch buffer if not */ if (__predict_false(m->m_len < ip_off + sizeof (*ip))) { m_copydata(m, 0, ip_off + sizeof (*ip), ss->scratch); ip = (struct ip *)(ss->scratch + ip_off); } else { ip = (struct ip *)(mtod(m, char *) + ip_off); } cksum_offset = ip_off + (ip->ip_hl << 2); pseudo_hdr_offset = cksum_offset + m->m_pkthdr.csum_data; pseudo_hdr_offset = htobe16(pseudo_hdr_offset); req->cksum_offset = cksum_offset; flags |= MXGEFW_FLAGS_CKSUM; odd_flag = MXGEFW_FLAGS_ALIGN_ODD; } else { odd_flag = 0; } if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE) flags |= MXGEFW_FLAGS_SMALL; /* convert segments into a request list */ cum_len = 0; seg = tx->seg_list; req->flags = MXGEFW_FLAGS_FIRST; for (i = 0; i < cnt; i++) { req->addr_low = htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); req->addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); req->length = htobe16(seg->ds_len); req->cksum_offset = cksum_offset; if (cksum_offset > seg->ds_len) cksum_offset -= seg->ds_len; else cksum_offset = 0; req->pseudo_hdr_offset = pseudo_hdr_offset; req->pad = 0; /* complete solid 16-byte block */ req->rdma_count = 1; req->flags |= flags | ((cum_len & 1) * odd_flag); cum_len += seg->ds_len; seg++; req++; req->flags = 0; } req--; /* pad runts to 60 bytes */ if (cum_len < 60) { req++; req->addr_low = htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr)); req->addr_high = htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr)); req->length = htobe16(60 - cum_len); req->cksum_offset = 0; req->pseudo_hdr_offset = pseudo_hdr_offset; req->pad = 0; /* complete solid 16-byte block */ req->rdma_count = 1; req->flags |= flags | ((cum_len & 1) * odd_flag); cnt++; } tx->req_list[0].rdma_count = cnt; #if 0 /* print what the firmware will see */ for (i = 0; i < cnt; i++) { printf("%d: addr: 0x%x 0x%x len:%d pso%d," "cso:%d, flags:0x%x, rdma:%d\n", i, (int)ntohl(tx->req_list[i].addr_high), (int)ntohl(tx->req_list[i].addr_low), (int)ntohs(tx->req_list[i].length), (int)ntohs(tx->req_list[i].pseudo_hdr_offset), tx->req_list[i].cksum_offset, tx->req_list[i].flags, tx->req_list[i].rdma_count); } printf("--------------\n"); #endif tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1; mxge_submit_req(tx, tx->req_list, cnt); #ifdef IFNET_BUF_RING if ((ss->sc->num_slices > 1) && tx->queue_active == 0) { /* tell the NIC to start polling this slice */ *tx->send_go = 1; tx->queue_active = 1; tx->activate++; wmb(); } #endif return; drop: m_freem(m); ss->oerrors++; return; } #ifdef IFNET_BUF_RING static void mxge_qflush(struct ifnet *ifp) { mxge_softc_t *sc = ifp->if_softc; mxge_tx_ring_t *tx; struct mbuf *m; int slice; for (slice = 0; slice < sc->num_slices; slice++) { tx = &sc->ss[slice].tx; mtx_lock(&tx->mtx); while ((m = buf_ring_dequeue_sc(tx->br)) != NULL) m_freem(m); mtx_unlock(&tx->mtx); } if_qflush(ifp); } static inline void mxge_start_locked(struct mxge_slice_state *ss) { mxge_softc_t *sc; struct mbuf *m; struct ifnet *ifp; mxge_tx_ring_t *tx; sc = ss->sc; ifp = sc->ifp; tx = &ss->tx; while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { m = drbr_dequeue(ifp, tx->br); if (m == NULL) { return; } /* let BPF see it */ BPF_MTAP(ifp, m); /* give it to the nic */ mxge_encap(ss, m); } /* ran out of transmit slots */ if (((ss->if_drv_flags & IFF_DRV_OACTIVE) == 0) && (!drbr_empty(ifp, tx->br))) { ss->if_drv_flags |= IFF_DRV_OACTIVE; tx->stall++; } } static int mxge_transmit_locked(struct mxge_slice_state *ss, struct mbuf *m) { mxge_softc_t *sc; struct ifnet *ifp; mxge_tx_ring_t *tx; int err; sc = ss->sc; ifp = sc->ifp; tx = &ss->tx; if ((ss->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) { err = drbr_enqueue(ifp, tx->br, m); return (err); } - if (drbr_empty(ifp, tx->br) && + if (!drbr_needs_enqueue(ifp, tx->br) && ((tx->mask - (tx->req - tx->done)) > tx->max_desc)) { /* let BPF see it */ BPF_MTAP(ifp, m); /* give it to the nic */ mxge_encap(ss, m); } else if ((err = drbr_enqueue(ifp, tx->br, m)) != 0) { return (err); } if (!drbr_empty(ifp, tx->br)) mxge_start_locked(ss); return (0); } static int mxge_transmit(struct ifnet *ifp, struct mbuf *m) { mxge_softc_t *sc = ifp->if_softc; struct mxge_slice_state *ss; mxge_tx_ring_t *tx; int err = 0; int slice; slice = m->m_pkthdr.flowid; slice &= (sc->num_slices - 1); /* num_slices always power of 2 */ ss = &sc->ss[slice]; tx = &ss->tx; if (mtx_trylock(&tx->mtx)) { err = mxge_transmit_locked(ss, m); mtx_unlock(&tx->mtx); } else { err = drbr_enqueue(ifp, tx->br, m); } return (err); } #else static inline void mxge_start_locked(struct mxge_slice_state *ss) { mxge_softc_t *sc; struct mbuf *m; struct ifnet *ifp; mxge_tx_ring_t *tx; sc = ss->sc; ifp = sc->ifp; tx = &ss->tx; while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) { IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) { return; } /* let BPF see it */ BPF_MTAP(ifp, m); /* give it to the nic */ mxge_encap(ss, m); } /* ran out of transmit slots */ if ((sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) { sc->ifp->if_drv_flags |= IFF_DRV_OACTIVE; tx->stall++; } } #endif static void mxge_start(struct ifnet *ifp) { mxge_softc_t *sc = ifp->if_softc; struct mxge_slice_state *ss; /* only use the first slice for now */ ss = &sc->ss[0]; mtx_lock(&ss->tx.mtx); mxge_start_locked(ss); mtx_unlock(&ss->tx.mtx); } /* * copy an array of mcp_kreq_ether_recv_t's to the mcp. Copy * at most 32 bytes at a time, so as to avoid involving the software * pio handler in the nic. We re-write the first segment's low * DMA address to mark it valid only after we write the entire chunk * in a burst */ static inline void mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst, mcp_kreq_ether_recv_t *src) { uint32_t low; low = src->addr_low; src->addr_low = 0xffffffff; mxge_pio_copy(dst, src, 4 * sizeof (*src)); wmb(); mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); wmb(); src->addr_low = low; dst->addr_low = low; wmb(); } static int mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) { bus_dma_segment_t seg; struct mbuf *m; mxge_rx_ring_t *rx = &ss->rx_small; int cnt, err; m = m_gethdr(M_DONTWAIT, MT_DATA); if (m == NULL) { rx->alloc_fail++; err = ENOBUFS; goto done; } m->m_len = MHLEN; err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, &seg, &cnt, BUS_DMA_NOWAIT); if (err != 0) { m_free(m); goto done; } rx->info[idx].m = m; rx->shadow[idx].addr_low = htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr)); rx->shadow[idx].addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr)); done: if ((idx & 7) == 7) mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]); return err; } static int mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx) { bus_dma_segment_t seg[3]; struct mbuf *m; mxge_rx_ring_t *rx = &ss->rx_big; int cnt, err, i; if (rx->cl_size == MCLBYTES) m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); else m = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, rx->cl_size); if (m == NULL) { rx->alloc_fail++; err = ENOBUFS; goto done; } m->m_len = rx->mlen; err = bus_dmamap_load_mbuf_sg(rx->dmat, map, m, seg, &cnt, BUS_DMA_NOWAIT); if (err != 0) { m_free(m); goto done; } rx->info[idx].m = m; rx->shadow[idx].addr_low = htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr)); rx->shadow[idx].addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr)); #if MXGE_VIRT_JUMBOS for (i = 1; i < cnt; i++) { rx->shadow[idx + i].addr_low = htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr)); rx->shadow[idx + i].addr_high = htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr)); } #endif done: for (i = 0; i < rx->nbufs; i++) { if ((idx & 7) == 7) { mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]); } idx++; } return err; } /* * Myri10GE hardware checksums are not valid if the sender * padded the frame with non-zero padding. This is because * the firmware just does a simple 16-bit 1s complement * checksum across the entire frame, excluding the first 14 * bytes. It is best to simply to check the checksum and * tell the stack about it only if the checksum is good */ static inline uint16_t mxge_rx_csum(struct mbuf *m, int csum) { struct ether_header *eh; struct ip *ip; uint16_t c; eh = mtod(m, struct ether_header *); /* only deal with IPv4 TCP & UDP for now */ if (__predict_false(eh->ether_type != htons(ETHERTYPE_IP))) return 1; ip = (struct ip *)(eh + 1); if (__predict_false(ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP)) return 1; #ifdef INET c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htonl(ntohs(csum) + ntohs(ip->ip_len) + - (ip->ip_hl << 2) + ip->ip_p)); #else c = 1; #endif c ^= 0xffff; return (c); } static void mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum) { struct ether_vlan_header *evl; struct ether_header *eh; uint32_t partial; evl = mtod(m, struct ether_vlan_header *); eh = mtod(m, struct ether_header *); /* * fix checksum by subtracting ETHER_VLAN_ENCAP_LEN bytes * after what the firmware thought was the end of the ethernet * header. */ /* put checksum into host byte order */ *csum = ntohs(*csum); partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN)); (*csum) += ~partial; (*csum) += ((*csum) < ~partial); (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF); /* restore checksum to network byte order; later consumers expect this */ *csum = htons(*csum); /* save the tag */ #ifdef MXGE_NEW_VLAN_API m->m_pkthdr.ether_vtag = ntohs(evl->evl_tag); #else { struct m_tag *mtag; mtag = m_tag_alloc(MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int), M_NOWAIT); if (mtag == NULL) return; VLAN_TAG_VALUE(mtag) = ntohs(evl->evl_tag); m_tag_prepend(m, mtag); } #endif m->m_flags |= M_VLANTAG; /* * Remove the 802.1q header by copying the Ethernet * addresses over it and adjusting the beginning of * the data in the mbuf. The encapsulated Ethernet * type field is already in place. */ bcopy((char *)evl, (char *)evl + ETHER_VLAN_ENCAP_LEN, ETHER_HDR_LEN - ETHER_TYPE_LEN); m_adj(m, ETHER_VLAN_ENCAP_LEN); } static inline void mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, uint32_t csum) { mxge_softc_t *sc; struct ifnet *ifp; struct mbuf *m; struct ether_header *eh; mxge_rx_ring_t *rx; bus_dmamap_t old_map; int idx; uint16_t tcpudp_csum; sc = ss->sc; ifp = sc->ifp; rx = &ss->rx_big; idx = rx->cnt & rx->mask; rx->cnt += rx->nbufs; /* save a pointer to the received mbuf */ m = rx->info[idx].m; /* try to replace the received mbuf */ if (mxge_get_buf_big(ss, rx->extra_map, idx)) { /* drop the frame -- the old mbuf is re-cycled */ ifp->if_ierrors++; return; } /* unmap the received buffer */ old_map = rx->info[idx].map; bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rx->dmat, old_map); /* swap the bus_dmamap_t's */ rx->info[idx].map = rx->extra_map; rx->extra_map = old_map; /* mcp implicitly skips 1st 2 bytes so that packet is properly * aligned */ m->m_data += MXGEFW_PAD; m->m_pkthdr.rcvif = ifp; m->m_len = m->m_pkthdr.len = len; ss->ipackets++; eh = mtod(m, struct ether_header *); if (eh->ether_type == htons(ETHERTYPE_VLAN)) { mxge_vlan_tag_remove(m, &csum); } /* if the checksum is valid, mark it in the mbuf header */ if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) { if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum))) return; /* otherwise, it was a UDP frame, or a TCP frame which we could not do LRO on. Tell the stack that the checksum is good */ m->m_pkthdr.csum_data = 0xffff; m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID; } /* flowid only valid if RSS hashing is enabled */ if (sc->num_slices > 1) { m->m_pkthdr.flowid = (ss - sc->ss); m->m_flags |= M_FLOWID; } /* pass the frame up the stack */ (*ifp->if_input)(ifp, m); } static inline void mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, uint32_t csum) { mxge_softc_t *sc; struct ifnet *ifp; struct ether_header *eh; struct mbuf *m; mxge_rx_ring_t *rx; bus_dmamap_t old_map; int idx; uint16_t tcpudp_csum; sc = ss->sc; ifp = sc->ifp; rx = &ss->rx_small; idx = rx->cnt & rx->mask; rx->cnt++; /* save a pointer to the received mbuf */ m = rx->info[idx].m; /* try to replace the received mbuf */ if (mxge_get_buf_small(ss, rx->extra_map, idx)) { /* drop the frame -- the old mbuf is re-cycled */ ifp->if_ierrors++; return; } /* unmap the received buffer */ old_map = rx->info[idx].map; bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(rx->dmat, old_map); /* swap the bus_dmamap_t's */ rx->info[idx].map = rx->extra_map; rx->extra_map = old_map; /* mcp implicitly skips 1st 2 bytes so that packet is properly * aligned */ m->m_data += MXGEFW_PAD; m->m_pkthdr.rcvif = ifp; m->m_len = m->m_pkthdr.len = len; ss->ipackets++; eh = mtod(m, struct ether_header *); if (eh->ether_type == htons(ETHERTYPE_VLAN)) { mxge_vlan_tag_remove(m, &csum); } /* if the checksum is valid, mark it in the mbuf header */ if (sc->csum_flag && (0 == (tcpudp_csum = mxge_rx_csum(m, csum)))) { if (sc->lro_cnt && (0 == mxge_lro_rx(ss, m, csum))) return; /* otherwise, it was a UDP frame, or a TCP frame which we could not do LRO on. Tell the stack that the checksum is good */ m->m_pkthdr.csum_data = 0xffff; m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR | CSUM_DATA_VALID; } /* flowid only valid if RSS hashing is enabled */ if (sc->num_slices > 1) { m->m_pkthdr.flowid = (ss - sc->ss); m->m_flags |= M_FLOWID; } /* pass the frame up the stack */ (*ifp->if_input)(ifp, m); } static inline void mxge_clean_rx_done(struct mxge_slice_state *ss) { mxge_rx_done_t *rx_done = &ss->rx_done; int limit = 0; uint16_t length; uint16_t checksum; while (rx_done->entry[rx_done->idx].length != 0) { length = ntohs(rx_done->entry[rx_done->idx].length); rx_done->entry[rx_done->idx].length = 0; checksum = rx_done->entry[rx_done->idx].checksum; if (length <= (MHLEN - MXGEFW_PAD)) mxge_rx_done_small(ss, length, checksum); else mxge_rx_done_big(ss, length, checksum); rx_done->cnt++; rx_done->idx = rx_done->cnt & rx_done->mask; /* limit potential for livelock */ if (__predict_false(++limit > rx_done->mask / 2)) break; } #ifdef INET while (!SLIST_EMPTY(&ss->lro_active)) { struct lro_entry *lro = SLIST_FIRST(&ss->lro_active); SLIST_REMOVE_HEAD(&ss->lro_active, next); mxge_lro_flush(ss, lro); } #endif } static inline void mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx) { struct ifnet *ifp; mxge_tx_ring_t *tx; struct mbuf *m; bus_dmamap_t map; int idx; int *flags; tx = &ss->tx; ifp = ss->sc->ifp; while (tx->pkt_done != mcp_idx) { idx = tx->done & tx->mask; tx->done++; m = tx->info[idx].m; /* mbuf and DMA map only attached to the first segment per-mbuf */ if (m != NULL) { ss->obytes += m->m_pkthdr.len; if (m->m_flags & M_MCAST) ss->omcasts++; ss->opackets++; tx->info[idx].m = NULL; map = tx->info[idx].map; bus_dmamap_unload(tx->dmat, map); m_freem(m); } if (tx->info[idx].flag) { tx->info[idx].flag = 0; tx->pkt_done++; } } /* If we have space, clear IFF_OACTIVE to tell the stack that its OK to send packets */ #ifdef IFNET_BUF_RING flags = &ss->if_drv_flags; #else flags = &ifp->if_drv_flags; #endif mtx_lock(&ss->tx.mtx); if ((*flags) & IFF_DRV_OACTIVE && tx->req - tx->done < (tx->mask + 1)/4) { *(flags) &= ~IFF_DRV_OACTIVE; ss->tx.wake++; mxge_start_locked(ss); } #ifdef IFNET_BUF_RING if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) { /* let the NIC stop polling this queue, since there * are no more transmits pending */ if (tx->req == tx->done) { *tx->send_stop = 1; tx->queue_active = 0; tx->deactivate++; wmb(); } } #endif mtx_unlock(&ss->tx.mtx); } static struct mxge_media_type mxge_xfp_media_types[] = { {IFM_10G_CX4, 0x7f, "10GBASE-CX4 (module)"}, {IFM_10G_SR, (1 << 7), "10GBASE-SR"}, {IFM_10G_LR, (1 << 6), "10GBASE-LR"}, {0, (1 << 5), "10GBASE-ER"}, {IFM_10G_LRM, (1 << 4), "10GBASE-LRM"}, {0, (1 << 3), "10GBASE-SW"}, {0, (1 << 2), "10GBASE-LW"}, {0, (1 << 1), "10GBASE-EW"}, {0, (1 << 0), "Reserved"} }; static struct mxge_media_type mxge_sfp_media_types[] = { {IFM_10G_TWINAX, 0, "10GBASE-Twinax"}, {0, (1 << 7), "Reserved"}, {IFM_10G_LRM, (1 << 6), "10GBASE-LRM"}, {IFM_10G_LR, (1 << 5), "10GBASE-LR"}, {IFM_10G_SR, (1 << 4), "10GBASE-SR"} }; static void mxge_set_media(mxge_softc_t *sc, int type) { sc->media_flags |= type; ifmedia_add(&sc->media, sc->media_flags, 0, NULL); ifmedia_set(&sc->media, sc->media_flags); } /* * Determine the media type for a NIC. Some XFPs will identify * themselves only when their link is up, so this is initiated via a * link up interrupt. However, this can potentially take up to * several milliseconds, so it is run via the watchdog routine, rather * than in the interrupt handler itself. This need only be done * once, not each time the link is up. */ static void mxge_media_probe(mxge_softc_t *sc) { mxge_cmd_t cmd; char *cage_type; char *ptr; struct mxge_media_type *mxge_media_types = NULL; int i, err, ms, mxge_media_type_entries; uint32_t byte; sc->need_media_probe = 0; /* if we've already set a media type, we're done */ if (sc->media_flags != (IFM_ETHER | IFM_AUTO)) return; /* * parse the product code to deterimine the interface type * (CX4, XFP, Quad Ribbon Fiber) by looking at the character * after the 3rd dash in the driver's cached copy of the * EEPROM's product code string. */ ptr = sc->product_code_string; if (ptr == NULL) { device_printf(sc->dev, "Missing product code\n"); } for (i = 0; i < 3; i++, ptr++) { ptr = index(ptr, '-'); if (ptr == NULL) { device_printf(sc->dev, "only %d dashes in PC?!?\n", i); return; } } if (*ptr == 'C') { /* -C is CX4 */ mxge_set_media(sc, IFM_10G_CX4); return; } else if (*ptr == 'Q') { /* -Q is Quad Ribbon Fiber */ device_printf(sc->dev, "Quad Ribbon Fiber Media\n"); /* FreeBSD has no media type for Quad ribbon fiber */ return; } if (*ptr == 'R') { /* -R is XFP */ mxge_media_types = mxge_xfp_media_types; mxge_media_type_entries = sizeof (mxge_xfp_media_types) / sizeof (mxge_xfp_media_types[0]); byte = MXGE_XFP_COMPLIANCE_BYTE; cage_type = "XFP"; } if (*ptr == 'S' || *(ptr +1) == 'S') { /* -S or -2S is SFP+ */ mxge_media_types = mxge_sfp_media_types; mxge_media_type_entries = sizeof (mxge_sfp_media_types) / sizeof (mxge_sfp_media_types[0]); cage_type = "SFP+"; byte = 3; } if (mxge_media_types == NULL) { device_printf(sc->dev, "Unknown media type: %c\n", *ptr); return; } /* * At this point we know the NIC has an XFP cage, so now we * try to determine what is in the cage by using the * firmware's XFP I2C commands to read the XFP 10GbE compilance * register. We read just one byte, which may take over * a millisecond */ cmd.data0 = 0; /* just fetch 1 byte, not all 256 */ cmd.data1 = byte; err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd); if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) { device_printf(sc->dev, "failed to read XFP\n"); } if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) { device_printf(sc->dev, "Type R/S with no XFP!?!?\n"); } if (err != MXGEFW_CMD_OK) { return; } /* now we wait for the data to be cached */ cmd.data0 = byte; err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); for (ms = 0; (err == EBUSY) && (ms < 50); ms++) { DELAY(1000); cmd.data0 = byte; err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd); } if (err != MXGEFW_CMD_OK) { device_printf(sc->dev, "failed to read %s (%d, %dms)\n", cage_type, err, ms); return; } if (cmd.data0 == mxge_media_types[0].bitmask) { if (mxge_verbose) device_printf(sc->dev, "%s:%s\n", cage_type, mxge_media_types[0].name); mxge_set_media(sc, mxge_media_types[0].flag); return; } for (i = 1; i < mxge_media_type_entries; i++) { if (cmd.data0 & mxge_media_types[i].bitmask) { if (mxge_verbose) device_printf(sc->dev, "%s:%s\n", cage_type, mxge_media_types[i].name); mxge_set_media(sc, mxge_media_types[i].flag); return; } } device_printf(sc->dev, "%s media 0x%x unknown\n", cage_type, cmd.data0); return; } static void mxge_intr(void *arg) { struct mxge_slice_state *ss = arg; mxge_softc_t *sc = ss->sc; mcp_irq_data_t *stats = ss->fw_stats; mxge_tx_ring_t *tx = &ss->tx; mxge_rx_done_t *rx_done = &ss->rx_done; uint32_t send_done_count; uint8_t valid; #ifndef IFNET_BUF_RING /* an interrupt on a non-zero slice is implicitly valid since MSI-X irqs are not shared */ if (ss != sc->ss) { mxge_clean_rx_done(ss); *ss->irq_claim = be32toh(3); return; } #endif /* make sure the DMA has finished */ if (!stats->valid) { return; } valid = stats->valid; if (sc->legacy_irq) { /* lower legacy IRQ */ *sc->irq_deassert = 0; if (!mxge_deassert_wait) /* don't wait for conf. that irq is low */ stats->valid = 0; } else { stats->valid = 0; } /* loop while waiting for legacy irq deassertion */ do { /* check for transmit completes and receives */ send_done_count = be32toh(stats->send_done_count); while ((send_done_count != tx->pkt_done) || (rx_done->entry[rx_done->idx].length != 0)) { if (send_done_count != tx->pkt_done) mxge_tx_done(ss, (int)send_done_count); mxge_clean_rx_done(ss); send_done_count = be32toh(stats->send_done_count); } if (sc->legacy_irq && mxge_deassert_wait) wmb(); } while (*((volatile uint8_t *) &stats->valid)); /* fw link & error stats meaningful only on the first slice */ if (__predict_false((ss == sc->ss) && stats->stats_updated)) { if (sc->link_state != stats->link_up) { sc->link_state = stats->link_up; if (sc->link_state) { if_link_state_change(sc->ifp, LINK_STATE_UP); if (mxge_verbose) device_printf(sc->dev, "link up\n"); } else { if_link_state_change(sc->ifp, LINK_STATE_DOWN); if (mxge_verbose) device_printf(sc->dev, "link down\n"); } sc->need_media_probe = 1; } if (sc->rdma_tags_available != be32toh(stats->rdma_tags_available)) { sc->rdma_tags_available = be32toh(stats->rdma_tags_available); device_printf(sc->dev, "RDMA timed out! %d tags " "left\n", sc->rdma_tags_available); } if (stats->link_down) { sc->down_cnt += stats->link_down; sc->link_state = 0; if_link_state_change(sc->ifp, LINK_STATE_DOWN); } } /* check to see if we have rx token to pass back */ if (valid & 0x1) *ss->irq_claim = be32toh(3); *(ss->irq_claim + 1) = be32toh(3); } static void mxge_init(void *arg) { } static void mxge_free_slice_mbufs(struct mxge_slice_state *ss) { struct lro_entry *lro_entry; int i; while (!SLIST_EMPTY(&ss->lro_free)) { lro_entry = SLIST_FIRST(&ss->lro_free); SLIST_REMOVE_HEAD(&ss->lro_free, next); free(lro_entry, M_DEVBUF); } for (i = 0; i <= ss->rx_big.mask; i++) { if (ss->rx_big.info[i].m == NULL) continue; bus_dmamap_unload(ss->rx_big.dmat, ss->rx_big.info[i].map); m_freem(ss->rx_big.info[i].m); ss->rx_big.info[i].m = NULL; } for (i = 0; i <= ss->rx_small.mask; i++) { if (ss->rx_small.info[i].m == NULL) continue; bus_dmamap_unload(ss->rx_small.dmat, ss->rx_small.info[i].map); m_freem(ss->rx_small.info[i].m); ss->rx_small.info[i].m = NULL; } /* transmit ring used only on the first slice */ if (ss->tx.info == NULL) return; for (i = 0; i <= ss->tx.mask; i++) { ss->tx.info[i].flag = 0; if (ss->tx.info[i].m == NULL) continue; bus_dmamap_unload(ss->tx.dmat, ss->tx.info[i].map); m_freem(ss->tx.info[i].m); ss->tx.info[i].m = NULL; } } static void mxge_free_mbufs(mxge_softc_t *sc) { int slice; for (slice = 0; slice < sc->num_slices; slice++) mxge_free_slice_mbufs(&sc->ss[slice]); } static void mxge_free_slice_rings(struct mxge_slice_state *ss) { int i; if (ss->rx_done.entry != NULL) mxge_dma_free(&ss->rx_done.dma); ss->rx_done.entry = NULL; if (ss->tx.req_bytes != NULL) free(ss->tx.req_bytes, M_DEVBUF); ss->tx.req_bytes = NULL; if (ss->tx.seg_list != NULL) free(ss->tx.seg_list, M_DEVBUF); ss->tx.seg_list = NULL; if (ss->rx_small.shadow != NULL) free(ss->rx_small.shadow, M_DEVBUF); ss->rx_small.shadow = NULL; if (ss->rx_big.shadow != NULL) free(ss->rx_big.shadow, M_DEVBUF); ss->rx_big.shadow = NULL; if (ss->tx.info != NULL) { if (ss->tx.dmat != NULL) { for (i = 0; i <= ss->tx.mask; i++) { bus_dmamap_destroy(ss->tx.dmat, ss->tx.info[i].map); } bus_dma_tag_destroy(ss->tx.dmat); } free(ss->tx.info, M_DEVBUF); } ss->tx.info = NULL; if (ss->rx_small.info != NULL) { if (ss->rx_small.dmat != NULL) { for (i = 0; i <= ss->rx_small.mask; i++) { bus_dmamap_destroy(ss->rx_small.dmat, ss->rx_small.info[i].map); } bus_dmamap_destroy(ss->rx_small.dmat, ss->rx_small.extra_map); bus_dma_tag_destroy(ss->rx_small.dmat); } free(ss->rx_small.info, M_DEVBUF); } ss->rx_small.info = NULL; if (ss->rx_big.info != NULL) { if (ss->rx_big.dmat != NULL) { for (i = 0; i <= ss->rx_big.mask; i++) { bus_dmamap_destroy(ss->rx_big.dmat, ss->rx_big.info[i].map); } bus_dmamap_destroy(ss->rx_big.dmat, ss->rx_big.extra_map); bus_dma_tag_destroy(ss->rx_big.dmat); } free(ss->rx_big.info, M_DEVBUF); } ss->rx_big.info = NULL; } static void mxge_free_rings(mxge_softc_t *sc) { int slice; for (slice = 0; slice < sc->num_slices; slice++) mxge_free_slice_rings(&sc->ss[slice]); } static int mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries, int tx_ring_entries) { mxge_softc_t *sc = ss->sc; size_t bytes; int err, i; err = ENOMEM; /* allocate per-slice receive resources */ ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; ss->rx_done.mask = (2 * rx_ring_entries) - 1; /* allocate the rx shadow rings */ bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow); ss->rx_small.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); if (ss->rx_small.shadow == NULL) return err; bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow); ss->rx_big.shadow = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); if (ss->rx_big.shadow == NULL) return err; /* allocate the rx host info rings */ bytes = rx_ring_entries * sizeof (*ss->rx_small.info); ss->rx_small.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); if (ss->rx_small.info == NULL) return err; bytes = rx_ring_entries * sizeof (*ss->rx_big.info); ss->rx_big.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); if (ss->rx_big.info == NULL) return err; /* allocate the rx busdma resources */ err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 1, /* alignment */ 4096, /* boundary */ BUS_SPACE_MAXADDR, /* low */ BUS_SPACE_MAXADDR, /* high */ NULL, NULL, /* filter */ MHLEN, /* maxsize */ 1, /* num segs */ MHLEN, /* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ NULL, NULL, /* lock */ &ss->rx_small.dmat); /* tag */ if (err != 0) { device_printf(sc->dev, "Err %d allocating rx_small dmat\n", err); return err; } err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 1, /* alignment */ #if MXGE_VIRT_JUMBOS 4096, /* boundary */ #else 0, /* boundary */ #endif BUS_SPACE_MAXADDR, /* low */ BUS_SPACE_MAXADDR, /* high */ NULL, NULL, /* filter */ 3*4096, /* maxsize */ #if MXGE_VIRT_JUMBOS 3, /* num segs */ 4096, /* maxsegsize*/ #else 1, /* num segs */ MJUM9BYTES, /* maxsegsize*/ #endif BUS_DMA_ALLOCNOW, /* flags */ NULL, NULL, /* lock */ &ss->rx_big.dmat); /* tag */ if (err != 0) { device_printf(sc->dev, "Err %d allocating rx_big dmat\n", err); return err; } for (i = 0; i <= ss->rx_small.mask; i++) { err = bus_dmamap_create(ss->rx_small.dmat, 0, &ss->rx_small.info[i].map); if (err != 0) { device_printf(sc->dev, "Err %d rx_small dmamap\n", err); return err; } } err = bus_dmamap_create(ss->rx_small.dmat, 0, &ss->rx_small.extra_map); if (err != 0) { device_printf(sc->dev, "Err %d extra rx_small dmamap\n", err); return err; } for (i = 0; i <= ss->rx_big.mask; i++) { err = bus_dmamap_create(ss->rx_big.dmat, 0, &ss->rx_big.info[i].map); if (err != 0) { device_printf(sc->dev, "Err %d rx_big dmamap\n", err); return err; } } err = bus_dmamap_create(ss->rx_big.dmat, 0, &ss->rx_big.extra_map); if (err != 0) { device_printf(sc->dev, "Err %d extra rx_big dmamap\n", err); return err; } /* now allocate TX resouces */ #ifndef IFNET_BUF_RING /* only use a single TX ring for now */ if (ss != ss->sc->ss) return 0; #endif ss->tx.mask = tx_ring_entries - 1; ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4); /* allocate the tx request copy block */ bytes = 8 + sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4); ss->tx.req_bytes = malloc(bytes, M_DEVBUF, M_WAITOK); if (ss->tx.req_bytes == NULL) return err; /* ensure req_list entries are aligned to 8 bytes */ ss->tx.req_list = (mcp_kreq_ether_send_t *) ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL); /* allocate the tx busdma segment list */ bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc; ss->tx.seg_list = (bus_dma_segment_t *) malloc(bytes, M_DEVBUF, M_WAITOK); if (ss->tx.seg_list == NULL) return err; /* allocate the tx host info ring */ bytes = tx_ring_entries * sizeof (*ss->tx.info); ss->tx.info = malloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK); if (ss->tx.info == NULL) return err; /* allocate the tx busdma resources */ err = bus_dma_tag_create(sc->parent_dmat, /* parent */ 1, /* alignment */ sc->tx_boundary, /* boundary */ BUS_SPACE_MAXADDR, /* low */ BUS_SPACE_MAXADDR, /* high */ NULL, NULL, /* filter */ 65536 + 256, /* maxsize */ ss->tx.max_desc - 2, /* num segs */ sc->tx_boundary, /* maxsegsz */ BUS_DMA_ALLOCNOW, /* flags */ NULL, NULL, /* lock */ &ss->tx.dmat); /* tag */ if (err != 0) { device_printf(sc->dev, "Err %d allocating tx dmat\n", err); return err; } /* now use these tags to setup dmamaps for each slot in the ring */ for (i = 0; i <= ss->tx.mask; i++) { err = bus_dmamap_create(ss->tx.dmat, 0, &ss->tx.info[i].map); if (err != 0) { device_printf(sc->dev, "Err %d tx dmamap\n", err); return err; } } return 0; } static int mxge_alloc_rings(mxge_softc_t *sc) { mxge_cmd_t cmd; int tx_ring_size; int tx_ring_entries, rx_ring_entries; int err, slice; /* get ring sizes */ err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd); tx_ring_size = cmd.data0; if (err != 0) { device_printf(sc->dev, "Cannot determine tx ring sizes\n"); goto abort; } tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t); rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t); IFQ_SET_MAXLEN(&sc->ifp->if_snd, tx_ring_entries - 1); sc->ifp->if_snd.ifq_drv_maxlen = sc->ifp->if_snd.ifq_maxlen; IFQ_SET_READY(&sc->ifp->if_snd); for (slice = 0; slice < sc->num_slices; slice++) { err = mxge_alloc_slice_rings(&sc->ss[slice], rx_ring_entries, tx_ring_entries); if (err != 0) goto abort; } return 0; abort: mxge_free_rings(sc); return err; } static void mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs) { int bufsize = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; if (bufsize < MCLBYTES) { /* easy, everything fits in a single buffer */ *big_buf_size = MCLBYTES; *cl_size = MCLBYTES; *nbufs = 1; return; } if (bufsize < MJUMPAGESIZE) { /* still easy, everything still fits in a single buffer */ *big_buf_size = MJUMPAGESIZE; *cl_size = MJUMPAGESIZE; *nbufs = 1; return; } #if MXGE_VIRT_JUMBOS /* now we need to use virtually contiguous buffers */ *cl_size = MJUM9BYTES; *big_buf_size = 4096; *nbufs = mtu / 4096 + 1; /* needs to be a power of two, so round up */ if (*nbufs == 3) *nbufs = 4; #else *cl_size = MJUM9BYTES; *big_buf_size = MJUM9BYTES; *nbufs = 1; #endif } static int mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size) { mxge_softc_t *sc; mxge_cmd_t cmd; bus_dmamap_t map; struct lro_entry *lro_entry; int err, i, slice; sc = ss->sc; slice = ss - sc->ss; SLIST_INIT(&ss->lro_free); SLIST_INIT(&ss->lro_active); for (i = 0; i < sc->lro_cnt; i++) { lro_entry = (struct lro_entry *) malloc(sizeof (*lro_entry), M_DEVBUF, M_NOWAIT | M_ZERO); if (lro_entry == NULL) { sc->lro_cnt = i; break; } SLIST_INSERT_HEAD(&ss->lro_free, lro_entry, next); } /* get the lanai pointers to the send and receive rings */ err = 0; #ifndef IFNET_BUF_RING /* We currently only send from the first slice */ if (slice == 0) { #endif cmd.data0 = slice; err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd); ss->tx.lanai = (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0); ss->tx.send_go = (volatile uint32_t *) (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice); ss->tx.send_stop = (volatile uint32_t *) (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); #ifndef IFNET_BUF_RING } #endif cmd.data0 = slice; err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd); ss->rx_small.lanai = (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); cmd.data0 = slice; err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd); ss->rx_big.lanai = (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0); if (err != 0) { device_printf(sc->dev, "failed to get ring sizes or locations\n"); return EIO; } /* stock receive rings */ for (i = 0; i <= ss->rx_small.mask; i++) { map = ss->rx_small.info[i].map; err = mxge_get_buf_small(ss, map, i); if (err) { device_printf(sc->dev, "alloced %d/%d smalls\n", i, ss->rx_small.mask + 1); return ENOMEM; } } for (i = 0; i <= ss->rx_big.mask; i++) { ss->rx_big.shadow[i].addr_low = 0xffffffff; ss->rx_big.shadow[i].addr_high = 0xffffffff; } ss->rx_big.nbufs = nbufs; ss->rx_big.cl_size = cl_size; ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + MXGEFW_PAD; for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) { map = ss->rx_big.info[i].map; err = mxge_get_buf_big(ss, map, i); if (err) { device_printf(sc->dev, "alloced %d/%d bigs\n", i, ss->rx_big.mask + 1); return ENOMEM; } } return 0; } static int mxge_open(mxge_softc_t *sc) { mxge_cmd_t cmd; int err, big_bytes, nbufs, slice, cl_size, i; bus_addr_t bus; volatile uint8_t *itable; struct mxge_slice_state *ss; /* Copy the MAC address in case it was overridden */ bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN); err = mxge_reset(sc, 1); if (err != 0) { device_printf(sc->dev, "failed to reset\n"); return EIO; } if (sc->num_slices > 1) { /* setup the indirection table */ cmd.data0 = sc->num_slices; err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE, &cmd); err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET, &cmd); if (err != 0) { device_printf(sc->dev, "failed to setup rss tables\n"); return err; } /* just enable an identity mapping */ itable = sc->sram + cmd.data0; for (i = 0; i < sc->num_slices; i++) itable[i] = (uint8_t)i; cmd.data0 = 1; cmd.data1 = mxge_rss_hash_type; err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd); if (err != 0) { device_printf(sc->dev, "failed to enable slices\n"); return err; } } mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs); cmd.data0 = nbufs; err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS, &cmd); /* error is only meaningful if we're trying to set MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */ if (err && nbufs > 1) { device_printf(sc->dev, "Failed to set alway-use-n to %d\n", nbufs); return EIO; } /* Give the firmware the mtu and the big and small buffer sizes. The firmware wants the big buf size to be a power of two. Luckily, FreeBSD's clusters are powers of two */ cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd); cmd.data0 = MHLEN - MXGEFW_PAD; err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd); cmd.data0 = big_bytes; err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd); if (err != 0) { device_printf(sc->dev, "failed to setup params\n"); goto abort; } /* Now give him the pointer to the stats block */ for (slice = 0; #ifdef IFNET_BUF_RING slice < sc->num_slices; #else slice < 1; #endif slice++) { ss = &sc->ss[slice]; cmd.data0 = MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr); cmd.data1 = MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr); cmd.data2 = sizeof(struct mcp_irq_data); cmd.data2 |= (slice << 16); err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd); } if (err != 0) { bus = sc->ss->fw_stats_dma.bus_addr; bus += offsetof(struct mcp_irq_data, send_done_count); cmd.data0 = MXGE_LOWPART_TO_U32(bus); cmd.data1 = MXGE_HIGHPART_TO_U32(bus); err = mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, &cmd); /* Firmware cannot support multicast without STATS_DMA_V2 */ sc->fw_multicast_support = 0; } else { sc->fw_multicast_support = 1; } if (err != 0) { device_printf(sc->dev, "failed to setup params\n"); goto abort; } for (slice = 0; slice < sc->num_slices; slice++) { err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size); if (err != 0) { device_printf(sc->dev, "couldn't open slice %d\n", slice); goto abort; } } /* Finally, start the firmware running */ err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd); if (err) { device_printf(sc->dev, "Couldn't bring up link\n"); goto abort; } #ifdef IFNET_BUF_RING for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; ss->if_drv_flags |= IFF_DRV_RUNNING; ss->if_drv_flags &= ~IFF_DRV_OACTIVE; } #endif sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; return 0; abort: mxge_free_mbufs(sc); return err; } static int mxge_close(mxge_softc_t *sc, int down) { mxge_cmd_t cmd; int err, old_down_cnt; #ifdef IFNET_BUF_RING struct mxge_slice_state *ss; int slice; #endif #ifdef IFNET_BUF_RING for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; ss->if_drv_flags &= ~IFF_DRV_RUNNING; } #endif sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; if (!down) { old_down_cnt = sc->down_cnt; wmb(); err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); if (err) { device_printf(sc->dev, "Couldn't bring down link\n"); } if (old_down_cnt == sc->down_cnt) { /* wait for down irq */ DELAY(10 * sc->intr_coal_delay); } wmb(); if (old_down_cnt == sc->down_cnt) { device_printf(sc->dev, "never got down irq\n"); } } mxge_free_mbufs(sc); return 0; } static void mxge_setup_cfg_space(mxge_softc_t *sc) { device_t dev = sc->dev; int reg; uint16_t cmd, lnk, pectl; /* find the PCIe link width and set max read request to 4KB*/ if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { lnk = pci_read_config(dev, reg + 0x12, 2); sc->link_width = (lnk >> 4) & 0x3f; if (sc->pectl == 0) { pectl = pci_read_config(dev, reg + 0x8, 2); pectl = (pectl & ~0x7000) | (5 << 12); pci_write_config(dev, reg + 0x8, pectl, 2); sc->pectl = pectl; } else { /* restore saved pectl after watchdog reset */ pci_write_config(dev, reg + 0x8, sc->pectl, 2); } } /* Enable DMA and Memory space access */ pci_enable_busmaster(dev); cmd = pci_read_config(dev, PCIR_COMMAND, 2); cmd |= PCIM_CMD_MEMEN; pci_write_config(dev, PCIR_COMMAND, cmd, 2); } static uint32_t mxge_read_reboot(mxge_softc_t *sc) { device_t dev = sc->dev; uint32_t vs; /* find the vendor specific offset */ if (pci_find_extcap(dev, PCIY_VENDOR, &vs) != 0) { device_printf(sc->dev, "could not find vendor specific offset\n"); return (uint32_t)-1; } /* enable read32 mode */ pci_write_config(dev, vs + 0x10, 0x3, 1); /* tell NIC which register to read */ pci_write_config(dev, vs + 0x18, 0xfffffff0, 4); return (pci_read_config(dev, vs + 0x14, 4)); } static void mxge_watchdog_reset(mxge_softc_t *sc) { struct pci_devinfo *dinfo; struct mxge_slice_state *ss; int err, running, s, num_tx_slices = 1; uint32_t reboot; uint16_t cmd; err = ENXIO; device_printf(sc->dev, "Watchdog reset!\n"); /* * check to see if the NIC rebooted. If it did, then all of * PCI config space has been reset, and things like the * busmaster bit will be zero. If this is the case, then we * must restore PCI config space before the NIC can be used * again */ cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); if (cmd == 0xffff) { /* * maybe the watchdog caught the NIC rebooting; wait * up to 100ms for it to finish. If it does not come * back, then give up */ DELAY(1000*100); cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); if (cmd == 0xffff) { device_printf(sc->dev, "NIC disappeared!\n"); } } if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { /* print the reboot status */ reboot = mxge_read_reboot(sc); device_printf(sc->dev, "NIC rebooted, status = 0x%x\n", reboot); running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; if (running) { /* * quiesce NIC so that TX routines will not try to * xmit after restoration of BAR */ /* Mark the link as down */ if (sc->link_state) { sc->link_state = 0; if_link_state_change(sc->ifp, LINK_STATE_DOWN); } #ifdef IFNET_BUF_RING num_tx_slices = sc->num_slices; #endif /* grab all TX locks to ensure no tx */ for (s = 0; s < num_tx_slices; s++) { ss = &sc->ss[s]; mtx_lock(&ss->tx.mtx); } mxge_close(sc, 1); } /* restore PCI configuration space */ dinfo = device_get_ivars(sc->dev); pci_cfg_restore(sc->dev, dinfo); /* and redo any changes we made to our config space */ mxge_setup_cfg_space(sc); /* reload f/w */ err = mxge_load_firmware(sc, 0); if (err) { device_printf(sc->dev, "Unable to re-load f/w\n"); } if (running) { if (!err) err = mxge_open(sc); /* release all TX locks */ for (s = 0; s < num_tx_slices; s++) { ss = &sc->ss[s]; #ifdef IFNET_BUF_RING mxge_start_locked(ss); #endif mtx_unlock(&ss->tx.mtx); } } sc->watchdog_resets++; } else { device_printf(sc->dev, "NIC did not reboot, not resetting\n"); err = 0; } if (err) { device_printf(sc->dev, "watchdog reset failed\n"); } else { if (sc->dying == 2) sc->dying = 0; callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); } } static void mxge_watchdog_task(void *arg, int pending) { mxge_softc_t *sc = arg; mtx_lock(&sc->driver_mtx); mxge_watchdog_reset(sc); mtx_unlock(&sc->driver_mtx); } static void mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice) { tx = &sc->ss[slice].tx; device_printf(sc->dev, "slice %d struck? ring state:\n", slice); device_printf(sc->dev, "tx.req=%d tx.done=%d, tx.queue_active=%d\n", tx->req, tx->done, tx->queue_active); device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n", tx->activate, tx->deactivate); device_printf(sc->dev, "pkt_done=%d fw=%d\n", tx->pkt_done, be32toh(sc->ss->fw_stats->send_done_count)); } static int mxge_watchdog(mxge_softc_t *sc) { mxge_tx_ring_t *tx; uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause); int i, err = 0; /* see if we have outstanding transmits, which have been pending for more than mxge_ticks */ for (i = 0; #ifdef IFNET_BUF_RING (i < sc->num_slices) && (err == 0); #else (i < 1) && (err == 0); #endif i++) { tx = &sc->ss[i].tx; if (tx->req != tx->done && tx->watchdog_req != tx->watchdog_done && tx->done == tx->watchdog_done) { /* check for pause blocking before resetting */ if (tx->watchdog_rx_pause == rx_pause) { mxge_warn_stuck(sc, tx, i); taskqueue_enqueue(sc->tq, &sc->watchdog_task); return (ENXIO); } else device_printf(sc->dev, "Flow control blocking " "xmits, check link partner\n"); } tx->watchdog_req = tx->req; tx->watchdog_done = tx->done; tx->watchdog_rx_pause = rx_pause; } if (sc->need_media_probe) mxge_media_probe(sc); return (err); } static u_long mxge_update_stats(mxge_softc_t *sc) { struct mxge_slice_state *ss; u_long pkts = 0; u_long ipackets = 0; u_long opackets = 0; #ifdef IFNET_BUF_RING u_long obytes = 0; u_long omcasts = 0; u_long odrops = 0; #endif u_long oerrors = 0; int slice; for (slice = 0; slice < sc->num_slices; slice++) { ss = &sc->ss[slice]; ipackets += ss->ipackets; opackets += ss->opackets; #ifdef IFNET_BUF_RING obytes += ss->obytes; omcasts += ss->omcasts; odrops += ss->tx.br->br_drops; #endif oerrors += ss->oerrors; } pkts = (ipackets - sc->ifp->if_ipackets); pkts += (opackets - sc->ifp->if_opackets); sc->ifp->if_ipackets = ipackets; sc->ifp->if_opackets = opackets; #ifdef IFNET_BUF_RING sc->ifp->if_obytes = obytes; sc->ifp->if_omcasts = omcasts; sc->ifp->if_snd.ifq_drops = odrops; #endif sc->ifp->if_oerrors = oerrors; return pkts; } static void mxge_tick(void *arg) { mxge_softc_t *sc = arg; u_long pkts = 0; int err = 0; int running, ticks; uint16_t cmd; ticks = mxge_ticks; running = sc->ifp->if_drv_flags & IFF_DRV_RUNNING; if (running) { /* aggregate stats from different slices */ pkts = mxge_update_stats(sc); if (!sc->watchdog_countdown) { err = mxge_watchdog(sc); sc->watchdog_countdown = 4; } sc->watchdog_countdown--; } if (pkts == 0) { /* ensure NIC did not suffer h/w fault while idle */ cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2); if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) { sc->dying = 2; taskqueue_enqueue(sc->tq, &sc->watchdog_task); err = ENXIO; } /* look less often if NIC is idle */ ticks *= 4; } if (err == 0) callout_reset(&sc->co_hdl, ticks, mxge_tick, sc); } static int mxge_media_change(struct ifnet *ifp) { return EINVAL; } static int mxge_change_mtu(mxge_softc_t *sc, int mtu) { struct ifnet *ifp = sc->ifp; int real_mtu, old_mtu; int err = 0; real_mtu = mtu + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; if ((real_mtu > sc->max_mtu) || real_mtu < 60) return EINVAL; mtx_lock(&sc->driver_mtx); old_mtu = ifp->if_mtu; ifp->if_mtu = mtu; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { mxge_close(sc, 0); err = mxge_open(sc); if (err != 0) { ifp->if_mtu = old_mtu; mxge_close(sc, 0); (void) mxge_open(sc); } } mtx_unlock(&sc->driver_mtx); return err; } static void mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) { mxge_softc_t *sc = ifp->if_softc; if (sc == NULL) return; ifmr->ifm_status = IFM_AVALID; ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0; ifmr->ifm_active = IFM_AUTO | IFM_ETHER; ifmr->ifm_active |= sc->link_state ? IFM_FDX : 0; } static int mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { mxge_softc_t *sc = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; int err, mask; err = 0; switch (command) { case SIOCSIFADDR: case SIOCGIFADDR: err = ether_ioctl(ifp, command, data); break; case SIOCSIFMTU: err = mxge_change_mtu(sc, ifr->ifr_mtu); break; case SIOCSIFFLAGS: mtx_lock(&sc->driver_mtx); if (sc->dying) { mtx_unlock(&sc->driver_mtx); return EINVAL; } if (ifp->if_flags & IFF_UP) { if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { err = mxge_open(sc); } else { /* take care of promis can allmulti flag chages */ mxge_change_promisc(sc, ifp->if_flags & IFF_PROMISC); mxge_set_multicast_list(sc); } } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) { mxge_close(sc, 0); } } mtx_unlock(&sc->driver_mtx); break; case SIOCADDMULTI: case SIOCDELMULTI: mtx_lock(&sc->driver_mtx); mxge_set_multicast_list(sc); mtx_unlock(&sc->driver_mtx); break; case SIOCSIFCAP: mtx_lock(&sc->driver_mtx); mask = ifr->ifr_reqcap ^ ifp->if_capenable; if (mask & IFCAP_TXCSUM) { if (IFCAP_TXCSUM & ifp->if_capenable) { ifp->if_capenable &= ~(IFCAP_TXCSUM|IFCAP_TSO4); ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP | CSUM_TSO); } else { ifp->if_capenable |= IFCAP_TXCSUM; ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); } } else if (mask & IFCAP_RXCSUM) { if (IFCAP_RXCSUM & ifp->if_capenable) { ifp->if_capenable &= ~IFCAP_RXCSUM; sc->csum_flag = 0; } else { ifp->if_capenable |= IFCAP_RXCSUM; sc->csum_flag = 1; } } if (mask & IFCAP_TSO4) { if (IFCAP_TSO4 & ifp->if_capenable) { ifp->if_capenable &= ~IFCAP_TSO4; ifp->if_hwassist &= ~CSUM_TSO; } else if (IFCAP_TXCSUM & ifp->if_capenable) { ifp->if_capenable |= IFCAP_TSO4; ifp->if_hwassist |= CSUM_TSO; } else { printf("mxge requires tx checksum offload" " be enabled to use TSO\n"); err = EINVAL; } } if (mask & IFCAP_LRO) { if (IFCAP_LRO & ifp->if_capenable) err = mxge_change_lro_locked(sc, 0); else err = mxge_change_lro_locked(sc, mxge_lro_cnt); } if (mask & IFCAP_VLAN_HWTAGGING) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; mtx_unlock(&sc->driver_mtx); VLAN_CAPABILITIES(ifp); break; case SIOCGIFMEDIA: err = ifmedia_ioctl(ifp, (struct ifreq *)data, &sc->media, command); break; default: err = ENOTTY; } return err; } static void mxge_fetch_tunables(mxge_softc_t *sc) { TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices); TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled", &mxge_flow_control); TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay", &mxge_intr_coal_delay); TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable", &mxge_nvidia_ecrc_enable); TUNABLE_INT_FETCH("hw.mxge.force_firmware", &mxge_force_firmware); TUNABLE_INT_FETCH("hw.mxge.deassert_wait", &mxge_deassert_wait); TUNABLE_INT_FETCH("hw.mxge.verbose", &mxge_verbose); TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks); TUNABLE_INT_FETCH("hw.mxge.lro_cnt", &sc->lro_cnt); TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc); TUNABLE_INT_FETCH("hw.mxge.rss_hash_type", &mxge_rss_hash_type); TUNABLE_INT_FETCH("hw.mxge.rss_hashtype", &mxge_rss_hash_type); TUNABLE_INT_FETCH("hw.mxge.initial_mtu", &mxge_initial_mtu); TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle); if (sc->lro_cnt != 0) mxge_lro_cnt = sc->lro_cnt; if (bootverbose) mxge_verbose = 1; if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000) mxge_intr_coal_delay = 30; if (mxge_ticks == 0) mxge_ticks = hz / 2; sc->pause = mxge_flow_control; if (mxge_rss_hash_type < MXGEFW_RSS_HASH_TYPE_IPV4 || mxge_rss_hash_type > MXGEFW_RSS_HASH_TYPE_MAX) { mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; } if (mxge_initial_mtu > ETHERMTU_JUMBO || mxge_initial_mtu < ETHER_MIN_LEN) mxge_initial_mtu = ETHERMTU_JUMBO; if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE) mxge_throttle = MXGE_MAX_THROTTLE; if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE) mxge_throttle = MXGE_MIN_THROTTLE; sc->throttle = mxge_throttle; } static void mxge_free_slices(mxge_softc_t *sc) { struct mxge_slice_state *ss; int i; if (sc->ss == NULL) return; for (i = 0; i < sc->num_slices; i++) { ss = &sc->ss[i]; if (ss->fw_stats != NULL) { mxge_dma_free(&ss->fw_stats_dma); ss->fw_stats = NULL; #ifdef IFNET_BUF_RING if (ss->tx.br != NULL) { drbr_free(ss->tx.br, M_DEVBUF); ss->tx.br = NULL; } #endif mtx_destroy(&ss->tx.mtx); } if (ss->rx_done.entry != NULL) { mxge_dma_free(&ss->rx_done.dma); ss->rx_done.entry = NULL; } } free(sc->ss, M_DEVBUF); sc->ss = NULL; } static int mxge_alloc_slices(mxge_softc_t *sc) { mxge_cmd_t cmd; struct mxge_slice_state *ss; size_t bytes; int err, i, max_intr_slots; err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); if (err != 0) { device_printf(sc->dev, "Cannot determine rx ring size\n"); return err; } sc->rx_ring_size = cmd.data0; max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t)); bytes = sizeof (*sc->ss) * sc->num_slices; sc->ss = malloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->ss == NULL) return (ENOMEM); for (i = 0; i < sc->num_slices; i++) { ss = &sc->ss[i]; ss->sc = sc; /* allocate per-slice rx interrupt queues */ bytes = max_intr_slots * sizeof (*ss->rx_done.entry); err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096); if (err != 0) goto abort; ss->rx_done.entry = ss->rx_done.dma.addr; bzero(ss->rx_done.entry, bytes); /* * allocate the per-slice firmware stats; stats * (including tx) are used used only on the first * slice for now */ #ifndef IFNET_BUF_RING if (i > 0) continue; #endif bytes = sizeof (*ss->fw_stats); err = mxge_dma_alloc(sc, &ss->fw_stats_dma, sizeof (*ss->fw_stats), 64); if (err != 0) goto abort; ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr; snprintf(ss->tx.mtx_name, sizeof(ss->tx.mtx_name), "%s:tx(%d)", device_get_nameunit(sc->dev), i); mtx_init(&ss->tx.mtx, ss->tx.mtx_name, NULL, MTX_DEF); #ifdef IFNET_BUF_RING ss->tx.br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, &ss->tx.mtx); #endif } return (0); abort: mxge_free_slices(sc); return (ENOMEM); } static void mxge_slice_probe(mxge_softc_t *sc) { mxge_cmd_t cmd; char *old_fw; int msix_cnt, status, max_intr_slots; sc->num_slices = 1; /* * don't enable multiple slices if they are not enabled, * or if this is not an SMP system */ if (mxge_max_slices == 0 || mxge_max_slices == 1 || mp_ncpus < 2) return; /* see how many MSI-X interrupts are available */ msix_cnt = pci_msix_count(sc->dev); if (msix_cnt < 2) return; /* now load the slice aware firmware see what it supports */ old_fw = sc->fw_name; if (old_fw == mxge_fw_aligned) sc->fw_name = mxge_fw_rss_aligned; else sc->fw_name = mxge_fw_rss_unaligned; status = mxge_load_firmware(sc, 0); if (status != 0) { device_printf(sc->dev, "Falling back to a single slice\n"); return; } /* try to send a reset command to the card to see if it is alive */ memset(&cmd, 0, sizeof (cmd)); status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd); if (status != 0) { device_printf(sc->dev, "failed reset\n"); goto abort_with_fw; } /* get rx ring size */ status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd); if (status != 0) { device_printf(sc->dev, "Cannot determine rx ring size\n"); goto abort_with_fw; } max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t)); /* tell it the size of the interrupt queues */ cmd.data0 = max_intr_slots * sizeof (struct mcp_slot); status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd); if (status != 0) { device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); goto abort_with_fw; } /* ask the maximum number of slices it supports */ status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd); if (status != 0) { device_printf(sc->dev, "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n"); goto abort_with_fw; } sc->num_slices = cmd.data0; if (sc->num_slices > msix_cnt) sc->num_slices = msix_cnt; if (mxge_max_slices == -1) { /* cap to number of CPUs in system */ if (sc->num_slices > mp_ncpus) sc->num_slices = mp_ncpus; } else { if (sc->num_slices > mxge_max_slices) sc->num_slices = mxge_max_slices; } /* make sure it is a power of two */ while (sc->num_slices & (sc->num_slices - 1)) sc->num_slices--; if (mxge_verbose) device_printf(sc->dev, "using %d slices\n", sc->num_slices); return; abort_with_fw: sc->fw_name = old_fw; (void) mxge_load_firmware(sc, 0); } static int mxge_add_msix_irqs(mxge_softc_t *sc) { size_t bytes; int count, err, i, rid; rid = PCIR_BAR(2); sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->msix_table_res == NULL) { device_printf(sc->dev, "couldn't alloc MSIX table res\n"); return ENXIO; } count = sc->num_slices; err = pci_alloc_msix(sc->dev, &count); if (err != 0) { device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d" "err = %d \n", sc->num_slices, err); goto abort_with_msix_table; } if (count < sc->num_slices) { device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n", count, sc->num_slices); device_printf(sc->dev, "Try setting hw.mxge.max_slices to %d\n", count); err = ENOSPC; goto abort_with_msix; } bytes = sizeof (*sc->msix_irq_res) * sc->num_slices; sc->msix_irq_res = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); if (sc->msix_irq_res == NULL) { err = ENOMEM; goto abort_with_msix; } for (i = 0; i < sc->num_slices; i++) { rid = i + 1; sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->msix_irq_res[i] == NULL) { device_printf(sc->dev, "couldn't allocate IRQ res" " for message %d\n", i); err = ENXIO; goto abort_with_res; } } bytes = sizeof (*sc->msix_ih) * sc->num_slices; sc->msix_ih = malloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO); for (i = 0; i < sc->num_slices; i++) { err = bus_setup_intr(sc->dev, sc->msix_irq_res[i], INTR_TYPE_NET | INTR_MPSAFE, #if __FreeBSD_version > 700030 NULL, #endif mxge_intr, &sc->ss[i], &sc->msix_ih[i]); if (err != 0) { device_printf(sc->dev, "couldn't setup intr for " "message %d\n", i); goto abort_with_intr; } } if (mxge_verbose) { device_printf(sc->dev, "using %d msix IRQs:", sc->num_slices); for (i = 0; i < sc->num_slices; i++) printf(" %ld", rman_get_start(sc->msix_irq_res[i])); printf("\n"); } return (0); abort_with_intr: for (i = 0; i < sc->num_slices; i++) { if (sc->msix_ih[i] != NULL) { bus_teardown_intr(sc->dev, sc->msix_irq_res[i], sc->msix_ih[i]); sc->msix_ih[i] = NULL; } } free(sc->msix_ih, M_DEVBUF); abort_with_res: for (i = 0; i < sc->num_slices; i++) { rid = i + 1; if (sc->msix_irq_res[i] != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, rid, sc->msix_irq_res[i]); sc->msix_irq_res[i] = NULL; } free(sc->msix_irq_res, M_DEVBUF); abort_with_msix: pci_release_msi(sc->dev); abort_with_msix_table: bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), sc->msix_table_res); return err; } static int mxge_add_single_irq(mxge_softc_t *sc) { int count, err, rid; count = pci_msi_count(sc->dev); if (count == 1 && pci_alloc_msi(sc->dev, &count) == 0) { rid = 1; } else { rid = 0; sc->legacy_irq = 1; } sc->irq_res = bus_alloc_resource(sc->dev, SYS_RES_IRQ, &rid, 0, ~0, 1, RF_SHAREABLE | RF_ACTIVE); if (sc->irq_res == NULL) { device_printf(sc->dev, "could not alloc interrupt\n"); return ENXIO; } if (mxge_verbose) device_printf(sc->dev, "using %s irq %ld\n", sc->legacy_irq ? "INTx" : "MSI", rman_get_start(sc->irq_res)); err = bus_setup_intr(sc->dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, #if __FreeBSD_version > 700030 NULL, #endif mxge_intr, &sc->ss[0], &sc->ih); if (err != 0) { bus_release_resource(sc->dev, SYS_RES_IRQ, sc->legacy_irq ? 0 : 1, sc->irq_res); if (!sc->legacy_irq) pci_release_msi(sc->dev); } return err; } static void mxge_rem_msix_irqs(mxge_softc_t *sc) { int i, rid; for (i = 0; i < sc->num_slices; i++) { if (sc->msix_ih[i] != NULL) { bus_teardown_intr(sc->dev, sc->msix_irq_res[i], sc->msix_ih[i]); sc->msix_ih[i] = NULL; } } free(sc->msix_ih, M_DEVBUF); for (i = 0; i < sc->num_slices; i++) { rid = i + 1; if (sc->msix_irq_res[i] != NULL) bus_release_resource(sc->dev, SYS_RES_IRQ, rid, sc->msix_irq_res[i]); sc->msix_irq_res[i] = NULL; } free(sc->msix_irq_res, M_DEVBUF); bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2), sc->msix_table_res); pci_release_msi(sc->dev); return; } static void mxge_rem_single_irq(mxge_softc_t *sc) { bus_teardown_intr(sc->dev, sc->irq_res, sc->ih); bus_release_resource(sc->dev, SYS_RES_IRQ, sc->legacy_irq ? 0 : 1, sc->irq_res); if (!sc->legacy_irq) pci_release_msi(sc->dev); } static void mxge_rem_irq(mxge_softc_t *sc) { if (sc->num_slices > 1) mxge_rem_msix_irqs(sc); else mxge_rem_single_irq(sc); } static int mxge_add_irq(mxge_softc_t *sc) { int err; if (sc->num_slices > 1) err = mxge_add_msix_irqs(sc); else err = mxge_add_single_irq(sc); if (0 && err == 0 && sc->num_slices > 1) { mxge_rem_msix_irqs(sc); err = mxge_add_msix_irqs(sc); } return err; } static int mxge_attach(device_t dev) { mxge_softc_t *sc = device_get_softc(dev); struct ifnet *ifp; int err, rid; sc->dev = dev; mxge_fetch_tunables(sc); TASK_INIT(&sc->watchdog_task, 1, mxge_watchdog_task, sc); sc->tq = taskqueue_create_fast("mxge_taskq", M_WAITOK, taskqueue_thread_enqueue, &sc->tq); if (sc->tq == NULL) { err = ENOMEM; goto abort_with_nothing; } taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq", device_get_nameunit(sc->dev)); err = bus_dma_tag_create(NULL, /* parent */ 1, /* alignment */ 0, /* boundary */ BUS_SPACE_MAXADDR, /* low */ BUS_SPACE_MAXADDR, /* high */ NULL, NULL, /* filter */ 65536 + 256, /* maxsize */ MXGE_MAX_SEND_DESC, /* num segs */ 65536, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lock */ &sc->parent_dmat); /* tag */ if (err != 0) { device_printf(sc->dev, "Err %d allocating parent dmat\n", err); goto abort_with_tq; } ifp = sc->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "can not if_alloc()\n"); err = ENOSPC; goto abort_with_parent_dmat; } if_initname(ifp, device_get_name(dev), device_get_unit(dev)); snprintf(sc->cmd_mtx_name, sizeof(sc->cmd_mtx_name), "%s:cmd", device_get_nameunit(dev)); mtx_init(&sc->cmd_mtx, sc->cmd_mtx_name, NULL, MTX_DEF); snprintf(sc->driver_mtx_name, sizeof(sc->driver_mtx_name), "%s:drv", device_get_nameunit(dev)); mtx_init(&sc->driver_mtx, sc->driver_mtx_name, MTX_NETWORK_LOCK, MTX_DEF); callout_init_mtx(&sc->co_hdl, &sc->driver_mtx, 0); mxge_setup_cfg_space(sc); /* Map the board into the kernel */ rid = PCIR_BARS; sc->mem_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 0, ~0, 1, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "could not map memory\n"); err = ENXIO; goto abort_with_lock; } sc->sram = rman_get_virtual(sc->mem_res); sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100; if (sc->sram_size > rman_get_size(sc->mem_res)) { device_printf(dev, "impossible memory region size %ld\n", rman_get_size(sc->mem_res)); err = ENXIO; goto abort_with_mem_res; } /* make NULL terminated copy of the EEPROM strings section of lanai SRAM */ bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE); bus_space_read_region_1(rman_get_bustag(sc->mem_res), rman_get_bushandle(sc->mem_res), sc->sram_size - MXGE_EEPROM_STRINGS_SIZE, sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE - 2); err = mxge_parse_strings(sc); if (err != 0) goto abort_with_mem_res; /* Enable write combining for efficient use of PCIe bus */ mxge_enable_wc(sc); /* Allocate the out of band dma memory */ err = mxge_dma_alloc(sc, &sc->cmd_dma, sizeof (mxge_cmd_t), 64); if (err != 0) goto abort_with_mem_res; sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr; err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64); if (err != 0) goto abort_with_cmd_dma; err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096); if (err != 0) goto abort_with_zeropad_dma; /* select & load the firmware */ err = mxge_select_firmware(sc); if (err != 0) goto abort_with_dmabench; sc->intr_coal_delay = mxge_intr_coal_delay; mxge_slice_probe(sc); err = mxge_alloc_slices(sc); if (err != 0) goto abort_with_dmabench; err = mxge_reset(sc, 0); if (err != 0) goto abort_with_slices; err = mxge_alloc_rings(sc); if (err != 0) { device_printf(sc->dev, "failed to allocate rings\n"); goto abort_with_dmabench; } err = mxge_add_irq(sc); if (err != 0) { device_printf(sc->dev, "failed to add irq\n"); goto abort_with_rings; } ifp->if_baudrate = IF_Gbps(10UL); ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO4 | IFCAP_VLAN_MTU; #ifdef INET ifp->if_capabilities |= IFCAP_LRO; #endif #ifdef MXGE_NEW_VLAN_API ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM; #endif sc->max_mtu = mxge_max_mtu(sc); if (sc->max_mtu >= 9000) ifp->if_capabilities |= IFCAP_JUMBO_MTU; else device_printf(dev, "MTU limited to %d. Install " "latest firmware for 9000 byte jumbo support\n", sc->max_mtu - ETHER_HDR_LEN); ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO; ifp->if_capenable = ifp->if_capabilities; if (sc->lro_cnt == 0) ifp->if_capenable &= ~IFCAP_LRO; sc->csum_flag = 1; ifp->if_init = mxge_init; ifp->if_softc = sc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = mxge_ioctl; ifp->if_start = mxge_start; /* Initialise the ifmedia structure */ ifmedia_init(&sc->media, 0, mxge_media_change, mxge_media_status); mxge_set_media(sc, IFM_ETHER | IFM_AUTO); mxge_media_probe(sc); sc->dying = 0; ether_ifattach(ifp, sc->mac_addr); /* ether_ifattach sets mtu to ETHERMTU */ if (mxge_initial_mtu != ETHERMTU) mxge_change_mtu(sc, mxge_initial_mtu); mxge_add_sysctls(sc); #ifdef IFNET_BUF_RING ifp->if_transmit = mxge_transmit; ifp->if_qflush = mxge_qflush; #endif callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc); return 0; abort_with_rings: mxge_free_rings(sc); abort_with_slices: mxge_free_slices(sc); abort_with_dmabench: mxge_dma_free(&sc->dmabench_dma); abort_with_zeropad_dma: mxge_dma_free(&sc->zeropad_dma); abort_with_cmd_dma: mxge_dma_free(&sc->cmd_dma); abort_with_mem_res: bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); abort_with_lock: pci_disable_busmaster(dev); mtx_destroy(&sc->cmd_mtx); mtx_destroy(&sc->driver_mtx); if_free(ifp); abort_with_parent_dmat: bus_dma_tag_destroy(sc->parent_dmat); abort_with_tq: if (sc->tq != NULL) { taskqueue_drain(sc->tq, &sc->watchdog_task); taskqueue_free(sc->tq); sc->tq = NULL; } abort_with_nothing: return err; } static int mxge_detach(device_t dev) { mxge_softc_t *sc = device_get_softc(dev); if (mxge_vlans_active(sc)) { device_printf(sc->dev, "Detach vlans before removing module\n"); return EBUSY; } mtx_lock(&sc->driver_mtx); sc->dying = 1; if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) mxge_close(sc, 0); mtx_unlock(&sc->driver_mtx); ether_ifdetach(sc->ifp); if (sc->tq != NULL) { taskqueue_drain(sc->tq, &sc->watchdog_task); taskqueue_free(sc->tq); sc->tq = NULL; } callout_drain(&sc->co_hdl); ifmedia_removeall(&sc->media); mxge_dummy_rdma(sc, 0); mxge_rem_sysctls(sc); mxge_rem_irq(sc); mxge_free_rings(sc); mxge_free_slices(sc); mxge_dma_free(&sc->dmabench_dma); mxge_dma_free(&sc->zeropad_dma); mxge_dma_free(&sc->cmd_dma); bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res); pci_disable_busmaster(dev); mtx_destroy(&sc->cmd_mtx); mtx_destroy(&sc->driver_mtx); if_free(sc->ifp); bus_dma_tag_destroy(sc->parent_dmat); return 0; } static int mxge_shutdown(device_t dev) { return 0; } /* This file uses Myri10GE driver indentation. Local Variables: c-file-style:"linux" tab-width:8 End: */ Index: user/imp/tbemd/sys/fs/msdosfs/bootsect.h =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/bootsect.h (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/bootsect.h (revision 203850) @@ -1,93 +1,97 @@ /* $FreeBSD$ */ /* $NetBSD: bootsect.h,v 1.9 1997/11/17 15:36:17 ws Exp $ */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ +#ifndef _FS_MSDOSFS_BOOTSECT_H_ +#define _FS_MSDOSFS_BOOTSECT_H_ /* * Format of a boot sector. This is the first sector on a DOS floppy disk * or the fist sector of a partition on a hard disk. But, it is not the * first sector of a partitioned hard disk. */ struct bootsector33 { u_int8_t bsJump[3]; /* jump inst E9xxxx or EBxx90 */ int8_t bsOemName[8]; /* OEM name and version */ int8_t bsBPB[19]; /* BIOS parameter block */ int8_t bsDriveNumber; /* drive number (0x80) */ int8_t bsBootCode[479]; /* pad so struct is 512b */ u_int8_t bsBootSectSig0; u_int8_t bsBootSectSig1; #define BOOTSIG0 0x55 #define BOOTSIG1 0xaa }; struct extboot { int8_t exDriveNumber; /* drive number (0x80) */ int8_t exReserved1; /* reserved */ int8_t exBootSignature; /* ext. boot signature (0x29) */ #define EXBOOTSIG 0x29 int8_t exVolumeID[4]; /* volume ID number */ int8_t exVolumeLabel[11]; /* volume label */ int8_t exFileSysType[8]; /* fs type (FAT12 or FAT16) */ }; struct bootsector50 { u_int8_t bsJump[3]; /* jump inst E9xxxx or EBxx90 */ int8_t bsOemName[8]; /* OEM name and version */ int8_t bsBPB[25]; /* BIOS parameter block */ int8_t bsExt[26]; /* Bootsector Extension */ int8_t bsBootCode[448]; /* pad so structure is 512b */ u_int8_t bsBootSectSig0; u_int8_t bsBootSectSig1; #define BOOTSIG0 0x55 #define BOOTSIG1 0xaa }; struct bootsector710 { u_int8_t bsJump[3]; /* jump inst E9xxxx or EBxx90 */ int8_t bsOEMName[8]; /* OEM name and version */ int8_t bsBPB[53]; /* BIOS parameter block */ int8_t bsExt[26]; /* Bootsector Extension */ int8_t bsBootCode[420]; /* pad so structure is 512b */ u_int8_t bsBootSectSig0; u_int8_t bsBootSectSig1; #define BOOTSIG0 0x55 #define BOOTSIG1 0xaa }; union bootsector { struct bootsector33 bs33; struct bootsector50 bs50; struct bootsector710 bs710; }; #if 0 /* * Shorthand for fields in the bpb. */ #define bsBytesPerSec bsBPB.bpbBytesPerSec #define bsSectPerClust bsBPB.bpbSectPerClust #define bsResSectors bsBPB.bpbResSectors #define bsFATS bsBPB.bpbFATS #define bsRootDirEnts bsBPB.bpbRootDirEnts #define bsSectors bsBPB.bpbSectors #define bsMedia bsBPB.bpbMedia #define bsFATsecs bsBPB.bpbFATsecs #define bsSectPerTrack bsBPB.bpbSectPerTrack #define bsHeads bsBPB.bpbHeads #define bsHiddenSecs bsBPB.bpbHiddenSecs #define bsHugeSectors bsBPB.bpbHugeSectors #endif + +#endif /* !_FS_MSDOSFS_BOOTSECT_H_ */ Index: user/imp/tbemd/sys/fs/msdosfs/bpb.h =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/bpb.h (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/bpb.h (revision 203850) @@ -1,170 +1,174 @@ /* $FreeBSD$ */ /* $NetBSD: bpb.h,v 1.7 1997/11/17 15:36:24 ws Exp $ */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ +#ifndef _FS_MSDOSFS_BPB_H_ +#define _FS_MSDOSFS_BPB_H_ + /* * BIOS Parameter Block (BPB) for DOS 3.3 */ struct bpb33 { u_int16_t bpbBytesPerSec; /* bytes per sector */ u_int8_t bpbSecPerClust; /* sectors per cluster */ u_int16_t bpbResSectors; /* number of reserved sectors */ u_int8_t bpbFATs; /* number of FATs */ u_int16_t bpbRootDirEnts; /* number of root directory entries */ u_int16_t bpbSectors; /* total number of sectors */ u_int8_t bpbMedia; /* media descriptor */ u_int16_t bpbFATsecs; /* number of sectors per FAT */ u_int16_t bpbSecPerTrack; /* sectors per track */ u_int16_t bpbHeads; /* number of heads */ u_int16_t bpbHiddenSecs; /* number of hidden sectors */ }; /* * BPB for DOS 5.0 The difference is bpbHiddenSecs is a short for DOS 3.3, * and bpbHugeSectors is not in the 3.3 bpb. */ struct bpb50 { u_int16_t bpbBytesPerSec; /* bytes per sector */ u_int8_t bpbSecPerClust; /* sectors per cluster */ u_int16_t bpbResSectors; /* number of reserved sectors */ u_int8_t bpbFATs; /* number of FATs */ u_int16_t bpbRootDirEnts; /* number of root directory entries */ u_int16_t bpbSectors; /* total number of sectors */ u_int8_t bpbMedia; /* media descriptor */ u_int16_t bpbFATsecs; /* number of sectors per FAT */ u_int16_t bpbSecPerTrack; /* sectors per track */ u_int16_t bpbHeads; /* number of heads */ u_int32_t bpbHiddenSecs; /* # of hidden sectors */ u_int32_t bpbHugeSectors; /* # of sectors if bpbSectors == 0 */ }; /* * BPB for DOS 7.10 (FAT32). This one has a few extensions to bpb50. */ struct bpb710 { u_int16_t bpbBytesPerSec; /* bytes per sector */ u_int8_t bpbSecPerClust; /* sectors per cluster */ u_int16_t bpbResSectors; /* number of reserved sectors */ u_int8_t bpbFATs; /* number of FATs */ u_int16_t bpbRootDirEnts; /* number of root directory entries */ u_int16_t bpbSectors; /* total number of sectors */ u_int8_t bpbMedia; /* media descriptor */ u_int16_t bpbFATsecs; /* number of sectors per FAT */ u_int16_t bpbSecPerTrack; /* sectors per track */ u_int16_t bpbHeads; /* number of heads */ u_int32_t bpbHiddenSecs; /* # of hidden sectors */ u_int32_t bpbHugeSectors; /* # of sectors if bpbSectors == 0 */ u_int32_t bpbBigFATsecs; /* like bpbFATsecs for FAT32 */ u_int16_t bpbExtFlags; /* extended flags: */ #define FATNUM 0xf /* mask for numbering active FAT */ #define FATMIRROR 0x80 /* FAT is mirrored (like it always was) */ u_int16_t bpbFSVers; /* filesystem version */ #define FSVERS 0 /* currently only 0 is understood */ u_int32_t bpbRootClust; /* start cluster for root directory */ u_int16_t bpbFSInfo; /* filesystem info structure sector */ u_int16_t bpbBackup; /* backup boot sector */ - /* There is a 12 byte filler here, but we ignore it */ + u_int8_t bpbReserved[12]; /* reserved for future expansion */ }; /* * The following structures represent how the bpb's look on disk. shorts * and longs are just character arrays of the appropriate length. This is * because the compiler forces shorts and longs to align on word or * halfword boundaries. */ #include #define getushort(x) le16dec(x) #define getulong(x) le32dec(x) #define putushort(p, v) le16enc(p, v) #define putulong(p, v) le32enc(p, v) /* * BIOS Parameter Block (BPB) for DOS 3.3 */ struct byte_bpb33 { int8_t bpbBytesPerSec[2]; /* bytes per sector */ int8_t bpbSecPerClust; /* sectors per cluster */ int8_t bpbResSectors[2]; /* number of reserved sectors */ int8_t bpbFATs; /* number of FATs */ int8_t bpbRootDirEnts[2]; /* number of root directory entries */ int8_t bpbSectors[2]; /* total number of sectors */ int8_t bpbMedia; /* media descriptor */ int8_t bpbFATsecs[2]; /* number of sectors per FAT */ int8_t bpbSecPerTrack[2]; /* sectors per track */ int8_t bpbHeads[2]; /* number of heads */ int8_t bpbHiddenSecs[2]; /* number of hidden sectors */ }; /* * BPB for DOS 5.0 The difference is bpbHiddenSecs is a short for DOS 3.3, * and bpbHugeSectors is not in the 3.3 bpb. */ struct byte_bpb50 { int8_t bpbBytesPerSec[2]; /* bytes per sector */ int8_t bpbSecPerClust; /* sectors per cluster */ int8_t bpbResSectors[2]; /* number of reserved sectors */ int8_t bpbFATs; /* number of FATs */ int8_t bpbRootDirEnts[2]; /* number of root directory entries */ int8_t bpbSectors[2]; /* total number of sectors */ int8_t bpbMedia; /* media descriptor */ int8_t bpbFATsecs[2]; /* number of sectors per FAT */ int8_t bpbSecPerTrack[2]; /* sectors per track */ int8_t bpbHeads[2]; /* number of heads */ int8_t bpbHiddenSecs[4]; /* number of hidden sectors */ int8_t bpbHugeSectors[4]; /* # of sectors if bpbSectors == 0 */ }; /* * BPB for DOS 7.10 (FAT32). This one has a few extensions to bpb50. */ struct byte_bpb710 { u_int8_t bpbBytesPerSec[2]; /* bytes per sector */ u_int8_t bpbSecPerClust; /* sectors per cluster */ u_int8_t bpbResSectors[2]; /* number of reserved sectors */ u_int8_t bpbFATs; /* number of FATs */ u_int8_t bpbRootDirEnts[2]; /* number of root directory entries */ u_int8_t bpbSectors[2]; /* total number of sectors */ u_int8_t bpbMedia; /* media descriptor */ u_int8_t bpbFATsecs[2]; /* number of sectors per FAT */ u_int8_t bpbSecPerTrack[2]; /* sectors per track */ u_int8_t bpbHeads[2]; /* number of heads */ u_int8_t bpbHiddenSecs[4]; /* # of hidden sectors */ u_int8_t bpbHugeSectors[4]; /* # of sectors if bpbSectors == 0 */ u_int8_t bpbBigFATsecs[4]; /* like bpbFATsecs for FAT32 */ u_int8_t bpbExtFlags[2]; /* extended flags: */ u_int8_t bpbFSVers[2]; /* filesystem version */ u_int8_t bpbRootClust[4]; /* start cluster for root directory */ u_int8_t bpbFSInfo[2]; /* filesystem info structure sector */ u_int8_t bpbBackup[2]; /* backup boot sector */ - /* There is a 12 byte filler here, but we ignore it */ + u_int8_t bpbReserved[12]; /* reserved for future expansion */ }; /* * FAT32 FSInfo block. */ struct fsinfo { u_int8_t fsisig1[4]; u_int8_t fsifill1[480]; u_int8_t fsisig2[4]; u_int8_t fsinfree[4]; u_int8_t fsinxtfree[4]; u_int8_t fsifill2[12]; u_int8_t fsisig3[4]; }; +#endif /* !_FS_MSDOSFS_BPB_H_ */ Index: user/imp/tbemd/sys/fs/msdosfs/denode.h =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/denode.h (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/denode.h (revision 203850) @@ -1,284 +1,281 @@ /* $FreeBSD$ */ /* $NetBSD: denode.h,v 1.25 1997/11/17 15:36:28 ws Exp $ */ /*- * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1994, 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ /* * This is the pc filesystem specific portion of the vnode structure. * * To describe a file uniquely the de_dirclust, de_diroffset, and * de_StartCluster fields are used. * * de_dirclust contains the cluster number of the directory cluster * containing the entry for a file or directory. * de_diroffset is the index into the cluster for the entry describing * a file or directory. * de_StartCluster is the number of the first cluster of the file or directory. * * Now to describe the quirks of the pc filesystem. * - Clusters 0 and 1 are reserved. * - The first allocatable cluster is 2. * - The root directory is of fixed size and all blocks that make it up * are contiguous. * - Cluster 0 refers to the root directory when it is found in the * startcluster field of a directory entry that points to another directory. * - Cluster 0 implies a 0 length file when found in the start cluster field * of a directory entry that points to a file. * - You can't use the cluster number 0 to derive the address of the root * directory. * - Multiple directory entries can point to a directory. The entry in the * parent directory points to a child directory. Any directories in the * child directory contain a ".." entry that points back to the parent. * The child directory itself contains a "." entry that points to itself. * - The root directory does not contain a "." or ".." entry. * - Directory entries for directories are never changed once they are created * (except when removed). The size stays 0, and the last modification time * is never changed. This is because so many directory entries can point to * the physical clusters that make up a directory. It would lead to an * update nightmare. * - The length field in a directory entry pointing to a directory contains 0 * (always). The only way to find the end of a directory is to follow the * cluster chain until the "last cluster" marker is found. * * My extensions to make this house of cards work. These apply only to the in * memory copy of the directory entry. * - A reference count for each denode will be kept since dos doesn't keep such * things. */ /* * Internal pseudo-offset for (nonexistent) directory entry for the root * dir in the root dir */ #define MSDOSFSROOT_OFS 0x1fffffff /* * The fat cache structure. fc_fsrcn is the filesystem relative cluster * number that corresponds to the file relative cluster number in this * structure (fc_frcn). */ struct fatcache { u_long fc_frcn; /* file relative cluster number */ u_long fc_fsrcn; /* filesystem relative cluster number */ }; /* * The fat entry cache as it stands helps make extending files a "quick" * operation by avoiding having to scan the fat to discover the last * cluster of the file. The cache also helps sequential reads by * remembering the last cluster read from the file. This also prevents us * from having to rescan the fat to find the next cluster to read. This * cache is probably pretty worthless if a file is opened by multiple * processes. */ #define FC_SIZE 3 /* number of entries in the cache */ #define FC_LASTMAP 0 /* entry the last call to pcbmap() resolved * to */ #define FC_LASTFC 1 /* entry for the last cluster in the file */ #define FC_NEXTTOLASTFC 2 /* entry for a close to the last cluster in * the file */ #define FCE_EMPTY 0xffffffff /* doesn't represent an actual cluster # */ /* * Set a slot in the fat cache. */ #define fc_setcache(dep, slot, frcn, fsrcn) \ (dep)->de_fc[(slot)].fc_frcn = (frcn); \ (dep)->de_fc[(slot)].fc_fsrcn = (fsrcn); /* * This is the in memory variant of a dos directory entry. It is usually * contained within a vnode. */ struct denode { struct vnode *de_vnode; /* addr of vnode we are part of */ u_long de_flag; /* flag bits */ u_long de_dirclust; /* cluster of the directory file containing this entry */ u_long de_diroffset; /* offset of this entry in the directory cluster */ u_long de_fndoffset; /* offset of found dir entry */ int de_fndcnt; /* number of slots before de_fndoffset */ long de_refcnt; /* reference count */ struct msdosfsmount *de_pmp; /* addr of our mount struct */ u_char de_Name[12]; /* name, from DOS directory entry */ u_char de_Attributes; /* attributes, from directory entry */ u_char de_LowerCase; /* NT VFAT lower case flags */ u_char de_CHun; /* Hundredth of second of CTime*/ u_short de_CTime; /* creation time */ u_short de_CDate; /* creation date */ u_short de_ADate; /* access date */ u_short de_MTime; /* modification time */ u_short de_MDate; /* modification date */ u_long de_StartCluster; /* starting cluster of file */ u_long de_FileSize; /* size of file in bytes */ struct fatcache de_fc[FC_SIZE]; /* fat cache */ u_quad_t de_modrev; /* Revision level for lease. */ u_int64_t de_inode; /* Inode number (really byte offset of direntry) */ }; /* * Values for the de_flag field of the denode. */ #define DE_UPDATE 0x0004 /* Modification time update request */ #define DE_CREATE 0x0008 /* Creation time update */ #define DE_ACCESS 0x0010 /* Access time update */ #define DE_MODIFIED 0x0020 /* Denode has been modified */ #define DE_RENAME 0x0040 /* Denode is in the process of being renamed */ /* * Transfer directory entries between internal and external form. * dep is a struct denode * (internal form), * dp is a struct direntry * (external form). */ #define DE_INTERNALIZE32(dep, dp) \ ((dep)->de_StartCluster |= getushort((dp)->deHighClust) << 16) #define DE_INTERNALIZE(dep, dp) \ (bcopy((dp)->deName, (dep)->de_Name, 11), \ (dep)->de_Attributes = (dp)->deAttributes, \ (dep)->de_LowerCase = (dp)->deLowerCase, \ (dep)->de_CHun = (dp)->deCHundredth, \ (dep)->de_CTime = getushort((dp)->deCTime), \ (dep)->de_CDate = getushort((dp)->deCDate), \ (dep)->de_ADate = getushort((dp)->deADate), \ (dep)->de_MTime = getushort((dp)->deMTime), \ (dep)->de_MDate = getushort((dp)->deMDate), \ (dep)->de_StartCluster = getushort((dp)->deStartCluster), \ (dep)->de_FileSize = getulong((dp)->deFileSize), \ (FAT32((dep)->de_pmp) ? DE_INTERNALIZE32((dep), (dp)) : 0)) #define DE_EXTERNALIZE(dp, dep) \ (bcopy((dep)->de_Name, (dp)->deName, 11), \ (dp)->deAttributes = (dep)->de_Attributes, \ (dp)->deLowerCase = (dep)->de_LowerCase, \ (dp)->deCHundredth = (dep)->de_CHun, \ putushort((dp)->deCTime, (dep)->de_CTime), \ putushort((dp)->deCDate, (dep)->de_CDate), \ putushort((dp)->deADate, (dep)->de_ADate), \ putushort((dp)->deMTime, (dep)->de_MTime), \ putushort((dp)->deMDate, (dep)->de_MDate), \ putushort((dp)->deStartCluster, (dep)->de_StartCluster), \ putulong((dp)->deFileSize, \ ((dep)->de_Attributes & ATTR_DIRECTORY) ? 0 : (dep)->de_FileSize), \ putushort((dp)->deHighClust, (dep)->de_StartCluster >> 16)) -#define de_forw de_chain[0] -#define de_back de_chain[1] - #ifdef _KERNEL #define VTODE(vp) ((struct denode *)(vp)->v_data) #define DETOV(de) ((de)->de_vnode) #define DETIMES(dep, acc, mod, cre) do { \ if ((dep)->de_flag & DE_UPDATE) { \ (dep)->de_flag |= DE_MODIFIED; \ timespec2fattime((mod), 0, &(dep)->de_MDate, \ &(dep)->de_MTime, NULL); \ (dep)->de_Attributes |= ATTR_ARCHIVE; \ } \ if ((dep)->de_pmp->pm_flags & MSDOSFSMNT_NOWIN95) { \ (dep)->de_flag &= ~(DE_UPDATE | DE_CREATE | DE_ACCESS); \ break; \ } \ if ((dep)->de_flag & DE_ACCESS) { \ u_int16_t adate; \ \ timespec2fattime((acc), 0, &adate, NULL, NULL); \ if (adate != (dep)->de_ADate) { \ (dep)->de_flag |= DE_MODIFIED; \ (dep)->de_ADate = adate; \ } \ } \ if ((dep)->de_flag & DE_CREATE) { \ timespec2fattime((cre), 0, &(dep)->de_CDate, \ &(dep)->de_CTime, &(dep)->de_CHun); \ (dep)->de_flag |= DE_MODIFIED; \ } \ (dep)->de_flag &= ~(DE_UPDATE | DE_CREATE | DE_ACCESS); \ } while (0) /* * This overlays the fid structure (see mount.h) */ struct defid { u_short defid_len; /* length of structure */ u_short defid_pad; /* force long alignment */ u_int32_t defid_dirclust; /* cluster this dir entry came from */ u_int32_t defid_dirofs; /* offset of entry within the cluster */ #if 0 u_int32_t defid_gen; /* generation number */ #endif }; extern struct vop_vector msdosfs_vnodeops; int msdosfs_lookup(struct vop_cachedlookup_args *); int msdosfs_inactive(struct vop_inactive_args *); int msdosfs_reclaim(struct vop_reclaim_args *); /* * Internal service routine prototypes. */ int deget(struct msdosfsmount *, u_long, u_long, struct denode **); int uniqdosname(struct denode *, struct componentname *, u_char *); int findwin95(struct denode *); int readep(struct msdosfsmount *pmp, u_long dirclu, u_long dirofs, struct buf **bpp, struct direntry **epp); int readde(struct denode *dep, struct buf **bpp, struct direntry **epp); int deextend(struct denode *dep, u_long length, struct ucred *cred); int fillinusemap(struct msdosfsmount *pmp); void reinsert(struct denode *dep); int dosdirempty(struct denode *dep); int createde(struct denode *dep, struct denode *ddep, struct denode **depp, struct componentname *cnp); int deupdat(struct denode *dep, int waitfor); int removede(struct denode *pdep, struct denode *dep); int detrunc(struct denode *dep, u_long length, int flags, struct ucred *cred, struct thread *td); int doscheckpath( struct denode *source, struct denode *target); #endif /* _KERNEL */ Index: user/imp/tbemd/sys/fs/msdosfs/direntry.h =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/direntry.h (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/direntry.h (revision 203850) @@ -1,161 +1,163 @@ /* $FreeBSD$ */ /* $NetBSD: direntry.h,v 1.14 1997/11/17 15:36:32 ws Exp $ */ /*- * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1994, 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ +#ifndef _FS_MSDOSFS_DIRENTRY_H_ +#define _FS_MSDOSFS_DIRENTRY_H_ /* * Structure of a dos directory entry. */ struct direntry { - u_int8_t deName[8]; /* filename, blank filled */ + u_int8_t deName[11]; /* filename, blank filled */ #define SLOT_EMPTY 0x00 /* slot has never been used */ #define SLOT_E5 0x05 /* the real value is 0xe5 */ #define SLOT_DELETED 0xe5 /* file in this slot deleted */ - u_int8_t deExtension[3]; /* extension, blank filled */ u_int8_t deAttributes; /* file attributes */ #define ATTR_NORMAL 0x00 /* normal file */ #define ATTR_READONLY 0x01 /* file is readonly */ #define ATTR_HIDDEN 0x02 /* file is hidden */ #define ATTR_SYSTEM 0x04 /* file is a system file */ #define ATTR_VOLUME 0x08 /* entry is a volume label */ #define ATTR_DIRECTORY 0x10 /* entry is a directory name */ #define ATTR_ARCHIVE 0x20 /* file is new or modified */ u_int8_t deLowerCase; /* NT VFAT lower case flags */ #define LCASE_BASE 0x08 /* filename base in lower case */ #define LCASE_EXT 0x10 /* filename extension in lower case */ u_int8_t deCHundredth; /* hundredth of seconds in CTime */ u_int8_t deCTime[2]; /* create time */ u_int8_t deCDate[2]; /* create date */ u_int8_t deADate[2]; /* access date */ u_int8_t deHighClust[2]; /* high bytes of cluster number */ u_int8_t deMTime[2]; /* last update time */ u_int8_t deMDate[2]; /* last update date */ u_int8_t deStartCluster[2]; /* starting cluster of file */ u_int8_t deFileSize[4]; /* size of file in bytes */ }; /* * Structure of a Win95 long name directory entry */ struct winentry { u_int8_t weCnt; #define WIN_LAST 0x40 #define WIN_CNT 0x3f u_int8_t wePart1[10]; u_int8_t weAttributes; #define ATTR_WIN95 0x0f u_int8_t weReserved1; u_int8_t weChksum; u_int8_t wePart2[12]; u_int16_t weReserved2; u_int8_t wePart3[4]; }; #define WIN_CHARS 13 /* Number of chars per winentry */ /* * Maximum number of winentries for a filename. */ #define WIN_MAXSUBENTRIES 20 /* * Maximum filename length in Win95 * Note: Must be < sizeof(dirent.d_name) */ #define WIN_MAXLEN 255 /* * This is the format of the contents of the deTime field in the direntry * structure. * We don't use bitfields because we don't know how compilers for * arbitrary machines will lay them out. */ #define DT_2SECONDS_MASK 0x1F /* seconds divided by 2 */ #define DT_2SECONDS_SHIFT 0 #define DT_MINUTES_MASK 0x7E0 /* minutes */ #define DT_MINUTES_SHIFT 5 #define DT_HOURS_MASK 0xF800 /* hours */ #define DT_HOURS_SHIFT 11 /* * This is the format of the contents of the deDate field in the direntry * structure. */ #define DD_DAY_MASK 0x1F /* day of month */ #define DD_DAY_SHIFT 0 #define DD_MONTH_MASK 0x1E0 /* month */ #define DD_MONTH_SHIFT 5 #define DD_YEAR_MASK 0xFE00 /* year - 1980 */ #define DD_YEAR_SHIFT 9 #ifdef _KERNEL struct mbnambuf { size_t nb_len; int nb_last_id; char nb_buf[WIN_MAXLEN + 1]; }; struct dirent; struct msdosfsmount; char *mbnambuf_flush(struct mbnambuf *nbp, struct dirent *dp); void mbnambuf_init(struct mbnambuf *nbp); void mbnambuf_write(struct mbnambuf *nbp, char *name, int id); int dos2unixfn(u_char dn[11], u_char *un, int lower, struct msdosfsmount *pmp); int unix2dosfn(const u_char *un, u_char dn[12], size_t unlen, u_int gen, struct msdosfsmount *pmp); int unix2winfn(const u_char *un, size_t unlen, struct winentry *wep, int cnt, int chksum, struct msdosfsmount *pmp); int winChkName(struct mbnambuf *nbp, const u_char *un, size_t unlen, int chksum, struct msdosfsmount *pmp); int win2unixfn(struct mbnambuf *nbp, struct winentry *wep, int chksum, struct msdosfsmount *pmp); -u_int8_t winChksum(struct direntry *dep); +u_int8_t winChksum(u_int8_t *name); int winSlotCnt(const u_char *un, size_t unlen, struct msdosfsmount *pmp); size_t winLenFixup(const u_char *un, size_t unlen); #endif /* _KERNEL */ +#endif /* !_FS_MSDOSFS_DIRENTRY_H_ */ Index: user/imp/tbemd/sys/fs/msdosfs/msdosfs_conv.c =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/msdosfs_conv.c (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/msdosfs_conv.c (revision 203850) @@ -1,1109 +1,1100 @@ /* $FreeBSD$ */ /* $NetBSD: msdosfs_conv.c,v 1.25 1997/11/17 15:36:40 ws Exp $ */ /*- * Copyright (C) 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ #include #include #include #include #include #include #include #include extern struct iconv_functions *msdosfs_iconv; static int mbsadjpos(const char **, size_t, size_t, int, int, void *handle); static u_int16_t dos2unixchr(const u_char **, size_t *, int, struct msdosfsmount *); static u_int16_t unix2doschr(const u_char **, size_t *, struct msdosfsmount *); static u_int16_t win2unixchr(u_int16_t, struct msdosfsmount *); static u_int16_t unix2winchr(const u_char **, size_t *, int, struct msdosfsmount *); /* * 0 - character disallowed in long file name. * 1 - character should be replaced by '_' in DOS file name, * and generation number inserted. * 2 - character ('.' and ' ') should be skipped in DOS file name, * and generation number inserted. */ static u_char unix2dos[256] = { /* iso8859-1 -> cp850 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 00-07 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 08-0f */ 0, 0, 0, 0, 0, 0, 0, 0, /* 10-17 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 18-1f */ 2, 0x21, 0, 0x23, 0x24, 0x25, 0x26, 0x27, /* 20-27 */ 0x28, 0x29, 0, 1, 1, 0x2d, 2, 0, /* 28-2f */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 30-37 */ 0x38, 0x39, 0, 1, 0, 1, 0, 0, /* 38-3f */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 40-47 */ 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 48-4f */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 50-57 */ 0x58, 0x59, 0x5a, 1, 0, 1, 0x5e, 0x5f, /* 58-5f */ 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 60-67 */ 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 68-6f */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 70-77 */ 0x58, 0x59, 0x5a, 0x7b, 0, 0x7d, 0x7e, 0, /* 78-7f */ 0, 0, 0, 0, 0, 0, 0, 0, /* 80-87 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 88-8f */ 0, 0, 0, 0, 0, 0, 0, 0, /* 90-97 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 98-9f */ 0, 0xad, 0xbd, 0x9c, 0xcf, 0xbe, 0xdd, 0xf5, /* a0-a7 */ 0xf9, 0xb8, 0xa6, 0xae, 0xaa, 0xf0, 0xa9, 0xee, /* a8-af */ 0xf8, 0xf1, 0xfd, 0xfc, 0xef, 0xe6, 0xf4, 0xfa, /* b0-b7 */ 0xf7, 0xfb, 0xa7, 0xaf, 0xac, 0xab, 0xf3, 0xa8, /* b8-bf */ 0xb7, 0xb5, 0xb6, 0xc7, 0x8e, 0x8f, 0x92, 0x80, /* c0-c7 */ 0xd4, 0x90, 0xd2, 0xd3, 0xde, 0xd6, 0xd7, 0xd8, /* c8-cf */ 0xd1, 0xa5, 0xe3, 0xe0, 0xe2, 0xe5, 0x99, 0x9e, /* d0-d7 */ 0x9d, 0xeb, 0xe9, 0xea, 0x9a, 0xed, 0xe8, 0xe1, /* d8-df */ 0xb7, 0xb5, 0xb6, 0xc7, 0x8e, 0x8f, 0x92, 0x80, /* e0-e7 */ 0xd4, 0x90, 0xd2, 0xd3, 0xde, 0xd6, 0xd7, 0xd8, /* e8-ef */ 0xd1, 0xa5, 0xe3, 0xe0, 0xe2, 0xe5, 0x99, 0xf6, /* f0-f7 */ 0x9d, 0xeb, 0xe9, 0xea, 0x9a, 0xed, 0xe8, 0x98, /* f8-ff */ }; static u_char dos2unix[256] = { /* cp850 -> iso8859-1 */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, /* 00-07 */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, /* 08-0f */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, /* 10-17 */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, /* 18-1f */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 20-27 */ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 28-2f */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 30-37 */ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 38-3f */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 40-47 */ 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 48-4f */ 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 50-57 */ 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 58-5f */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 60-67 */ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 68-6f */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 70-77 */ 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 78-7f */ 0xc7, 0xfc, 0xe9, 0xe2, 0xe4, 0xe0, 0xe5, 0xe7, /* 80-87 */ 0xea, 0xeb, 0xe8, 0xef, 0xee, 0xec, 0xc4, 0xc5, /* 88-8f */ 0xc9, 0xe6, 0xc6, 0xf4, 0xf6, 0xf2, 0xfb, 0xf9, /* 90-97 */ 0xff, 0xd6, 0xdc, 0xf8, 0xa3, 0xd8, 0xd7, 0x3f, /* 98-9f */ 0xe1, 0xed, 0xf3, 0xfa, 0xf1, 0xd1, 0xaa, 0xba, /* a0-a7 */ 0xbf, 0xae, 0xac, 0xbd, 0xbc, 0xa1, 0xab, 0xbb, /* a8-af */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0xc1, 0xc2, 0xc0, /* b0-b7 */ 0xa9, 0x3f, 0x3f, 0x3f, 0x3f, 0xa2, 0xa5, 0x3f, /* b8-bf */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0xe3, 0xc3, /* c0-c7 */ 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0xa4, /* c8-cf */ 0xf0, 0xd0, 0xca, 0xcb, 0xc8, 0x3f, 0xcd, 0xce, /* d0-d7 */ 0xcf, 0x3f, 0x3f, 0x3f, 0x3f, 0xa6, 0xcc, 0x3f, /* d8-df */ 0xd3, 0xdf, 0xd4, 0xd2, 0xf5, 0xd5, 0xb5, 0xfe, /* e0-e7 */ 0xde, 0xda, 0xdb, 0xd9, 0xfd, 0xdd, 0xaf, 0x3f, /* e8-ef */ 0xad, 0xb1, 0x3f, 0xbe, 0xb6, 0xa7, 0xf7, 0xb8, /* f0-f7 */ 0xb0, 0xa8, 0xb7, 0xb9, 0xb3, 0xb2, 0x3f, 0x3f, /* f8-ff */ }; static u_char u2l[256] = { /* tolower */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 00-07 */ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 08-0f */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 10-17 */ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 18-1f */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 20-27 */ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 28-2f */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 30-37 */ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 38-3f */ 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 40-47 */ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 48-4f */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 50-57 */ 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 58-5f */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 60-67 */ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 68-6f */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 70-77 */ 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 78-7f */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 80-87 */ 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 88-8f */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 90-97 */ 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 98-9f */ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* a0-a7 */ 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* a8-af */ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* b0-b7 */ 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* b8-bf */ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* c0-c7 */ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* c8-cf */ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* d0-d7 */ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* d8-df */ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* e0-e7 */ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* e8-ef */ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* f0-f7 */ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* f8-ff */ }; static u_char l2u[256] = { /* toupper */ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 00-07 */ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 08-0f */ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 10-17 */ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 18-1f */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 20-27 */ 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 28-2f */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 30-37 */ 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 38-3f */ 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 40-47 */ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 48-4f */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 50-57 */ 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 58-5f */ 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 60-67 */ 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 68-6f */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 70-77 */ 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 78-7f */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 80-87 */ 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 88-8f */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 90-97 */ 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 98-9f */ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* a0-a7 */ 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* a8-af */ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* b0-b7 */ 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* b8-bf */ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* c0-c7 */ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* c8-cf */ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* d0-d7 */ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* d8-df */ 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* e0-e7 */ 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* e8-ef */ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* f0-f7 */ 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* f8-ff */ }; /* * DOS filenames are made of 2 parts, the name part and the extension part. * The name part is 8 characters long and the extension part is 3 * characters long. They may contain trailing blanks if the name or * extension are not long enough to fill their respective fields. */ /* * Convert a DOS filename to a unix filename. And, return the number of * characters in the resulting unix filename excluding the terminating * null. */ int dos2unixfn(dn, un, lower, pmp) u_char dn[11]; u_char *un; int lower; struct msdosfsmount *pmp; { size_t i; int thislong = 0; u_int16_t c; /* * If first char of the filename is SLOT_E5 (0x05), then the real * first char of the filename should be 0xe5. But, they couldn't * just have a 0xe5 mean 0xe5 because that is used to mean a freed * directory slot. Another dos quirk. */ if (*dn == SLOT_E5) *dn = 0xe5; /* * Copy the name portion into the unix filename string. */ for (i = 8; i > 0 && *dn != ' ';) { c = dos2unixchr((const u_char **)&dn, &i, lower & LCASE_BASE, pmp); if (c & 0xff00) { *un++ = c >> 8; thislong++; } *un++ = c; thislong++; } dn += i; /* * Now, if there is an extension then put in a period and copy in * the extension. */ if (*dn != ' ') { *un++ = '.'; thislong++; for (i = 3; i > 0 && *dn != ' ';) { c = dos2unixchr((const u_char **)&dn, &i, lower & LCASE_EXT, pmp); if (c & 0xff00) { *un++ = c >> 8; thislong++; } *un++ = c; thislong++; } } *un++ = 0; return (thislong); } /* * Convert a unix filename to a DOS filename according to Win95 rules. * If applicable and gen is not 0, it is inserted into the converted * filename as a generation number. * Returns * 0 if name couldn't be converted * 1 if the converted name is the same as the original * (no long filename entry necessary for Win95) * 2 if conversion was successful * 3 if conversion was successful and generation number was inserted */ int unix2dosfn(un, dn, unlen, gen, pmp) const u_char *un; u_char dn[12]; size_t unlen; u_int gen; struct msdosfsmount *pmp; { ssize_t i, j; int l; int conv = 1; const u_char *cp, *dp, *dp1; u_char gentext[6], *wcp; u_int16_t c; /* * Fill the dos filename string with blanks. These are DOS's pad * characters. */ for (i = 0; i < 11; i++) dn[i] = ' '; dn[11] = 0; /* * The filenames "." and ".." are handled specially, since they * don't follow dos filename rules. */ if (un[0] == '.' && unlen == 1) { dn[0] = '.'; return gen <= 1; } if (un[0] == '.' && un[1] == '.' && unlen == 2) { dn[0] = '.'; dn[1] = '.'; return gen <= 1; } /* * Filenames with only blanks and dots are not allowed! */ for (cp = un, i = unlen; --i >= 0; cp++) if (*cp != ' ' && *cp != '.') break; if (i < 0) return 0; /* * Filenames with some characters are not allowed! */ for (cp = un, i = unlen; i > 0;) if (unix2doschr(&cp, (size_t *)&i, pmp) == 0) return 0; /* * Now find the extension * Note: dot as first char doesn't start extension * and trailing dots and blanks are ignored * Note(2003/7): It seems recent Windows has * defferent rule than this code, that Windows * ignores all dots before extension, and use all * chars as filename except for dots. */ dp = dp1 = 0; for (cp = un + 1, i = unlen - 1; --i >= 0;) { switch (*cp++) { case '.': if (!dp1) dp1 = cp; break; case ' ': break; default: if (dp1) dp = dp1; dp1 = 0; break; } } /* * Now convert it (this part is for extension). * As Windows XP do, if it's not ascii char, * this function should return 2 or 3, so that checkng out Unicode name. */ if (dp) { if (dp1) l = dp1 - dp; else l = unlen - (dp - un); for (cp = dp, i = l, j = 8; i > 0 && j < 11; j++) { c = unix2doschr(&cp, (size_t *)&i, pmp); if (c & 0xff00) { dn[j] = c >> 8; if (++j < 11) { dn[j] = c; if (conv != 3) conv = 2; continue; } else { conv = 3; dn[j-1] = ' '; break; } } else { dn[j] = c; } if (((dn[j] & 0x80) || *(cp - 1) != dn[j]) && conv != 3) conv = 2; if (dn[j] == 1) { conv = 3; dn[j] = '_'; } if (dn[j] == 2) { conv = 3; dn[j--] = ' '; } } if (i > 0) conv = 3; dp--; } else { for (dp = cp; *--dp == ' ' || *dp == '.';); dp++; } /* * Now convert the rest of the name */ for (i = dp - un, j = 0; un < dp && j < 8; j++) { c = unix2doschr(&un, &i, pmp); if (c & 0xff00) { dn[j] = c >> 8; if (++j < 8) { dn[j] = c; if (conv != 3) conv = 2; continue; } else { conv = 3; dn[j-1] = ' '; break; } } else { dn[j] = c; } if (((dn[j] & 0x80) || *(un - 1) != dn[j]) && conv != 3) conv = 2; if (dn[j] == 1) { conv = 3; dn[j] = '_'; } if (dn[j] == 2) { conv = 3; dn[j--] = ' '; } } if (un < dp) conv = 3; /* * If we didn't have any chars in filename, * generate a default */ if (!j) dn[0] = '_'; /* * If there wasn't any char dropped, * there is no place for generation numbers */ if (conv != 3) { if (gen > 1) conv = 0; goto done; } /* * Now insert the generation number into the filename part */ if (gen == 0) goto done; for (wcp = gentext + sizeof(gentext); wcp > gentext && gen; gen /= 10) *--wcp = gen % 10 + '0'; if (gen) { conv = 0; goto done; } for (i = 8; dn[--i] == ' ';); i++; if (gentext + sizeof(gentext) - wcp + 1 > 8 - i) i = 8 - (gentext + sizeof(gentext) - wcp + 1); /* * Correct posision to where insert the generation number */ cp = dn; i -= mbsadjpos((const char**)&cp, i, unlen, 1, pmp->pm_flags, pmp->pm_d2u); dn[i++] = '~'; while (wcp < gentext + sizeof(gentext)) dn[i++] = *wcp++; /* * Tail of the filename should be space */ while (i < 8) dn[i++] = ' '; conv = 3; done: /* * The first character cannot be E5, * because that means a deleted entry */ if (dn[0] == 0xe5) dn[0] = SLOT_E5; return conv; } /* * Create a Win95 long name directory entry * Note: assumes that the filename is valid, * i.e. doesn't consist solely of blanks and dots */ int unix2winfn(un, unlen, wep, cnt, chksum, pmp) const u_char *un; size_t unlen; struct winentry *wep; int cnt; int chksum; struct msdosfsmount *pmp; { u_int8_t *wcp; int i, end; u_int16_t code; /* * Drop trailing blanks and dots */ unlen = winLenFixup(un, unlen); /* * Cut *un for this slot */ unlen = mbsadjpos((const char **)&un, unlen, (cnt - 1) * WIN_CHARS, 2, pmp->pm_flags, pmp->pm_u2w); /* * Initialize winentry to some useful default */ for (wcp = (u_int8_t *)wep, i = sizeof(*wep); --i >= 0; *wcp++ = 0xff); wep->weCnt = cnt; wep->weAttributes = ATTR_WIN95; wep->weReserved1 = 0; wep->weChksum = chksum; wep->weReserved2 = 0; /* * Now convert the filename parts */ end = 0; for (wcp = wep->wePart1, i = sizeof(wep->wePart1)/2; --i >= 0 && !end;) { code = unix2winchr(&un, &unlen, 0, pmp); *wcp++ = code; *wcp++ = code >> 8; if (!code) end = WIN_LAST; } for (wcp = wep->wePart2, i = sizeof(wep->wePart2)/2; --i >= 0 && !end;) { code = unix2winchr(&un, &unlen, 0, pmp); *wcp++ = code; *wcp++ = code >> 8; if (!code) end = WIN_LAST; } for (wcp = wep->wePart3, i = sizeof(wep->wePart3)/2; --i >= 0 && !end;) { code = unix2winchr(&un, &unlen, 0, pmp); *wcp++ = code; *wcp++ = code >> 8; if (!code) end = WIN_LAST; } if (*un == '\0') end = WIN_LAST; wep->weCnt |= end; return !end; } /* * Compare our filename to the one in the Win95 entry * Returns the checksum or -1 if no match */ int winChkName(nbp, un, unlen, chksum, pmp) struct mbnambuf *nbp; const u_char *un; size_t unlen; int chksum; struct msdosfsmount *pmp; { size_t len; u_int16_t c1, c2; u_char *np; struct dirent dirbuf; /* * We already have winentry in *nbp. */ if (!mbnambuf_flush(nbp, &dirbuf) || dirbuf.d_namlen == 0) return -1; #ifdef MSDOSFS_DEBUG printf("winChkName(): un=%s:%d,d_name=%s:%d\n", un, unlen, dirbuf.d_name, dirbuf.d_namlen); #endif /* * Compare the name parts */ len = dirbuf.d_namlen; if (unlen != len) return -2; for (np = dirbuf.d_name; unlen > 0 && len > 0;) { /* * Comparison must be case insensitive, because FAT disallows * to look up or create files in case sensitive even when * it's a long file name. */ c1 = unix2winchr((const u_char **)&np, &len, LCASE_BASE, pmp); c2 = unix2winchr(&un, &unlen, LCASE_BASE, pmp); if (c1 != c2) return -2; } return chksum; } /* * Convert Win95 filename to dirbuf. * Returns the checksum or -1 if impossible */ int win2unixfn(nbp, wep, chksum, pmp) struct mbnambuf *nbp; struct winentry *wep; int chksum; struct msdosfsmount *pmp; { u_int8_t *cp; u_int8_t *np, name[WIN_CHARS * 2 + 1]; u_int16_t code; int i; if ((wep->weCnt&WIN_CNT) > howmany(WIN_MAXLEN, WIN_CHARS) || !(wep->weCnt&WIN_CNT)) return -1; /* * First compare checksums */ if (wep->weCnt&WIN_LAST) { chksum = wep->weChksum; } else if (chksum != wep->weChksum) chksum = -1; if (chksum == -1) return -1; /* * Convert the name parts */ np = name; for (cp = wep->wePart1, i = sizeof(wep->wePart1)/2; --i >= 0;) { code = (cp[1] << 8) | cp[0]; switch (code) { case 0: *np = '\0'; mbnambuf_write(nbp, name, (wep->weCnt & WIN_CNT) - 1); return chksum; case '/': *np = '\0'; return -1; default: code = win2unixchr(code, pmp); if (code & 0xff00) *np++ = code >> 8; *np++ = code; break; } cp += 2; } for (cp = wep->wePart2, i = sizeof(wep->wePart2)/2; --i >= 0;) { code = (cp[1] << 8) | cp[0]; switch (code) { case 0: *np = '\0'; mbnambuf_write(nbp, name, (wep->weCnt & WIN_CNT) - 1); return chksum; case '/': *np = '\0'; return -1; default: code = win2unixchr(code, pmp); if (code & 0xff00) *np++ = code >> 8; *np++ = code; break; } cp += 2; } for (cp = wep->wePart3, i = sizeof(wep->wePart3)/2; --i >= 0;) { code = (cp[1] << 8) | cp[0]; switch (code) { case 0: *np = '\0'; mbnambuf_write(nbp, name, (wep->weCnt & WIN_CNT) - 1); return chksum; case '/': *np = '\0'; return -1; default: code = win2unixchr(code, pmp); if (code & 0xff00) *np++ = code >> 8; *np++ = code; break; } cp += 2; } *np = '\0'; mbnambuf_write(nbp, name, (wep->weCnt & WIN_CNT) - 1); return chksum; } /* * Compute the unrolled checksum of a DOS filename for Win95 LFN use. */ u_int8_t -winChksum(struct direntry *dep) +winChksum(u_int8_t *name) { + int i; u_int8_t s; - s = dep->deName[0]; - s = ((s << 7) | (s >> 1)) + dep->deName[1]; - s = ((s << 7) | (s >> 1)) + dep->deName[2]; - s = ((s << 7) | (s >> 1)) + dep->deName[3]; - s = ((s << 7) | (s >> 1)) + dep->deName[4]; - s = ((s << 7) | (s >> 1)) + dep->deName[5]; - s = ((s << 7) | (s >> 1)) + dep->deName[6]; - s = ((s << 7) | (s >> 1)) + dep->deName[7]; - s = ((s << 7) | (s >> 1)) + dep->deExtension[0]; - s = ((s << 7) | (s >> 1)) + dep->deExtension[1]; - s = ((s << 7) | (s >> 1)) + dep->deExtension[2]; - + for (s = 0, i = 11; --i >= 0; s += *name++) + s = (s << 7)|(s >> 1); return (s); } /* * Determine the number of slots necessary for Win95 names */ int winSlotCnt(un, unlen, pmp) const u_char *un; size_t unlen; struct msdosfsmount *pmp; { size_t wlen; char wn[WIN_MAXLEN * 2 + 1], *wnp; unlen = winLenFixup(un, unlen); if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) { wlen = WIN_MAXLEN * 2; wnp = wn; msdosfs_iconv->conv(pmp->pm_u2w, (const char **)&un, &unlen, &wnp, &wlen); if (unlen > 0) return 0; return howmany(WIN_MAXLEN - wlen/2, WIN_CHARS); } if (unlen > WIN_MAXLEN) return 0; return howmany(unlen, WIN_CHARS); } /* * Determine the number of bytes neccesary for Win95 names */ size_t winLenFixup(un, unlen) const u_char* un; size_t unlen; { for (un += unlen; unlen > 0; unlen--) if (*--un != ' ' && *un != '.') break; return unlen; } /* * Store an area with multi byte string instr, and reterns left * byte of instr and moves pointer forward. The area's size is * inlen or outlen. */ static int mbsadjpos(const char **instr, size_t inlen, size_t outlen, int weight, int flag, void *handle) { char *outp, outstr[outlen * weight + 1]; if (flag & MSDOSFSMNT_KICONV && msdosfs_iconv) { outp = outstr; outlen *= weight; msdosfs_iconv->conv(handle, instr, &inlen, &outp, &outlen); return (inlen); } (*instr) += min(inlen, outlen); return (inlen - min(inlen, outlen)); } /* * Convert DOS char to Local char */ static u_int16_t dos2unixchr(const u_char **instr, size_t *ilen, int lower, struct msdosfsmount *pmp) { u_char c; char *outp, outbuf[3]; u_int16_t wc; size_t len, olen; if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) { olen = len = 2; outp = outbuf; if (lower & (LCASE_BASE | LCASE_EXT)) msdosfs_iconv->convchr_case(pmp->pm_d2u, (const char **)instr, ilen, &outp, &olen, KICONV_LOWER); else msdosfs_iconv->convchr(pmp->pm_d2u, (const char **)instr, ilen, &outp, &olen); len -= olen; /* * return '?' if failed to convert */ if (len == 0) { (*ilen)--; (*instr)++; return ('?'); } wc = 0; while(len--) wc |= (*(outp - len - 1) & 0xff) << (len << 3); return (wc); } (*ilen)--; c = *(*instr)++; c = dos2unix[c]; if (lower & (LCASE_BASE | LCASE_EXT)) c = u2l[c]; return ((u_int16_t)c); } /* * Convert Local char to DOS char */ static u_int16_t unix2doschr(const u_char **instr, size_t *ilen, struct msdosfsmount *pmp) { u_char c; char *up, *outp, unicode[3], outbuf[3]; u_int16_t wc; size_t len, ucslen, unixlen, olen; if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) { /* * to hide an invisible character, using a unicode filter */ ucslen = 2; len = *ilen; up = unicode; msdosfs_iconv->convchr(pmp->pm_u2w, (const char **)instr, ilen, &up, &ucslen); unixlen = len - *ilen; /* * cannot be converted */ if (unixlen == 0) { (*ilen)--; (*instr)++; return (0); } /* * return magic number for ascii char */ if (unixlen == 1) { c = *(*instr -1); if (! (c & 0x80)) { c = unix2dos[c]; if (c <= 2) return (c); } } /* * now convert using libiconv */ *instr -= unixlen; *ilen = len; olen = len = 2; outp = outbuf; msdosfs_iconv->convchr_case(pmp->pm_u2d, (const char **)instr, ilen, &outp, &olen, KICONV_FROM_UPPER); len -= olen; /* * cannot be converted, but has unicode char, should return magic number */ if (len == 0) { (*ilen) -= unixlen; (*instr) += unixlen; return (1); } wc = 0; while(len--) wc |= (*(outp - len - 1) & 0xff) << (len << 3); return (wc); } (*ilen)--; c = *(*instr)++; c = l2u[c]; c = unix2dos[c]; return ((u_int16_t)c); } /* * Convert Windows char to Local char */ static u_int16_t win2unixchr(u_int16_t wc, struct msdosfsmount *pmp) { u_char *inp, *outp, inbuf[3], outbuf[3]; size_t ilen, olen, len; if (wc == 0) return (0); if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) { inbuf[0] = (u_char)(wc>>8); inbuf[1] = (u_char)wc; inbuf[2] = '\0'; ilen = olen = len = 2; inp = inbuf; outp = outbuf; msdosfs_iconv->convchr(pmp->pm_w2u, (const char **)&inp, &ilen, (char **)&outp, &olen); len -= olen; /* * return '?' if failed to convert */ if (len == 0) { wc = '?'; return (wc); } wc = 0; while(len--) wc |= (*(outp - len - 1) & 0xff) << (len << 3); return (wc); } if (wc & 0xff00) wc = '?'; return (wc); } /* * Convert Local char to Windows char */ static u_int16_t unix2winchr(const u_char **instr, size_t *ilen, int lower, struct msdosfsmount *pmp) { u_char *outp, outbuf[3]; u_int16_t wc; size_t olen; if (*ilen == 0) return (0); if (pmp->pm_flags & MSDOSFSMNT_KICONV && msdosfs_iconv) { outp = outbuf; olen = 2; if (lower & (LCASE_BASE | LCASE_EXT)) msdosfs_iconv->convchr_case(pmp->pm_u2w, (const char **)instr, ilen, (char **)&outp, &olen, KICONV_FROM_LOWER); else msdosfs_iconv->convchr(pmp->pm_u2w, (const char **)instr, ilen, (char **)&outp, &olen); /* * return '0' if end of filename */ if (olen == 2) return (0); wc = (outbuf[0]<<8) | outbuf[1]; return (wc); } (*ilen)--; wc = (*instr)[0]; if (lower & (LCASE_BASE | LCASE_EXT)) wc = u2l[wc]; (*instr)++; return (wc); } /* * Initialize the temporary concatenation buffer. */ void mbnambuf_init(struct mbnambuf *nbp) { nbp->nb_len = 0; nbp->nb_last_id = -1; nbp->nb_buf[sizeof(nbp->nb_buf) - 1] = '\0'; } /* * Fill out our concatenation buffer with the given substring, at the offset * specified by its id. Since this function must be called with ids in * descending order, we take advantage of the fact that ASCII substrings are * exactly WIN_CHARS in length. For non-ASCII substrings, we shift all * previous (i.e. higher id) substrings upwards to make room for this one. * This only penalizes portions of substrings that contain more than * WIN_CHARS bytes when they are first encountered. */ void mbnambuf_write(struct mbnambuf *nbp, char *name, int id) { char *slot; size_t count, newlen; if (nbp->nb_len != 0 && id != nbp->nb_last_id - 1) { printf("msdosfs: non-decreasing id: id %d, last id %d\n", id, nbp->nb_last_id); return; } /* Will store this substring in a WIN_CHARS-aligned slot. */ slot = &nbp->nb_buf[id * WIN_CHARS]; count = strlen(name); newlen = nbp->nb_len + count; if (newlen > WIN_MAXLEN || newlen > MAXNAMLEN) { printf("msdosfs: file name length %zu too large\n", newlen); return; } /* Shift suffix upwards by the amount length exceeds WIN_CHARS. */ if (count > WIN_CHARS && nbp->nb_len != 0) bcopy(slot + WIN_CHARS, slot + count, nbp->nb_len); /* Copy in the substring to its slot and update length so far. */ bcopy(name, slot, count); nbp->nb_len = newlen; nbp->nb_last_id = id; } /* * Take the completed string and use it to setup the struct dirent. * Be sure to always nul-terminate the d_name and then copy the string * from our buffer. Note that this function assumes the full string has * been reassembled in the buffer. If it's called before all substrings * have been written via mbnambuf_write(), the result will be incorrect. */ char * mbnambuf_flush(struct mbnambuf *nbp, struct dirent *dp) { if (nbp->nb_len > sizeof(dp->d_name) - 1) { mbnambuf_init(nbp); return (NULL); } bcopy(&nbp->nb_buf[0], dp->d_name, nbp->nb_len); dp->d_name[nbp->nb_len] = '\0'; dp->d_namlen = nbp->nb_len; mbnambuf_init(nbp); return (dp->d_name); } Index: user/imp/tbemd/sys/fs/msdosfs/msdosfs_denode.c =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/msdosfs_denode.c (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/msdosfs_denode.c (revision 203850) @@ -1,629 +1,628 @@ /* $FreeBSD$ */ /* $NetBSD: msdosfs_denode.c,v 1.28 1998/02/10 14:10:00 mrg Exp $ */ /*- * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1994, 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_MSDOSFSNODE, "msdosfs_node", "MSDOSFS vnode private part"); static int de_vncmpf(struct vnode *vp, void *arg) { struct denode *de; uint64_t *a; a = arg; de = VTODE(vp); return (de->de_inode != *a); } /* * If deget() succeeds it returns with the gotten denode locked(). * * pmp - address of msdosfsmount structure of the filesystem containing * the denode of interest. The address of * the msdosfsmount structure are used. * dirclust - which cluster bp contains, if dirclust is 0 (root directory) * diroffset is relative to the beginning of the root directory, * otherwise it is cluster relative. * diroffset - offset past begin of cluster of denode we want * depp - returns the address of the gotten denode. */ int deget(pmp, dirclust, diroffset, depp) struct msdosfsmount *pmp; /* so we know the maj/min number */ u_long dirclust; /* cluster this dir entry came from */ u_long diroffset; /* index of entry within the cluster */ struct denode **depp; /* returns the addr of the gotten denode */ { int error; uint64_t inode; struct mount *mntp = pmp->pm_mountp; struct direntry *direntptr; struct denode *ldep; struct vnode *nvp, *xvp; struct buf *bp; #ifdef MSDOSFS_DEBUG printf("deget(pmp %p, dirclust %lu, diroffset %lx, depp %p)\n", pmp, dirclust, diroffset, depp); #endif /* * On FAT32 filesystems, root is a (more or less) normal * directory */ if (FAT32(pmp) && dirclust == MSDOSFSROOT) dirclust = pmp->pm_rootdirblk; /* * See if the denode is in the denode cache. Use the location of * the directory entry to compute the hash value. For subdir use * address of "." entry. For root dir (if not FAT32) use cluster * MSDOSFSROOT, offset MSDOSFSROOT_OFS * * NOTE: The check for de_refcnt > 0 below insures the denode being * examined does not represent an unlinked but still open file. * These files are not to be accessible even when the directory * entry that represented the file happens to be reused while the * deleted file is still open. */ inode = (uint64_t)pmp->pm_bpcluster * dirclust + diroffset; error = vfs_hash_get(mntp, inode, LK_EXCLUSIVE, curthread, &nvp, de_vncmpf, &inode); if (error) return (error); if (nvp != NULL) { *depp = VTODE(nvp); KASSERT((*depp)->de_dirclust == dirclust, ("wrong dirclust")); KASSERT((*depp)->de_diroffset == diroffset, ("wrong diroffset")); return (0); } /* - * Do the MALLOC before the getnewvnode since doing so afterward + * Do the malloc before the getnewvnode since doing so afterward * might cause a bogus v_data pointer to get dereferenced - * elsewhere if MALLOC should block. + * elsewhere if malloc should block. */ - ldep = malloc(sizeof(struct denode), M_MSDOSFSNODE, M_WAITOK); + ldep = malloc(sizeof(struct denode), M_MSDOSFSNODE, M_WAITOK | M_ZERO); /* * Directory entry was not in cache, have to create a vnode and * copy it from the passed disk buffer. */ /* getnewvnode() does a VREF() on the vnode */ error = getnewvnode("msdosfs", mntp, &msdosfs_vnodeops, &nvp); if (error) { *depp = NULL; free(ldep, M_MSDOSFSNODE); return error; } - bzero((caddr_t)ldep, sizeof *ldep); nvp->v_data = ldep; ldep->de_vnode = nvp; ldep->de_flag = 0; ldep->de_dirclust = dirclust; ldep->de_diroffset = diroffset; ldep->de_inode = inode; fc_purge(ldep, 0); /* init the fat cache for this denode */ lockmgr(nvp->v_vnlock, LK_EXCLUSIVE, NULL); error = insmntque(nvp, mntp); if (error != 0) { free(ldep, M_MSDOSFSNODE); *depp = NULL; return (error); } error = vfs_hash_insert(nvp, inode, LK_EXCLUSIVE, curthread, &xvp, de_vncmpf, &inode); if (error) { *depp = NULL; return (error); } if (xvp != NULL) { /* XXX: Not sure this is right */ nvp = xvp; ldep->de_vnode = nvp; } ldep->de_pmp = pmp; ldep->de_refcnt = 1; /* * Copy the directory entry into the denode area of the vnode. */ if ((dirclust == MSDOSFSROOT || (FAT32(pmp) && dirclust == pmp->pm_rootdirblk)) && diroffset == MSDOSFSROOT_OFS) { /* * Directory entry for the root directory. There isn't one, * so we manufacture one. We should probably rummage * through the root directory and find a label entry (if it * exists), and then use the time and date from that entry * as the time and date for the root denode. */ nvp->v_vflag |= VV_ROOT; /* should be further down XXX */ ldep->de_Attributes = ATTR_DIRECTORY; ldep->de_LowerCase = 0; if (FAT32(pmp)) ldep->de_StartCluster = pmp->pm_rootdirblk; /* de_FileSize will be filled in further down */ else { ldep->de_StartCluster = MSDOSFSROOT; ldep->de_FileSize = pmp->pm_rootdirsize * DEV_BSIZE; } /* * fill in time and date so that fattime2timespec() doesn't * spit up when called from msdosfs_getattr() with root * denode */ ldep->de_CHun = 0; ldep->de_CTime = 0x0000; /* 00:00:00 */ ldep->de_CDate = (0 << DD_YEAR_SHIFT) | (1 << DD_MONTH_SHIFT) | (1 << DD_DAY_SHIFT); /* Jan 1, 1980 */ ldep->de_ADate = ldep->de_CDate; ldep->de_MTime = ldep->de_CTime; ldep->de_MDate = ldep->de_CDate; /* leave the other fields as garbage */ } else { error = readep(pmp, dirclust, diroffset, &bp, &direntptr); if (error) { /* * The denode does not contain anything useful, so * it would be wrong to leave it on its hash chain. * Arrange for vput() to just forget about it. */ ldep->de_Name[0] = SLOT_DELETED; vput(nvp); *depp = NULL; return (error); } DE_INTERNALIZE(ldep, direntptr); brelse(bp); } /* * Fill in a few fields of the vnode and finish filling in the * denode. Then return the address of the found denode. */ if (ldep->de_Attributes & ATTR_DIRECTORY) { /* * Since DOS directory entries that describe directories * have 0 in the filesize field, we take this opportunity * to find out the length of the directory and plug it into * the denode structure. */ u_long size; /* * XXX it sometimes happens that the "." entry has cluster * number 0 when it shouldn't. Use the actual cluster number * instead of what is written in directory entry. */ if (diroffset == 0 && ldep->de_StartCluster != dirclust) { printf("deget(): \".\" entry at clust %lu != %lu\n", dirclust, ldep->de_StartCluster); ldep->de_StartCluster = dirclust; } nvp->v_type = VDIR; if (ldep->de_StartCluster != MSDOSFSROOT) { error = pcbmap(ldep, 0xffff, 0, &size, 0); if (error == E2BIG) { ldep->de_FileSize = de_cn2off(pmp, size); error = 0; } else printf("deget(): pcbmap returned %d\n", error); } } else nvp->v_type = VREG; ldep->de_modrev = init_va_filerev(); *depp = ldep; return (0); } int deupdat(dep, waitfor) struct denode *dep; int waitfor; { int error; struct buf *bp; struct direntry *dirp; struct timespec ts; if (DETOV(dep)->v_mount->mnt_flag & MNT_RDONLY) return (0); getnanotime(&ts); DETIMES(dep, &ts, &ts, &ts); if ((dep->de_flag & DE_MODIFIED) == 0) return (0); dep->de_flag &= ~DE_MODIFIED; if (dep->de_Attributes & ATTR_DIRECTORY) return (0); if (dep->de_refcnt <= 0) return (0); error = readde(dep, &bp, &dirp); if (error) return (error); DE_EXTERNALIZE(dirp, dep); if (waitfor) return (bwrite(bp)); else { bdwrite(bp); return (0); } } /* * Truncate the file described by dep to the length specified by length. */ int detrunc(dep, length, flags, cred, td) struct denode *dep; u_long length; int flags; struct ucred *cred; struct thread *td; { int error; int allerror; u_long eofentry; u_long chaintofree; daddr_t bn; int boff; int isadir = dep->de_Attributes & ATTR_DIRECTORY; struct buf *bp; struct msdosfsmount *pmp = dep->de_pmp; #ifdef MSDOSFS_DEBUG printf("detrunc(): file %s, length %lu, flags %x\n", dep->de_Name, length, flags); #endif /* * Disallow attempts to truncate the root directory since it is of * fixed size. That's just the way dos filesystems are. We use * the VROOT bit in the vnode because checking for the directory * bit and a startcluster of 0 in the denode is not adequate to * recognize the root directory at this point in a file or * directory's life. */ if ((DETOV(dep)->v_vflag & VV_ROOT) && !FAT32(pmp)) { printf("detrunc(): can't truncate root directory, clust %ld, offset %ld\n", dep->de_dirclust, dep->de_diroffset); return (EINVAL); } if (dep->de_FileSize < length) { vnode_pager_setsize(DETOV(dep), length); return deextend(dep, length, cred); } /* * If the desired length is 0 then remember the starting cluster of * the file and set the StartCluster field in the directory entry * to 0. If the desired length is not zero, then get the number of * the last cluster in the shortened file. Then get the number of * the first cluster in the part of the file that is to be freed. * Then set the next cluster pointer in the last cluster of the * file to CLUST_EOFE. */ if (length == 0) { chaintofree = dep->de_StartCluster; dep->de_StartCluster = 0; eofentry = ~0; } else { error = pcbmap(dep, de_clcount(pmp, length) - 1, 0, &eofentry, 0); if (error) { #ifdef MSDOSFS_DEBUG printf("detrunc(): pcbmap fails %d\n", error); #endif return (error); } } fc_purge(dep, de_clcount(pmp, length)); /* * If the new length is not a multiple of the cluster size then we * must zero the tail end of the new last cluster in case it * becomes part of the file again because of a seek. */ if ((boff = length & pmp->pm_crbomask) != 0) { if (isadir) { bn = cntobn(pmp, eofentry); error = bread(pmp->pm_devvp, bn, pmp->pm_bpcluster, NOCRED, &bp); if (error) { brelse(bp); #ifdef MSDOSFS_DEBUG printf("detrunc(): bread fails %d\n", error); #endif return (error); } bzero(bp->b_data + boff, pmp->pm_bpcluster - boff); if (flags & IO_SYNC) bwrite(bp); else bdwrite(bp); } } /* * Write out the updated directory entry. Even if the update fails * we free the trailing clusters. */ dep->de_FileSize = length; if (!isadir) dep->de_flag |= DE_UPDATE | DE_MODIFIED; allerror = vtruncbuf(DETOV(dep), cred, td, length, pmp->pm_bpcluster); #ifdef MSDOSFS_DEBUG if (allerror) printf("detrunc(): vtruncbuf error %d\n", allerror); #endif error = deupdat(dep, !(DETOV(dep)->v_mount->mnt_flag & MNT_ASYNC)); if (error != 0 && allerror == 0) allerror = error; #ifdef MSDOSFS_DEBUG printf("detrunc(): allerror %d, eofentry %lu\n", allerror, eofentry); #endif /* * If we need to break the cluster chain for the file then do it * now. */ if (eofentry != ~0) { error = fatentry(FAT_GET_AND_SET, pmp, eofentry, &chaintofree, CLUST_EOFE); if (error) { #ifdef MSDOSFS_DEBUG printf("detrunc(): fatentry errors %d\n", error); #endif return (error); } fc_setcache(dep, FC_LASTFC, de_cluster(pmp, length - 1), eofentry); } /* * Now free the clusters removed from the file because of the * truncation. */ if (chaintofree != 0 && !MSDOSFSEOF(pmp, chaintofree)) freeclusterchain(pmp, chaintofree); return (allerror); } /* * Extend the file described by dep to length specified by length. */ int deextend(dep, length, cred) struct denode *dep; u_long length; struct ucred *cred; { struct msdosfsmount *pmp = dep->de_pmp; u_long count; int error; /* * The root of a DOS filesystem cannot be extended. */ if ((DETOV(dep)->v_vflag & VV_ROOT) && !FAT32(pmp)) return (EINVAL); /* * Directories cannot be extended. */ if (dep->de_Attributes & ATTR_DIRECTORY) return (EISDIR); if (length <= dep->de_FileSize) panic("deextend: file too large"); /* * Compute the number of clusters to allocate. */ count = de_clcount(pmp, length) - de_clcount(pmp, dep->de_FileSize); if (count > 0) { if (count > pmp->pm_freeclustercount) return (ENOSPC); error = extendfile(dep, count, NULL, NULL, DE_CLEAR); if (error) { /* truncate the added clusters away again */ (void) detrunc(dep, dep->de_FileSize, 0, cred, NULL); return (error); } } dep->de_FileSize = length; dep->de_flag |= DE_UPDATE | DE_MODIFIED; return (deupdat(dep, !(DETOV(dep)->v_mount->mnt_flag & MNT_ASYNC))); } /* * Move a denode to its correct hash queue after the file it represents has * been moved to a new directory. */ void reinsert(dep) struct denode *dep; { struct vnode *vp; /* * Fix up the denode cache. If the denode is for a directory, * there is nothing to do since the hash is based on the starting * cluster of the directory file and that hasn't changed. If for a * file the hash is based on the location of the directory entry, * so we must remove it from the cache and re-enter it with the * hash based on the new location of the directory entry. */ #if 0 if (dep->de_Attributes & ATTR_DIRECTORY) return; #endif vp = DETOV(dep); dep->de_inode = (uint64_t)dep->de_pmp->pm_bpcluster * dep->de_dirclust + dep->de_diroffset; vfs_hash_rehash(vp, dep->de_inode); } int msdosfs_reclaim(ap) struct vop_reclaim_args /* { struct vnode *a_vp; } */ *ap; { struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(vp); #ifdef MSDOSFS_DEBUG printf("msdosfs_reclaim(): dep %p, file %s, refcnt %ld\n", dep, dep->de_Name, dep->de_refcnt); #endif if (prtactive && vrefcnt(vp) != 0) vprint("msdosfs_reclaim(): pushing active", vp); /* * Destroy the vm object and flush associated pages. */ vnode_destroy_vobject(vp); /* * Remove the denode from its hash chain. */ vfs_hash_remove(vp); /* * Purge old data structures associated with the denode. */ #if 0 /* XXX */ dep->de_flag = 0; #endif free(dep, M_MSDOSFSNODE); vp->v_data = NULL; return (0); } int msdosfs_inactive(ap) struct vop_inactive_args /* { struct vnode *a_vp; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(vp); struct thread *td = ap->a_td; int error = 0; #ifdef MSDOSFS_DEBUG printf("msdosfs_inactive(): dep %p, de_Name[0] %x\n", dep, dep->de_Name[0]); #endif if (prtactive && vrefcnt(vp) != 0) vprint("msdosfs_inactive(): pushing active", vp); /* * Ignore denodes related to stale file handles. */ if (dep->de_Name[0] == SLOT_DELETED) goto out; /* * If the file has been deleted and it is on a read/write * filesystem, then truncate the file, and mark the directory slot * as empty. (This may not be necessary for the dos filesystem.) */ #ifdef MSDOSFS_DEBUG printf("msdosfs_inactive(): dep %p, refcnt %ld, mntflag %x, MNT_RDONLY %x\n", dep, dep->de_refcnt, vp->v_mount->mnt_flag, MNT_RDONLY); #endif if (dep->de_refcnt <= 0 && (vp->v_mount->mnt_flag & MNT_RDONLY) == 0) { error = detrunc(dep, (u_long) 0, 0, NOCRED, td); dep->de_flag |= DE_UPDATE; dep->de_Name[0] = SLOT_DELETED; } deupdat(dep, 0); out: /* * If we are done with the denode, reclaim it * so that it can be reused immediately. */ #ifdef MSDOSFS_DEBUG printf("msdosfs_inactive(): v_usecount %d, de_Name[0] %x\n", vrefcnt(vp), dep->de_Name[0]); #endif if (dep->de_Name[0] == SLOT_DELETED) vrecycle(vp, td); return (error); } Index: user/imp/tbemd/sys/fs/msdosfs/msdosfs_lookup.c =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/msdosfs_lookup.c (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/msdosfs_lookup.c (revision 203850) @@ -1,1087 +1,1087 @@ /* $FreeBSD$ */ /* $NetBSD: msdosfs_lookup.c,v 1.37 1997/11/17 15:36:54 ws Exp $ */ /*- * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1994, 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ #include #include #include #include #include #include #include #include #include #include #include /* * When we search a directory the blocks containing directory entries are * read and examined. The directory entries contain information that would * normally be in the inode of a unix filesystem. This means that some of * a directory's contents may also be in memory resident denodes (sort of * an inode). This can cause problems if we are searching while some other * process is modifying a directory. To prevent one process from accessing * incompletely modified directory information we depend upon being the * sole owner of a directory block. bread/brelse provide this service. * This being the case, when a process modifies a directory it must first * acquire the disk block that contains the directory entry to be modified. * Then update the disk block and the denode, and then write the disk block * out to disk. This way disk blocks containing directory entries and in * memory denode's will be in synch. */ int msdosfs_lookup(ap) struct vop_cachedlookup_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; } */ *ap; { struct mbnambuf nb; struct vnode *vdp = ap->a_dvp; struct vnode **vpp = ap->a_vpp; struct componentname *cnp = ap->a_cnp; daddr_t bn; int error; int slotcount; int slotoffset = 0; int frcn; u_long cluster; int blkoff; int diroff; int blsize; int isadir; /* ~0 if found direntry is a directory */ u_long scn; /* starting cluster number */ struct vnode *pdp; struct denode *dp; struct denode *tdp; struct msdosfsmount *pmp; struct buf *bp = 0; struct direntry *dep = NULL; u_char dosfilename[12]; int flags = cnp->cn_flags; int nameiop = cnp->cn_nameiop; int unlen; int wincnt = 1; int chksum = -1, chksum_ok; int olddos = 1; #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): looking for %s\n", cnp->cn_nameptr); #endif dp = VTODE(vdp); pmp = dp->de_pmp; *vpp = NULL; #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): vdp %p, dp %p, Attr %02x\n", vdp, dp, dp->de_Attributes); #endif /* * If they are going after the . or .. entry in the root directory, * they won't find it. DOS filesystems don't have them in the root * directory. So, we fake it. deget() is in on this scam too. */ if ((vdp->v_vflag & VV_ROOT) && cnp->cn_nameptr[0] == '.' && (cnp->cn_namelen == 1 || (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.'))) { isadir = ATTR_DIRECTORY; scn = MSDOSFSROOT; #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): looking for . or .. in root directory\n"); #endif cluster = MSDOSFSROOT; blkoff = MSDOSFSROOT_OFS; goto foundroot; } switch (unix2dosfn((const u_char *)cnp->cn_nameptr, dosfilename, cnp->cn_namelen, 0, pmp)) { case 0: return (EINVAL); case 1: break; case 2: wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr, cnp->cn_namelen, pmp) + 1; break; case 3: olddos = 0; wincnt = winSlotCnt((const u_char *)cnp->cn_nameptr, cnp->cn_namelen, pmp) + 1; break; } if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME) { wincnt = 1; olddos = 1; } unlen = winLenFixup(cnp->cn_nameptr, cnp->cn_namelen); /* * Suppress search for slots unless creating * file and at end of pathname, in which case * we watch for a place to put the new file in * case it doesn't already exist. */ slotcount = wincnt; if ((nameiop == CREATE || nameiop == RENAME) && (flags & ISLASTCN)) slotcount = 0; #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): dos version of filename %s, length %ld\n", dosfilename, cnp->cn_namelen); #endif /* * Search the directory pointed at by vdp for the name pointed at * by cnp->cn_nameptr. */ tdp = NULL; mbnambuf_init(&nb); /* * The outer loop ranges over the clusters that make up the * directory. Note that the root directory is different from all * other directories. It has a fixed number of blocks that are not * part of the pool of allocatable clusters. So, we treat it a * little differently. The root directory starts at "cluster" 0. */ diroff = 0; for (frcn = 0;; frcn++) { error = pcbmap(dp, frcn, &bn, &cluster, &blsize); if (error) { if (error == E2BIG) break; return (error); } error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return (error); } for (blkoff = 0; blkoff < blsize; blkoff += sizeof(struct direntry), diroff += sizeof(struct direntry)) { dep = (struct direntry *)(bp->b_data + blkoff); /* * If the slot is empty and we are still looking * for an empty then remember this one. If the * slot is not empty then check to see if it * matches what we are looking for. If the slot * has never been filled with anything, then the * remainder of the directory has never been used, * so there is no point in searching it. */ if (dep->deName[0] == SLOT_EMPTY || dep->deName[0] == SLOT_DELETED) { /* * Drop memory of previous long matches */ chksum = -1; mbnambuf_init(&nb); if (slotcount < wincnt) { slotcount++; slotoffset = diroff; } if (dep->deName[0] == SLOT_EMPTY) { brelse(bp); goto notfound; } } else { /* * If there wasn't enough space for our winentries, * forget about the empty space */ if (slotcount < wincnt) slotcount = 0; /* * Check for Win95 long filename entry */ if (dep->deAttributes == ATTR_WIN95) { if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME) continue; chksum = win2unixfn(&nb, (struct winentry *)dep, chksum, pmp); continue; } chksum = winChkName(&nb, (const u_char *)cnp->cn_nameptr, unlen, chksum, pmp); if (chksum == -2) { chksum = -1; continue; } /* * Ignore volume labels (anywhere, not just * the root directory). */ if (dep->deAttributes & ATTR_VOLUME) { chksum = -1; continue; } /* * Check for a checksum or name match */ - chksum_ok = (chksum == winChksum(dep)); + chksum_ok = (chksum == winChksum(dep->deName)); if (!chksum_ok && (!olddos || bcmp(dosfilename, dep->deName, 11))) { chksum = -1; continue; } #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): match blkoff %d, diroff %d\n", blkoff, diroff); #endif /* * Remember where this directory * entry came from for whoever did * this lookup. */ dp->de_fndoffset = diroff; if (chksum_ok && nameiop == RENAME) { /* * Target had correct long name * directory entries, reuse them * as needed. */ dp->de_fndcnt = wincnt - 1; } else { /* * Long name directory entries * not present or corrupt, can only * reuse dos directory entry. */ dp->de_fndcnt = 0; } goto found; } } /* for (blkoff = 0; .... */ /* * Release the buffer holding the directory cluster just * searched. */ brelse(bp); } /* for (frcn = 0; ; frcn++) */ notfound: /* * We hold no disk buffers at this point. */ /* * Fixup the slot description to point to the place where * we might put the new DOS direntry (putting the Win95 * long name entries before that) */ if (!slotcount) { slotcount = 1; slotoffset = diroff; } if (wincnt > slotcount) slotoffset += sizeof(struct direntry) * (wincnt - slotcount); /* * If we get here we didn't find the entry we were looking for. But * that's ok if we are creating or renaming and are at the end of * the pathname and the directory hasn't been removed. */ #ifdef MSDOSFS_DEBUG printf("msdosfs_lookup(): op %d, refcnt %ld\n", nameiop, dp->de_refcnt); printf(" slotcount %d, slotoffset %d\n", slotcount, slotoffset); #endif if ((nameiop == CREATE || nameiop == RENAME) && (flags & ISLASTCN) && dp->de_refcnt != 0) { /* * Access for write is interpreted as allowing * creation of files in the directory. */ error = VOP_ACCESS(vdp, VWRITE, cnp->cn_cred, cnp->cn_thread); if (error) return (error); /* * Return an indication of where the new directory * entry should be put. */ dp->de_fndoffset = slotoffset; dp->de_fndcnt = wincnt - 1; /* * We return with the directory locked, so that * the parameters we set up above will still be * valid if we actually decide to do a direnter(). * We return ni_vp == NULL to indicate that the entry * does not currently exist; we leave a pointer to * the (locked) directory inode in ndp->ni_dvp. * The pathname buffer is saved so that the name * can be obtained later. * * NB - if the directory is unlocked, then this * information cannot be used. */ cnp->cn_flags |= SAVENAME; return (EJUSTRETURN); } #if 0 /* * Insert name into cache (as non-existent) if appropriate. * * XXX Negative caching is broken for msdosfs because the name * cache doesn't understand peculiarities such as case insensitivity * and 8.3 filenames. Hence, it may not invalidate all negative * entries if a file with this name is later created. */ if ((cnp->cn_flags & MAKEENTRY) && nameiop != CREATE) cache_enter(vdp, *vpp, cnp); #endif return (ENOENT); found: /* * NOTE: We still have the buffer with matched directory entry at * this point. */ isadir = dep->deAttributes & ATTR_DIRECTORY; scn = getushort(dep->deStartCluster); if (FAT32(pmp)) { scn |= getushort(dep->deHighClust) << 16; if (scn == pmp->pm_rootdirblk) { /* * There should actually be 0 here. * Just ignore the error. */ scn = MSDOSFSROOT; } } if (isadir) { cluster = scn; if (cluster == MSDOSFSROOT) blkoff = MSDOSFSROOT_OFS; else blkoff = 0; } else if (cluster == MSDOSFSROOT) blkoff = diroff; /* * Now release buf to allow deget to read the entry again. * Reserving it here and giving it to deget could result * in a deadlock. */ brelse(bp); bp = 0; foundroot: /* * If we entered at foundroot, then we are looking for the . or .. * entry of the filesystems root directory. isadir and scn were * setup before jumping here. And, bp is already null. */ if (FAT32(pmp) && scn == MSDOSFSROOT) scn = pmp->pm_rootdirblk; /* * If deleting, and at end of pathname, return * parameters which can be used to remove file. */ if (nameiop == DELETE && (flags & ISLASTCN)) { /* * Don't allow deleting the root. */ if (blkoff == MSDOSFSROOT_OFS) return EROFS; /* really? XXX */ /* * Write access to directory required to delete files. */ error = VOP_ACCESS(vdp, VWRITE, cnp->cn_cred, cnp->cn_thread); if (error) return (error); /* * Return pointer to current entry in dp->i_offset. * Save directory inode pointer in ndp->ni_dvp for dirremove(). */ if (dp->de_StartCluster == scn && isadir) { /* "." */ VREF(vdp); *vpp = vdp; return (0); } error = deget(pmp, cluster, blkoff, &tdp); if (error) return (error); *vpp = DETOV(tdp); return (0); } /* * If rewriting (RENAME), return the inode and the * information required to rewrite the present directory * Must get inode of directory entry to verify it's a * regular file, or empty directory. */ if (nameiop == RENAME && (flags & ISLASTCN)) { if (blkoff == MSDOSFSROOT_OFS) return EROFS; /* really? XXX */ error = VOP_ACCESS(vdp, VWRITE, cnp->cn_cred, cnp->cn_thread); if (error) return (error); /* * Careful about locking second inode. * This can only occur if the target is ".". */ if (dp->de_StartCluster == scn && isadir) return (EISDIR); if ((error = deget(pmp, cluster, blkoff, &tdp)) != 0) return (error); *vpp = DETOV(tdp); cnp->cn_flags |= SAVENAME; return (0); } /* * Step through the translation in the name. We do not `vput' the * directory because we may need it again if a symbolic link * is relative to the current directory. Instead we save it * unlocked as "pdp". We must get the target inode before unlocking * the directory to insure that the inode will not be removed * before we get it. We prevent deadlock by always fetching * inodes from the root, moving down the directory tree. Thus * when following backward pointers ".." we must unlock the * parent directory before getting the requested directory. * There is a potential race condition here if both the current * and parent directories are removed before the VFS_VGET for the * inode associated with ".." returns. We hope that this occurs * infrequently since we cannot avoid this race condition without * implementing a sophisticated deadlock detection algorithm. * Note also that this simple deadlock detection scheme will not * work if the filesystem has any hard links other than ".." * that point backwards in the directory structure. */ pdp = vdp; if (flags & ISDOTDOT) { VOP_UNLOCK(pdp, 0); error = deget(pmp, cluster, blkoff, &tdp); vn_lock(pdp, LK_EXCLUSIVE | LK_RETRY); if (error) return (error); *vpp = DETOV(tdp); } else if (dp->de_StartCluster == scn && isadir) { VREF(vdp); /* we want ourself, ie "." */ *vpp = vdp; } else { if ((error = deget(pmp, cluster, blkoff, &tdp)) != 0) return (error); *vpp = DETOV(tdp); } /* * Insert name into cache if appropriate. */ if (cnp->cn_flags & MAKEENTRY) cache_enter(vdp, *vpp, cnp); return (0); } /* * dep - directory entry to copy into the directory * ddep - directory to add to * depp - return the address of the denode for the created directory entry * if depp != 0 * cnp - componentname needed for Win95 long filenames */ int createde(dep, ddep, depp, cnp) struct denode *dep; struct denode *ddep; struct denode **depp; struct componentname *cnp; { int error; u_long dirclust, diroffset; struct direntry *ndep; struct msdosfsmount *pmp = ddep->de_pmp; struct buf *bp; daddr_t bn; int blsize; #ifdef MSDOSFS_DEBUG printf("createde(dep %p, ddep %p, depp %p, cnp %p)\n", dep, ddep, depp, cnp); #endif /* * If no space left in the directory then allocate another cluster * and chain it onto the end of the file. There is one exception * to this. That is, if the root directory has no more space it * can NOT be expanded. extendfile() checks for and fails attempts * to extend the root directory. We just return an error in that * case. */ if (ddep->de_fndoffset >= ddep->de_FileSize) { diroffset = ddep->de_fndoffset + sizeof(struct direntry) - ddep->de_FileSize; dirclust = de_clcount(pmp, diroffset); error = extendfile(ddep, dirclust, 0, 0, DE_CLEAR); if (error) { (void)detrunc(ddep, ddep->de_FileSize, 0, NOCRED, NULL); return error; } /* * Update the size of the directory */ ddep->de_FileSize += de_cn2off(pmp, dirclust); } /* * We just read in the cluster with space. Copy the new directory * entry in. Then write it to disk. NOTE: DOS directories * do not get smaller as clusters are emptied. */ error = pcbmap(ddep, de_cluster(pmp, ddep->de_fndoffset), &bn, &dirclust, &blsize); if (error) return error; diroffset = ddep->de_fndoffset; if (dirclust != MSDOSFSROOT) diroffset &= pmp->pm_crbomask; if ((error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp)) != 0) { brelse(bp); return error; } ndep = bptoep(pmp, bp, ddep->de_fndoffset); DE_EXTERNALIZE(ndep, dep); /* * Now write the Win95 long name */ if (ddep->de_fndcnt > 0) { - u_int8_t chksum = winChksum(ndep); + u_int8_t chksum = winChksum(ndep->deName); const u_char *un = (const u_char *)cnp->cn_nameptr; int unlen = cnp->cn_namelen; int cnt = 1; while (--ddep->de_fndcnt >= 0) { if (!(ddep->de_fndoffset & pmp->pm_crbomask)) { if (DETOV(ddep)->v_mount->mnt_flag & MNT_ASYNC) bdwrite(bp); else if ((error = bwrite(bp)) != 0) return error; ddep->de_fndoffset -= sizeof(struct direntry); error = pcbmap(ddep, de_cluster(pmp, ddep->de_fndoffset), &bn, 0, &blsize); if (error) return error; error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return error; } ndep = bptoep(pmp, bp, ddep->de_fndoffset); } else { ndep--; ddep->de_fndoffset -= sizeof(struct direntry); } if (!unix2winfn(un, unlen, (struct winentry *)ndep, cnt++, chksum, pmp)) break; } } if (DETOV(ddep)->v_mount->mnt_flag & MNT_ASYNC) bdwrite(bp); else if ((error = bwrite(bp)) != 0) return error; /* * If they want us to return with the denode gotten. */ if (depp) { if (dep->de_Attributes & ATTR_DIRECTORY) { dirclust = dep->de_StartCluster; if (FAT32(pmp) && dirclust == pmp->pm_rootdirblk) dirclust = MSDOSFSROOT; if (dirclust == MSDOSFSROOT) diroffset = MSDOSFSROOT_OFS; else diroffset = 0; } return deget(pmp, dirclust, diroffset, depp); } return 0; } /* * Be sure a directory is empty except for "." and "..". Return 1 if empty, * return 0 if not empty or error. */ int dosdirempty(dep) struct denode *dep; { int blsize; int error; u_long cn; daddr_t bn; struct buf *bp; struct msdosfsmount *pmp = dep->de_pmp; struct direntry *dentp; /* * Since the filesize field in directory entries for a directory is * zero, we just have to feel our way through the directory until * we hit end of file. */ for (cn = 0;; cn++) { if ((error = pcbmap(dep, cn, &bn, 0, &blsize)) != 0) { if (error == E2BIG) return (1); /* it's empty */ return (0); } error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return (0); } for (dentp = (struct direntry *)bp->b_data; (char *)dentp < bp->b_data + blsize; dentp++) { if (dentp->deName[0] != SLOT_DELETED && (dentp->deAttributes & ATTR_VOLUME) == 0) { /* * In dos directories an entry whose name * starts with SLOT_EMPTY (0) starts the * beginning of the unused part of the * directory, so we can just return that it * is empty. */ if (dentp->deName[0] == SLOT_EMPTY) { brelse(bp); return (1); } /* * Any names other than "." and ".." in a * directory mean it is not empty. */ if (bcmp(dentp->deName, ". ", 11) && bcmp(dentp->deName, ".. ", 11)) { brelse(bp); #ifdef MSDOSFS_DEBUG printf("dosdirempty(): entry found %02x, %02x\n", dentp->deName[0], dentp->deName[1]); #endif return (0); /* not empty */ } } } brelse(bp); } /* NOTREACHED */ } /* * Check to see if the directory described by target is in some * subdirectory of source. This prevents something like the following from * succeeding and leaving a bunch or files and directories orphaned. mv * /a/b/c /a/b/c/d/e/f Where c and f are directories. * * source - the inode for /a/b/c * target - the inode for /a/b/c/d/e/f * * Returns 0 if target is NOT a subdirectory of source. * Otherwise returns a non-zero error number. * The target inode is always unlocked on return. */ int doscheckpath(source, target) struct denode *source; struct denode *target; { daddr_t scn; struct msdosfsmount *pmp; struct direntry *ep; struct denode *dep; struct buf *bp = NULL; int error = 0; dep = target; if ((target->de_Attributes & ATTR_DIRECTORY) == 0 || (source->de_Attributes & ATTR_DIRECTORY) == 0) { error = ENOTDIR; goto out; } if (dep->de_StartCluster == source->de_StartCluster) { error = EEXIST; goto out; } if (dep->de_StartCluster == MSDOSFSROOT) goto out; pmp = dep->de_pmp; #ifdef DIAGNOSTIC if (pmp != source->de_pmp) panic("doscheckpath: source and target on different filesystems"); #endif if (FAT32(pmp) && dep->de_StartCluster == pmp->pm_rootdirblk) goto out; for (;;) { if ((dep->de_Attributes & ATTR_DIRECTORY) == 0) { error = ENOTDIR; break; } scn = dep->de_StartCluster; error = bread(pmp->pm_devvp, cntobn(pmp, scn), pmp->pm_bpcluster, NOCRED, &bp); if (error) break; ep = (struct direntry *) bp->b_data + 1; if ((ep->deAttributes & ATTR_DIRECTORY) == 0 || bcmp(ep->deName, ".. ", 11) != 0) { error = ENOTDIR; break; } scn = getushort(ep->deStartCluster); if (FAT32(pmp)) scn |= getushort(ep->deHighClust) << 16; if (scn == source->de_StartCluster) { error = EINVAL; break; } if (scn == MSDOSFSROOT) break; if (FAT32(pmp) && scn == pmp->pm_rootdirblk) { /* * scn should be 0 in this case, * but we silently ignore the error. */ break; } vput(DETOV(dep)); brelse(bp); bp = NULL; /* NOTE: deget() clears dep on error */ if ((error = deget(pmp, scn, 0, &dep)) != 0) break; } out:; if (bp) brelse(bp); if (error == ENOTDIR) printf("doscheckpath(): .. not a directory?\n"); if (dep != NULL) vput(DETOV(dep)); return (error); } /* * Read in the disk block containing the directory entry (dirclu, dirofs) * and return the address of the buf header, and the address of the * directory entry within the block. */ int readep(pmp, dirclust, diroffset, bpp, epp) struct msdosfsmount *pmp; u_long dirclust, diroffset; struct buf **bpp; struct direntry **epp; { int error; daddr_t bn; int blsize; blsize = pmp->pm_bpcluster; if (dirclust == MSDOSFSROOT && de_blk(pmp, diroffset + blsize) > pmp->pm_rootdirsize) blsize = de_bn2off(pmp, pmp->pm_rootdirsize) & pmp->pm_crbomask; bn = detobn(pmp, dirclust, diroffset); if ((error = bread(pmp->pm_devvp, bn, blsize, NOCRED, bpp)) != 0) { brelse(*bpp); *bpp = NULL; return (error); } if (epp) *epp = bptoep(pmp, *bpp, diroffset); return (0); } /* * Read in the disk block containing the directory entry dep came from and * return the address of the buf header, and the address of the directory * entry within the block. */ int readde(dep, bpp, epp) struct denode *dep; struct buf **bpp; struct direntry **epp; { return (readep(dep->de_pmp, dep->de_dirclust, dep->de_diroffset, bpp, epp)); } /* * Remove a directory entry. At this point the file represented by the * directory entry to be removed is still full length until noone has it * open. When the file no longer being used msdosfs_inactive() is called * and will truncate the file to 0 length. When the vnode containing the * denode is needed for some other purpose by VFS it will call * msdosfs_reclaim() which will remove the denode from the denode cache. */ int removede(pdep, dep) struct denode *pdep; /* directory where the entry is removed */ struct denode *dep; /* file to be removed */ { int error; struct direntry *ep; struct buf *bp; daddr_t bn; int blsize; struct msdosfsmount *pmp = pdep->de_pmp; u_long offset = pdep->de_fndoffset; #ifdef MSDOSFS_DEBUG printf("removede(): filename %s, dep %p, offset %08lx\n", dep->de_Name, dep, offset); #endif dep->de_refcnt--; offset += sizeof(struct direntry); do { offset -= sizeof(struct direntry); error = pcbmap(pdep, de_cluster(pmp, offset), &bn, 0, &blsize); if (error) return error; error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return error; } ep = bptoep(pmp, bp, offset); /* * Check whether, if we came here the second time, i.e. * when underflowing into the previous block, the last * entry in this block is a longfilename entry, too. */ if (ep->deAttributes != ATTR_WIN95 && offset != pdep->de_fndoffset) { brelse(bp); break; } offset += sizeof(struct direntry); while (1) { /* * We are a bit agressive here in that we delete any Win95 * entries preceding this entry, not just the ones we "own". * Since these presumably aren't valid anyway, * there should be no harm. */ offset -= sizeof(struct direntry); ep--->deName[0] = SLOT_DELETED; if ((pmp->pm_flags & MSDOSFSMNT_NOWIN95) || !(offset & pmp->pm_crbomask) || ep->deAttributes != ATTR_WIN95) break; } if (DETOV(pdep)->v_mount->mnt_flag & MNT_ASYNC) bdwrite(bp); else if ((error = bwrite(bp)) != 0) return error; } while (!(pmp->pm_flags & MSDOSFSMNT_NOWIN95) && !(offset & pmp->pm_crbomask) && offset); return 0; } /* * Create a unique DOS name in dvp */ int uniqdosname(dep, cnp, cp) struct denode *dep; struct componentname *cnp; u_char *cp; { struct msdosfsmount *pmp = dep->de_pmp; struct direntry *dentp; int gen; int blsize; u_long cn; daddr_t bn; struct buf *bp; int error; if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME) return (unix2dosfn((const u_char *)cnp->cn_nameptr, cp, cnp->cn_namelen, 0, pmp) ? 0 : EINVAL); for (gen = 1;; gen++) { /* * Generate DOS name with generation number */ if (!unix2dosfn((const u_char *)cnp->cn_nameptr, cp, cnp->cn_namelen, gen, pmp)) return gen == 1 ? EINVAL : EEXIST; /* * Now look for a dir entry with this exact name */ for (cn = error = 0; !error; cn++) { if ((error = pcbmap(dep, cn, &bn, 0, &blsize)) != 0) { if (error == E2BIG) /* EOF reached and not found */ return 0; return error; } error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return error; } for (dentp = (struct direntry *)bp->b_data; (char *)dentp < bp->b_data + blsize; dentp++) { if (dentp->deName[0] == SLOT_EMPTY) { /* * Last used entry and not found */ brelse(bp); return 0; } /* * Ignore volume labels and Win95 entries */ if (dentp->deAttributes & ATTR_VOLUME) continue; if (!bcmp(dentp->deName, cp, 11)) { error = EEXIST; break; } } brelse(bp); } } } /* * Find any Win'95 long filename entry in directory dep */ int findwin95(dep) struct denode *dep; { struct msdosfsmount *pmp = dep->de_pmp; struct direntry *dentp; int blsize, win95; u_long cn; daddr_t bn; struct buf *bp; win95 = 1; /* * Read through the directory looking for Win'95 entries * Note: Error currently handled just as EOF XXX */ for (cn = 0;; cn++) { if (pcbmap(dep, cn, &bn, 0, &blsize)) return (win95); if (bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp)) { brelse(bp); return (win95); } for (dentp = (struct direntry *)bp->b_data; (char *)dentp < bp->b_data + blsize; dentp++) { if (dentp->deName[0] == SLOT_EMPTY) { /* * Last used entry and not found */ brelse(bp); return (win95); } if (dentp->deName[0] == SLOT_DELETED) { /* * Ignore deleted files * Note: might be an indication of Win'95 anyway XXX */ continue; } if (dentp->deAttributes == ATTR_WIN95) { brelse(bp); return 1; } win95 = 0; } brelse(bp); } } Index: user/imp/tbemd/sys/fs/msdosfs/msdosfs_vnops.c =================================================================== --- user/imp/tbemd/sys/fs/msdosfs/msdosfs_vnops.c (revision 203849) +++ user/imp/tbemd/sys/fs/msdosfs/msdosfs_vnops.c (revision 203850) @@ -1,1995 +1,1995 @@ /* $FreeBSD$ */ /* $NetBSD: msdosfs_vnops.c,v 1.68 1998/02/10 14:10:04 mrg Exp $ */ /*- * Copyright (C) 1994, 1995, 1997 Wolfgang Solfrank. * Copyright (C) 1994, 1995, 1997 TooLs GmbH. * All rights reserved. * Original code by Paul Popelka (paulp@uts.amdahl.com) (see below). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by TooLs GmbH. * 4. The name of TooLs GmbH may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*- * Written by Paul Popelka (paulp@uts.amdahl.com) * * You can do anything you want with this software, just don't say you wrote * it, and don't remove this notice. * * This software is provided "as is". * * The author supplies this software to be publicly redistributed on the * understanding that the author is not responsible for the correct * functioning of this software in any circumstances and is not liable for * any damages caused by this software. * * October 1992 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DOS_FILESIZE_MAX 0xffffffff /* * Prototypes for MSDOSFS vnode operations */ static vop_create_t msdosfs_create; static vop_mknod_t msdosfs_mknod; static vop_open_t msdosfs_open; static vop_close_t msdosfs_close; static vop_access_t msdosfs_access; static vop_getattr_t msdosfs_getattr; static vop_setattr_t msdosfs_setattr; static vop_read_t msdosfs_read; static vop_write_t msdosfs_write; static vop_fsync_t msdosfs_fsync; static vop_remove_t msdosfs_remove; static vop_link_t msdosfs_link; static vop_rename_t msdosfs_rename; static vop_mkdir_t msdosfs_mkdir; static vop_rmdir_t msdosfs_rmdir; static vop_symlink_t msdosfs_symlink; static vop_readdir_t msdosfs_readdir; static vop_bmap_t msdosfs_bmap; static vop_strategy_t msdosfs_strategy; static vop_print_t msdosfs_print; static vop_pathconf_t msdosfs_pathconf; static vop_vptofh_t msdosfs_vptofh; /* * Some general notes: * * In the ufs filesystem the inodes, superblocks, and indirect blocks are * read/written using the vnode for the filesystem. Blocks that represent * the contents of a file are read/written using the vnode for the file * (including directories when they are read/written as files). This * presents problems for the dos filesystem because data that should be in * an inode (if dos had them) resides in the directory itself. Since we * must update directory entries without the benefit of having the vnode * for the directory we must use the vnode for the filesystem. This means * that when a directory is actually read/written (via read, write, or * readdir, or seek) we must use the vnode for the filesystem instead of * the vnode for the directory as would happen in ufs. This is to insure we * retreive the correct block from the buffer cache since the hash value is * based upon the vnode address and the desired block number. */ /* * Create a regular file. On entry the directory to contain the file being * created is locked. We must release before we return. We must also free * the pathname buffer pointed at by cnp->cn_pnbuf, always on error, or * only if the SAVESTART bit in cn_flags is clear on success. */ static int msdosfs_create(ap) struct vop_create_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; } */ *ap; { struct componentname *cnp = ap->a_cnp; struct denode ndirent; struct denode *dep; struct denode *pdep = VTODE(ap->a_dvp); struct timespec ts; int error; #ifdef MSDOSFS_DEBUG printf("msdosfs_create(cnp %p, vap %p\n", cnp, ap->a_vap); #endif /* * If this is the root directory and there is no space left we * can't do anything. This is because the root directory can not * change size. */ if (pdep->de_StartCluster == MSDOSFSROOT && pdep->de_fndoffset >= pdep->de_FileSize) { error = ENOSPC; goto bad; } /* * Create a directory entry for the file, then call createde() to * have it installed. NOTE: DOS files are always executable. We * use the absence of the owner write bit to make the file * readonly. */ #ifdef DIAGNOSTIC if ((cnp->cn_flags & HASBUF) == 0) panic("msdosfs_create: no name"); #endif bzero(&ndirent, sizeof(ndirent)); error = uniqdosname(pdep, cnp, ndirent.de_Name); if (error) goto bad; ndirent.de_Attributes = (ap->a_vap->va_mode & VWRITE) ? ATTR_ARCHIVE : ATTR_ARCHIVE | ATTR_READONLY; ndirent.de_LowerCase = 0; ndirent.de_StartCluster = 0; ndirent.de_FileSize = 0; ndirent.de_pmp = pdep->de_pmp; ndirent.de_flag = DE_ACCESS | DE_CREATE | DE_UPDATE; getnanotime(&ts); DETIMES(&ndirent, &ts, &ts, &ts); error = createde(&ndirent, pdep, &dep, cnp); if (error) goto bad; *ap->a_vpp = DETOV(dep); return (0); bad: return (error); } static int msdosfs_mknod(ap) struct vop_mknod_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; } */ *ap; { return (EINVAL); } static int msdosfs_open(ap) struct vop_open_args /* { struct vnode *a_vp; int a_mode; struct ucred *a_cred; struct thread *a_td; struct file *a_fp; } */ *ap; { struct denode *dep = VTODE(ap->a_vp); vnode_create_vobject(ap->a_vp, dep->de_FileSize, ap->a_td); return 0; } static int msdosfs_close(ap) struct vop_close_args /* { struct vnode *a_vp; int a_fflag; struct ucred *a_cred; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(vp); struct timespec ts; VI_LOCK(vp); if (vp->v_usecount > 1) { getnanotime(&ts); DETIMES(dep, &ts, &ts, &ts); } VI_UNLOCK(vp); return 0; } static int msdosfs_access(ap) struct vop_access_args /* { struct vnode *a_vp; accmode_t a_accmode; struct ucred *a_cred; struct thread *a_td; } */ *ap; { struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(ap->a_vp); struct msdosfsmount *pmp = dep->de_pmp; mode_t file_mode; accmode_t accmode = ap->a_accmode; file_mode = (S_IXUSR|S_IXGRP|S_IXOTH) | (S_IRUSR|S_IRGRP|S_IROTH) | ((dep->de_Attributes & ATTR_READONLY) ? 0 : (S_IWUSR|S_IWGRP|S_IWOTH)); file_mode &= (vp->v_type == VDIR ? pmp->pm_dirmask : pmp->pm_mask); /* * Disallow writing to directories and regular files if the * filesystem is read-only. */ if (accmode & VWRITE) { switch (vp->v_type) { case VDIR: case VREG: if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); break; default: break; } } return (vaccess(vp->v_type, file_mode, pmp->pm_uid, pmp->pm_gid, ap->a_accmode, ap->a_cred, NULL)); } static int msdosfs_getattr(ap) struct vop_getattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { struct denode *dep = VTODE(ap->a_vp); struct msdosfsmount *pmp = dep->de_pmp; struct vattr *vap = ap->a_vap; mode_t mode; struct timespec ts; u_long dirsperblk = pmp->pm_BytesPerSec / sizeof(struct direntry); uint64_t fileid; getnanotime(&ts); DETIMES(dep, &ts, &ts, &ts); vap->va_fsid = dev2udev(pmp->pm_dev); /* * The following computation of the fileid must be the same as that * used in msdosfs_readdir() to compute d_fileno. If not, pwd * doesn't work. */ if (dep->de_Attributes & ATTR_DIRECTORY) { fileid = (uint64_t)cntobn(pmp, dep->de_StartCluster) * dirsperblk; if (dep->de_StartCluster == MSDOSFSROOT) fileid = 1; } else { fileid = (uint64_t)cntobn(pmp, dep->de_dirclust) * dirsperblk; if (dep->de_dirclust == MSDOSFSROOT) fileid = (uint64_t)roottobn(pmp, 0) * dirsperblk; fileid += (uoff_t)dep->de_diroffset / sizeof(struct direntry); } if (pmp->pm_flags & MSDOSFS_LARGEFS) vap->va_fileid = msdosfs_fileno_map(pmp->pm_mountp, fileid); else vap->va_fileid = (long)fileid; if ((dep->de_Attributes & ATTR_READONLY) == 0) mode = S_IRWXU|S_IRWXG|S_IRWXO; else mode = S_IRUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; vap->va_mode = mode & (ap->a_vp->v_type == VDIR ? pmp->pm_dirmask : pmp->pm_mask); vap->va_uid = pmp->pm_uid; vap->va_gid = pmp->pm_gid; vap->va_nlink = 1; vap->va_rdev = NODEV; vap->va_size = dep->de_FileSize; fattime2timespec(dep->de_MDate, dep->de_MTime, 0, 0, &vap->va_mtime); vap->va_ctime = vap->va_mtime; if (pmp->pm_flags & MSDOSFSMNT_LONGNAME) { fattime2timespec(dep->de_ADate, 0, 0, 0, &vap->va_atime); fattime2timespec(dep->de_CDate, dep->de_CTime, dep->de_CHun, 0, &vap->va_birthtime); } else { vap->va_atime = vap->va_mtime; vap->va_birthtime.tv_sec = -1; vap->va_birthtime.tv_nsec = 0; } vap->va_flags = 0; if ((dep->de_Attributes & ATTR_ARCHIVE) == 0) vap->va_flags |= SF_ARCHIVED; vap->va_gen = 0; vap->va_blocksize = pmp->pm_bpcluster; vap->va_bytes = (dep->de_FileSize + pmp->pm_crbomask) & ~pmp->pm_crbomask; vap->va_type = ap->a_vp->v_type; vap->va_filerev = dep->de_modrev; return (0); } static int msdosfs_setattr(ap) struct vop_setattr_args /* { struct vnode *a_vp; struct vattr *a_vap; struct ucred *a_cred; } */ *ap; { struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(ap->a_vp); struct msdosfsmount *pmp = dep->de_pmp; struct vattr *vap = ap->a_vap; struct ucred *cred = ap->a_cred; struct thread *td = curthread; int error = 0; #ifdef MSDOSFS_DEBUG printf("msdosfs_setattr(): vp %p, vap %p, cred %p\n", ap->a_vp, vap, cred); #endif /* * Check for unsettable attributes. */ if ((vap->va_type != VNON) || (vap->va_nlink != VNOVAL) || (vap->va_fsid != VNOVAL) || (vap->va_fileid != VNOVAL) || (vap->va_blocksize != VNOVAL) || (vap->va_rdev != VNOVAL) || (vap->va_bytes != VNOVAL) || (vap->va_gen != VNOVAL)) { #ifdef MSDOSFS_DEBUG printf("msdosfs_setattr(): returning EINVAL\n"); printf(" va_type %d, va_nlink %x, va_fsid %lx, va_fileid %lx\n", vap->va_type, vap->va_nlink, vap->va_fsid, vap->va_fileid); printf(" va_blocksize %lx, va_rdev %x, va_bytes %qx, va_gen %lx\n", vap->va_blocksize, vap->va_rdev, vap->va_bytes, vap->va_gen); printf(" va_uid %x, va_gid %x\n", vap->va_uid, vap->va_gid); #endif return (EINVAL); } if (vap->va_flags != VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if (cred->cr_uid != pmp->pm_uid) { error = priv_check_cred(cred, PRIV_VFS_ADMIN, 0); if (error) return (error); } /* * We are very inconsistent about handling unsupported * attributes. We ignored the access time and the * read and execute bits. We were strict for the other * attributes. * * Here we are strict, stricter than ufs in not allowing * users to attempt to set SF_SETTABLE bits or anyone to * set unsupported bits. However, we ignore attempts to * set ATTR_ARCHIVE for directories `cp -pr' from a more * sensible filesystem attempts it a lot. */ if (vap->va_flags & SF_SETTABLE) { error = priv_check_cred(cred, PRIV_VFS_SYSFLAGS, 0); if (error) return (error); } if (vap->va_flags & ~SF_ARCHIVED) return EOPNOTSUPP; if (vap->va_flags & SF_ARCHIVED) dep->de_Attributes &= ~ATTR_ARCHIVE; else if (!(dep->de_Attributes & ATTR_DIRECTORY)) dep->de_Attributes |= ATTR_ARCHIVE; dep->de_flag |= DE_MODIFIED; } if (vap->va_uid != (uid_t)VNOVAL || vap->va_gid != (gid_t)VNOVAL) { uid_t uid; gid_t gid; if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); uid = vap->va_uid; if (uid == (uid_t)VNOVAL) uid = pmp->pm_uid; gid = vap->va_gid; if (gid == (gid_t)VNOVAL) gid = pmp->pm_gid; if (cred->cr_uid != pmp->pm_uid || uid != pmp->pm_uid || (gid != pmp->pm_gid && !groupmember(gid, cred))) { error = priv_check_cred(cred, PRIV_VFS_CHOWN, 0); if (error) return (error); } if (uid != pmp->pm_uid || gid != pmp->pm_gid) return EINVAL; } if (vap->va_size != VNOVAL) { switch (vp->v_type) { case VDIR: return (EISDIR); case VREG: /* * Truncation is only supported for regular files, * Disallow it if the filesystem is read-only. */ if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); break; default: /* * According to POSIX, the result is unspecified * for file types other than regular files, * directories and shared memory objects. We * don't support any file types except regular * files and directories in this file system, so * this (default) case is unreachable and can do * anything. Keep falling through to detrunc() * for now. */ break; } error = detrunc(dep, vap->va_size, 0, cred, td); if (error) return error; } if (vap->va_atime.tv_sec != VNOVAL || vap->va_mtime.tv_sec != VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if (vap->va_vaflags & VA_UTIMES_NULL) { error = VOP_ACCESS(vp, VADMIN, cred, td); if (error) error = VOP_ACCESS(vp, VWRITE, cred, td); } else error = VOP_ACCESS(vp, VADMIN, cred, td); if (vp->v_type != VDIR) { if ((pmp->pm_flags & MSDOSFSMNT_NOWIN95) == 0 && vap->va_atime.tv_sec != VNOVAL) { dep->de_flag &= ~DE_ACCESS; timespec2fattime(&vap->va_atime, 0, &dep->de_ADate, NULL, NULL); } if (vap->va_mtime.tv_sec != VNOVAL) { dep->de_flag &= ~DE_UPDATE; timespec2fattime(&vap->va_mtime, 0, &dep->de_MDate, &dep->de_MTime, NULL); } dep->de_Attributes |= ATTR_ARCHIVE; dep->de_flag |= DE_MODIFIED; } } /* * DOS files only have the ability to have their writability * attribute set, so we use the owner write bit to set the readonly * attribute. */ if (vap->va_mode != (mode_t)VNOVAL) { if (vp->v_mount->mnt_flag & MNT_RDONLY) return (EROFS); if (cred->cr_uid != pmp->pm_uid) { error = priv_check_cred(cred, PRIV_VFS_ADMIN, 0); if (error) return (error); } if (vp->v_type != VDIR) { /* We ignore the read and execute bits. */ if (vap->va_mode & VWRITE) dep->de_Attributes &= ~ATTR_READONLY; else dep->de_Attributes |= ATTR_READONLY; dep->de_Attributes |= ATTR_ARCHIVE; dep->de_flag |= DE_MODIFIED; } } return (deupdat(dep, 0)); } static int msdosfs_read(ap) struct vop_read_args /* { struct vnode *a_vp; struct uio *a_uio; int a_ioflag; struct ucred *a_cred; } */ *ap; { int error = 0; int blsize; int isadir; int orig_resid; u_int n; u_long diff; u_long on; daddr_t lbn; daddr_t rablock; int rasize; int seqcount; struct buf *bp; struct vnode *vp = ap->a_vp; struct denode *dep = VTODE(vp); struct msdosfsmount *pmp = dep->de_pmp; struct uio *uio = ap->a_uio; /* * If they didn't ask for any data, then we are done. */ orig_resid = uio->uio_resid; if (orig_resid == 0) return (0); /* * The caller is supposed to ensure that * uio->uio_offset >= 0 and uio->uio_resid >= 0. * We don't need to check for large offsets as in ffs because * dep->de_FileSize <= DOS_FILESIZE_MAX < OFF_MAX, so large * offsets cannot cause overflow even in theory. */ seqcount = ap->a_ioflag >> IO_SEQSHIFT; isadir = dep->de_Attributes & ATTR_DIRECTORY; do { if (uio->uio_offset >= dep->de_FileSize) break; lbn = de_cluster(pmp, uio->uio_offset); rablock = lbn + 1; blsize = pmp->pm_bpcluster; on = uio->uio_offset & pmp->pm_crbomask; /* * If we are operating on a directory file then be sure to * do i/o with the vnode for the filesystem instead of the * vnode for the directory. */ if (isadir) { /* convert cluster # to block # */ error = pcbmap(dep, lbn, &lbn, 0, &blsize); if (error == E2BIG) { error = EINVAL; break; } else if (error) break; error = bread(pmp->pm_devvp, lbn, blsize, NOCRED, &bp); } else if (de_cn2off(pmp, rablock) >= dep->de_FileSize) { error = bread(vp, lbn, blsize, NOCRED, &bp); } else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { error = cluster_read(vp, dep->de_FileSize, lbn, blsize, NOCRED, on + uio->uio_resid, seqcount, &bp); } else if (seqcount > 1) { rasize = blsize; error = breadn(vp, lbn, blsize, &rablock, &rasize, 1, NOCRED, &bp); } else { error = bread(vp, lbn, blsize, NOCRED, &bp); } if (error) { brelse(bp); break; } diff = pmp->pm_bpcluster - on; n = diff > uio->uio_resid ? uio->uio_resid : diff; diff = dep->de_FileSize - uio->uio_offset; if (diff < n) n = diff; diff = blsize - bp->b_resid; if (diff < n) n = diff; error = uiomove(bp->b_data + on, (int) n, uio); brelse(bp); } while (error == 0 && uio->uio_resid > 0 && n != 0); if (!isadir && (error == 0 || uio->uio_resid != orig_resid) && (vp->v_mount->mnt_flag & MNT_NOATIME) == 0) dep->de_flag |= DE_ACCESS; return (error); } /* * Write data to a file or directory. */ static int msdosfs_write(ap) struct vop_write_args /* { struct vnode *a_vp; struct uio *a_uio; int a_ioflag; struct ucred *a_cred; } */ *ap; { int n; int croffset; int resid; u_long osize; int error = 0; u_long count; int seqcount; daddr_t bn, lastcn; struct buf *bp; int ioflag = ap->a_ioflag; struct uio *uio = ap->a_uio; struct thread *td = uio->uio_td; struct vnode *vp = ap->a_vp; struct vnode *thisvp; struct denode *dep = VTODE(vp); struct msdosfsmount *pmp = dep->de_pmp; struct ucred *cred = ap->a_cred; #ifdef MSDOSFS_DEBUG printf("msdosfs_write(vp %p, uio %p, ioflag %x, cred %p\n", vp, uio, ioflag, cred); printf("msdosfs_write(): diroff %lu, dirclust %lu, startcluster %lu\n", dep->de_diroffset, dep->de_dirclust, dep->de_StartCluster); #endif switch (vp->v_type) { case VREG: if (ioflag & IO_APPEND) uio->uio_offset = dep->de_FileSize; thisvp = vp; break; case VDIR: return EISDIR; default: panic("msdosfs_write(): bad file type"); } /* * This is needed (unlike in ffs_write()) because we extend the * file outside of the loop but we don't want to extend the file * for writes of 0 bytes. */ if (uio->uio_resid == 0) return (0); /* * The caller is supposed to ensure that * uio->uio_offset >= 0 and uio->uio_resid >= 0. */ if ((uoff_t)uio->uio_offset + uio->uio_resid > DOS_FILESIZE_MAX) return (EFBIG); /* * If they've exceeded their filesize limit, tell them about it. */ if (td != NULL) { PROC_LOCK(td->td_proc); if ((uoff_t)uio->uio_offset + uio->uio_resid > lim_cur(td->td_proc, RLIMIT_FSIZE)) { psignal(td->td_proc, SIGXFSZ); PROC_UNLOCK(td->td_proc); return (EFBIG); } PROC_UNLOCK(td->td_proc); } /* * If the offset we are starting the write at is beyond the end of * the file, then they've done a seek. Unix filesystems allow * files with holes in them, DOS doesn't so we must fill the hole * with zeroed blocks. */ if (uio->uio_offset > dep->de_FileSize) { error = deextend(dep, uio->uio_offset, cred); if (error) return (error); } /* * Remember some values in case the write fails. */ resid = uio->uio_resid; osize = dep->de_FileSize; /* * If we write beyond the end of the file, extend it to its ultimate * size ahead of the time to hopefully get a contiguous area. */ if (uio->uio_offset + resid > osize) { count = de_clcount(pmp, uio->uio_offset + resid) - de_clcount(pmp, osize); error = extendfile(dep, count, NULL, NULL, 0); if (error && (error != ENOSPC || (ioflag & IO_UNIT))) goto errexit; lastcn = dep->de_fc[FC_LASTFC].fc_frcn; } else lastcn = de_clcount(pmp, osize) - 1; seqcount = ioflag >> IO_SEQSHIFT; do { if (de_cluster(pmp, uio->uio_offset) > lastcn) { error = ENOSPC; break; } croffset = uio->uio_offset & pmp->pm_crbomask; n = min(uio->uio_resid, pmp->pm_bpcluster - croffset); if (uio->uio_offset + n > dep->de_FileSize) { dep->de_FileSize = uio->uio_offset + n; /* The object size needs to be set before buffer is allocated */ vnode_pager_setsize(vp, dep->de_FileSize); } bn = de_cluster(pmp, uio->uio_offset); if ((uio->uio_offset & pmp->pm_crbomask) == 0 && (de_cluster(pmp, uio->uio_offset + uio->uio_resid) > de_cluster(pmp, uio->uio_offset) || uio->uio_offset + uio->uio_resid >= dep->de_FileSize)) { /* * If either the whole cluster gets written, * or we write the cluster from its start beyond EOF, * then no need to read data from disk. */ bp = getblk(thisvp, bn, pmp->pm_bpcluster, 0, 0, 0); vfs_bio_clrbuf(bp); /* * Do the bmap now, since pcbmap needs buffers * for the fat table. (see msdosfs_strategy) */ if (bp->b_blkno == bp->b_lblkno) { error = pcbmap(dep, bp->b_lblkno, &bn, 0, 0); if (error) bp->b_blkno = -1; else bp->b_blkno = bn; } if (bp->b_blkno == -1) { brelse(bp); if (!error) error = EIO; /* XXX */ break; } } else { /* * The block we need to write into exists, so read it in. */ error = bread(thisvp, bn, pmp->pm_bpcluster, cred, &bp); if (error) { brelse(bp); break; } } /* * Should these vnode_pager_* functions be done on dir * files? */ /* * Copy the data from user space into the buf header. */ error = uiomove(bp->b_data + croffset, n, uio); if (error) { brelse(bp); break; } /* Prepare for clustered writes in some else clauses. */ if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) bp->b_flags |= B_CLUSTEROK; /* * If IO_SYNC, then each buffer is written synchronously. * Otherwise, if we have a severe page deficiency then * write the buffer asynchronously. Otherwise, if on a * cluster boundary then write the buffer asynchronously, * combining it with contiguous clusters if permitted and * possible, since we don't expect more writes into this * buffer soon. Otherwise, do a delayed write because we * expect more writes into this buffer soon. */ if (ioflag & IO_SYNC) (void)bwrite(bp); else if (vm_page_count_severe() || buf_dirty_count_severe()) bawrite(bp); else if (n + croffset == pmp->pm_bpcluster) { if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) cluster_write(vp, bp, dep->de_FileSize, seqcount); else bawrite(bp); } else bdwrite(bp); dep->de_flag |= DE_UPDATE; } while (error == 0 && uio->uio_resid > 0); /* * If the write failed and they want us to, truncate the file back * to the size it was before the write was attempted. */ errexit: if (error) { if (ioflag & IO_UNIT) { detrunc(dep, osize, ioflag & IO_SYNC, NOCRED, NULL); uio->uio_offset -= resid - uio->uio_resid; uio->uio_resid = resid; } else { detrunc(dep, dep->de_FileSize, ioflag & IO_SYNC, NOCRED, NULL); if (uio->uio_resid != resid) error = 0; } } else if (ioflag & IO_SYNC) error = deupdat(dep, 1); return (error); } /* * Flush the blocks of a file to disk. * * This function is worthless for vnodes that represent directories. Maybe we * could just do a sync if they try an fsync on a directory file. */ static int msdosfs_fsync(ap) struct vop_fsync_args /* { struct vnode *a_vp; struct ucred *a_cred; int a_waitfor; struct thread *a_td; } */ *ap; { vop_stdfsync(ap); return (deupdat(VTODE(ap->a_vp), ap->a_waitfor == MNT_WAIT)); } static int msdosfs_remove(ap) struct vop_remove_args /* { struct vnode *a_dvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { struct denode *dep = VTODE(ap->a_vp); struct denode *ddep = VTODE(ap->a_dvp); int error; if (ap->a_vp->v_type == VDIR) error = EPERM; else error = removede(ddep, dep); #ifdef MSDOSFS_DEBUG printf("msdosfs_remove(), dep %p, v_usecount %d\n", dep, ap->a_vp->v_usecount); #endif return (error); } /* * DOS filesystems don't know what links are. */ static int msdosfs_link(ap) struct vop_link_args /* { struct vnode *a_tdvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { return (EOPNOTSUPP); } /* * Renames on files require moving the denode to a new hash queue since the * denode's location is used to compute which hash queue to put the file * in. Unless it is a rename in place. For example "mv a b". * * What follows is the basic algorithm: * * if (file move) { * if (dest file exists) { * remove dest file * } * if (dest and src in same directory) { * rewrite name in existing directory slot * } else { * write new entry in dest directory * update offset and dirclust in denode * move denode to new hash chain * clear old directory entry * } * } else { * directory move * if (dest directory exists) { * if (dest is not empty) { * return ENOTEMPTY * } * remove dest directory * } * if (dest and src in same directory) { * rewrite name in existing entry * } else { * be sure dest is not a child of src directory * write entry in dest directory * update "." and ".." in moved directory * clear old directory entry for moved directory * } * } * * On entry: * source's parent directory is unlocked * source file or directory is unlocked * destination's parent directory is locked * destination file or directory is locked if it exists * * On exit: * all denodes should be released */ static int msdosfs_rename(ap) struct vop_rename_args /* { struct vnode *a_fdvp; struct vnode *a_fvp; struct componentname *a_fcnp; struct vnode *a_tdvp; struct vnode *a_tvp; struct componentname *a_tcnp; } */ *ap; { struct vnode *tdvp = ap->a_tdvp; struct vnode *fvp = ap->a_fvp; struct vnode *fdvp = ap->a_fdvp; struct vnode *tvp = ap->a_tvp; struct componentname *tcnp = ap->a_tcnp; struct componentname *fcnp = ap->a_fcnp; struct denode *ip, *xp, *dp, *zp; u_char toname[12], oldname[11]; u_long from_diroffset, to_diroffset; u_char to_count; int doingdirectory = 0, newparent = 0; int error; u_long cn; daddr_t bn; struct denode *fddep; /* from file's parent directory */ struct msdosfsmount *pmp; struct direntry *dotdotp; struct buf *bp; fddep = VTODE(ap->a_fdvp); pmp = fddep->de_pmp; pmp = VFSTOMSDOSFS(fdvp->v_mount); #ifdef DIAGNOSTIC if ((tcnp->cn_flags & HASBUF) == 0 || (fcnp->cn_flags & HASBUF) == 0) panic("msdosfs_rename: no name"); #endif /* * Check for cross-device rename. */ if (fvp->v_mount != tdvp->v_mount || (tvp && fvp->v_mount != tvp->v_mount)) { error = EXDEV; abortit: if (tdvp == tvp) vrele(tdvp); else vput(tdvp); if (tvp) vput(tvp); vrele(fdvp); vrele(fvp); return (error); } /* * If source and dest are the same, do nothing. */ if (tvp == fvp) { error = 0; goto abortit; } error = vn_lock(fvp, LK_EXCLUSIVE); if (error) goto abortit; dp = VTODE(fdvp); ip = VTODE(fvp); /* * Be sure we are not renaming ".", "..", or an alias of ".". This * leads to a crippled directory tree. It's pretty tough to do a * "ls" or "pwd" with the "." directory entry missing, and "cd .." * doesn't work if the ".." entry is missing. */ if (ip->de_Attributes & ATTR_DIRECTORY) { /* * Avoid ".", "..", and aliases of "." for obvious reasons. */ if ((fcnp->cn_namelen == 1 && fcnp->cn_nameptr[0] == '.') || dp == ip || (fcnp->cn_flags & ISDOTDOT) || (tcnp->cn_flags & ISDOTDOT) || (ip->de_flag & DE_RENAME)) { VOP_UNLOCK(fvp, 0); error = EINVAL; goto abortit; } ip->de_flag |= DE_RENAME; doingdirectory++; } /* * When the target exists, both the directory * and target vnodes are returned locked. */ dp = VTODE(tdvp); xp = tvp ? VTODE(tvp) : NULL; /* * Remember direntry place to use for destination */ to_diroffset = dp->de_fndoffset; to_count = dp->de_fndcnt; /* * If ".." must be changed (ie the directory gets a new * parent) then the source directory must not be in the * directory heirarchy above the target, as this would * orphan everything below the source directory. Also * the user must have write permission in the source so * as to be able to change "..". We must repeat the call * to namei, as the parent directory is unlocked by the * call to doscheckpath(). */ error = VOP_ACCESS(fvp, VWRITE, tcnp->cn_cred, tcnp->cn_thread); VOP_UNLOCK(fvp, 0); if (VTODE(fdvp)->de_StartCluster != VTODE(tdvp)->de_StartCluster) newparent = 1; if (doingdirectory && newparent) { if (error) /* write access check above */ goto bad; if (xp != NULL) vput(tvp); /* * doscheckpath() vput()'s dp, * so we have to do a relookup afterwards */ error = doscheckpath(ip, dp); if (error) goto out; if ((tcnp->cn_flags & SAVESTART) == 0) panic("msdosfs_rename: lost to startdir"); error = relookup(tdvp, &tvp, tcnp); if (error) goto out; dp = VTODE(tdvp); xp = tvp ? VTODE(tvp) : NULL; } if (xp != NULL) { /* * Target must be empty if a directory and have no links * to it. Also, ensure source and target are compatible * (both directories, or both not directories). */ if (xp->de_Attributes & ATTR_DIRECTORY) { if (!dosdirempty(xp)) { error = ENOTEMPTY; goto bad; } if (!doingdirectory) { error = ENOTDIR; goto bad; } cache_purge(tdvp); } else if (doingdirectory) { error = EISDIR; goto bad; } error = removede(dp, xp); if (error) goto bad; vput(tvp); xp = NULL; } /* * Convert the filename in tcnp into a dos filename. We copy this * into the denode and directory entry for the destination * file/directory. */ error = uniqdosname(VTODE(tdvp), tcnp, toname); if (error) goto abortit; /* * Since from wasn't locked at various places above, * have to do a relookup here. */ fcnp->cn_flags &= ~MODMASK; fcnp->cn_flags |= LOCKPARENT | LOCKLEAF; if ((fcnp->cn_flags & SAVESTART) == 0) panic("msdosfs_rename: lost from startdir"); if (!newparent) VOP_UNLOCK(tdvp, 0); if (relookup(fdvp, &fvp, fcnp) == 0) vrele(fdvp); if (fvp == NULL) { /* * From name has disappeared. */ if (doingdirectory) panic("rename: lost dir entry"); if (newparent) VOP_UNLOCK(tdvp, 0); vrele(tdvp); vrele(ap->a_fvp); return 0; } xp = VTODE(fvp); zp = VTODE(fdvp); from_diroffset = zp->de_fndoffset; /* * Ensure that the directory entry still exists and has not * changed till now. If the source is a file the entry may * have been unlinked or renamed. In either case there is * no further work to be done. If the source is a directory * then it cannot have been rmdir'ed or renamed; this is * prohibited by the DE_RENAME flag. */ if (xp != ip) { if (doingdirectory) panic("rename: lost dir entry"); VOP_UNLOCK(fvp, 0); if (newparent) VOP_UNLOCK(fdvp, 0); vrele(ap->a_fvp); xp = NULL; } else { vrele(fvp); xp = NULL; /* * First write a new entry in the destination * directory and mark the entry in the source directory * as deleted. Then move the denode to the correct hash * chain for its new location in the filesystem. And, if * we moved a directory, then update its .. entry to point * to the new parent directory. */ bcopy(ip->de_Name, oldname, 11); bcopy(toname, ip->de_Name, 11); /* update denode */ dp->de_fndoffset = to_diroffset; dp->de_fndcnt = to_count; error = createde(ip, dp, (struct denode **)0, tcnp); if (error) { bcopy(oldname, ip->de_Name, 11); if (newparent) VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(fvp, 0); goto bad; } ip->de_refcnt++; zp->de_fndoffset = from_diroffset; error = removede(zp, ip); if (error) { /* XXX should downgrade to ro here, fs is corrupt */ if (newparent) VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(fvp, 0); goto bad; } if (!doingdirectory) { error = pcbmap(dp, de_cluster(pmp, to_diroffset), 0, &ip->de_dirclust, 0); if (error) { /* XXX should downgrade to ro here, fs is corrupt */ if (newparent) VOP_UNLOCK(fdvp, 0); VOP_UNLOCK(fvp, 0); goto bad; } if (ip->de_dirclust == MSDOSFSROOT) ip->de_diroffset = to_diroffset; else ip->de_diroffset = to_diroffset & pmp->pm_crbomask; } reinsert(ip); if (newparent) VOP_UNLOCK(fdvp, 0); } /* * If we moved a directory to a new parent directory, then we must * fixup the ".." entry in the moved directory. */ if (doingdirectory && newparent) { cn = ip->de_StartCluster; if (cn == MSDOSFSROOT) { /* this should never happen */ panic("msdosfs_rename(): updating .. in root directory?"); } else bn = cntobn(pmp, cn); error = bread(pmp->pm_devvp, bn, pmp->pm_bpcluster, NOCRED, &bp); if (error) { /* XXX should downgrade to ro here, fs is corrupt */ brelse(bp); VOP_UNLOCK(fvp, 0); goto bad; } dotdotp = (struct direntry *)bp->b_data + 1; putushort(dotdotp->deStartCluster, dp->de_StartCluster); if (FAT32(pmp)) putushort(dotdotp->deHighClust, dp->de_StartCluster >> 16); if (fvp->v_mount->mnt_flag & MNT_ASYNC) bdwrite(bp); else if ((error = bwrite(bp)) != 0) { /* XXX should downgrade to ro here, fs is corrupt */ VOP_UNLOCK(fvp, 0); goto bad; } } VOP_UNLOCK(fvp, 0); bad: if (xp) vput(tvp); vput(tdvp); out: ip->de_flag &= ~DE_RENAME; vrele(fdvp); vrele(fvp); return (error); } static struct { struct direntry dot; struct direntry dotdot; } dosdirtemplate = { - { ". ", " ", /* the . entry */ + { ". ", /* the . entry */ ATTR_DIRECTORY, /* file attribute */ 0, /* reserved */ 0, { 0, 0 }, { 0, 0 }, /* create time & date */ { 0, 0 }, /* access date */ { 0, 0 }, /* high bits of start cluster */ { 210, 4 }, { 210, 4 }, /* modify time & date */ { 0, 0 }, /* startcluster */ { 0, 0, 0, 0 } /* filesize */ }, - { ".. ", " ", /* the .. entry */ + { ".. ", /* the .. entry */ ATTR_DIRECTORY, /* file attribute */ 0, /* reserved */ 0, { 0, 0 }, { 0, 0 }, /* create time & date */ { 0, 0 }, /* access date */ { 0, 0 }, /* high bits of start cluster */ { 210, 4 }, { 210, 4 }, /* modify time & date */ { 0, 0 }, /* startcluster */ { 0, 0, 0, 0 } /* filesize */ } }; static int msdosfs_mkdir(ap) struct vop_mkdir_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struvt componentname *a_cnp; struct vattr *a_vap; } */ *ap; { struct componentname *cnp = ap->a_cnp; struct denode *dep; struct denode *pdep = VTODE(ap->a_dvp); struct direntry *denp; struct msdosfsmount *pmp = pdep->de_pmp; struct buf *bp; u_long newcluster, pcl; int bn; int error; struct denode ndirent; struct timespec ts; /* * If this is the root directory and there is no space left we * can't do anything. This is because the root directory can not * change size. */ if (pdep->de_StartCluster == MSDOSFSROOT && pdep->de_fndoffset >= pdep->de_FileSize) { error = ENOSPC; goto bad2; } /* * Allocate a cluster to hold the about to be created directory. */ error = clusteralloc(pmp, 0, 1, CLUST_EOFE, &newcluster, NULL); if (error) goto bad2; bzero(&ndirent, sizeof(ndirent)); ndirent.de_pmp = pmp; ndirent.de_flag = DE_ACCESS | DE_CREATE | DE_UPDATE; getnanotime(&ts); DETIMES(&ndirent, &ts, &ts, &ts); /* * Now fill the cluster with the "." and ".." entries. And write * the cluster to disk. This way it is there for the parent * directory to be pointing at if there were a crash. */ bn = cntobn(pmp, newcluster); /* always succeeds */ bp = getblk(pmp->pm_devvp, bn, pmp->pm_bpcluster, 0, 0, 0); bzero(bp->b_data, pmp->pm_bpcluster); bcopy(&dosdirtemplate, bp->b_data, sizeof dosdirtemplate); denp = (struct direntry *)bp->b_data; putushort(denp[0].deStartCluster, newcluster); putushort(denp[0].deCDate, ndirent.de_CDate); putushort(denp[0].deCTime, ndirent.de_CTime); denp[0].deCHundredth = ndirent.de_CHun; putushort(denp[0].deADate, ndirent.de_ADate); putushort(denp[0].deMDate, ndirent.de_MDate); putushort(denp[0].deMTime, ndirent.de_MTime); pcl = pdep->de_StartCluster; if (FAT32(pmp) && pcl == pmp->pm_rootdirblk) pcl = 0; putushort(denp[1].deStartCluster, pcl); putushort(denp[1].deCDate, ndirent.de_CDate); putushort(denp[1].deCTime, ndirent.de_CTime); denp[1].deCHundredth = ndirent.de_CHun; putushort(denp[1].deADate, ndirent.de_ADate); putushort(denp[1].deMDate, ndirent.de_MDate); putushort(denp[1].deMTime, ndirent.de_MTime); if (FAT32(pmp)) { putushort(denp[0].deHighClust, newcluster >> 16); putushort(denp[1].deHighClust, pdep->de_StartCluster >> 16); } if (ap->a_dvp->v_mount->mnt_flag & MNT_ASYNC) bdwrite(bp); else if ((error = bwrite(bp)) != 0) goto bad; /* * Now build up a directory entry pointing to the newly allocated * cluster. This will be written to an empty slot in the parent * directory. */ #ifdef DIAGNOSTIC if ((cnp->cn_flags & HASBUF) == 0) panic("msdosfs_mkdir: no name"); #endif error = uniqdosname(pdep, cnp, ndirent.de_Name); if (error) goto bad; ndirent.de_Attributes = ATTR_DIRECTORY; ndirent.de_LowerCase = 0; ndirent.de_StartCluster = newcluster; ndirent.de_FileSize = 0; error = createde(&ndirent, pdep, &dep, cnp); if (error) goto bad; *ap->a_vpp = DETOV(dep); return (0); bad: clusterfree(pmp, newcluster, NULL); bad2: return (error); } static int msdosfs_rmdir(ap) struct vop_rmdir_args /* { struct vnode *a_dvp; struct vnode *a_vp; struct componentname *a_cnp; } */ *ap; { struct vnode *vp = ap->a_vp; struct vnode *dvp = ap->a_dvp; struct componentname *cnp = ap->a_cnp; struct denode *ip, *dp; struct thread *td = cnp->cn_thread; int error; ip = VTODE(vp); dp = VTODE(dvp); /* * Verify the directory is empty (and valid). * (Rmdir ".." won't be valid since * ".." will contain a reference to * the current directory and thus be * non-empty.) */ error = 0; if (!dosdirempty(ip) || ip->de_flag & DE_RENAME) { error = ENOTEMPTY; goto out; } /* * Delete the entry from the directory. For dos filesystems this * gets rid of the directory entry on disk, the in memory copy * still exists but the de_refcnt is <= 0. This prevents it from * being found by deget(). When the vput() on dep is done we give * up access and eventually msdosfs_reclaim() will be called which * will remove it from the denode cache. */ error = removede(dp, ip); if (error) goto out; /* * This is where we decrement the link count in the parent * directory. Since dos filesystems don't do this we just purge * the name cache. */ cache_purge(dvp); VOP_UNLOCK(dvp, 0); /* * Truncate the directory that is being deleted. */ error = detrunc(ip, (u_long)0, IO_SYNC, cnp->cn_cred, td); cache_purge(vp); vn_lock(dvp, LK_EXCLUSIVE | LK_RETRY); out: return (error); } /* * DOS filesystems don't know what symlinks are. */ static int msdosfs_symlink(ap) struct vop_symlink_args /* { struct vnode *a_dvp; struct vnode **a_vpp; struct componentname *a_cnp; struct vattr *a_vap; char *a_target; } */ *ap; { return (EOPNOTSUPP); } static int msdosfs_readdir(ap) struct vop_readdir_args /* { struct vnode *a_vp; struct uio *a_uio; struct ucred *a_cred; int *a_eofflag; int *a_ncookies; u_long **a_cookies; } */ *ap; { struct mbnambuf nb; int error = 0; int diff; long n; int blsize; long on; u_long cn; uint64_t fileno; u_long dirsperblk; long bias = 0; daddr_t bn, lbn; struct buf *bp; struct denode *dep = VTODE(ap->a_vp); struct msdosfsmount *pmp = dep->de_pmp; struct direntry *dentp; struct dirent dirbuf; struct uio *uio = ap->a_uio; u_long *cookies = NULL; int ncookies = 0; off_t offset, off; int chksum = -1; #ifdef MSDOSFS_DEBUG printf("msdosfs_readdir(): vp %p, uio %p, cred %p, eofflagp %p\n", ap->a_vp, uio, ap->a_cred, ap->a_eofflag); #endif /* * msdosfs_readdir() won't operate properly on regular files since * it does i/o only with the the filesystem vnode, and hence can * retrieve the wrong block from the buffer cache for a plain file. * So, fail attempts to readdir() on a plain file. */ if ((dep->de_Attributes & ATTR_DIRECTORY) == 0) return (ENOTDIR); /* * To be safe, initialize dirbuf */ bzero(dirbuf.d_name, sizeof(dirbuf.d_name)); /* * If the user buffer is smaller than the size of one dos directory * entry or the file offset is not a multiple of the size of a * directory entry, then we fail the read. */ off = offset = uio->uio_offset; if (uio->uio_resid < sizeof(struct direntry) || (offset & (sizeof(struct direntry) - 1))) return (EINVAL); if (ap->a_ncookies) { ncookies = uio->uio_resid / 16; cookies = malloc(ncookies * sizeof(u_long), M_TEMP, M_WAITOK); *ap->a_cookies = cookies; *ap->a_ncookies = ncookies; } dirsperblk = pmp->pm_BytesPerSec / sizeof(struct direntry); /* * If they are reading from the root directory then, we simulate * the . and .. entries since these don't exist in the root * directory. We also set the offset bias to make up for having to * simulate these entries. By this I mean that at file offset 64 we * read the first entry in the root directory that lives on disk. */ if (dep->de_StartCluster == MSDOSFSROOT || (FAT32(pmp) && dep->de_StartCluster == pmp->pm_rootdirblk)) { #if 0 printf("msdosfs_readdir(): going after . or .. in root dir, offset %d\n", offset); #endif bias = 2 * sizeof(struct direntry); if (offset < bias) { for (n = (int)offset / sizeof(struct direntry); n < 2; n++) { if (FAT32(pmp)) fileno = (uint64_t)cntobn(pmp, pmp->pm_rootdirblk) * dirsperblk; else fileno = 1; if (pmp->pm_flags & MSDOSFS_LARGEFS) { dirbuf.d_fileno = msdosfs_fileno_map(pmp->pm_mountp, fileno); } else { dirbuf.d_fileno = (uint32_t)fileno; } dirbuf.d_type = DT_DIR; switch (n) { case 0: dirbuf.d_namlen = 1; strcpy(dirbuf.d_name, "."); break; case 1: dirbuf.d_namlen = 2; strcpy(dirbuf.d_name, ".."); break; } dirbuf.d_reclen = GENERIC_DIRSIZ(&dirbuf); if (uio->uio_resid < dirbuf.d_reclen) goto out; error = uiomove(&dirbuf, dirbuf.d_reclen, uio); if (error) goto out; offset += sizeof(struct direntry); off = offset; if (cookies) { *cookies++ = offset; if (--ncookies <= 0) goto out; } } } } mbnambuf_init(&nb); off = offset; while (uio->uio_resid > 0) { lbn = de_cluster(pmp, offset - bias); on = (offset - bias) & pmp->pm_crbomask; n = min(pmp->pm_bpcluster - on, uio->uio_resid); diff = dep->de_FileSize - (offset - bias); if (diff <= 0) break; n = min(n, diff); error = pcbmap(dep, lbn, &bn, &cn, &blsize); if (error) break; error = bread(pmp->pm_devvp, bn, blsize, NOCRED, &bp); if (error) { brelse(bp); return (error); } n = min(n, blsize - bp->b_resid); if (n == 0) { brelse(bp); return (EIO); } /* * Convert from dos directory entries to fs-independent * directory entries. */ for (dentp = (struct direntry *)(bp->b_data + on); (char *)dentp < bp->b_data + on + n; dentp++, offset += sizeof(struct direntry)) { #if 0 printf("rd: dentp %08x prev %08x crnt %08x deName %02x attr %02x\n", dentp, prev, crnt, dentp->deName[0], dentp->deAttributes); #endif /* * If this is an unused entry, we can stop. */ if (dentp->deName[0] == SLOT_EMPTY) { brelse(bp); goto out; } /* * Skip deleted entries. */ if (dentp->deName[0] == SLOT_DELETED) { chksum = -1; mbnambuf_init(&nb); continue; } /* * Handle Win95 long directory entries */ if (dentp->deAttributes == ATTR_WIN95) { if (pmp->pm_flags & MSDOSFSMNT_SHORTNAME) continue; chksum = win2unixfn(&nb, (struct winentry *)dentp, chksum, pmp); continue; } /* * Skip volume labels */ if (dentp->deAttributes & ATTR_VOLUME) { chksum = -1; mbnambuf_init(&nb); continue; } /* * This computation of d_fileno must match * the computation of va_fileid in * msdosfs_getattr. */ if (dentp->deAttributes & ATTR_DIRECTORY) { fileno = getushort(dentp->deStartCluster); if (FAT32(pmp)) fileno |= getushort(dentp->deHighClust) << 16; /* if this is the root directory */ if (fileno == MSDOSFSROOT) if (FAT32(pmp)) fileno = (uint64_t)cntobn(pmp, pmp->pm_rootdirblk) * dirsperblk; else fileno = 1; else fileno = (uint64_t)cntobn(pmp, fileno) * dirsperblk; dirbuf.d_type = DT_DIR; } else { fileno = (uoff_t)offset / sizeof(struct direntry); dirbuf.d_type = DT_REG; } if (pmp->pm_flags & MSDOSFS_LARGEFS) { dirbuf.d_fileno = msdosfs_fileno_map(pmp->pm_mountp, fileno); } else dirbuf.d_fileno = (uint32_t)fileno; - if (chksum != winChksum(dentp)) { + if (chksum != winChksum(dentp->deName)) { dirbuf.d_namlen = dos2unixfn(dentp->deName, (u_char *)dirbuf.d_name, dentp->deLowerCase | ((pmp->pm_flags & MSDOSFSMNT_SHORTNAME) ? (LCASE_BASE | LCASE_EXT) : 0), pmp); mbnambuf_init(&nb); } else mbnambuf_flush(&nb, &dirbuf); chksum = -1; dirbuf.d_reclen = GENERIC_DIRSIZ(&dirbuf); if (uio->uio_resid < dirbuf.d_reclen) { brelse(bp); goto out; } error = uiomove(&dirbuf, dirbuf.d_reclen, uio); if (error) { brelse(bp); goto out; } if (cookies) { *cookies++ = offset + sizeof(struct direntry); if (--ncookies <= 0) { brelse(bp); goto out; } } off = offset + sizeof(struct direntry); } brelse(bp); } out: /* Subtract unused cookies */ if (ap->a_ncookies) *ap->a_ncookies -= ncookies; uio->uio_offset = off; /* * Set the eofflag (NFS uses it) */ if (ap->a_eofflag) { if (dep->de_FileSize - (offset - bias) <= 0) *ap->a_eofflag = 1; else *ap->a_eofflag = 0; } return (error); } /*- * a_vp - pointer to the file's vnode * a_bn - logical block number within the file (cluster number for us) * a_bop - where to return the bufobj of the special file containing the fs * a_bnp - where to return the "physical" block number corresponding to a_bn * (relative to the special file; units are blocks of size DEV_BSIZE) * a_runp - where to return the "run past" a_bn. This is the count of logical * blocks whose physical blocks (together with a_bn's physical block) * are contiguous. * a_runb - where to return the "run before" a_bn. */ static int msdosfs_bmap(ap) struct vop_bmap_args /* { struct vnode *a_vp; daddr_t a_bn; struct bufobj **a_bop; daddr_t *a_bnp; int *a_runp; int *a_runb; } */ *ap; { struct denode *dep; struct mount *mp; struct msdosfsmount *pmp; struct vnode *vp; daddr_t runbn; u_long cn; int bnpercn, error, maxio, maxrun, run; vp = ap->a_vp; dep = VTODE(vp); pmp = dep->de_pmp; if (ap->a_bop != NULL) *ap->a_bop = &pmp->pm_devvp->v_bufobj; if (ap->a_bnp == NULL) return (0); if (ap->a_runp != NULL) *ap->a_runp = 0; if (ap->a_runb != NULL) *ap->a_runb = 0; cn = ap->a_bn; if (cn != ap->a_bn) return (EFBIG); error = pcbmap(dep, cn, ap->a_bnp, NULL, NULL); if (error != 0 || (ap->a_runp == NULL && ap->a_runb == NULL)) return (error); mp = vp->v_mount; maxio = mp->mnt_iosize_max / mp->mnt_stat.f_iosize; bnpercn = de_cn2bn(pmp, 1); if (ap->a_runp != NULL) { maxrun = ulmin(maxio - 1, pmp->pm_maxcluster - cn); for (run = 1; run <= maxrun; run++) { if (pcbmap(dep, cn + run, &runbn, NULL, NULL) != 0 || runbn != *ap->a_bnp + run * bnpercn) break; } *ap->a_runp = run - 1; } if (ap->a_runb != NULL) { maxrun = ulmin(maxio - 1, cn); for (run = 1; run < maxrun; run++) { if (pcbmap(dep, cn - run, &runbn, NULL, NULL) != 0 || runbn != *ap->a_bnp - run * bnpercn) break; } *ap->a_runb = run - 1; } return (0); } static int msdosfs_strategy(ap) struct vop_strategy_args /* { struct vnode *a_vp; struct buf *a_bp; } */ *ap; { struct buf *bp = ap->a_bp; struct denode *dep = VTODE(ap->a_vp); struct bufobj *bo; int error = 0; daddr_t blkno; /* * If we don't already know the filesystem relative block number * then get it using pcbmap(). If pcbmap() returns the block * number as -1 then we've got a hole in the file. DOS filesystems * don't allow files with holes, so we shouldn't ever see this. */ if (bp->b_blkno == bp->b_lblkno) { error = pcbmap(dep, bp->b_lblkno, &blkno, 0, 0); bp->b_blkno = blkno; if (error) { bp->b_error = error; bp->b_ioflags |= BIO_ERROR; bufdone(bp); return (0); } if ((long)bp->b_blkno == -1) vfs_bio_clrbuf(bp); } if (bp->b_blkno == -1) { bufdone(bp); return (0); } /* * Read/write the block from/to the disk that contains the desired * file block. */ bp->b_iooffset = dbtob(bp->b_blkno); bo = dep->de_pmp->pm_bo; BO_STRATEGY(bo, bp); return (0); } static int msdosfs_print(ap) struct vop_print_args /* { struct vnode *vp; } */ *ap; { struct denode *dep = VTODE(ap->a_vp); printf("\tstartcluster %lu, dircluster %lu, diroffset %lu, ", dep->de_StartCluster, dep->de_dirclust, dep->de_diroffset); printf("on dev %s\n", devtoname(dep->de_pmp->pm_dev)); return (0); } static int msdosfs_pathconf(ap) struct vop_pathconf_args /* { struct vnode *a_vp; int a_name; int *a_retval; } */ *ap; { struct msdosfsmount *pmp = VTODE(ap->a_vp)->de_pmp; switch (ap->a_name) { case _PC_LINK_MAX: *ap->a_retval = 1; return (0); case _PC_NAME_MAX: *ap->a_retval = pmp->pm_flags & MSDOSFSMNT_LONGNAME ? WIN_MAXLEN : 12; return (0); case _PC_PATH_MAX: *ap->a_retval = PATH_MAX; return (0); case _PC_CHOWN_RESTRICTED: *ap->a_retval = 1; return (0); case _PC_NO_TRUNC: *ap->a_retval = 0; return (0); default: return (EINVAL); } /* NOTREACHED */ } static int msdosfs_vptofh(ap) struct vop_vptofh_args /* { struct vnode *a_vp; struct fid *a_fhp; } */ *ap; { struct denode *dep; struct defid *defhp; dep = VTODE(ap->a_vp); defhp = (struct defid *)ap->a_fhp; defhp->defid_len = sizeof(struct defid); defhp->defid_dirclust = dep->de_dirclust; defhp->defid_dirofs = dep->de_diroffset; /* defhp->defid_gen = dep->de_gen; */ return (0); } /* Global vfs data structures for msdosfs */ struct vop_vector msdosfs_vnodeops = { .vop_default = &default_vnodeops, .vop_access = msdosfs_access, .vop_bmap = msdosfs_bmap, .vop_cachedlookup = msdosfs_lookup, .vop_open = msdosfs_open, .vop_close = msdosfs_close, .vop_create = msdosfs_create, .vop_fsync = msdosfs_fsync, .vop_getattr = msdosfs_getattr, .vop_inactive = msdosfs_inactive, .vop_link = msdosfs_link, .vop_lookup = vfs_cache_lookup, .vop_mkdir = msdosfs_mkdir, .vop_mknod = msdosfs_mknod, .vop_pathconf = msdosfs_pathconf, .vop_print = msdosfs_print, .vop_read = msdosfs_read, .vop_readdir = msdosfs_readdir, .vop_reclaim = msdosfs_reclaim, .vop_remove = msdosfs_remove, .vop_rename = msdosfs_rename, .vop_rmdir = msdosfs_rmdir, .vop_setattr = msdosfs_setattr, .vop_strategy = msdosfs_strategy, .vop_symlink = msdosfs_symlink, .vop_write = msdosfs_write, .vop_vptofh = msdosfs_vptofh, }; Index: user/imp/tbemd/sys/net/if_var.h =================================================================== --- user/imp/tbemd/sys/net/if_var.h (revision 203849) +++ user/imp/tbemd/sys/net/if_var.h (revision 203850) @@ -1,892 +1,902 @@ /*- * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * From: @(#)if.h 8.1 (Berkeley) 6/10/93 * $FreeBSD$ */ #ifndef _NET_IF_VAR_H_ #define _NET_IF_VAR_H_ /* * Structures defining a network interface, providing a packet * transport mechanism (ala level 0 of the PUP protocols). * * Each interface accepts output datagrams of a specified maximum * length, and provides higher level routines with input datagrams * received from its medium. * * Output occurs when the routine if_output is called, with three parameters: * (*ifp->if_output)(ifp, m, dst, rt) * Here m is the mbuf chain to be sent and dst is the destination address. * The output routine encapsulates the supplied datagram if necessary, * and then transmits it on its medium. * * On input, each interface unwraps the data received by it, and either * places it on the input queue of an internetwork datagram routine * and posts the associated software interrupt, or passes the datagram to a raw * packet input routine. * * Routines exist for locating interfaces by their addresses * or for locating an interface on a certain network, as well as more general * routing and gateway routines maintaining information used to locate * interfaces. These routines live in the files if.c and route.c */ #ifdef __STDC__ /* * Forward structure declarations for function prototypes [sic]. */ struct mbuf; struct thread; struct rtentry; struct rt_addrinfo; struct socket; struct ether_header; struct carp_if; struct ifvlantrunk; struct route; struct vnet; #endif #include /* get TAILQ macros */ #ifdef _KERNEL #include #include #include #include #endif /* _KERNEL */ #include /* XXX */ #include /* XXX */ #include /* XXX */ #include /* XXX */ #include /* XXX */ #include #define IF_DUNIT_NONE -1 #include TAILQ_HEAD(ifnethead, ifnet); /* we use TAILQs so that the order of */ TAILQ_HEAD(ifaddrhead, ifaddr); /* instantiation is preserved in the list */ TAILQ_HEAD(ifprefixhead, ifprefix); TAILQ_HEAD(ifmultihead, ifmultiaddr); TAILQ_HEAD(ifgrouphead, ifg_group); /* * Structure defining a queue for a network interface. */ struct ifqueue { struct mbuf *ifq_head; struct mbuf *ifq_tail; int ifq_len; int ifq_maxlen; int ifq_drops; struct mtx ifq_mtx; }; /* * Structure defining a network interface. * * (Would like to call this struct ``if'', but C isn't PL/1.) */ struct ifnet { void *if_softc; /* pointer to driver state */ void *if_l2com; /* pointer to protocol bits */ struct vnet *if_vnet; /* pointer to network stack instance */ TAILQ_ENTRY(ifnet) if_link; /* all struct ifnets are chained */ char if_xname[IFNAMSIZ]; /* external name (name + unit) */ const char *if_dname; /* driver name */ int if_dunit; /* unit or IF_DUNIT_NONE */ u_int if_refcount; /* reference count */ struct ifaddrhead if_addrhead; /* linked list of addresses per if */ /* * if_addrhead is the list of all addresses associated to * an interface. * Some code in the kernel assumes that first element * of the list has type AF_LINK, and contains sockaddr_dl * addresses which store the link-level address and the name * of the interface. * However, access to the AF_LINK address through this * field is deprecated. Use if_addr or ifaddr_byindex() instead. */ int if_pcount; /* number of promiscuous listeners */ struct carp_if *if_carp; /* carp interface structure */ struct bpf_if *if_bpf; /* packet filter structure */ u_short if_index; /* numeric abbreviation for this if */ short if_index_reserved; /* spare space to grow if_index */ struct ifvlantrunk *if_vlantrunk; /* pointer to 802.1q data */ int if_flags; /* up/down, broadcast, etc. */ int if_capabilities; /* interface features & capabilities */ int if_capenable; /* enabled features & capabilities */ void *if_linkmib; /* link-type-specific MIB data */ size_t if_linkmiblen; /* length of above data */ struct if_data if_data; struct ifmultihead if_multiaddrs; /* multicast addresses configured */ int if_amcount; /* number of all-multicast requests */ /* procedure handles */ int (*if_output) /* output routine (enqueue) */ (struct ifnet *, struct mbuf *, struct sockaddr *, struct route *); void (*if_input) /* input routine (from h/w driver) */ (struct ifnet *, struct mbuf *); void (*if_start) /* initiate output routine */ (struct ifnet *); int (*if_ioctl) /* ioctl routine */ (struct ifnet *, u_long, caddr_t); void (*if_init) /* Init routine */ (void *); int (*if_resolvemulti) /* validate/resolve multicast */ (struct ifnet *, struct sockaddr **, struct sockaddr *); void (*if_qflush) /* flush any queues */ (struct ifnet *); int (*if_transmit) /* initiate output routine */ (struct ifnet *, struct mbuf *); void (*if_reassign) /* reassign to vnet routine */ (struct ifnet *, struct vnet *, char *); struct vnet *if_home_vnet; /* where this ifnet originates from */ struct ifaddr *if_addr; /* pointer to link-level address */ void *if_llsoftc; /* link layer softc */ int if_drv_flags; /* driver-managed status flags */ struct ifaltq if_snd; /* output queue (includes altq) */ const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */ void *if_bridge; /* bridge glue */ struct label *if_label; /* interface MAC label */ /* these are only used by IPv6 */ struct ifprefixhead if_prefixhead; /* list of prefixes per if */ void *if_afdata[AF_MAX]; int if_afdata_initialized; struct rwlock if_afdata_lock; struct task if_linktask; /* task for link change events */ struct mtx if_addr_mtx; /* mutex to protect address lists */ LIST_ENTRY(ifnet) if_clones; /* interfaces of a cloner */ TAILQ_HEAD(, ifg_list) if_groups; /* linked list of groups per if */ /* protected by if_addr_mtx */ void *if_pf_kif; void *if_lagg; /* lagg glue */ u_char if_alloctype; /* if_type at time of allocation */ /* * Spare fields are added so that we can modify sensitive data * structures without changing the kernel binary interface, and must * be used with care where binary compatibility is required. */ char if_cspare[3]; char *if_description; /* interface description */ void *if_pspare[7]; int if_ispare[4]; }; typedef void if_init_f_t(void *); /* * XXX These aliases are terribly dangerous because they could apply * to anything. */ #define if_mtu if_data.ifi_mtu #define if_type if_data.ifi_type #define if_physical if_data.ifi_physical #define if_addrlen if_data.ifi_addrlen #define if_hdrlen if_data.ifi_hdrlen #define if_metric if_data.ifi_metric #define if_link_state if_data.ifi_link_state #define if_baudrate if_data.ifi_baudrate #define if_hwassist if_data.ifi_hwassist #define if_ipackets if_data.ifi_ipackets #define if_ierrors if_data.ifi_ierrors #define if_opackets if_data.ifi_opackets #define if_oerrors if_data.ifi_oerrors #define if_collisions if_data.ifi_collisions #define if_ibytes if_data.ifi_ibytes #define if_obytes if_data.ifi_obytes #define if_imcasts if_data.ifi_imcasts #define if_omcasts if_data.ifi_omcasts #define if_iqdrops if_data.ifi_iqdrops #define if_noproto if_data.ifi_noproto #define if_lastchange if_data.ifi_lastchange /* for compatibility with other BSDs */ #define if_addrlist if_addrhead #define if_list if_link #define if_name(ifp) ((ifp)->if_xname) /* * Locks for address lists on the network interface. */ #define IF_ADDR_LOCK_INIT(if) mtx_init(&(if)->if_addr_mtx, \ "if_addr_mtx", NULL, MTX_DEF) #define IF_ADDR_LOCK_DESTROY(if) mtx_destroy(&(if)->if_addr_mtx) #define IF_ADDR_LOCK(if) mtx_lock(&(if)->if_addr_mtx) #define IF_ADDR_UNLOCK(if) mtx_unlock(&(if)->if_addr_mtx) #define IF_ADDR_LOCK_ASSERT(if) mtx_assert(&(if)->if_addr_mtx, MA_OWNED) /* * Function variations on locking macros intended to be used by loadable * kernel modules in order to divorce them from the internals of address list * locking. */ void if_addr_rlock(struct ifnet *ifp); /* if_addrhead */ void if_addr_runlock(struct ifnet *ifp); /* if_addrhead */ void if_maddr_rlock(struct ifnet *ifp); /* if_multiaddrs */ void if_maddr_runlock(struct ifnet *ifp); /* if_multiaddrs */ /* * Output queues (ifp->if_snd) and slow device input queues (*ifp->if_slowq) * are queues of messages stored on ifqueue structures * (defined above). Entries are added to and deleted from these structures * by these macros, which should be called with ipl raised to splimp(). */ #define IF_LOCK(ifq) mtx_lock(&(ifq)->ifq_mtx) #define IF_UNLOCK(ifq) mtx_unlock(&(ifq)->ifq_mtx) #define IF_LOCK_ASSERT(ifq) mtx_assert(&(ifq)->ifq_mtx, MA_OWNED) #define _IF_QFULL(ifq) ((ifq)->ifq_len >= (ifq)->ifq_maxlen) #define _IF_DROP(ifq) ((ifq)->ifq_drops++) #define _IF_QLEN(ifq) ((ifq)->ifq_len) #define _IF_ENQUEUE(ifq, m) do { \ (m)->m_nextpkt = NULL; \ if ((ifq)->ifq_tail == NULL) \ (ifq)->ifq_head = m; \ else \ (ifq)->ifq_tail->m_nextpkt = m; \ (ifq)->ifq_tail = m; \ (ifq)->ifq_len++; \ } while (0) #define IF_ENQUEUE(ifq, m) do { \ IF_LOCK(ifq); \ _IF_ENQUEUE(ifq, m); \ IF_UNLOCK(ifq); \ } while (0) #define _IF_PREPEND(ifq, m) do { \ (m)->m_nextpkt = (ifq)->ifq_head; \ if ((ifq)->ifq_tail == NULL) \ (ifq)->ifq_tail = (m); \ (ifq)->ifq_head = (m); \ (ifq)->ifq_len++; \ } while (0) #define IF_PREPEND(ifq, m) do { \ IF_LOCK(ifq); \ _IF_PREPEND(ifq, m); \ IF_UNLOCK(ifq); \ } while (0) #define _IF_DEQUEUE(ifq, m) do { \ (m) = (ifq)->ifq_head; \ if (m) { \ if (((ifq)->ifq_head = (m)->m_nextpkt) == NULL) \ (ifq)->ifq_tail = NULL; \ (m)->m_nextpkt = NULL; \ (ifq)->ifq_len--; \ } \ } while (0) #define IF_DEQUEUE(ifq, m) do { \ IF_LOCK(ifq); \ _IF_DEQUEUE(ifq, m); \ IF_UNLOCK(ifq); \ } while (0) #define _IF_POLL(ifq, m) ((m) = (ifq)->ifq_head) #define IF_POLL(ifq, m) _IF_POLL(ifq, m) #define _IF_DRAIN(ifq) do { \ struct mbuf *m; \ for (;;) { \ _IF_DEQUEUE(ifq, m); \ if (m == NULL) \ break; \ m_freem(m); \ } \ } while (0) #define IF_DRAIN(ifq) do { \ IF_LOCK(ifq); \ _IF_DRAIN(ifq); \ IF_UNLOCK(ifq); \ } while(0) #ifdef _KERNEL /* interface link layer address change event */ typedef void (*iflladdr_event_handler_t)(void *, struct ifnet *); EVENTHANDLER_DECLARE(iflladdr_event, iflladdr_event_handler_t); /* interface address change event */ typedef void (*ifaddr_event_handler_t)(void *, struct ifnet *); EVENTHANDLER_DECLARE(ifaddr_event, ifaddr_event_handler_t); /* new interface arrival event */ typedef void (*ifnet_arrival_event_handler_t)(void *, struct ifnet *); EVENTHANDLER_DECLARE(ifnet_arrival_event, ifnet_arrival_event_handler_t); /* interface departure event */ typedef void (*ifnet_departure_event_handler_t)(void *, struct ifnet *); EVENTHANDLER_DECLARE(ifnet_departure_event, ifnet_departure_event_handler_t); /* * interface groups */ struct ifg_group { char ifg_group[IFNAMSIZ]; u_int ifg_refcnt; void *ifg_pf_kif; TAILQ_HEAD(, ifg_member) ifg_members; TAILQ_ENTRY(ifg_group) ifg_next; }; struct ifg_member { TAILQ_ENTRY(ifg_member) ifgm_next; struct ifnet *ifgm_ifp; }; struct ifg_list { struct ifg_group *ifgl_group; TAILQ_ENTRY(ifg_list) ifgl_next; }; /* group attach event */ typedef void (*group_attach_event_handler_t)(void *, struct ifg_group *); EVENTHANDLER_DECLARE(group_attach_event, group_attach_event_handler_t); /* group detach event */ typedef void (*group_detach_event_handler_t)(void *, struct ifg_group *); EVENTHANDLER_DECLARE(group_detach_event, group_detach_event_handler_t); /* group change event */ typedef void (*group_change_event_handler_t)(void *, const char *); EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t); #define IF_AFDATA_LOCK_INIT(ifp) \ rw_init(&(ifp)->if_afdata_lock, "if_afdata") #define IF_AFDATA_WLOCK(ifp) rw_wlock(&(ifp)->if_afdata_lock) #define IF_AFDATA_RLOCK(ifp) rw_rlock(&(ifp)->if_afdata_lock) #define IF_AFDATA_WUNLOCK(ifp) rw_wunlock(&(ifp)->if_afdata_lock) #define IF_AFDATA_RUNLOCK(ifp) rw_runlock(&(ifp)->if_afdata_lock) #define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp) #define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp) #define IF_AFDATA_TRYLOCK(ifp) rw_try_wlock(&(ifp)->if_afdata_lock) #define IF_AFDATA_DESTROY(ifp) rw_destroy(&(ifp)->if_afdata_lock) #define IF_AFDATA_LOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED) #define IF_AFDATA_UNLOCK_ASSERT(ifp) rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED) int if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust); #define IF_HANDOFF(ifq, m, ifp) \ if_handoff((struct ifqueue *)ifq, m, ifp, 0) #define IF_HANDOFF_ADJ(ifq, m, ifp, adj) \ if_handoff((struct ifqueue *)ifq, m, ifp, adj) void if_start(struct ifnet *); #define IFQ_ENQUEUE(ifq, m, err) \ do { \ IF_LOCK(ifq); \ if (ALTQ_IS_ENABLED(ifq)) \ ALTQ_ENQUEUE(ifq, m, NULL, err); \ else { \ if (_IF_QFULL(ifq)) { \ m_freem(m); \ (err) = ENOBUFS; \ } else { \ _IF_ENQUEUE(ifq, m); \ (err) = 0; \ } \ } \ if (err) \ (ifq)->ifq_drops++; \ IF_UNLOCK(ifq); \ } while (0) #define IFQ_DEQUEUE_NOLOCK(ifq, m) \ do { \ if (TBR_IS_ENABLED(ifq)) \ (m) = tbr_dequeue_ptr(ifq, ALTDQ_REMOVE); \ else if (ALTQ_IS_ENABLED(ifq)) \ ALTQ_DEQUEUE(ifq, m); \ else \ _IF_DEQUEUE(ifq, m); \ } while (0) #define IFQ_DEQUEUE(ifq, m) \ do { \ IF_LOCK(ifq); \ IFQ_DEQUEUE_NOLOCK(ifq, m); \ IF_UNLOCK(ifq); \ } while (0) #define IFQ_POLL_NOLOCK(ifq, m) \ do { \ if (TBR_IS_ENABLED(ifq)) \ (m) = tbr_dequeue_ptr(ifq, ALTDQ_POLL); \ else if (ALTQ_IS_ENABLED(ifq)) \ ALTQ_POLL(ifq, m); \ else \ _IF_POLL(ifq, m); \ } while (0) #define IFQ_POLL(ifq, m) \ do { \ IF_LOCK(ifq); \ IFQ_POLL_NOLOCK(ifq, m); \ IF_UNLOCK(ifq); \ } while (0) #define IFQ_PURGE_NOLOCK(ifq) \ do { \ if (ALTQ_IS_ENABLED(ifq)) { \ ALTQ_PURGE(ifq); \ } else \ _IF_DRAIN(ifq); \ } while (0) #define IFQ_PURGE(ifq) \ do { \ IF_LOCK(ifq); \ IFQ_PURGE_NOLOCK(ifq); \ IF_UNLOCK(ifq); \ } while (0) #define IFQ_SET_READY(ifq) \ do { ((ifq)->altq_flags |= ALTQF_READY); } while (0) #define IFQ_LOCK(ifq) IF_LOCK(ifq) #define IFQ_UNLOCK(ifq) IF_UNLOCK(ifq) #define IFQ_LOCK_ASSERT(ifq) IF_LOCK_ASSERT(ifq) #define IFQ_IS_EMPTY(ifq) ((ifq)->ifq_len == 0) #define IFQ_INC_LEN(ifq) ((ifq)->ifq_len++) #define IFQ_DEC_LEN(ifq) (--(ifq)->ifq_len) #define IFQ_INC_DROPS(ifq) ((ifq)->ifq_drops++) #define IFQ_SET_MAXLEN(ifq, len) ((ifq)->ifq_maxlen = (len)) /* * The IFF_DRV_OACTIVE test should really occur in the device driver, not in * the handoff logic, as that flag is locked by the device driver. */ #define IFQ_HANDOFF_ADJ(ifp, m, adj, err) \ do { \ int len; \ short mflags; \ \ len = (m)->m_pkthdr.len; \ mflags = (m)->m_flags; \ IFQ_ENQUEUE(&(ifp)->if_snd, m, err); \ if ((err) == 0) { \ (ifp)->if_obytes += len + (adj); \ if (mflags & M_MCAST) \ (ifp)->if_omcasts++; \ if (((ifp)->if_drv_flags & IFF_DRV_OACTIVE) == 0) \ if_start(ifp); \ } \ } while (0) #define IFQ_HANDOFF(ifp, m, err) \ IFQ_HANDOFF_ADJ(ifp, m, 0, err) #define IFQ_DRV_DEQUEUE(ifq, m) \ do { \ (m) = (ifq)->ifq_drv_head; \ if (m) { \ if (((ifq)->ifq_drv_head = (m)->m_nextpkt) == NULL) \ (ifq)->ifq_drv_tail = NULL; \ (m)->m_nextpkt = NULL; \ (ifq)->ifq_drv_len--; \ } else { \ IFQ_LOCK(ifq); \ IFQ_DEQUEUE_NOLOCK(ifq, m); \ while ((ifq)->ifq_drv_len < (ifq)->ifq_drv_maxlen) { \ struct mbuf *m0; \ IFQ_DEQUEUE_NOLOCK(ifq, m0); \ if (m0 == NULL) \ break; \ m0->m_nextpkt = NULL; \ if ((ifq)->ifq_drv_tail == NULL) \ (ifq)->ifq_drv_head = m0; \ else \ (ifq)->ifq_drv_tail->m_nextpkt = m0; \ (ifq)->ifq_drv_tail = m0; \ (ifq)->ifq_drv_len++; \ } \ IFQ_UNLOCK(ifq); \ } \ } while (0) #define IFQ_DRV_PREPEND(ifq, m) \ do { \ (m)->m_nextpkt = (ifq)->ifq_drv_head; \ if ((ifq)->ifq_drv_tail == NULL) \ (ifq)->ifq_drv_tail = (m); \ (ifq)->ifq_drv_head = (m); \ (ifq)->ifq_drv_len++; \ } while (0) #define IFQ_DRV_IS_EMPTY(ifq) \ (((ifq)->ifq_drv_len == 0) && ((ifq)->ifq_len == 0)) #define IFQ_DRV_PURGE(ifq) \ do { \ struct mbuf *m, *n = (ifq)->ifq_drv_head; \ while((m = n) != NULL) { \ n = m->m_nextpkt; \ m_freem(m); \ } \ (ifq)->ifq_drv_head = (ifq)->ifq_drv_tail = NULL; \ (ifq)->ifq_drv_len = 0; \ IFQ_PURGE(ifq); \ } while (0) #ifdef _KERNEL static __inline void drbr_stats_update(struct ifnet *ifp, int len, int mflags) { #ifndef NO_SLOW_STATS ifp->if_obytes += len; if (mflags & M_MCAST) ifp->if_omcasts++; #endif } static __inline int drbr_enqueue(struct ifnet *ifp, struct buf_ring *br, struct mbuf *m) { int error = 0; int len = m->m_pkthdr.len; int mflags = m->m_flags; #ifdef ALTQ if (ALTQ_IS_ENABLED(&ifp->if_snd)) { IFQ_ENQUEUE(&ifp->if_snd, m, error); return (error); } #endif if ((error = buf_ring_enqueue_bytes(br, m, len)) == ENOBUFS) { br->br_drops++; m_freem(m); } else drbr_stats_update(ifp, len, mflags); return (error); } static __inline void drbr_flush(struct ifnet *ifp, struct buf_ring *br) { struct mbuf *m; #ifdef ALTQ - if (ifp != NULL && ALTQ_IS_ENABLED(&ifp->if_snd)) { - while (!IFQ_IS_EMPTY(&ifp->if_snd)) { - IFQ_DRV_DEQUEUE(&ifp->if_snd, m); - m_freem(m); - } - } + if (ifp != NULL && ALTQ_IS_ENABLED(&ifp->if_snd)) + IFQ_PURGE(&ifp->if_snd); #endif while ((m = buf_ring_dequeue_sc(br)) != NULL) m_freem(m); } static __inline void drbr_free(struct buf_ring *br, struct malloc_type *type) { drbr_flush(NULL, br); buf_ring_free(br, type); } static __inline struct mbuf * drbr_dequeue(struct ifnet *ifp, struct buf_ring *br) { #ifdef ALTQ struct mbuf *m; if (ALTQ_IS_ENABLED(&ifp->if_snd)) { - IFQ_DRV_DEQUEUE(&ifp->if_snd, m); + IFQ_DEQUEUE(&ifp->if_snd, m); return (m); } #endif return (buf_ring_dequeue_sc(br)); } static __inline struct mbuf * drbr_dequeue_cond(struct ifnet *ifp, struct buf_ring *br, int (*func) (struct mbuf *, void *), void *arg) { struct mbuf *m; #ifdef ALTQ - /* - * XXX need to evaluate / requeue - */ - if (ALTQ_IS_ENABLED(&ifp->if_snd)) { - IFQ_DRV_DEQUEUE(&ifp->if_snd, m); + if (ALTQ_IS_ENABLED(&ifp->if_snd)) { + IFQ_LOCK(&ifp->if_snd); + IFQ_POLL_NOLOCK(&ifp->if_snd, m); + if (m != NULL && func(m, arg) == 0) { + IFQ_UNLOCK(&ifp->if_snd); + return (NULL); + } + IFQ_DEQUEUE(&ifp->if_snd, m); + IFQ_UNLOCK(&ifp->if_snd); return (m); } #endif m = buf_ring_peek(br); if (m == NULL || func(m, arg) == 0) return (NULL); return (buf_ring_dequeue_sc(br)); } static __inline int drbr_empty(struct ifnet *ifp, struct buf_ring *br) { #ifdef ALTQ if (ALTQ_IS_ENABLED(&ifp->if_snd)) - return (IFQ_DRV_IS_EMPTY(&ifp->if_snd)); + return (IFQ_IS_EMPTY(&ifp->if_snd)); #endif return (buf_ring_empty(br)); +} + +static __inline int +drbr_needs_enqueue(struct ifnet *ifp, struct buf_ring *br) +{ +#ifdef ALTQ + if (ALTQ_IS_ENABLED(&ifp->if_snd)) + return (1); +#endif + return (!buf_ring_empty(br)); } static __inline int drbr_inuse(struct ifnet *ifp, struct buf_ring *br) { #ifdef ALTQ if (ALTQ_IS_ENABLED(&ifp->if_snd)) return (ifp->if_snd.ifq_len); #endif return (buf_ring_count(br)); } #endif /* * 72 was chosen below because it is the size of a TCP/IP * header (40) + the minimum mss (32). */ #define IF_MINMTU 72 #define IF_MAXMTU 65535 #endif /* _KERNEL */ /* * The ifaddr structure contains information about one address * of an interface. They are maintained by the different address families, * are allocated and attached when an address is set, and are linked * together so all addresses for an interface can be located. * * NOTE: a 'struct ifaddr' is always at the beginning of a larger * chunk of malloc'ed memory, where we store the three addresses * (ifa_addr, ifa_dstaddr and ifa_netmask) referenced here. */ struct ifaddr { struct sockaddr *ifa_addr; /* address of interface */ struct sockaddr *ifa_dstaddr; /* other end of p-to-p link */ #define ifa_broadaddr ifa_dstaddr /* broadcast address interface */ struct sockaddr *ifa_netmask; /* used to determine subnet */ struct if_data if_data; /* not all members are meaningful */ struct ifnet *ifa_ifp; /* back-pointer to interface */ TAILQ_ENTRY(ifaddr) ifa_link; /* queue macro glue */ void (*ifa_rtrequest) /* check or clean routes (+ or -)'d */ (int, struct rtentry *, struct rt_addrinfo *); u_short ifa_flags; /* mostly rt_flags for cloning */ u_int ifa_refcnt; /* references to this structure */ int ifa_metric; /* cost of going out this interface */ int (*ifa_claim_addr) /* check if an addr goes to this if */ (struct ifaddr *, struct sockaddr *); struct mtx ifa_mtx; }; #define IFA_ROUTE RTF_UP /* route installed */ #define IFA_RTSELF RTF_HOST /* loopback route to self installed */ /* for compatibility with other BSDs */ #define ifa_list ifa_link #ifdef _KERNEL #define IFA_LOCK(ifa) mtx_lock(&(ifa)->ifa_mtx) #define IFA_UNLOCK(ifa) mtx_unlock(&(ifa)->ifa_mtx) void ifa_free(struct ifaddr *ifa); void ifa_init(struct ifaddr *ifa); void ifa_ref(struct ifaddr *ifa); #endif /* * The prefix structure contains information about one prefix * of an interface. They are maintained by the different address families, * are allocated and attached when a prefix or an address is set, * and are linked together so all prefixes for an interface can be located. */ struct ifprefix { struct sockaddr *ifpr_prefix; /* prefix of interface */ struct ifnet *ifpr_ifp; /* back-pointer to interface */ TAILQ_ENTRY(ifprefix) ifpr_list; /* queue macro glue */ u_char ifpr_plen; /* prefix length in bits */ u_char ifpr_type; /* protocol dependent prefix type */ }; /* * Multicast address structure. This is analogous to the ifaddr * structure except that it keeps track of multicast addresses. */ struct ifmultiaddr { TAILQ_ENTRY(ifmultiaddr) ifma_link; /* queue macro glue */ struct sockaddr *ifma_addr; /* address this membership is for */ struct sockaddr *ifma_lladdr; /* link-layer translation, if any */ struct ifnet *ifma_ifp; /* back-pointer to interface */ u_int ifma_refcount; /* reference count */ void *ifma_protospec; /* protocol-specific state, if any */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ }; #ifdef _KERNEL extern struct rwlock ifnet_rwlock; extern struct sx ifnet_sxlock; #define IFNET_LOCK_INIT() do { \ rw_init_flags(&ifnet_rwlock, "ifnet_rw", RW_RECURSE); \ sx_init_flags(&ifnet_sxlock, "ifnet_sx", SX_RECURSE); \ } while(0) #define IFNET_WLOCK() do { \ sx_xlock(&ifnet_sxlock); \ rw_wlock(&ifnet_rwlock); \ } while (0) #define IFNET_WUNLOCK() do { \ rw_wunlock(&ifnet_rwlock); \ sx_xunlock(&ifnet_sxlock); \ } while (0) /* * To assert the ifnet lock, you must know not only whether it's for read or * write, but also whether it was acquired with sleep support or not. */ #define IFNET_RLOCK_ASSERT() sx_assert(&ifnet_sxlock, SA_SLOCKED) #define IFNET_RLOCK_NOSLEEP_ASSERT() rw_assert(&ifnet_rwlock, RA_RLOCKED) #define IFNET_WLOCK_ASSERT() do { \ sx_assert(&ifnet_sxlock, SA_XLOCKED); \ rw_assert(&ifnet_rwlock, RA_WLOCKED); \ } while (0) #define IFNET_RLOCK() sx_slock(&ifnet_sxlock) #define IFNET_RLOCK_NOSLEEP() rw_rlock(&ifnet_rwlock) #define IFNET_RUNLOCK() sx_sunlock(&ifnet_sxlock) #define IFNET_RUNLOCK_NOSLEEP() rw_runlock(&ifnet_rwlock) /* * Look up an ifnet given its index; the _ref variant also acquires a * reference that must be freed using if_rele(). It is almost always a bug * to call ifnet_byindex() instead if ifnet_byindex_ref(). */ struct ifnet *ifnet_byindex(u_short idx); struct ifnet *ifnet_byindex_locked(u_short idx); struct ifnet *ifnet_byindex_ref(u_short idx); /* * Given the index, ifaddr_byindex() returns the one and only * link-level ifaddr for the interface. You are not supposed to use * it to traverse the list of addresses associated to the interface. */ struct ifaddr *ifaddr_byindex(u_short idx); VNET_DECLARE(struct ifnethead, ifnet); VNET_DECLARE(struct ifgrouphead, ifg_head); VNET_DECLARE(int, if_index); VNET_DECLARE(struct ifnet *, loif); /* first loopback interface */ VNET_DECLARE(int, useloopback); #define V_ifnet VNET(ifnet) #define V_ifg_head VNET(ifg_head) #define V_if_index VNET(if_index) #define V_loif VNET(loif) #define V_useloopback VNET(useloopback) extern int ifqmaxlen; int if_addgroup(struct ifnet *, const char *); int if_delgroup(struct ifnet *, const char *); int if_addmulti(struct ifnet *, struct sockaddr *, struct ifmultiaddr **); int if_allmulti(struct ifnet *, int); struct ifnet* if_alloc(u_char); void if_attach(struct ifnet *); void if_dead(struct ifnet *); int if_delmulti(struct ifnet *, struct sockaddr *); void if_delmulti_ifma(struct ifmultiaddr *); void if_detach(struct ifnet *); void if_vmove(struct ifnet *, struct vnet *); void if_purgeaddrs(struct ifnet *); void if_delallmulti(struct ifnet *); void if_down(struct ifnet *); struct ifmultiaddr * if_findmulti(struct ifnet *, struct sockaddr *); void if_free(struct ifnet *); void if_free_type(struct ifnet *, u_char); void if_initname(struct ifnet *, const char *, int); void if_link_state_change(struct ifnet *, int); int if_printf(struct ifnet *, const char *, ...) __printflike(2, 3); void if_qflush(struct ifnet *); void if_ref(struct ifnet *); void if_rele(struct ifnet *); int if_setlladdr(struct ifnet *, const u_char *, int); void if_up(struct ifnet *); int ifioctl(struct socket *, u_long, caddr_t, struct thread *); int ifpromisc(struct ifnet *, int); struct ifnet *ifunit(const char *); struct ifnet *ifunit_ref(const char *); void ifq_init(struct ifaltq *, struct ifnet *ifp); void ifq_delete(struct ifaltq *); int ifa_add_loopback_route(struct ifaddr *, struct sockaddr *); int ifa_del_loopback_route(struct ifaddr *, struct sockaddr *); struct ifaddr *ifa_ifwithaddr(struct sockaddr *); int ifa_ifwithaddr_check(struct sockaddr *); struct ifaddr *ifa_ifwithbroadaddr(struct sockaddr *); struct ifaddr *ifa_ifwithdstaddr(struct sockaddr *); struct ifaddr *ifa_ifwithnet(struct sockaddr *); struct ifaddr *ifa_ifwithroute(int, struct sockaddr *, struct sockaddr *); struct ifaddr *ifa_ifwithroute_fib(int, struct sockaddr *, struct sockaddr *, u_int); struct ifaddr *ifaof_ifpforaddr(struct sockaddr *, struct ifnet *); int if_simloop(struct ifnet *ifp, struct mbuf *m, int af, int hlen); typedef void *if_com_alloc_t(u_char type, struct ifnet *ifp); typedef void if_com_free_t(void *com, u_char type); void if_register_com_alloc(u_char type, if_com_alloc_t *a, if_com_free_t *f); void if_deregister_com_alloc(u_char type); #define IF_LLADDR(ifp) \ LLADDR((struct sockaddr_dl *)((ifp)->if_addr->ifa_addr)) #ifdef DEVICE_POLLING enum poll_cmd { POLL_ONLY, POLL_AND_CHECK_STATUS }; typedef int poll_handler_t(struct ifnet *ifp, enum poll_cmd cmd, int count); int ether_poll_register(poll_handler_t *h, struct ifnet *ifp); int ether_poll_deregister(struct ifnet *ifp); #endif /* DEVICE_POLLING */ #endif /* _KERNEL */ #endif /* !_NET_IF_VAR_H_ */ Index: user/imp/tbemd/sys/sparc64/include/asi.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/asi.h (revision 203849) +++ user/imp/tbemd/sys/sparc64/include/asi.h (revision 203850) @@ -1,247 +1,255 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: BSDI: asi.h,v 1.3 1997/08/08 14:31:42 torek * $FreeBSD$ */ #ifndef _MACHINE_ASI_H_ #define _MACHINE_ASI_H_ /* - * Standard v9 asis + * Standard v9 ASIs */ #define ASI_N 0x4 #define ASI_NL 0xc #define ASI_AIUP 0x10 #define ASI_AIUS 0x11 #define ASI_AIUPL 0x18 #define ASI_AIUSL 0x19 #define ASI_P 0x80 #define ASI_S 0x81 #define ASI_PNF 0x82 #define ASI_SNF 0x83 #define ASI_PL 0x88 #define ASI_SL 0x89 #define ASI_PNFL 0x8a #define ASI_SNFL 0x8b /* - * UltraSPARC extensions. ASIs limited to a certain family are annotated. + * UltraSPARC extensions - ASIs limited to a certain family are annotated. */ #define ASI_PHYS_USE_EC 0x14 #define ASI_PHYS_BYPASS_EC_WITH_EBIT 0x15 #define ASI_PHYS_USE_EC_L 0x1c #define ASI_PHYS_BYPASS_EC_WITH_EBIT_L 0x1d #define ASI_NUCLEUS_QUAD_LDD 0x24 #define ASI_NUCLEUS_QUAD_LDD_L 0x2c #define ASI_PCACHE_STATUS_DATA 0x30 /* US-III Cu */ #define ASI_PCACHE_DATA 0x31 /* US-III Cu */ #define ASI_PCACHE_TAG 0x32 /* US-III Cu */ #define ASI_PCACHE_SNOOP_TAG 0x33 /* US-III Cu */ #define ASI_ATOMIC_QUAD_LDD_PHYS 0x34 /* US-III Cu */ #define ASI_WCACHE_VALID_BITS 0x38 /* US-III Cu */ #define ASI_WCACHE_DATA 0x39 /* US-III Cu */ #define ASI_WCACHE_TAG 0x3a /* US-III Cu */ #define ASI_WCACHE_SNOOP_TAG 0x3b /* US-III Cu */ #define ASI_ATOMIC_QUAD_LDD_PHYS_L 0x3c /* US-III Cu */ #define ASI_SRAM_FAST_INIT 0x40 /* US-III Cu */ #define ASI_DCACHE_INVALIDATE 0x42 /* US-III Cu */ #define ASI_DCACHE_UTAG 0x43 /* US-III Cu */ #define ASI_DCACHE_SNOOP_TAG 0x44 /* US-III Cu */ /* Named ASI_DCUCR on US-III, but is mostly identical except for added bits. */ #define ASI_LSU_CTL_REG 0x45 #define ASI_DCACHE_DATA 0x46 #define ASI_DCACHE_TAG 0x47 #define ASI_INTR_DISPATCH_STATUS 0x48 #define ASI_INTR_RECEIVE 0x49 #define ASI_UPA_CONFIG_REG 0x4a /* US-I, II */ -#define ASI_FIREPLANE_CONFIG_REG 0x4a /* US-III Cu */ -#define AA_FIREPLANE_CONFIG 0x0 /* US-III Cu */ -#define AA_FIREPLANE_ADDRESS 0x8 /* US-III Cu */ +#define ASI_FIREPLANE_CONFIG_REG 0x4a /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_CONFIG 0x0 /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_ADDRESS 0x8 /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_CONFIG_2 0x10 /* US-IV{,+} */ +#define ASI_JBUS_CONFIG_REG 0x4a /* US-IIIi{,+} */ + #define ASI_ESTATE_ERROR_EN_REG 0x4b #define AA_ESTATE_CEEN 0x1 #define AA_ESTATE_NCEEN 0x2 #define AA_ESTATE_ISAPEN 0x4 #define ASI_AFSR 0x4c #define ASI_AFAR 0x4d #define ASI_ECACHE_TAG_DATA 0x4e #define ASI_IMMU_TAG_TARGET_REG 0x50 #define ASI_IMMU 0x50 #define AA_IMMU_TTR 0x0 #define AA_IMMU_SFSR 0x18 #define AA_IMMU_TSB 0x28 #define AA_IMMU_TAR 0x30 #define AA_IMMU_TSB_PEXT_REG 0x48 /* US-III family */ #define AA_IMMU_TSB_SEXT_REG 0x50 /* US-III family */ #define AA_IMMU_TSB_NEXT_REG 0x58 /* US-III family */ #define ASI_IMMU_TSB_8KB_PTR_REG 0x51 #define ASI_IMMU_TSB_64KB_PTR_REG 0x52 #define ASI_SERIAL_ID 0x53 /* US-III family */ #define ASI_ITLB_DATA_IN_REG 0x54 /* US-III Cu: also ASI_ITLB_CAM_ADDRESS_REG */ #define ASI_ITLB_DATA_ACCESS_REG 0x55 #define ASI_ITLB_TAG_READ_REG 0x56 #define ASI_IMMU_DEMAP 0x57 #define ASI_DMMU_TAG_TARGET_REG 0x58 #define ASI_DMMU 0x58 #define AA_DMMU_TTR 0x0 #define AA_DMMU_PCXR 0x8 #define AA_DMMU_SCXR 0x10 #define AA_DMMU_SFSR 0x18 #define AA_DMMU_SFAR 0x20 #define AA_DMMU_TSB 0x28 #define AA_DMMU_TAR 0x30 #define AA_DMMU_VWPR 0x38 #define AA_DMMU_PWPR 0x40 #define AA_DMMU_TSB_PEXT_REG 0x48 #define AA_DMMU_TSB_SEXT_REG 0x50 #define AA_DMMU_TSB_NEXT_REG 0x58 #define AA_DMMU_TAG_ACCESS_EXT 0x60 /* US-III family */ #define ASI_DMMU_TSB_8KB_PTR_REG 0x59 #define ASI_DMMU_TSB_64KB_PTR_REG 0x5a #define ASI_DMMU_TSB_DIRECT_PTR_REG 0x5b #define ASI_DTLB_DATA_IN_REG 0x5c /* US-III Cu: also ASI_DTLB_CAM_ADDRESS_REG */ #define ASI_DTLB_DATA_ACCESS_REG 0x5d #define ASI_DTLB_TAG_READ_REG 0x5e #define ASI_DMMU_DEMAP 0x5f #define ASI_IIU_INST_TRAP 0x60 /* US-III family */ +#define ASI_INTR_ID 0x63 /* US-IV{,+} */ +#define AA_INTR_ID 0x0 /* US-IV{,+} */ +#define AA_CORE_ID 0x10 /* US-IV{,+} */ +#define AA_CESR_ID 0x40 /* US-IV{,+} */ + #define ASI_ICACHE_INSTR 0x66 #define ASI_ICACHE_TAG 0x67 #define ASI_ICACHE_SNOOP_TAG 0x68 /* US-III family */ #define ASI_ICACHE_PRE_DECODE 0x6e /* US-I, II */ #define ASI_ICACHE_PRE_NEXT_FIELD 0x6f /* US-I, II */ #define ASI_BLK_AUIP 0x70 #define ASI_BLK_AIUS 0x71 #define ASI_MCU_CONFIG_REG 0x72 /* US-III Cu */ #define AA_MCU_TIMING1_REG 0x0 /* US-III Cu */ #define AA_MCU_TIMING2_REG 0x8 /* US-III Cu */ #define AA_MCU_TIMING3_REG 0x10 /* US-III Cu */ #define AA_MCU_TIMING4_REG 0x18 /* US-III Cu */ #define AA_MCU_DEC1_REG 0x20 /* US-III Cu */ #define AA_MCU_DEC2_REG 0x28 /* US-III Cu */ #define AA_MCU_DEC3_REG 0x30 /* US-III Cu */ #define AA_MCU_DEC4_REG 0x38 /* US-III Cu */ #define AA_MCU_ADDR_CNTL_REG 0x40 /* US-III Cu */ #define ASI_ECACHE_DATA 0x74 /* US-III Cu */ #define ASI_ECACHE_CONTROL 0x75 /* US-III Cu */ #define ASI_ECACHE_W 0x76 /* * With the advent of the US-III, the numbering has changed, as additional - * registers were inserted in between. We retain the original ordering for + * registers were inserted in between. We retain the original ordering for * now, and append an A to the inserted registers. * Exceptions are AA_SDB_INTR_D6 and AA_SDB_INTR_D7, which were appended * at the end. */ #define ASI_SDB_ERROR_W 0x77 #define ASI_SDB_CONTROL_W 0x77 #define ASI_SDB_INTR_W 0x77 #define AA_SDB_ERR_HIGH 0x0 #define AA_SDB_ERR_LOW 0x18 #define AA_SDB_CNTL_HIGH 0x20 #define AA_SDB_CNTL_LOW 0x38 #define AA_SDB_INTR_D0 0x40 #define AA_SDB_INTR_D0A 0x48 /* US-III family */ #define AA_SDB_INTR_D1 0x50 #define AA_SDB_INTR_D1A 0x5A /* US-III family */ #define AA_SDB_INTR_D2 0x60 #define AA_SDB_INTR_D2A 0x68 /* US-III family */ #define AA_INTR_SEND 0x70 #define AA_SDB_INTR_D6 0x80 /* US-III family */ #define AA_SDB_INTR_D7 0x88 /* US-III family */ #define ASI_BLK_AIUPL 0x78 #define ASI_BLK_AIUSL 0x79 #define ASI_ECACHE_R 0x7e /* * These have the same registers as their corresponding write versions * except for AA_INTR_SEND. */ #define ASI_SDB_ERROR_R 0x7f #define ASI_SDB_CONTROL_R 0x7f #define ASI_SDB_INTR_R 0x7f #define ASI_PST8_P 0xc0 #define ASI_PST8_S 0xc1 #define ASI_PST16_P 0xc2 #define ASI_PST16_S 0xc3 #define ASI_PST32_P 0xc4 #define ASI_PST32_S 0xc5 #define ASI_PST8_PL 0xc8 #define ASI_PST8_SL 0xc9 #define ASI_PST16_PL 0xca #define ASI_PST16_SL 0xcb #define ASI_PST32_PL 0xcc #define ASI_PST32_SL 0xcd #define ASI_FL8_P 0xd0 #define ASI_FL8_S 0xd1 #define ASI_FL16_P 0xd2 #define ASI_FL16_S 0xd3 #define ASI_FL8_PL 0xd8 #define ASI_FL8_SL 0xd9 #define ASI_FL16_PL 0xda #define ASI_FL16_SL 0xdb #define ASI_BLK_COMMIT_P 0xe0 #define ASI_BLK_COMMIT_S 0xe1 #define ASI_BLK_P 0xf0 #define ASI_BLK_S 0xf1 #define ASI_BLK_PL 0xf8 #define ASI_BLK_SL 0xf9 #endif /* !_MACHINE_ASI_H_ */ Index: user/imp/tbemd/sys/sparc64/include/cmt.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/cmt.h (nonexistent) +++ user/imp/tbemd/sys/sparc64/include/cmt.h (revision 203850) @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2010 Marius Strobl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_CMT_H_ +#define _MACHINE_CMT_H_ + +#define INTR_ID_ID_SHIFT (0) +#define INTR_ID_ID_SIZE (10) +#define INTR_ID_ID_MASK \ + (((1 << INTR_ID_ID_SIZE) - 1) << INTR_ID_ID_SHIFT) + +#define INTR_ID_GET_ID(cr) ((cr & INTR_ID_ID_MASK) >> INTR_ID_ID_SHIFT) + +#endif /* _MACHINE_CMT_H_ */ Property changes on: user/imp/tbemd/sys/sparc64/include/cmt.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/sparc64/include/fireplane.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/fireplane.h (nonexistent) +++ user/imp/tbemd/sys/sparc64/include/fireplane.h (revision 203850) @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2010 Marius Strobl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_FIREPLANE_H_ +#define _MACHINE_FIREPLANE_H_ + +#define FIREPLANE_CR_AID_SHIFT (17) +#define FIREPLANE_CR_AID_SIZE (10) +#define FIREPLANE_CR_AID_MASK \ + (((1 << FIREPLANE_CR_AID_SIZE) - 1) << FIREPLANE_CR_AID_SHIFT) + +#define FIREPLANE_CR_GET_AID(cr) \ + ((cr & FIREPLANE_CR_AID_MASK) >> FIREPLANE_CR_AID_SHIFT) + +#endif /* _MACHINE_FIREPLANE_H_ */ Property changes on: user/imp/tbemd/sys/sparc64/include/fireplane.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/sparc64/include/jbus.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/jbus.h (nonexistent) +++ user/imp/tbemd/sys/sparc64/include/jbus.h (revision 203850) @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2010 Marius Strobl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_JBUS_H_ +#define _MACHINE_JBUS_H_ + +#define JBUS_CR_JID_SHIFT (17) +#define JBUS_CR_JID_SIZE (3) +#define JBUS_CR_JID_MASK \ + (((1 << JBUS_CR_JID_SIZE) - 1) << JBUS_CR_JID_SHIFT) + +#define JBUS_CR_GET_JID(cr) ((cr & JBUS_CR_JID_MASK) >> JBUS_CR_JID_SHIFT) + +#endif /* _MACHINE_JBUS_H_ */ Property changes on: user/imp/tbemd/sys/sparc64/include/jbus.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/sparc64/include/ktr.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/ktr.h (revision 203849) +++ user/imp/tbemd/sys/sparc64/include/ktr.h (revision 203850) @@ -1,95 +1,93 @@ /*- * Copyright (c) 1996 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI $Id: ktr.h,v 1.10.2.7 2000/03/16 21:44:42 cp Exp $ * $FreeBSD$ */ #ifndef _MACHINE_KTR_H_ #define _MACHINE_KTR_H_ #include -#include - #ifndef LOCORE -#define KTR_CPU UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG)) +#define KTR_CPU PCPU_GET(mid) #else #define AND(var, mask, r1, r2) \ SET(var, r2, r1) ; \ lduw [r1], r2 ; \ and r2, mask, r1 #define TEST(var, mask, r1, r2, l1) \ AND(var, mask, r1, r2) ; \ brz r1, l1 ## f ; \ nop /* * XXX could really use another register... */ #define ATR(desc, r1, r2, r3, l1, l2) \ .sect .rodata ; \ l1: .asciz desc ; \ .previous ; \ SET(ktr_idx, r2, r1) ; \ lduw [r1], r2 ; \ l2: add r2, 1, r3 ; \ set KTR_ENTRIES - 1, r1 ; \ and r3, r1, r3 ; \ set ktr_idx, r1 ; \ casa [r1] ASI_N, r2, r3 ; \ cmp r2, r3 ; \ bne %icc, l2 ## b ; \ mov r3, r2 ; \ SET(ktr_buf, r3, r1) ; \ mulx r2, KTR_SIZEOF, r2 ; \ add r1, r2, r1 ; \ rd %tick, r2 ; \ stx r2, [r1 + KTR_TIMESTAMP] ; \ - UPA_GET_MID(r2) ; \ + lduw [PCPU(MID)], r2 ; \ stw r2, [r1 + KTR_CPU] ; \ stw %g0, [r1 + KTR_LINE] ; \ stx %g0, [r1 + KTR_FILE] ; \ SET(l1 ## b, r3, r2) ; \ stx r2, [r1 + KTR_DESC] #define CATR(mask, desc, r1, r2, r3, l1, l2, l3) \ set mask, r1 ; \ TEST(ktr_mask, r1, r2, r2, l3) ; \ - UPA_GET_MID(r1) ; \ + lduw [PCPU(MID)], r1 ; \ mov 1, r2 ; \ sllx r2, r1, r1 ; \ TEST(ktr_cpumask, r1, r2, r3, l3) ; \ ATR(desc, r1, r2, r3, l1, l2) #endif /* LOCORE */ #endif /* !_MACHINE_KTR_H_ */ Index: user/imp/tbemd/sys/sparc64/include/md_var.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/md_var.h (revision 203849) +++ user/imp/tbemd/sys/sparc64/include/md_var.h (revision 203850) @@ -1,76 +1,78 @@ /*- * Copyright (c) 1995 Bruce D. Evans. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the author nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/include/md_var.h,v 1.40 2001/07/12 * $FreeBSD$ */ #ifndef _MACHINE_MD_VAR_H_ #define _MACHINE_MD_VAR_H_ typedef void cpu_block_copy_t(const void *src, void *dst, size_t len); typedef void cpu_block_zero_t(void *dst, size_t len); extern char tl0_base[]; extern char _end[]; extern long Maxmem; extern vm_offset_t kstack0; extern vm_paddr_t kstack0_phys; struct pcpu; struct md_utrap; +const char *cpu_cpuid_prop(void); +uint32_t cpu_get_mid(void); void cpu_identify(u_long vers, u_int clock, u_int id); void cpu_setregs(struct pcpu *pc); int is_physical_memory(vm_paddr_t addr); struct md_utrap *utrap_alloc(void); void utrap_free(struct md_utrap *ut); struct md_utrap *utrap_hold(struct md_utrap *ut); cpu_block_copy_t spitfire_block_copy; cpu_block_zero_t spitfire_block_zero; extern cpu_block_copy_t *cpu_block_copy; extern cpu_block_zero_t *cpu_block_zero; /* * Given that the Sun disk label only uses 16-bit fields for cylinders, * heads and sectors we might need to adjust the geometry of large IDE * disks. * We have to have a knowledge that a device_t is a struct device * here * to avoid including too many things from this file. */ struct disk; struct device; void sparc64_ad_firmware_geom_adjust(struct device *dev, struct disk *disk); #define ad_firmware_geom_adjust(dev, dsk) \ sparc64_ad_firmware_geom_adjust(dev, dsk) #endif /* !_MACHINE_MD_VAR_H_ */ Index: user/imp/tbemd/sys/sparc64/include/nexusvar.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/nexusvar.h (nonexistent) +++ user/imp/tbemd/sys/sparc64/include/nexusvar.h (revision 203850) @@ -0,0 +1,34 @@ +/*- + * Copyright (c) 2010 Marius Strobl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_NEXUSVAR_H_ +#define _MACHINE_NEXUSVAR_H_ + +DECLARE_CLASS(nexus_driver); + +#endif /* _MACHINE_NEXUSVAR_H_ */ Property changes on: user/imp/tbemd/sys/sparc64/include/nexusvar.h ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: user/imp/tbemd/sys/sparc64/include/upa.h =================================================================== --- user/imp/tbemd/sys/sparc64/include/upa.h (revision 203849) +++ user/imp/tbemd/sys/sparc64/include/upa.h (revision 203850) @@ -1,50 +1,41 @@ /*- * Copyright (c) 2001 by Thomas Moestl . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_UPA_H_ -#define _MACHINE_UPA_H_ +#define _MACHINE_UPA_H_ #define UPA_MEMSTART 0x1c000000000UL #define UPA_MEMEND 0x1ffffffffffUL #define UPA_CR_MID_SHIFT (17) #define UPA_CR_MID_SIZE (5) -#define UPA_CR_MID_MASK \ +#define UPA_CR_MID_MASK \ (((1 << UPA_CR_MID_SIZE) - 1) << UPA_CR_MID_SHIFT) #define UPA_CR_GET_MID(cr) ((cr & UPA_CR_MID_MASK) >> UPA_CR_MID_SHIFT) - -#ifdef LOCORE - -#define UPA_GET_MID(r1) \ - ldxa [%g0] ASI_UPA_CONFIG_REG, r1 ; \ - srlx r1, UPA_CR_MID_SHIFT, r1 ; \ - and r1, (1 << UPA_CR_MID_SIZE) - 1, r1 - -#endif #endif /* _MACHINE_UPA_H_ */ Index: user/imp/tbemd/sys/sparc64/sparc64/locore.S =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/locore.S (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/locore.S (revision 203850) @@ -1,150 +1,152 @@ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include "assym.s" .register %g2,#ignore .globl kernbase .set kernbase, KERNBASE /* * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3, * u_long ofw_vec) */ ENTRY(btext) ENTRY(_start) /* * Initialize misc. state to known values: interrupts disabled, normal - * globals, windows flushed (cr = 0, cs = nwindows - 1), no clean - * windows, pil 0, and floating point disabled. + * globals, windows flushed (cr = 0, cs = nwindows - 1), PIL 0 and + * floating point disabled. + * Note that some firmware versions don't implement a clean window + * trap handler so we unfortunately can't clear the windows by setting + * %cleanwin to zero here. */ wrpr %g0, PSTATE_NORMAL, %pstate flushw - wrpr %g0, 0, %cleanwin wrpr %g0, 0, %pil wr %g0, 0, %fprs /* * Get onto our per-CPU panic stack, which precedes the struct pcpu in * the per-CPU page. */ SET(pcpu0 + (PCPU_PAGES * PAGE_SIZE) - PC_SIZEOF, %l1, %l0) sub %l0, SPOFF + CCFSZ, %sp /* * Enable interrupts. */ wrpr %g0, PSTATE_KERNEL, %pstate /* * Do initial bootstrap to setup pmap and thread0. */ call sparc64_init nop /* * Get onto thread0's kstack. */ sub PCB_REG, SPOFF + CCFSZ, %sp /* * And away we go. This doesn't return. */ call mi_startup nop sir ! NOTREACHED END(_start) /* * void cpu_setregs(struct pcpu *pc) */ ENTRY(cpu_setregs) ldx [%o0 + PC_CURPCB], %o1 /* * Disable interrupts, normal globals. */ wrpr %g0, PSTATE_NORMAL, %pstate /* * Normal %g6 points to the current thread's PCB, and %g7 points to * the per-CPU data structure. */ mov %o1, PCB_REG mov %o0, PCPU_REG /* * Switch to alternate globals. */ wrpr %g0, PSTATE_ALT, %pstate /* * Alternate %g5 points to a per-CPU panic stack, %g6 points to the * current thread's PCB, and %g7 points to the per-CPU data structure. */ mov %o0, ASP_REG mov %o1, PCB_REG mov %o0, PCPU_REG /* * Switch to interrupt globals. */ wrpr %g0, PSTATE_INTR, %pstate /* * Interrupt %g7 points to the per-CPU data structure. */ mov %o0, PCPU_REG /* * Switch to normal globals again. */ wrpr %g0, PSTATE_NORMAL, %pstate /* * Force trap level 1 and take over the trap table. */ SET(tl0_base, %o2, %o1) wrpr %g0, 1, %tl wrpr %o1, 0, %tba /* * Re-enable interrupts. */ wrpr %g0, PSTATE_KERNEL, %pstate retl nop END(cpu_setregs) Index: user/imp/tbemd/sys/sparc64/sparc64/machdep.c =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/machdep.c (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/machdep.c (revision 203850) @@ -1,1032 +1,1099 @@ /*- * Copyright (c) 2001 Jake Burkholder. * Copyright (c) 1992 Terrence R. Lambert. * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 * from: FreeBSD: src/sys/i386/i386/machdep.c,v 1.477 2001/08/27 */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_msgbuf.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include +#include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef int ofw_vec_t(void *); #ifdef DDB extern vm_offset_t ksym_start, ksym_end; #endif int dtlb_slots; int itlb_slots; struct tlb_entry *kernel_tlbs; int kernel_tlb_slots; int cold = 1; long Maxmem; long realmem; void *dpcpu0; char pcpu0[PCPU_PAGES * PAGE_SIZE]; struct trapframe frame0; vm_offset_t kstack0; vm_paddr_t kstack0_phys; struct kva_md_info kmi; u_long ofw_vec; u_long ofw_tba; char sparc64_model[32]; static int cpu_use_vis = 1; cpu_block_copy_t *cpu_block_copy; cpu_block_zero_t *cpu_block_zero; +static phandle_t find_bsp(phandle_t node, uint32_t bspid); void sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec); -void sparc64_shutdown_final(void *dummy, int howto); +static void sparc64_shutdown_final(void *dummy, int howto); -static void cpu_startup(void *); +static void cpu_startup(void *arg); SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); CTASSERT((1 << INT_SHIFT) == sizeof(int)); CTASSERT((1 << PTR_SHIFT) == sizeof(char *)); CTASSERT(sizeof(struct reg) == 256); CTASSERT(sizeof(struct fpreg) == 272); CTASSERT(sizeof(struct __mcontext) == 512); CTASSERT((sizeof(struct pcb) & (64 - 1)) == 0); CTASSERT((offsetof(struct pcb, pcb_kfp) & (64 - 1)) == 0); CTASSERT((offsetof(struct pcb, pcb_ufp) & (64 - 1)) == 0); CTASSERT(sizeof(struct pcb) <= ((KSTACK_PAGES * PAGE_SIZE) / 8)); CTASSERT(sizeof(struct pcpu) <= ((PCPU_PAGES * PAGE_SIZE) / 2)); static void cpu_startup(void *arg) { vm_paddr_t physsz; int i; physsz = 0; for (i = 0; i < sparc64_nmemreg; i++) physsz += sparc64_memreg[i].mr_size; printf("real memory = %lu (%lu MB)\n", physsz, physsz / (1024 * 1024)); realmem = (long)physsz / PAGE_SIZE; vm_ksubmap_init(&kmi); bufinit(); vm_pager_bufferinit(); EVENTHANDLER_REGISTER(shutdown_final, sparc64_shutdown_final, NULL, SHUTDOWN_PRI_LAST); printf("avail memory = %lu (%lu MB)\n", cnt.v_free_count * PAGE_SIZE, cnt.v_free_count / ((1024 * 1024) / PAGE_SIZE)); if (bootverbose) printf("machine: %s\n", sparc64_model); cpu_identify(rdpr(ver), PCPU_GET(clock), curcpu); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { struct intr_request *ir; int i; pcpu->pc_irtail = &pcpu->pc_irhead; for (i = 0; i < IR_FREE; i++) { ir = &pcpu->pc_irpool[i]; ir->ir_next = pcpu->pc_irfree; pcpu->pc_irfree = ir; } } void spinlock_enter(void) { struct thread *td; register_t pil; td = curthread; if (td->td_md.md_spinlock_count == 0) { pil = rdpr(pil); wrpr(pil, 0, PIL_TICK); td->td_md.md_saved_pil = pil; } td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; td = curthread; critical_exit(); td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) wrpr(pil, td->td_md.md_saved_pil, 0); } +static phandle_t +find_bsp(phandle_t node, uint32_t bspid) +{ + char type[sizeof("cpu")]; + phandle_t child; + uint32_t cpuid; + + for (; node != 0; node = OF_peer(node)) { + child = OF_child(node); + if (child > 0) { + child = find_bsp(child, bspid); + if (child > 0) + return (child); + } else { + if (OF_getprop(node, "device_type", type, + sizeof(type)) <= 0) + continue; + if (strcmp(type, "cpu") != 0) + continue; + if (OF_getprop(node, cpu_cpuid_prop(), &cpuid, + sizeof(cpuid)) <= 0) + continue; + if (cpuid == bspid) + return (node); + } + } + return (0); +} + +const char * +cpu_cpuid_prop(void) +{ + + switch (cpu_impl) { + case CPU_IMPL_SPARC64: + case CPU_IMPL_ULTRASPARCI: + case CPU_IMPL_ULTRASPARCII: + case CPU_IMPL_ULTRASPARCIIi: + case CPU_IMPL_ULTRASPARCIIe: + return ("upa-portid"); + case CPU_IMPL_ULTRASPARCIII: + case CPU_IMPL_ULTRASPARCIIIp: + case CPU_IMPL_ULTRASPARCIIIi: + case CPU_IMPL_ULTRASPARCIIIip: + return ("portid"); + case CPU_IMPL_ULTRASPARCIV: + case CPU_IMPL_ULTRASPARCIVp: + return ("cpuid"); + default: + return (""); + } +} + +uint32_t +cpu_get_mid(void) +{ + + switch (cpu_impl) { + case CPU_IMPL_SPARC64: + case CPU_IMPL_ULTRASPARCI: + case CPU_IMPL_ULTRASPARCII: + case CPU_IMPL_ULTRASPARCIIi: + case CPU_IMPL_ULTRASPARCIIe: + return (UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIII: + case CPU_IMPL_ULTRASPARCIIIp: + return (FIREPLANE_CR_GET_AID(ldxa(AA_FIREPLANE_CONFIG, + ASI_FIREPLANE_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIIIi: + case CPU_IMPL_ULTRASPARCIIIip: + return (JBUS_CR_GET_JID(ldxa(0, ASI_JBUS_CONFIG_REG))); + case CPU_IMPL_ULTRASPARCIV: + case CPU_IMPL_ULTRASPARCIVp: + return (INTR_ID_GET_ID(ldxa(AA_INTR_ID, ASI_INTR_ID))); + default: + return (0); + } +} + void sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec) { - char type[8]; char *env; struct pcpu *pc; vm_offset_t end; vm_offset_t va; caddr_t kmdp; - phandle_t child; phandle_t root; - uint32_t portid; end = 0; kmdp = NULL; /* * Find out what kind of CPU we have first, for anything that changes * behaviour. */ cpu_impl = VER_IMPL(rdpr(ver)); /* * Do CPU-specific Initialization. */ if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(); /* * Clear (S)TICK timer (including NPT). */ tick_clear(); /* * UltraSparc II[e,i] based systems come up with the tick interrupt * enabled and a handler that resets the tick counter, causing DELAY() * to not work properly when used early in boot. * UltraSPARC III based systems come up with the system tick interrupt * enabled, causing an interrupt storm on startup since they are not * handled. */ tick_stop(); /* * Set up Open Firmware entry points. */ ofw_tba = rdpr(tba); ofw_vec = (u_long)vec; /* * Parse metadata if present and fetch parameters. Must be before the * console is inited so cninit gets the right value of boothowto. */ if (mdp != NULL) { preload_metadata = mdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp != NULL) { boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *); end = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); kernel_tlb_slots = MD_FETCH(kmdp, MODINFOMD_DTLB_SLOTS, int); kernel_tlbs = (void *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_DTLB); } } init_param1(); /* * Initialize Open Firmware (needed for console). */ OF_install(OFW_STD_DIRECT, 0); OF_init(ofw_entry); /* * Prime our per-CPU data page for use. Note, we are using it for * our stack, so don't pass the real size (PAGE_SIZE) to pcpu_init * or it'll zero it out from under us. */ pc = (struct pcpu *)(pcpu0 + (PCPU_PAGES * PAGE_SIZE)) - 1; pcpu_init(pc, 0, sizeof(struct pcpu)); pc->pc_addr = (vm_offset_t)pcpu0; - pc->pc_mid = UPA_CR_GET_MID(ldxa(0, ASI_UPA_CONFIG_REG)); + pc->pc_mid = cpu_get_mid(); pc->pc_tlb_ctx = TLB_CTX_USER_MIN; pc->pc_tlb_ctx_min = TLB_CTX_USER_MIN; pc->pc_tlb_ctx_max = TLB_CTX_USER_MAX; /* * Determine the OFW node and frequency of the BSP (and ensure the * BSP is in the device tree in the first place). */ - pc->pc_node = 0; root = OF_peer(0); - for (child = OF_child(root); child != 0; child = OF_peer(child)) { - if (OF_getprop(child, "device_type", type, sizeof(type)) <= 0) - continue; - if (strcmp(type, "cpu") != 0) - continue; - if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ? - "upa-portid" : "portid", &portid, sizeof(portid)) <= 0) - continue; - if (portid == pc->pc_mid) { - pc->pc_node = child; - break; - } - } + pc->pc_node = find_bsp(root, pc->pc_mid); if (pc->pc_node == 0) OF_exit(); - if (OF_getprop(child, "clock-frequency", &pc->pc_clock, + if (OF_getprop(pc->pc_node, "clock-frequency", &pc->pc_clock, sizeof(pc->pc_clock)) <= 0) OF_exit(); /* * Provide a DELAY() that works before PCPU_REG is set. We can't * set PCPU_REG without also taking over the trap table or the * firmware will overwrite it. Unfortunately, it's way to early * to also take over the trap table at this point. */ clock_boot = pc->pc_clock; delay_func = delay_boot; /* * Initialize the console before printing anything. * NB: the low-level console drivers require a working DELAY() at * this point. */ cninit(); /* * Panic if there is no metadata. Most likely the kernel was booted * directly, instead of through loader(8). */ if (mdp == NULL || kmdp == NULL || end == 0 || kernel_tlb_slots == 0 || kernel_tlbs == NULL) { printf("sparc64_init: missing loader metadata.\n" "This probably means you are not using loader(8).\n"); panic("sparc64_init"); } /* * Work around the broken loader behavior of not demapping no * longer used kernel TLB slots when unloading the kernel or * modules. */ for (va = KERNBASE + (kernel_tlb_slots - 1) * PAGE_SIZE_4M; va >= roundup2(end, PAGE_SIZE_4M); va -= PAGE_SIZE_4M) { if (bootverbose) printf("demapping unused kernel TLB slot " "(va %#lx - %#lx)\n", va, va + PAGE_SIZE_4M - 1); stxa(TLB_DEMAP_VA(va) | TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(va) | TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, ASI_IMMU_DEMAP, 0); flush(KERNBASE); kernel_tlb_slots--; } /* * Determine the TLB slot maxima, which are expected to be * equal across all CPUs. * NB: for Cheetah-class CPUs, these properties only refer * to the t16s. */ if (OF_getprop(pc->pc_node, "#dtlb-entries", &dtlb_slots, sizeof(dtlb_slots)) == -1) panic("sparc64_init: cannot determine number of dTLB slots"); if (OF_getprop(pc->pc_node, "#itlb-entries", &itlb_slots, sizeof(itlb_slots)) == -1) panic("sparc64_init: cannot determine number of iTLB slots"); cache_init(pc); cache_enable(); uma_set_align(pc->pc_cache.dc_linesize - 1); cpu_block_copy = bcopy; cpu_block_zero = bzero; getenv_int("machdep.use_vis", &cpu_use_vis); if (cpu_use_vis) { switch (cpu_impl) { case CPU_IMPL_SPARC64: case CPU_IMPL_ULTRASPARCI: case CPU_IMPL_ULTRASPARCII: case CPU_IMPL_ULTRASPARCIIi: case CPU_IMPL_ULTRASPARCIIe: case CPU_IMPL_ULTRASPARCIII: /* NB: we've disabled P$. */ case CPU_IMPL_ULTRASPARCIIIp: case CPU_IMPL_ULTRASPARCIIIi: case CPU_IMPL_ULTRASPARCIV: case CPU_IMPL_ULTRASPARCIVp: case CPU_IMPL_ULTRASPARCIIIip: cpu_block_copy = spitfire_block_copy; cpu_block_zero = spitfire_block_zero; break; } } #ifdef SMP mp_init(); #endif /* * Initialize virtual memory and calculate physmem. */ pmap_bootstrap(); /* * Initialize tunables. */ init_param2(physmem); env = getenv("kernelname"); if (env != NULL) { strlcpy(kernelname, env, sizeof(kernelname)); freeenv(env); } /* * Initialize the interrupt tables. */ intr_init1(); /* * Initialize proc0, set kstack0, frame0, curthread and curpcb. */ proc_linkup0(&proc0, &thread0); proc0.p_md.md_sigtramp = NULL; proc0.p_md.md_utrap = NULL; thread0.td_kstack = kstack0; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; frame0.tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_PRIV; thread0.td_frame = &frame0; pc->pc_curthread = &thread0; pc->pc_curpcb = thread0.td_pcb; /* * Initialize global registers. */ cpu_setregs(pc); /* * Take over the trap table via the PROM. Using the PROM for this * is necessary in order to set obp-control-relinquished to true * within the PROM so obtaining /virtual-memory/translations doesn't * trigger a fatal reset error or worse things further down the road. * XXX it should be possible to use this soley instead of writing * %tba in cpu_setregs(). Doing so causes a hang however. */ sun4u_set_traptable(tl0_base); /* * It's now safe to use the real DELAY(). */ delay_func = delay_tick; /* * Initialize the dynamic per-CPU area for the BSP and the message * buffer (after setting the trap table). */ dpcpu_init(dpcpu0, 0); msgbufinit(msgbufp, MSGBUF_SIZE); mutex_init(); intr_init2(); /* * Finish pmap initialization now that we're ready for mutexes. */ PMAP_LOCK_INIT(kernel_pmap); OF_getprop(root, "name", sparc64_model, sizeof(sparc64_model) - 1); kdb_init(); #ifdef KDB if (boothowto & RB_KDB) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); #endif } void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct trapframe *tf; struct sigframe *sfp; struct sigacts *psp; struct sigframe sf; struct thread *td; struct frame *fp; struct proc *p; u_long sp; int oonstack; int sig; oonstack = 0; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; sp = tf->tf_sp + SPOFF; oonstack = sigonstack(sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Make sure we have a signal trampoline to return to. */ if (p->p_md.md_sigtramp == NULL) { /* * No signal trampoline... kill the process. */ CTR0(KTR_SIG, "sendsig: no sigtramp"); printf("sendsig: %s is too old, rebuild it\n", p->p_comm); sigexit(td, sig); /* NOTREACHED */ } /* Save user context. */ bzero(&sf, sizeof(sf)); get_mcontext(td, &sf.sf_uc.uc_mcontext, 0); sf.sf_uc.uc_sigmask = *mask; sf.sf_uc.uc_stack = td->td_sigstk; sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE; /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack && SIGISMEMBER(psp->ps_sigonstack, sig)) { sfp = (struct sigframe *)(td->td_sigstk.ss_sp + td->td_sigstk.ss_size - sizeof(struct sigframe)); } else sfp = (struct sigframe *)sp - 1; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(p); fp = (struct frame *)sfp - 1; /* Translate the signal if appropriate. */ if (p->p_sysent->sv_sigtbl && sig <= p->p_sysent->sv_sigsize) sig = p->p_sysent->sv_sigtbl[_SIG_IDX(sig)]; /* Build the argument list for the signal handler. */ tf->tf_out[0] = sig; tf->tf_out[2] = (register_t)&sfp->sf_uc; tf->tf_out[4] = (register_t)catcher; if (SIGISMEMBER(psp->ps_siginfo, sig)) { /* Signal handler installed with SA_SIGINFO. */ tf->tf_out[1] = (register_t)&sfp->sf_si; /* Fill in POSIX parts. */ sf.sf_si = ksi->ksi_info; sf.sf_si.si_signo = sig; /* maybe a translated signal */ } else { /* Old FreeBSD-style arguments. */ tf->tf_out[1] = ksi->ksi_code; tf->tf_out[3] = (register_t)ksi->ksi_addr; } /* Copy the sigframe out to the user's stack. */ if (rwindow_save(td) != 0 || copyout(&sf, sfp, sizeof(*sfp)) != 0 || suword(&fp->fr_in[6], tf->tf_out[6]) != 0) { /* * Something is wrong with the stack pointer. * ...Kill the process. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p sfp=%p", td, sfp); PROC_LOCK(p); sigexit(td, SIGILL); /* NOTREACHED */ } tf->tf_tpc = (u_long)p->p_md.md_sigtramp; tf->tf_tnpc = tf->tf_tpc + 4; tf->tf_sp = (u_long)fp - SPOFF; CTR3(KTR_SIG, "sendsig: return td=%p pc=%#lx sp=%#lx", td, tf->tf_tpc, tf->tf_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } #ifndef _SYS_SYSPROTO_H_ struct sigreturn_args { ucontext_t *ucp; }; #endif /* * MPSAFE */ int sigreturn(struct thread *td, struct sigreturn_args *uap) { struct proc *p; mcontext_t *mc; ucontext_t uc; int error; p = td->td_proc; if (rwindow_save(td)) { PROC_LOCK(p); sigexit(td, SIGILL); } CTR2(KTR_SIG, "sigreturn: td=%p ucp=%p", td, uap->sigcntxp); if (copyin(uap->sigcntxp, &uc, sizeof(uc)) != 0) { CTR1(KTR_SIG, "sigreturn: efault td=%p", td); return (EFAULT); } mc = &uc.uc_mcontext; error = set_mcontext(td, mc); if (error != 0) return (error); kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); CTR4(KTR_SIG, "sigreturn: return td=%p pc=%#lx sp=%#lx tstate=%#lx", td, mc->mc_tpc, mc->mc_sp, mc->mc_tstate); return (EJUSTRETURN); } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { pcb->pcb_pc = tf->tf_tpc; pcb->pcb_sp = tf->tf_sp; } int get_mcontext(struct thread *td, mcontext_t *mc, int flags) { struct trapframe *tf; struct pcb *pcb; tf = td->td_frame; pcb = td->td_pcb; /* * Copy the registers which will be restored by tl0_ret() from the * trapframe. * Note that we skip %g7 which is used as the userland TLS register * and %wstate. */ mc->mc_flags = _MC_VERSION; mc->mc_global[1] = tf->tf_global[1]; mc->mc_global[2] = tf->tf_global[2]; mc->mc_global[3] = tf->tf_global[3]; mc->mc_global[4] = tf->tf_global[4]; mc->mc_global[5] = tf->tf_global[5]; mc->mc_global[6] = tf->tf_global[6]; if (flags & GET_MC_CLEAR_RET) { mc->mc_out[0] = 0; mc->mc_out[1] = 0; } else { mc->mc_out[0] = tf->tf_out[0]; mc->mc_out[1] = tf->tf_out[1]; } mc->mc_out[2] = tf->tf_out[2]; mc->mc_out[3] = tf->tf_out[3]; mc->mc_out[4] = tf->tf_out[4]; mc->mc_out[5] = tf->tf_out[5]; mc->mc_out[6] = tf->tf_out[6]; mc->mc_out[7] = tf->tf_out[7]; mc->mc_fprs = tf->tf_fprs; mc->mc_fsr = tf->tf_fsr; mc->mc_gsr = tf->tf_gsr; mc->mc_tnpc = tf->tf_tnpc; mc->mc_tpc = tf->tf_tpc; mc->mc_tstate = tf->tf_tstate; mc->mc_y = tf->tf_y; critical_enter(); if ((tf->tf_fprs & FPRS_FEF) != 0) { savefpctx(pcb->pcb_ufp); tf->tf_fprs &= ~FPRS_FEF; pcb->pcb_flags |= PCB_FEF; } if ((pcb->pcb_flags & PCB_FEF) != 0) { bcopy(pcb->pcb_ufp, mc->mc_fp, sizeof(mc->mc_fp)); mc->mc_fprs |= FPRS_FEF; } critical_exit(); return (0); } int set_mcontext(struct thread *td, const mcontext_t *mc) { struct trapframe *tf; struct pcb *pcb; if (!TSTATE_SECURE(mc->mc_tstate) || (mc->mc_flags & ((1L << _MC_VERSION_BITS) - 1)) != _MC_VERSION) return (EINVAL); tf = td->td_frame; pcb = td->td_pcb; /* Make sure the windows are spilled first. */ flushw(); /* * Copy the registers which will be restored by tl0_ret() to the * trapframe. * Note that we skip %g7 which is used as the userland TLS register * and %wstate. */ tf->tf_global[1] = mc->mc_global[1]; tf->tf_global[2] = mc->mc_global[2]; tf->tf_global[3] = mc->mc_global[3]; tf->tf_global[4] = mc->mc_global[4]; tf->tf_global[5] = mc->mc_global[5]; tf->tf_global[6] = mc->mc_global[6]; tf->tf_out[0] = mc->mc_out[0]; tf->tf_out[1] = mc->mc_out[1]; tf->tf_out[2] = mc->mc_out[2]; tf->tf_out[3] = mc->mc_out[3]; tf->tf_out[4] = mc->mc_out[4]; tf->tf_out[5] = mc->mc_out[5]; tf->tf_out[6] = mc->mc_out[6]; tf->tf_out[7] = mc->mc_out[7]; tf->tf_fprs = mc->mc_fprs; tf->tf_fsr = mc->mc_fsr; tf->tf_gsr = mc->mc_gsr; tf->tf_tnpc = mc->mc_tnpc; tf->tf_tpc = mc->mc_tpc; tf->tf_tstate = mc->mc_tstate; tf->tf_y = mc->mc_y; if ((mc->mc_fprs & FPRS_FEF) != 0) { tf->tf_fprs = 0; bcopy(mc->mc_fp, pcb->pcb_ufp, sizeof(pcb->pcb_ufp)); pcb->pcb_flags |= PCB_FEF; } return (0); } /* * Exit the kernel and execute a firmware call that will not return, as * specified by the arguments. */ void cpu_shutdown(void *args) { #ifdef SMP cpu_mp_shutdown(); #endif ofw_exit(args); } /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { /* TBD */ } /* Get current clock frequency for the given CPU ID. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { struct pcpu *pc; pc = pcpu_find(cpu_id); if (pc == NULL || rate == NULL) return (EINVAL); *rate = pc->pc_clock; return (0); } /* * Duplicate OF_exit() with a different firmware call function that restores * the trap table, otherwise a RED state exception is triggered in at least * some firmware versions. */ void cpu_halt(void) { static struct { cell_t name; cell_t nargs; cell_t nreturns; } args = { (cell_t)"exit", 0, 0 }; cpu_shutdown(&args); } -void +static void sparc64_shutdown_final(void *dummy, int howto) { static struct { cell_t name; cell_t nargs; cell_t nreturns; } args = { (cell_t)"SUNW,power-off", 0, 0 }; /* Turn the power off? */ if ((howto & RB_POWEROFF) != 0) cpu_shutdown(&args); /* In case of halt, return to the firmware. */ if ((howto & RB_HALT) != 0) cpu_halt(); } void cpu_idle(int busy) { /* Insert code to halt (until next interrupt) for the idle loop. */ } int cpu_idle_wakeup(int cpu) { return (0); } int ptrace_set_pc(struct thread *td, u_long addr) { td->td_frame->tf_tpc = addr; td->td_frame->tf_tnpc = addr + 4; return (0); } int ptrace_single_step(struct thread *td) { /* TODO; */ return (0); } int ptrace_clear_single_step(struct thread *td) { /* TODO; */ return (0); } void exec_setregs(struct thread *td, u_long entry, u_long stack, u_long ps_strings) { struct trapframe *tf; struct pcb *pcb; struct proc *p; u_long sp; /* XXX no cpu_exec */ p = td->td_proc; p->p_md.md_sigtramp = NULL; if (p->p_md.md_utrap != NULL) { utrap_free(p->p_md.md_utrap); p->p_md.md_utrap = NULL; } pcb = td->td_pcb; tf = td->td_frame; sp = rounddown(stack, 16); bzero(pcb, sizeof(*pcb)); bzero(tf, sizeof(*tf)); tf->tf_out[0] = stack; tf->tf_out[3] = p->p_sysent->sv_psstrings; tf->tf_out[6] = sp - SPOFF - sizeof(struct frame); tf->tf_tnpc = entry + 4; tf->tf_tpc = entry; tf->tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_MM_TSO; td->td_retval[0] = tf->tf_out[0]; td->td_retval[1] = tf->tf_out[1]; } int fill_regs(struct thread *td, struct reg *regs) { bcopy(td->td_frame, regs, sizeof(*regs)); return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *tf; if (!TSTATE_SECURE(regs->r_tstate)) return (EINVAL); tf = td->td_frame; regs->r_wstate = tf->tf_wstate; bcopy(regs, tf, sizeof(*regs)); return (0); } int fill_dbregs(struct thread *td, struct dbreg *dbregs) { return (ENOSYS); } int set_dbregs(struct thread *td, struct dbreg *dbregs) { return (ENOSYS); } int fill_fpregs(struct thread *td, struct fpreg *fpregs) { struct trapframe *tf; struct pcb *pcb; pcb = td->td_pcb; tf = td->td_frame; bcopy(pcb->pcb_ufp, fpregs->fr_regs, sizeof(fpregs->fr_regs)); fpregs->fr_fsr = tf->tf_fsr; fpregs->fr_gsr = tf->tf_gsr; return (0); } int set_fpregs(struct thread *td, struct fpreg *fpregs) { struct trapframe *tf; struct pcb *pcb; pcb = td->td_pcb; tf = td->td_frame; tf->tf_fprs &= ~FPRS_FEF; bcopy(fpregs->fr_regs, pcb->pcb_ufp, sizeof(pcb->pcb_ufp)); tf->tf_fsr = fpregs->fr_fsr; tf->tf_gsr = fpregs->fr_gsr; return (0); } struct md_utrap * utrap_alloc(void) { struct md_utrap *ut; ut = malloc(sizeof(struct md_utrap), M_SUBPROC, M_WAITOK | M_ZERO); ut->ut_refcnt = 1; return (ut); } void utrap_free(struct md_utrap *ut) { int refcnt; if (ut == NULL) return; mtx_pool_lock(mtxpool_sleep, ut); ut->ut_refcnt--; refcnt = ut->ut_refcnt; mtx_pool_unlock(mtxpool_sleep, ut); if (refcnt == 0) free(ut, M_SUBPROC); } struct md_utrap * utrap_hold(struct md_utrap *ut) { if (ut == NULL) return (NULL); mtx_pool_lock(mtxpool_sleep, ut); ut->ut_refcnt++; mtx_pool_unlock(mtxpool_sleep, ut); return (ut); } Index: user/imp/tbemd/sys/sparc64/sparc64/mp_locore.S =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/mp_locore.S (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/mp_locore.S (revision 203850) @@ -1,302 +1,290 @@ /*- * Copyright (c) 2002 Jake Burkholder. * Copyright (c) 2008 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include -#include #include #include "assym.s" .register %g2, #ignore .register %g3, #ignore .text _ALIGN_TEXT /* - * Initialize misc. state to known values: interrupts disabled, - * normal globals, no clean windows, PIL 0, and floating point - * disabled. + * Initialize misc. state to known values: interrupts disabled, normal + * globals, windows flushed (cr = 0, cs = nwindows - 1), PIL 0 and + * floating point disabled. + * Note that some firmware versions don't implement a clean window + * trap handler so we unfortunately can't clear the windows by setting + * %cleanwin to zero here. */ 1: wrpr %g0, PSTATE_NORMAL, %pstate - wrpr %g0, 0, %cleanwin wrpr %g0, 0, %pil wr %g0, 0, %fprs rdpr %ver, %l7 srlx %l7, VER_IMPL_SHIFT, %l7 sll %l7, VER_IMPL_SIZE, %l7 srl %l7, VER_IMPL_SIZE, %l7 cmp %l7, CPU_IMPL_ULTRASPARCIIIp bne %icc, 3f nop /* * Relocate the locked entry in it16 slot 0 (if existent) * as part of working around Cheetah+ erratum 34. */ setx TD_V | TD_L, %l1, %l0 /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work + * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ ldxa [%g0] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%g0] ASI_ITLB_DATA_ACCESS_REG, %l6 and %l6, %l0, %l1 cmp %l0, %l1 bne %xcc, 3f nop /* Flush the mapping of slot 0. */ ldxa [%g0] ASI_ITLB_TAG_READ_REG, %l5 srlx %l5, TAR_VPN_SHIFT, %l0 sllx %l0, TAR_VPN_SHIFT, %l0 or %l0, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %l0 stxa %g0, [%l0] ASI_IMMU_DEMAP /* The USIII-family ignores the address. */ flush %g0 /* * Search a replacement slot != 0 and enter the data and tag * that formerly were in slot 0. */ mov (1 << TLB_DAR_SLOT_SHIFT), %l4 setx TD_V, %l1, %l0 /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work + * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ 2: ldxa [%l4] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%l4] ASI_ITLB_DATA_ACCESS_REG, %l1 and %l1, %l0, %l1 cmp %l0, %l1 be,a %xcc, 2b add %l4, (1 << TLB_DAR_SLOT_SHIFT), %l4 wr %g0, ASI_IMMU, %asi stxa %l5, [%g0 + AA_IMMU_TAR] %asi stxa %l6, [%l4] ASI_ITLB_DATA_ACCESS_REG /* The USIII-family ignores the address. */ flush %g0 3: rd %pc, %l6 ldx [%l6 + (9f-3b)], %l1 add %l6, (11f-3b), %l2 clr %l3 4: cmp %l3, %l1 be %xcc, 8f nop ldx [%l2 + TTE_VPN], %l4 ldx [%l2 + TTE_DATA], %l5 srlx %l4, TV_SIZE_BITS, %l4 sllx %l4, PAGE_SHIFT_4M, %l4 wr %g0, ASI_DMMU, %asi stxa %l4, [%g0 + AA_DMMU_TAR] %asi stxa %l5, [%g0] ASI_DTLB_DATA_IN_REG membar #Sync cmp %l7, CPU_IMPL_ULTRASPARCIIIp bne %icc, 6f wr %g0, ASI_IMMU, %asi /* * Search an unused slot != 0 and explicitly enter the data * and tag there in order to avoid Cheetah+ erratum 34. */ mov (1 << TLB_DAR_SLOT_SHIFT), %l0 setx TD_V, %o1, %o0 /* - * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work + * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work * around errata of USIII and beyond. */ 5: ldxa [%l0] ASI_ITLB_DATA_ACCESS_REG, %g0 ldxa [%l0] ASI_ITLB_DATA_ACCESS_REG, %o1 and %o1, %o0, %o1 cmp %o0, %o1 be,a %xcc, 5b add %l0, (1 << TLB_DAR_SLOT_SHIFT), %l0 sethi %hi(KERNBASE), %o0 stxa %l4, [%g0 + AA_IMMU_TAR] %asi stxa %l5, [%l0] ASI_ITLB_DATA_ACCESS_REG flush %o0 ba %xcc, 7f nop 6: sethi %hi(KERNBASE), %l0 stxa %l4, [%g0 + AA_IMMU_TAR] %asi stxa %l5, [%g0] ASI_ITLB_DATA_IN_REG flush %l0 7: add %l2, 1 << TTE_SHIFT, %l2 add %l3, 1, %l3 ba %xcc, 4b nop 8: ldx [%l6 + (10f-3b)], %l1 jmpl %l1, %g0 nop _ALIGN_DATA 9: .xword 0x0 10: .xword 0x0 11: DATA(mp_tramp_code) .xword 1b DATA(mp_tramp_code_len) .xword 11b-1b DATA(mp_tramp_tlb_slots) .xword 9b-1b DATA(mp_tramp_func) .xword 10b-1b /* * void mp_startup(void) */ ENTRY(mp_startup) SET(cpu_start_args, %l1, %l0) mov CPU_TICKSYNC, %l1 membar #StoreLoad stw %l1, [%l0 + CSA_STATE] 1: ldx [%l0 + CSA_TICK], %l1 brz %l1, 1b nop wrpr %l1, 0, %tick rdpr %ver, %l1 stx %l1, [%l0 + CSA_VER] srlx %l1, VER_IMPL_SHIFT, %l1 sll %l1, VER_IMPL_SIZE, %l1 srl %l1, VER_IMPL_SIZE, %l1 cmp %l1, CPU_IMPL_ULTRASPARCIII bl %icc, 3f nop mov CPU_STICKSYNC, %l1 membar #StoreLoad stw %l1, [%l0 + CSA_STATE] 2: ldx [%l0 + CSA_STICK], %l1 brz %l1, 2b nop wr %l1, 0, %asr24 -3: UPA_GET_MID(%o0) +3: call cpu_get_mid + nop -#if KTR_COMPILE & KTR_SMP - CATR(KTR_SMP, "mp_start: CPU %d entered kernel" - , %g1, %g2, %g3, 7, 8, 9) - stx %o0, [%g1 + KTR_PARM1] -9: -#endif - /* * Inform the boot processor we have inited. */ mov CPU_INIT, %l1 membar #LoadStore stw %l1, [%l0 + CSA_STATE] /* * Wait till its our turn to bootstrap. */ 4: lduw [%l0 + CSA_MID], %l1 cmp %l1, %o0 bne %xcc, 4b nop -#if KTR_COMPILE & KTR_SMP - CATR(KTR_SMP, "_mp_start: CPU %d got start signal" - , %g1, %g2, %g3, 7, 8, 9) - stx %o0, [%g1 + KTR_PARM1] -9: -#endif - add %l0, CSA_TTES, %l1 clr %l2 /* * Map the per-CPU pages. */ 5: sllx %l2, TTE_SHIFT, %l3 add %l1, %l3, %l3 ldx [%l3 + TTE_VPN], %l4 ldx [%l3 + TTE_DATA], %l5 wr %g0, ASI_DMMU, %asi srlx %l4, TV_SIZE_BITS, %l4 sllx %l4, PAGE_SHIFT_8K, %l4 stxa %l4, [%g0 + AA_DMMU_TAR] %asi stxa %l5, [%g0] ASI_DTLB_DATA_IN_REG membar #Sync add %l2, 1, %l2 cmp %l2, PCPU_PAGES bne %xcc, 5b nop /* * Get onto our per-CPU panic stack, which precedes the struct pcpu * in the per-CPU page. */ ldx [%l0 + CSA_PCPU], %l1 set PCPU_PAGES * PAGE_SIZE - PC_SIZEOF, %l2 add %l1, %l2, %l1 sub %l1, SPOFF + CCFSZ, %sp /* * Enable interrupts. */ wrpr %g0, PSTATE_KERNEL, %pstate #if KTR_COMPILE & KTR_SMP CATR(KTR_SMP, - "_mp_start: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx" + "mp_startup: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx" , %g1, %g2, %g3, 7, 8, 9) lduw [%l1 + PC_CPUID], %g2 stx %g2, [%g1 + KTR_PARM1] lduw [%l1 + PC_MID], %g2 stx %g2, [%g1 + KTR_PARM2] stx %l1, [%g1 + KTR_PARM3] stx %sp, [%g1 + KTR_PARM5] 9: #endif /* * And away we go. This doesn't return. */ call cpu_mp_bootstrap mov %l1, %o0 sir ! NOTREACHED END(mp_startup) Index: user/imp/tbemd/sys/sparc64/sparc64/mp_machdep.c =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/mp_machdep.c (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/mp_machdep.c (revision 203850) @@ -1,586 +1,615 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp */ /*- * Copyright (c) 2002 Jake Burkholder. * Copyright (c) 2007 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SUNW_STARTCPU "SUNW,start-cpu" #define SUNW_STOPSELF "SUNW,stop-self" static ih_func_t cpu_ipi_ast; static ih_func_t cpu_ipi_preempt; static ih_func_t cpu_ipi_stop; /* * Argument area used to pass data to non-boot processors as they start up. * This must be statically initialized with a known invalid CPU module ID, * since the other processors will use it before the boot CPU enters the * kernel. */ struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; struct ipi_cache_args ipi_cache_args; struct ipi_tlb_args ipi_tlb_args; struct pcb stoppcbs[MAXCPU]; struct mtx ipi_mtx; cpu_ipi_selected_t *cpu_ipi_selected; static vm_offset_t mp_tramp; static u_int cpuid_to_mid[MAXCPU]; static int isjbus; static volatile u_int shutdown_cpus; +static void ap_count(phandle_t node, u_int mid); +static void ap_start(phandle_t node, u_int mid); static void cpu_mp_unleash(void *v); +static void foreach_ap(phandle_t node, void (*func)(phandle_t node, + u_int mid)); static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2); static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); static cpu_ipi_selected_t cheetah_ipi_selected; static cpu_ipi_selected_t spitfire_ipi_selected; SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); CTASSERT(MAXCPU <= IDR_CHEETAH_MAX_BN_PAIRS); CTASSERT(MAXCPU <= sizeof(u_int) * NBBY); CTASSERT(MAXCPU <= sizeof(int) * NBBY); void mp_init(void) { struct tte *tp; int i; mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); if (mp_tramp == (vm_offset_t)-1) panic("%s", __func__); bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; tp = (struct tte *)(mp_tramp + mp_tramp_code_len); for (i = 0; i < kernel_tlb_slots; i++) { tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; } for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) flush(mp_tramp + i); /* * On UP systems cpu_ipi_selected() can be called while * cpu_mp_start() wasn't so initialize these here. */ if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || cpu_impl == CPU_IMPL_ULTRASPARCIIIip) isjbus = 1; if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) cpu_ipi_selected = cheetah_ipi_selected; else cpu_ipi_selected = spitfire_ipi_selected; } +static void +foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid)) +{ + char type[sizeof("cpu")]; + phandle_t child; + u_int cpuid; + + /* There's no need to traverse the whole OFW tree twice. */ + if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) + return; + + for (; node != 0; node = OF_peer(node)) { + child = OF_child(node); + if (child > 0) + foreach_ap(child, func); + else { + if (OF_getprop(node, "device_type", type, + sizeof(type)) <= 0) + continue; + if (strcmp(type, "cpu") != 0) + continue; + if (OF_getprop(node, cpu_cpuid_prop(), &cpuid, + sizeof(cpuid)) <= 0) + panic("%s: can't get module ID", __func__); + if (cpuid == PCPU_GET(mid)) + continue; + (*func)(node, cpuid); + } + } +} + /* * Probe for other CPUs. */ void -cpu_mp_setmaxid(void) +cpu_mp_setmaxid() { - char buf[128]; - phandle_t child; - u_int cpus; all_cpus = 1 << curcpu; mp_ncpus = 1; + mp_maxid = 0; - cpus = 0; - for (child = OF_child(OF_peer(0)); child != 0; child = OF_peer(child)) - if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 && - strcmp(buf, "cpu") == 0) - cpus++; - mp_maxid = cpus - 1; + foreach_ap(OF_child(OF_peer(0)), ap_count); } +static void +ap_count(phandle_t node __unused, u_int mid __unused) +{ + + mp_maxid++; +} + int cpu_mp_probe(void) { return (mp_maxid > 0); } struct cpu_group * cpu_topo(void) { return (smp_topo_none()); } static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg) { static struct { cell_t name; cell_t nargs; cell_t nreturns; cell_t cpu; cell_t func; cell_t arg; } args = { (cell_t)SUNW_STARTCPU, 3, }; args.cpu = cpu; args.func = (cell_t)func; args.arg = (cell_t)arg; ofw_entry(&args); } /* * Fire up any non-boot processors. */ void cpu_mp_start(void) { - char buf[128]; - volatile struct cpu_start_args *csa; - struct pcpu *pc; - register_t s; - vm_offset_t va; - phandle_t child; - u_int mid; - u_int clock; - u_int cpuid; mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, -1, NULL, NULL); intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); cpuid_to_mid[curcpu] = PCPU_GET(mid); - csa = &cpu_start_args; - for (child = OF_child(OF_peer(0)); child != 0 && mp_ncpus <= MAXCPU; - child = OF_peer(child)) { - if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 || - strcmp(buf, "cpu") != 0) - continue; - if (OF_getprop(child, cpu_impl < CPU_IMPL_ULTRASPARCIII ? - "upa-portid" : "portid", &mid, sizeof(mid)) <= 0) - panic("%s: can't get module ID", __func__); - if (mid == PCPU_GET(mid)) - continue; - if (OF_getprop(child, "clock-frequency", &clock, - sizeof(clock)) <= 0) - panic("%s: can't get clock", __func__); - if (clock != PCPU_GET(clock)) - hardclock_use_stick = 1; + foreach_ap(OF_child(OF_peer(0)), ap_start); + KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, + ("%s: can only IPI a maximum of %d JBus-CPUs", + __func__, IDR_JALAPENO_MAX_BN_PAIRS)); + PCPU_SET(other_cpus, all_cpus & ~(1 << curcpu)); + smp_active = 1; +} - csa->csa_state = 0; - sun4u_startcpu(child, (void *)mp_tramp, 0); - s = intr_disable(); - while (csa->csa_state != CPU_TICKSYNC) +static void +ap_start(phandle_t node, u_int mid) +{ + volatile struct cpu_start_args *csa; + struct pcpu *pc; + register_t s; + vm_offset_t va; + u_int clock; + u_int cpuid; + + if (mp_ncpus > MAXCPU) + return; + + if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) + panic("%s: can't get clock", __func__); + if (clock != PCPU_GET(clock)) + hardclock_use_stick = 1; + + csa = &cpu_start_args; + csa->csa_state = 0; + sun4u_startcpu(node, (void *)mp_tramp, 0); + s = intr_disable(); + while (csa->csa_state != CPU_TICKSYNC) + ; + membar(StoreLoad); + csa->csa_tick = rd(tick); + if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) { + while (csa->csa_state != CPU_STICKSYNC) ; membar(StoreLoad); - csa->csa_tick = rd(tick); - if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) { - while (csa->csa_state != CPU_STICKSYNC) - ; - membar(StoreLoad); - csa->csa_stick = rdstick(); - } - while (csa->csa_state != CPU_INIT) - ; - csa->csa_tick = csa->csa_stick = 0; - intr_restore(s); + csa->csa_stick = rdstick(); + } + while (csa->csa_state != CPU_INIT) + ; + csa->csa_tick = csa->csa_stick = 0; + intr_restore(s); - cpuid = mp_ncpus++; - cpuid_to_mid[cpuid] = mid; - cpu_identify(csa->csa_ver, clock, cpuid); + cpuid = mp_ncpus++; + cpuid_to_mid[cpuid] = mid; + cpu_identify(csa->csa_ver, clock, cpuid); - va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE); - pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; - pcpu_init(pc, cpuid, sizeof(*pc)); - dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE), - cpuid); - pc->pc_addr = va; - pc->pc_clock = clock; - pc->pc_mid = mid; - pc->pc_node = child; + va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE); + pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; + pcpu_init(pc, cpuid, sizeof(*pc)); + dpcpu_init((void *)kmem_alloc(kernel_map, DPCPU_SIZE), cpuid); + pc->pc_addr = va; + pc->pc_clock = clock; + pc->pc_mid = mid; + pc->pc_node = node; - cache_init(pc); + cache_init(pc); - all_cpus |= 1 << cpuid; - intr_add_cpu(cpuid); - } - KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, - ("%s: can only IPI a maximum of %d JBus-CPUs", - __func__, IDR_JALAPENO_MAX_BN_PAIRS)); - PCPU_SET(other_cpus, all_cpus & ~(1 << curcpu)); - smp_active = 1; + all_cpus |= 1 << cpuid; + intr_add_cpu(cpuid); } void cpu_mp_announce(void) { } static void cpu_mp_unleash(void *v) { volatile struct cpu_start_args *csa; struct pcpu *pc; register_t s; vm_offset_t va; vm_paddr_t pa; u_int ctx_inc; u_int ctx_min; int i; ctx_min = TLB_CTX_USER_MIN; ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; csa = &cpu_start_args; csa->csa_count = mp_ncpus; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { pc->pc_tlb_ctx = ctx_min; pc->pc_tlb_ctx_min = ctx_min; pc->pc_tlb_ctx_max = ctx_min + ctx_inc; ctx_min += ctx_inc; if (pc->pc_cpuid == curcpu) continue; KASSERT(pc->pc_idlethread != NULL, ("%s: idlethread", __func__)); pc->pc_curthread = pc->pc_idlethread; pc->pc_curpcb = pc->pc_curthread->td_pcb; for (i = 0; i < PCPU_PAGES; i++) { va = pc->pc_addr + i * PAGE_SIZE; pa = pmap_kextract(va); if (pa == 0) panic("%s: pmap_kextract", __func__); csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; } csa->csa_state = 0; csa->csa_pcpu = pc->pc_addr; csa->csa_mid = pc->pc_mid; s = intr_disable(); while (csa->csa_state != CPU_BOOTSTRAP) ; intr_restore(s); } membar(StoreLoad); csa->csa_count = 0; smp_started = 1; } void cpu_mp_bootstrap(struct pcpu *pc) { volatile struct cpu_start_args *csa; csa = &cpu_start_args; if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) cheetah_init(); cache_enable(); pmap_map_tsb(); /* * Flush all non-locked TLB entries possibly left over by the * firmware. */ tlb_flush_nonlocked(); cpu_setregs(pc); tick_start(); smp_cpus++; KASSERT(curthread != NULL, ("%s: curthread", __func__)); PCPU_SET(other_cpus, all_cpus & ~(1 << curcpu)); printf("SMP: AP CPU #%d Launched!\n", curcpu); csa->csa_count--; membar(StoreLoad); csa->csa_state = CPU_BOOTSTRAP; while (csa->csa_count != 0) ; /* Ok, now enter the scheduler. */ sched_throw(NULL); } void cpu_mp_shutdown(void) { int i; critical_enter(); shutdown_cpus = PCPU_GET(other_cpus); if (stopped_cpus != PCPU_GET(other_cpus)) /* XXX */ stop_cpus(stopped_cpus ^ PCPU_GET(other_cpus)); i = 0; while (shutdown_cpus != 0) { if (i++ > 100000) { printf("timeout shutting down CPUs.\n"); break; } } critical_exit(); } static void cpu_ipi_ast(struct trapframe *tf) { } static void cpu_ipi_stop(struct trapframe *tf) { CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); savectx(&stoppcbs[curcpu]); atomic_set_acq_int(&stopped_cpus, PCPU_GET(cpumask)); while ((started_cpus & PCPU_GET(cpumask)) == 0) { if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) { atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask)); (void)intr_disable(); for (;;) ; } } atomic_clear_rel_int(&started_cpus, PCPU_GET(cpumask)); atomic_clear_rel_int(&stopped_cpus, PCPU_GET(cpumask)); CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); } static void cpu_ipi_preempt(struct trapframe *tf) { sched_preempt(curthread); } static void spitfire_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) { u_int cpu; KASSERT((cpus & (1 << curcpu)) == 0, ("%s: CPU can't IPI itself", __func__)); while (cpus) { cpu = ffs(cpus) - 1; cpus &= ~(1 << cpu); spitfire_ipi_send(cpuid_to_mid[cpu], d0, d1, d2); } } static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2) { register_t s; u_long ids; int i; KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, ("%s: outstanding dispatch", __func__)); for (i = 0; i < IPI_RETRIES; i++) { s = intr_disable(); stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); membar(Sync); stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); /* * Workaround for SpitFire erratum #54; do a dummy read * from a SDB internal register before the MEMBAR #Sync * for the write to ASI_SDB_INTR_W (requiring another * MEMBAR #Sync in order to make sure the write has * occurred before the load). */ membar(Sync); (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); membar(Sync); while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & IDR_BUSY) != 0) ; intr_restore(s); if ((ids & (IDR_BUSY | IDR_NACK)) == 0) return; /* * Leave interrupts enabled for a bit before retrying * in order to avoid deadlocks if the other CPU is also * trying to send an IPI. */ DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", __func__, mid); else panic("%s: couldn't send IPI to module 0x%u", __func__, mid); } static void cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) { register_t s; u_long ids; u_int bnp; u_int cpu; int i; KASSERT((cpus & (1 << curcpu)) == 0, ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); if (cpus == 0) return; ids = 0; for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { s = intr_disable(); stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); membar(Sync); bnp = 0; for (cpu = 0; cpu < mp_ncpus; cpu++) { if ((cpus & (1 << cpu)) != 0) { stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << IDC_ITID_SHIFT) | (isjbus ? 0 : bnp << IDC_BN_SHIFT), ASI_SDB_INTR_W, 0); membar(Sync); bnp++; } } while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & IDR_CHEETAH_ALL_BUSY) != 0) ; intr_restore(s); if ((ids & (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) return; bnp = 0; for (cpu = 0; cpu < mp_ncpus; cpu++) { if ((cpus & (1 << cpu)) != 0) { if ((ids & (IDR_NACK << (isjbus ? (2 * cpuid_to_mid[cpu]) : (2 * bnp)))) == 0) cpus &= ~(1 << cpu); bnp++; } } /* * On at least Fire V880 we may receive IDR_NACKs for * CPUs we actually haven't tried to send an IPI to, * but which apparently can be safely ignored. */ if (cpus == 0) return; /* * Leave interrupts enabled for a bit before retrying * in order to avoid deadlocks if the other CPUs are * also trying to send IPIs. */ DELAY(2 * mp_ncpus); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n", __func__, cpus, ids); else panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)", __func__, cpus, ids); } Index: user/imp/tbemd/sys/sparc64/sparc64/nexus.c =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/nexus.c (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/nexus.c (revision 203850) @@ -1,554 +1,570 @@ /*- * Copyright 1998 Massachusetts Institute of Technology * Copyright 2001 by Thomas Moestl . * Copyright 2006 by Marius Strobl . * All rights reserved. * * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby * granted, provided that both the above copyright notice and this * permission notice appear in all copies, that both the above * copyright notice and this permission notice appear in all * supporting documentation, and that the name of M.I.T. not be used * in advertising or publicity pertaining to distribution of the * software without specific, written prior permission. M.I.T. makes * no representations about the suitability of this software for any * purpose. It is provided "as is" without express or implied * warranty. * * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: FreeBSD: src/sys/i386/i386/nexus.c,v 1.43 2001/02/09 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include /* * The nexus (which is a pseudo-bus actually) iterates over the nodes that * hang from the Open Firmware root node and adds them as devices to this bus * (except some special nodes which are excluded) so that drivers can be * attached to them. * * Additionally, interrupt setup/teardown and some resource management are * done at this level. * * Maybe this code should get into dev/ofw to some extent, as some of it should * work for all Open Firmware based machines... */ struct nexus_devinfo { struct ofw_bus_devinfo ndi_obdinfo; struct resource_list ndi_rl; }; struct nexus_softc { struct rman sc_intr_rman; struct rman sc_mem_rman; }; static device_probe_t nexus_probe; static device_attach_t nexus_attach; static bus_print_child_t nexus_print_child; static bus_add_child_t nexus_add_child; static bus_probe_nomatch_t nexus_probe_nomatch; static bus_setup_intr_t nexus_setup_intr; static bus_teardown_intr_t nexus_teardown_intr; static bus_alloc_resource_t nexus_alloc_resource; static bus_activate_resource_t nexus_activate_resource; static bus_deactivate_resource_t nexus_deactivate_resource; static bus_release_resource_t nexus_release_resource; static bus_get_resource_list_t nexus_get_resource_list; #ifdef SMP static bus_bind_intr_t nexus_bind_intr; #endif static bus_describe_intr_t nexus_describe_intr; static bus_get_dma_tag_t nexus_get_dma_tag; static ofw_bus_get_devinfo_t nexus_get_devinfo; static int nexus_inlist(const char *, const char *const *); static struct nexus_devinfo * nexus_setup_dinfo(device_t, phandle_t); static void nexus_destroy_dinfo(struct nexus_devinfo *); static int nexus_print_res(struct nexus_devinfo *); static device_method_t nexus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nexus_probe), DEVMETHOD(device_attach, nexus_attach), DEVMETHOD(device_detach, bus_generic_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_print_child, nexus_print_child), DEVMETHOD(bus_probe_nomatch, nexus_probe_nomatch), DEVMETHOD(bus_read_ivar, bus_generic_read_ivar), DEVMETHOD(bus_write_ivar, bus_generic_write_ivar), DEVMETHOD(bus_add_child, nexus_add_child), DEVMETHOD(bus_alloc_resource, nexus_alloc_resource), DEVMETHOD(bus_activate_resource, nexus_activate_resource), DEVMETHOD(bus_deactivate_resource, nexus_deactivate_resource), DEVMETHOD(bus_release_resource, nexus_release_resource), DEVMETHOD(bus_setup_intr, nexus_setup_intr), DEVMETHOD(bus_teardown_intr, nexus_teardown_intr), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_get_resource_list, nexus_get_resource_list), #ifdef SMP DEVMETHOD(bus_bind_intr, nexus_bind_intr), #endif DEVMETHOD(bus_describe_intr, nexus_describe_intr), DEVMETHOD(bus_get_dma_tag, nexus_get_dma_tag), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, nexus_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), KOBJMETHOD_END }; static devclass_t nexus_devclass; DEFINE_CLASS_0(nexus, nexus_driver, nexus_methods, sizeof(struct nexus_softc)); EARLY_DRIVER_MODULE(nexus, root, nexus_driver, nexus_devclass, 0, 0, BUS_PASS_BUS); MODULE_VERSION(nexus, 1); static const char *const nexus_excl_name[] = { "aliases", "associations", "chosen", + "cmp", "counter-timer", /* No separate device; handled by psycho/sbus */ + "failsafe", "memory", "openprom", "options", "packages", "rsc", + "sgcn", + "todsg", "virtual-memory", NULL }; static const char *const nexus_excl_type[] = { + "core", "cpu", NULL }; extern struct bus_space_tag nexus_bustag; extern struct bus_dma_tag nexus_dmatag; static int nexus_inlist(const char *name, const char *const *list) { int i; if (name == NULL) return (0); for (i = 0; list[i] != NULL; i++) if (strcmp(name, list[i]) == 0) return (1); return (0); } #define NEXUS_EXCLUDED(name, type) \ (nexus_inlist((name), nexus_excl_name) || \ ((type) != NULL && nexus_inlist((type), nexus_excl_type))) static int nexus_probe(device_t dev) { /* Nexus does always match. */ device_set_desc(dev, "Open Firmware Nexus device"); return (0); } static int nexus_attach(device_t dev) { struct nexus_devinfo *ndi; struct nexus_softc *sc; device_t cdev; phandle_t node; - node = OF_peer(0); - if (node == -1) - panic("%s: OF_peer failed.", __func__); + if (strcmp(device_get_name(device_get_parent(dev)), "root") == 0) { + node = OF_peer(0); + if (node == -1) + panic("%s: OF_peer failed.", __func__); - sc = device_get_softc(dev); - sc->sc_intr_rman.rm_type = RMAN_ARRAY; - sc->sc_intr_rman.rm_descr = "Interrupts"; - sc->sc_mem_rman.rm_type = RMAN_ARRAY; - sc->sc_mem_rman.rm_descr = "Device Memory"; - if (rman_init(&sc->sc_intr_rman) != 0 || - rman_init(&sc->sc_mem_rman) != 0 || - rman_manage_region(&sc->sc_intr_rman, 0, IV_MAX - 1) != 0 || - rman_manage_region(&sc->sc_mem_rman, 0ULL, ~0ULL) != 0) - panic("%s: failed to set up rmans.", __func__); + sc = device_get_softc(dev); + sc->sc_intr_rman.rm_type = RMAN_ARRAY; + sc->sc_intr_rman.rm_descr = "Interrupts"; + sc->sc_mem_rman.rm_type = RMAN_ARRAY; + sc->sc_mem_rman.rm_descr = "Device Memory"; + if (rman_init(&sc->sc_intr_rman) != 0 || + rman_init(&sc->sc_mem_rman) != 0 || + rman_manage_region(&sc->sc_intr_rman, 0, + IV_MAX - 1) != 0 || + rman_manage_region(&sc->sc_mem_rman, 0ULL, ~0ULL) != 0) + panic("%s: failed to set up rmans.", __func__); + } else + node = ofw_bus_get_node(dev); /* * Allow devices to identify. */ bus_generic_probe(dev); /* * Now walk the OFW tree and attach top-level devices. */ for (node = OF_child(node); node > 0; node = OF_peer(node)) { if ((ndi = nexus_setup_dinfo(dev, node)) == NULL) continue; cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child failed\n", ndi->ndi_obdinfo.obd_name); nexus_destroy_dinfo(ndi); continue; } device_set_ivars(cdev, ndi); } return (bus_generic_attach(dev)); } static device_t nexus_add_child(device_t dev, int order, const char *name, int unit) { device_t cdev; struct nexus_devinfo *ndi; cdev = device_add_child_ordered(dev, order, name, unit); if (cdev == NULL) return (NULL); ndi = malloc(sizeof(*ndi), M_DEVBUF, M_WAITOK | M_ZERO); ndi->ndi_obdinfo.obd_node = -1; ndi->ndi_obdinfo.obd_name = strdup(name, M_OFWPROP); resource_list_init(&ndi->ndi_rl); device_set_ivars(cdev, ndi); return (cdev); } static int nexus_print_child(device_t dev, device_t child) { int rv; rv = bus_print_child_header(dev, child); rv += nexus_print_res(device_get_ivars(child)); rv += bus_print_child_footer(dev, child); return (rv); } static void nexus_probe_nomatch(device_t dev, device_t child) { const char *type; device_printf(dev, "<%s>", ofw_bus_get_name(child)); nexus_print_res(device_get_ivars(child)); type = ofw_bus_get_type(child); printf(" type %s (no driver attached)\n", type != NULL ? type : "unknown"); } static int nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { int error; if (res == NULL) panic("%s: NULL interrupt resource!", __func__); if ((rman_get_flags(res) & RF_SHAREABLE) == 0) flags |= INTR_EXCL; /* We depend here on rman_activate_resource() being idempotent. */ error = rman_activate_resource(res); if (error) return (error); error = inthand_add(device_get_nameunit(child), rman_get_start(res), filt, intr, arg, flags, cookiep); /* * XXX in case of the AFB/FFB interrupt and a Psycho, Sabre or U2S * bridge enable the interrupt in the respective bridge. */ return (error); } static int nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih) { inthand_remove(rman_get_start(r), ih); return (0); } #ifdef SMP static int nexus_bind_intr(device_t dev, device_t child, struct resource *r, int cpu) { return (intr_bind(rman_get_start(r), cpu)); } #endif static int nexus_describe_intr(device_t dev, device_t child, struct resource *r, void *cookie, const char *descr) { return (intr_describe(rman_get_start(r), cookie, descr)); } static struct resource * nexus_alloc_resource(device_t bus, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct nexus_softc *sc; struct rman *rm; struct resource *rv; struct resource_list_entry *rle; + device_t nexus; int isdefault, needactivate, passthrough; isdefault = (start == 0UL && end == ~0UL); needactivate = flags & RF_ACTIVE; passthrough = (device_get_parent(child) != bus); - sc = device_get_softc(bus); + nexus = bus; + while (strcmp(device_get_name(device_get_parent(nexus)), "root") != 0) + nexus = device_get_parent(nexus); + sc = device_get_softc(nexus); rle = NULL; if (!passthrough) { rle = resource_list_find(BUS_GET_RESOURCE_LIST(bus, child), type, *rid); if (rle == NULL) return (NULL); if (rle->res != NULL) panic("%s: resource entry is busy", __func__); if (isdefault) { start = rle->start; count = ulmax(count, rle->count); end = ulmax(rle->end, start + count - 1); } } switch (type) { case SYS_RES_IRQ: rm = &sc->sc_intr_rman; break; case SYS_RES_MEMORY: rm = &sc->sc_mem_rman; break; default: return (NULL); } flags &= ~RF_ACTIVE; rv = rman_reserve_resource(rm, start, end, count, flags, child); if (rv == NULL) return (NULL); rman_set_rid(rv, *rid); if (type == SYS_RES_MEMORY) { rman_set_bustag(rv, &nexus_bustag); rman_set_bushandle(rv, rman_get_start(rv)); } if (needactivate) { if (bus_activate_resource(child, type, *rid, rv) != 0) { rman_release_resource(rv); return (NULL); } } if (!passthrough) { rle->res = rv; rle->start = rman_get_start(rv); rle->end = rman_get_end(rv); rle->count = rle->end - rle->start + 1; } return (rv); } static int nexus_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { /* Not much to be done yet... */ return (rman_activate_resource(r)); } static int nexus_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { /* Not much to be done yet... */ return (rman_deactivate_resource(r)); } static int nexus_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { int error; if (rman_get_flags(r) & RF_ACTIVE) { error = bus_deactivate_resource(child, type, rid, r); if (error) return (error); } return (rman_release_resource(r)); } static struct resource_list * nexus_get_resource_list(device_t dev, device_t child) { struct nexus_devinfo *ndi; ndi = device_get_ivars(child); return (&ndi->ndi_rl); } static bus_dma_tag_t nexus_get_dma_tag(device_t bus, device_t child) { return (&nexus_dmatag); } static const struct ofw_bus_devinfo * nexus_get_devinfo(device_t dev, device_t child) { struct nexus_devinfo *ndi; ndi = device_get_ivars(child); return (&ndi->ndi_obdinfo); } static struct nexus_devinfo * nexus_setup_dinfo(device_t dev, phandle_t node) { struct nexus_devinfo *ndi; struct nexus_regs *reg; bus_addr_t phys; bus_size_t size; uint32_t ign; uint32_t *intr; int i; int nintr; int nreg; ndi = malloc(sizeof(*ndi), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&ndi->ndi_obdinfo, node) != 0) { free(ndi, M_DEVBUF); return (NULL); } if (NEXUS_EXCLUDED(ndi->ndi_obdinfo.obd_name, ndi->ndi_obdinfo.obd_type)) { ofw_bus_gen_destroy_devinfo(&ndi->ndi_obdinfo); free(ndi, M_DEVBUF); return (NULL); } resource_list_init(&ndi->ndi_rl); nreg = OF_getprop_alloc(node, "reg", sizeof(*reg), (void **)®); if (nreg == -1) { device_printf(dev, "<%s>: incomplete\n", ndi->ndi_obdinfo.obd_name); goto fail; } for (i = 0; i < nreg; i++) { phys = NEXUS_REG_PHYS(®[i]); size = NEXUS_REG_SIZE(®[i]); - resource_list_add(&ndi->ndi_rl, SYS_RES_MEMORY, i, phys, - phys + size - 1, size); + /* Skip the dummy reg property of glue devices like ssm(4). */ + if (size != 0) + resource_list_add(&ndi->ndi_rl, SYS_RES_MEMORY, i, + phys, phys + size - 1, size); } free(reg, M_OFWPROP); nintr = OF_getprop_alloc(node, "interrupts", sizeof(*intr), (void **)&intr); if (nintr > 0) { if (OF_getprop(node, cpu_impl < CPU_IMPL_ULTRASPARCIII ? "upa-portid" : "portid", &ign, sizeof(ign)) <= 0) { device_printf(dev, "<%s>: could not determine portid\n", ndi->ndi_obdinfo.obd_name); free(intr, M_OFWPROP); goto fail; } /* XXX 7-bit MID on Starfire */ ign = (ign << INTMAP_IGN_SHIFT) & INTMAP_IGN_MASK; for (i = 0; i < nintr; i++) { intr[i] |= ign; resource_list_add(&ndi->ndi_rl, SYS_RES_IRQ, i, intr[i], intr[i], 1); } free(intr, M_OFWPROP); } return (ndi); fail: nexus_destroy_dinfo(ndi); return (NULL); } static void nexus_destroy_dinfo(struct nexus_devinfo *ndi) { resource_list_free(&ndi->ndi_rl); ofw_bus_gen_destroy_devinfo(&ndi->ndi_obdinfo); free(ndi, M_DEVBUF); } static int nexus_print_res(struct nexus_devinfo *ndi) { int rv; rv = 0; rv += resource_list_print_type(&ndi->ndi_rl, "mem", SYS_RES_MEMORY, "%#lx"); rv += resource_list_print_type(&ndi->ndi_rl, "irq", SYS_RES_IRQ, "%ld"); return (rv); } Index: user/imp/tbemd/sys/sparc64/sparc64/pmap.c =================================================================== --- user/imp/tbemd/sys/sparc64/sparc64/pmap.c (revision 203849) +++ user/imp/tbemd/sys/sparc64/sparc64/pmap.c (revision 203850) @@ -1,2018 +1,2013 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * In addition to hardware address maps, this module is called upon to * provide software-use-only maps which may or may not be stored in the * same form as hardware maps. These pseudo-maps are used to store * intermediate results from copy operations to and from address spaces. * * Since the information managed by this module is also stored by the * logical address mapping module, this module may throw away valid virtual * to physical mappings at almost any time. However, invalidations of * mappings must be done as requested. * * In order to cope with hardware architectures which make virtual to * physical map invalidates expensive, this module may delay invalidate * reduced protection operations until such time as they are actually * necessary. This module is given full information as to which processors * are currently using which maps, and to when physical maps must be made * correct. */ #include "opt_kstack_pages.h" #include "opt_msgbuf.h" #include "opt_pmap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define PMAP_DEBUG #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif /* XXX */ #include "opt_sched.h" #ifndef SCHED_4BSD #error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." #endif extern struct mtx sched_lock; /* * Virtual address of message buffer */ struct msgbuf *msgbufp; /* * Map of physical memory reagions */ vm_paddr_t phys_avail[128]; static struct ofw_mem_region mra[128]; struct ofw_mem_region sparc64_memreg[128]; int sparc64_nmemreg; static struct ofw_map translations[128]; static int translations_size; static vm_offset_t pmap_idle_map; static vm_offset_t pmap_temp_map_1; static vm_offset_t pmap_temp_map_2; /* * First and last available kernel virtual addresses */ vm_offset_t virtual_avail; vm_offset_t virtual_end; vm_offset_t kernel_vm_end; vm_offset_t vm_max_kernel_address; /* * Kernel pmap */ struct pmap kernel_pmap_store; /* * Allocate physical memory for use in pmap_bootstrap. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired); extern int tl1_immu_miss_patch_1[]; extern int tl1_immu_miss_patch_2[]; extern int tl1_dmmu_miss_patch_1[]; extern int tl1_dmmu_miss_patch_2[]; extern int tl1_dmmu_prot_patch_1[]; extern int tl1_dmmu_prot_patch_2[]; /* * If user pmap is processed with pmap_remove and with pmap_remove and the * resident count drops to 0, there are no more pages to remove, so we * need not continue. */ #define PMAP_REMOVE_DONE(pm) \ ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) /* * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() * and pmap_protect() instead of trying each virtual address. */ #define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); PMAP_STATS_VAR(pmap_nenter); PMAP_STATS_VAR(pmap_nenter_update); PMAP_STATS_VAR(pmap_nenter_replace); PMAP_STATS_VAR(pmap_nenter_new); PMAP_STATS_VAR(pmap_nkenter); PMAP_STATS_VAR(pmap_nkenter_oc); PMAP_STATS_VAR(pmap_nkenter_stupid); PMAP_STATS_VAR(pmap_nkremove); PMAP_STATS_VAR(pmap_nqenter); PMAP_STATS_VAR(pmap_nqremove); PMAP_STATS_VAR(pmap_ncache_enter); PMAP_STATS_VAR(pmap_ncache_enter_c); PMAP_STATS_VAR(pmap_ncache_enter_oc); PMAP_STATS_VAR(pmap_ncache_enter_cc); PMAP_STATS_VAR(pmap_ncache_enter_coc); PMAP_STATS_VAR(pmap_ncache_enter_nc); PMAP_STATS_VAR(pmap_ncache_enter_cnc); PMAP_STATS_VAR(pmap_ncache_remove); PMAP_STATS_VAR(pmap_ncache_remove_c); PMAP_STATS_VAR(pmap_ncache_remove_oc); PMAP_STATS_VAR(pmap_ncache_remove_cc); PMAP_STATS_VAR(pmap_ncache_remove_coc); PMAP_STATS_VAR(pmap_ncache_remove_nc); PMAP_STATS_VAR(pmap_nzero_page); PMAP_STATS_VAR(pmap_nzero_page_c); PMAP_STATS_VAR(pmap_nzero_page_oc); PMAP_STATS_VAR(pmap_nzero_page_nc); PMAP_STATS_VAR(pmap_nzero_page_area); PMAP_STATS_VAR(pmap_nzero_page_area_c); PMAP_STATS_VAR(pmap_nzero_page_area_oc); PMAP_STATS_VAR(pmap_nzero_page_area_nc); PMAP_STATS_VAR(pmap_nzero_page_idle); PMAP_STATS_VAR(pmap_nzero_page_idle_c); PMAP_STATS_VAR(pmap_nzero_page_idle_oc); PMAP_STATS_VAR(pmap_nzero_page_idle_nc); PMAP_STATS_VAR(pmap_ncopy_page); PMAP_STATS_VAR(pmap_ncopy_page_c); PMAP_STATS_VAR(pmap_ncopy_page_oc); PMAP_STATS_VAR(pmap_ncopy_page_nc); PMAP_STATS_VAR(pmap_ncopy_page_dc); PMAP_STATS_VAR(pmap_ncopy_page_doc); PMAP_STATS_VAR(pmap_ncopy_page_sc); PMAP_STATS_VAR(pmap_ncopy_page_soc); PMAP_STATS_VAR(pmap_nnew_thread); PMAP_STATS_VAR(pmap_nnew_thread_oc); /* * Quick sort callout for comparing memory regions */ static int mr_cmp(const void *a, const void *b); static int om_cmp(const void *a, const void *b); + static int mr_cmp(const void *a, const void *b) { const struct ofw_mem_region *mra; const struct ofw_mem_region *mrb; mra = a; mrb = b; if (mra->mr_start < mrb->mr_start) return (-1); else if (mra->mr_start > mrb->mr_start) return (1); else return (0); } + static int om_cmp(const void *a, const void *b) { const struct ofw_map *oma; const struct ofw_map *omb; oma = a; omb = b; if (oma->om_start < omb->om_start) return (-1); else if (oma->om_start > omb->om_start) return (1); else return (0); } /* * Bootstrap the system enough to run with virtual memory. */ void pmap_bootstrap(void) { struct pmap *pm; struct tte *tp; vm_offset_t off; vm_offset_t va; vm_paddr_t pa; vm_size_t physsz; vm_size_t virtsz; phandle_t pmem; phandle_t vmem; int sz; int i; int j; /* * Find out what physical memory is available from the PROM and * initialize the phys_avail array. This must be done before * pmap_bootstrap_alloc is called. */ if ((pmem = OF_finddevice("/memory")) == -1) panic("pmap_bootstrap: finddevice /memory"); if ((sz = OF_getproplen(pmem, "available")) == -1) panic("pmap_bootstrap: getproplen /memory/available"); if (sizeof(phys_avail) < sz) panic("pmap_bootstrap: phys_avail too small"); if (sizeof(mra) < sz) panic("pmap_bootstrap: mra too small"); bzero(mra, sz); if (OF_getprop(pmem, "available", mra, sz) == -1) panic("pmap_bootstrap: getprop /memory/available"); sz /= sizeof(*mra); CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); qsort(mra, sz, sizeof (*mra), mr_cmp); physsz = 0; getenv_quad("hw.physmem", &physmem); physmem = btoc(physmem); for (i = 0, j = 0; i < sz; i++, j += 2) { CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, mra[i].mr_size); if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { if (btoc(physsz) < physmem) { phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + (ctob(physmem) - physsz); physsz = ctob(physmem); } break; } phys_avail[j] = mra[i].mr_start; phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; physsz += mra[i].mr_size; } physmem = btoc(physsz); /* * Calculate the size of kernel virtual memory, and the size and mask * for the kernel TSB based on the phsyical memory size but limited * by letting the kernel TSB take up no more than half of the dTLB * slots available for locked entries. */ virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); virtsz = MIN(virtsz, (dtlb_slots / 2 * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; if (kernel_tlb_slots + PCPU_PAGES + tsb_kernel_size / PAGE_SIZE_4M + 1 /* PROM page */ + 1 /* spare */ > dtlb_slots) panic("pmap_bootstrap: insufficient dTLB entries"); if (kernel_tlb_slots + 1 /* PROM page */ + 1 /* spare */ > itlb_slots) panic("pmap_bootstrap: insufficient iTLB entries"); /* * Allocate the kernel TSB and lock it in the TLB. */ pa = pmap_bootstrap_alloc(tsb_kernel_size); if (pa & PAGE_MASK_4M) panic("pmap_bootstrap: tsb unaligned\n"); tsb_kernel_phys = pa; tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); pmap_map_tsb(); bzero(tsb_kernel, tsb_kernel_size); /* * Allocate and map the dynamic per-CPU area for the BSP. */ pa = pmap_bootstrap_alloc(DPCPU_SIZE); dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); /* * Allocate and map the message buffer. */ pa = pmap_bootstrap_alloc(MSGBUF_SIZE); msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); /* * Patch the virtual address and the tsb mask into the trap table. */ #define SETHI(rd, imm22) \ (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ EIF_IMM((imm22) >> 10, 22)) #define OR_R_I_R(rd, imm13, rs1) \ (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) #define PATCH(addr) do { \ if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ panic("pmap_boostrap: patched instructions have changed"); \ addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ flush(addr); \ flush(addr + 1); \ flush(addr + 2); \ } while (0) PATCH(tl1_immu_miss_patch_1); PATCH(tl1_immu_miss_patch_2); PATCH(tl1_dmmu_miss_patch_1); PATCH(tl1_dmmu_miss_patch_2); PATCH(tl1_dmmu_prot_patch_1); PATCH(tl1_dmmu_prot_patch_2); /* * Enter fake 8k pages for the 4MB kernel pages, so that * pmap_kextract() will work for them. */ for (i = 0; i < kernel_tlb_slots; i++) { pa = kernel_tlbs[i].te_pa; va = kernel_tlbs[i].te_va; for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { tp = tsb_kvtotte(va + off); tp->tte_vpn = TV_VPN(va + off, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } } /* * Set the start and end of KVA. The kernel is loaded starting * at the first available 4MB super page, so we advance to the * end of the last one used for it. */ virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; virtual_end = vm_max_kernel_address; kernel_vm_end = vm_max_kernel_address; /* * Allocate kva space for temporary mappings. */ pmap_idle_map = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; pmap_temp_map_1 = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; pmap_temp_map_2 = virtual_avail; virtual_avail += PAGE_SIZE * DCACHE_COLORS; /* * Allocate a kernel stack with guard page for thread0 and map it * into the kernel TSB. We must ensure that the virtual address is * coloured properly, since we're allocating from phys_avail so the * memory won't have an associated vm_page_t. */ pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); kstack0_phys = pa; virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * PAGE_SIZE; kstack0 = virtual_avail; virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), ("pmap_bootstrap: kstack0 miscoloured")); for (i = 0; i < KSTACK_PAGES; i++) { pa = kstack0_phys + i * PAGE_SIZE; va = kstack0 + i * PAGE_SIZE; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; } /* * Calculate the last available physical address. */ for (i = 0; phys_avail[i + 2] != 0; i += 2) ; Maxmem = sparc64_btop(phys_avail[i + 1]); /* * Add the PROM mappings to the kernel TSB. */ if ((vmem = OF_finddevice("/virtual-memory")) == -1) panic("pmap_bootstrap: finddevice /virtual-memory"); if ((sz = OF_getproplen(vmem, "translations")) == -1) panic("pmap_bootstrap: getproplen translations"); if (sizeof(translations) < sz) panic("pmap_bootstrap: translations too small"); bzero(translations, sz); if (OF_getprop(vmem, "translations", translations, sz) == -1) panic("pmap_bootstrap: getprop /virtual-memory/translations"); sz /= sizeof(*translations); translations_size = sz; CTR0(KTR_PMAP, "pmap_bootstrap: translations"); qsort(translations, sz, sizeof (*translations), om_cmp); for (i = 0; i < sz; i++) { CTR3(KTR_PMAP, "translation: start=%#lx size=%#lx tte=%#lx", translations[i].om_start, translations[i].om_size, translations[i].om_tte); if ((translations[i].om_tte & TD_V) == 0) continue; if (translations[i].om_start < VM_MIN_PROM_ADDRESS || translations[i].om_start > VM_MAX_PROM_ADDRESS) continue; for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { va = translations[i].om_start + off; tp = tsb_kvtotte(va); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = ((translations[i].om_tte & ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | (cpu_impl < CPU_IMPL_ULTRASPARCIII ? (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + off; } } /* * Get the available physical memory ranges from /memory/reg. These * are only used for kernel dumps, but it may not be wise to do PROM * calls in that situation. */ if ((sz = OF_getproplen(pmem, "reg")) == -1) panic("pmap_bootstrap: getproplen /memory/reg"); if (sizeof(sparc64_memreg) < sz) panic("pmap_bootstrap: sparc64_memreg too small"); if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) panic("pmap_bootstrap: getprop /memory/reg"); sparc64_nmemreg = sz / sizeof(*sparc64_memreg); /* * Initialize the kernel pmap (which is statically allocated). * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization * but sparc64 start up is not ready to initialize mutexes yet. * It is called in machdep.c. */ pm = kernel_pmap; for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = TLB_CTX_KERNEL; pm->pm_active = ~0; /* * Flush all non-locked TLB entries possibly left over by the * firmware. */ tlb_flush_nonlocked(); } void pmap_map_tsb(void) { vm_offset_t va; vm_paddr_t pa; u_long data; register_t s; int i; s = intr_disable(); /* * Map the 4MB TSB pages. */ for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { va = (vm_offset_t)tsb_kernel + i; pa = tsb_kernel_phys + i; data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); } /* * Set the secondary context to be the kernel context (needed for * FP block operations in the kernel). */ stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & TLB_SCXR_PGSZ_MASK) | TLB_CTX_KERNEL); flush(KERNBASE); intr_restore(s); } /* * Allocate a physical page of memory directly from the phys_avail map. * Can only be called from pmap_bootstrap before avail start and end are * calculated. */ static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size) { vm_paddr_t pa; int i; size = roundup(size, PAGE_SIZE * DCACHE_COLORS); for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (phys_avail[i + 1] - phys_avail[i] < size) continue; pa = phys_avail[i]; phys_avail[i] += size; return (pa); } panic("pmap_bootstrap_alloc"); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.tte_list); m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); m->md.flags = 0; m->md.pmap = NULL; } /* * Initialize the pmap module. */ void pmap_init(void) { vm_offset_t addr; vm_size_t size; int result; int i; for (i = 0; i < translations_size; i++) { addr = translations[i].om_start; size = translations[i].om_size; if ((translations[i].om_tte & TD_V) == 0) continue; if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) continue; result = vm_map_find(kernel_map, NULL, 0, &addr, size, VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); if (result != KERN_SUCCESS || addr != translations[i].om_start) panic("pmap_init: vm_map_find"); } } /* * Extract the physical page address associated with the given * map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pm, vm_offset_t va) { struct tte *tp; vm_paddr_t pa; if (pm == kernel_pmap) return (pmap_kextract(va)); PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); if (tp == NULL) pa = 0; else pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); PMAP_UNLOCK(pm); return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. */ vm_page_t pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) { struct tte *tp; vm_page_t m; m = NULL; vm_page_lock_queues(); if (pm == kernel_pmap) { if (va >= VM_MIN_DIRECT_ADDRESS) { tp = NULL; m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); vm_page_hold(m); } else { tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) tp = NULL; } } else { PMAP_LOCK(pm); tp = tsb_tte_lookup(pm, va); } if (tp != NULL && ((tp->tte_data & TD_SW) || (prot & VM_PROT_WRITE) == 0)) { m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); vm_page_hold(m); } vm_page_unlock_queues(); if (pm != kernel_pmap) PMAP_UNLOCK(pm); return (m); } /* * Extract the physical page address associated with the given kernel virtual * address. */ vm_paddr_t pmap_kextract(vm_offset_t va) { struct tte *tp; if (va >= VM_MIN_DIRECT_ADDRESS) return (TLB_DIRECT_TO_PHYS(va)); tp = tsb_kvtotte(va); if ((tp->tte_data & TD_V) == 0) return (0); return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); } int pmap_cache_enter(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_enter: fake page")); PMAP_STATS_INC(pmap_ncache_enter); /* * Find the color for this virtual address and note the added mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]++; /* * If all existing mappings have the same color, the mapping is * cacheable. */ if (m->md.color == color) { KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, ("pmap_cache_enter: cacheable, mappings of other color")); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_c); else PMAP_STATS_INC(pmap_ncache_enter_oc); return (1); } /* * If there are no mappings of the other color, and the page still has * the wrong color, this must be a new mapping. Change the color to * match the new mapping, which is cacheable. We must flush the page * from the cache now. */ if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { KASSERT(m->md.colors[color] == 1, ("pmap_cache_enter: changing color, not new mapping")); dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = color; if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_enter_cc); else PMAP_STATS_INC(pmap_ncache_enter_coc); return (1); } /* * If the mapping is already non-cacheable, just return. */ if (m->md.color == -1) { PMAP_STATS_INC(pmap_ncache_enter_nc); return (0); } PMAP_STATS_INC(pmap_ncache_enter_cnc); /* * Mark all mappings as uncacheable, flush any lines with the other * color out of the dcache, and set the color to none (-1). */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_clear_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } dcache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.color = -1; return (0); } void pmap_cache_remove(vm_page_t m, vm_offset_t va) { struct tte *tp; int color; mtx_assert(&vm_page_queue_mtx, MA_OWNED); CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, m->md.colors[DCACHE_COLOR(va)]); KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_cache_remove: fake page")); KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, ("pmap_cache_remove: no mappings %d <= 0", m->md.colors[DCACHE_COLOR(va)])); PMAP_STATS_INC(pmap_ncache_remove); /* * Find the color for this virtual address and note the removal of * the mapping. */ color = DCACHE_COLOR(va); m->md.colors[color]--; /* * If the page is cacheable, just return and keep the same color, even * if there are no longer any mappings. */ if (m->md.color != -1) { if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_c); else PMAP_STATS_INC(pmap_ncache_remove_oc); return; } KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, ("pmap_cache_remove: uncacheable, no mappings of other color")); /* * If the page is not cacheable (color is -1), and the number of * mappings for this color is not zero, just return. There are * mappings of the other color still, so remain non-cacheable. */ if (m->md.colors[color] != 0) { PMAP_STATS_INC(pmap_ncache_remove_nc); return; } /* * The number of mappings for this color is now zero. Recache the * other colored mappings, and change the page color to the other * color. There should be no lines in the data cache for this page, * so flushing should not be needed. */ TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { atomic_set_long(&tp->tte_data, TD_CV); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } m->md.color = DCACHE_OTHER_COLOR(color); if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) PMAP_STATS_INC(pmap_ncache_remove_cc); else PMAP_STATS_INC(pmap_ncache_remove_coc); } /* * Map a wired page into kernel virtual address space. */ void pmap_kenter(vm_offset_t va, vm_page_t m) { vm_offset_t ova; struct tte *tp; vm_page_t om; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_STATS_INC(pmap_nkenter); tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { CTR5(KTR_CT2, "pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", va, VM_PAGE_TO_PHYS(m), m->object, m->object ? m->object->type : -1, m->pindex); PMAP_STATS_INC(pmap_nkenter_oc); } if ((tp->tte_data & TD_V) != 0) { om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); ova = TTE_GET_VA(tp); if (m == om && va == ova) { PMAP_STATS_INC(pmap_nkenter_stupid); return; } TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); pmap_cache_remove(om, ova); if (va != ova) tlb_page_demap(kernel_pmap, ova); } data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; if (pmap_cache_enter(m, va) != 0) data |= TD_CV; tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = data; TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); } /* * Map a wired page into kernel virtual address space. This additionally * takes a flag argument wich is or'ed to the TTE data. This is used by * sparc64_bus_mem_map(). * NOTE: if the mapping is non-cacheable, it's the caller's responsibility * to flush entries that might still be in the cache, if applicable. */ void pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) { struct tte *tp; tp = tsb_kvtotte(va); CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", va, pa, tp, tp->tte_data); tp->tte_vpn = TV_VPN(va, TS_8K); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; } /* * Remove a wired page from kernel virtual address space. */ void pmap_kremove(vm_offset_t va) { struct tte *tp; vm_page_t m; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_STATS_INC(pmap_nkremove); tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); if ((tp->tte_data & TD_V) == 0) return; m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pmap_cache_remove(m, va); TTE_ZERO(tp); } /* * Inverse of pmap_kenter_flags, used by bus_space_unmap(). */ void pmap_kremove_flags(vm_offset_t va) { struct tte *tp; tp = tsb_kvtotte(va); CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, tp->tte_data); TTE_ZERO(tp); } /* * Map a range of physical addresses into kernel virtual address space. * * The value passed in *virt is a suggested virtual address for the mapping. * Architectures which can support a direct-mapped physical to virtual region * can return the appropriate address within that region, leaving '*virt' * unchanged. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return (TLB_PHYS_TO_DIRECT(start)); } /* * Map a list of wired pages into kernel virtual address space. This is * intended for temporary mappings which do not need page modification or * references recorded. Existing mappings in the region are overwritten. */ void pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) { vm_offset_t va; int locked; PMAP_STATS_INC(pmap_nqenter); va = sva; if (!(locked = mtx_owned(&vm_page_queue_mtx))) vm_page_lock_queues(); while (count-- > 0) { pmap_kenter(va, *m); va += PAGE_SIZE; m++; } if (!locked) vm_page_unlock_queues(); tlb_range_demap(kernel_pmap, sva, va); } /* * Remove page mappings from kernel virtual address space. Intended for * temporary mappings entered by pmap_qenter. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; int locked; PMAP_STATS_INC(pmap_nqremove); va = sva; if (!(locked = mtx_owned(&vm_page_queue_mtx))) vm_page_lock_queues(); while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } if (!locked) vm_page_unlock_queues(); tlb_range_demap(kernel_pmap, sva, va); } /* * Initialize the pmap associated with process 0. */ void pmap_pinit0(pmap_t pm) { int i; PMAP_LOCK_INIT(pm); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = 0; pm->pm_active = 0; pm->pm_tsb = NULL; pm->pm_tsb_obj = NULL; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); } /* * Initialize a preallocated and zeroed pmap structure, such as one in a * vmspace structure. */ int pmap_pinit(pmap_t pm) { vm_page_t ma[TSB_PAGES]; vm_page_t m; int i; PMAP_LOCK_INIT(pm); /* * Allocate KVA space for the TSB. */ if (pm->pm_tsb == NULL) { pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, TSB_BSIZE); if (pm->pm_tsb == NULL) { PMAP_LOCK_DESTROY(pm); return (0); } } /* * Allocate an object for it. */ if (pm->pm_tsb_obj == NULL) pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); VM_OBJECT_LOCK(pm->pm_tsb_obj); for (i = 0; i < TSB_PAGES; i++) { m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); m->valid = VM_PAGE_BITS_ALL; m->md.pmap = pm; ma[i] = m; } VM_OBJECT_UNLOCK(pm->pm_tsb_obj); pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); for (i = 0; i < MAXCPU; i++) pm->pm_context[i] = -1; pm->pm_active = 0; bzero(&pm->pm_stats, sizeof(pm->pm_stats)); return (1); } /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pm) { vm_object_t obj; vm_page_t m; struct pcpu *pc; CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", pm->pm_context[curcpu], pm->pm_tsb); KASSERT(pmap_resident_count(pm) == 0, ("pmap_release: resident pages %ld != 0", pmap_resident_count(pm))); /* * After the pmap was freed, it might be reallocated to a new process. * When switching, this might lead us to wrongly assume that we need * not switch contexts because old and new pmap pointer are equal. * Therefore, make sure that this pmap is not referenced by any PCPU * pointer any more. This could happen in two cases: * - A process that referenced the pmap is currently exiting on a CPU. * However, it is guaranteed to not switch in any more after setting * its state to PRS_ZOMBIE. * - A process that referenced this pmap ran on a CPU, but we switched * to a kernel thread, leaving the pmap pointer unchanged. */ mtx_lock_spin(&sched_lock); - SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) if (pc->pc_pmap == pm) pc->pc_pmap = NULL; - } mtx_unlock_spin(&sched_lock); obj = pm->pm_tsb_obj; VM_OBJECT_LOCK(obj); KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); while (!TAILQ_EMPTY(&obj->memq)) { m = TAILQ_FIRST(&obj->memq); vm_page_lock_queues(); if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) continue; KASSERT(m->hold_count == 0, ("pmap_release: freeing held tsb page")); m->md.pmap = NULL; m->wire_count--; atomic_subtract_int(&cnt.v_wire_count, 1); vm_page_free_zero(m); vm_page_unlock_queues(); } VM_OBJECT_UNLOCK(obj); pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); PMAP_LOCK_DESTROY(pm); } /* * Grow the number of kernel page table entries. Unneeded. */ void pmap_growkernel(vm_offset_t addr) { panic("pmap_growkernel: can't grow kernel"); } int pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, - vm_offset_t va) + vm_offset_t va) { vm_page_t m; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); data = atomic_readandclear_long(&tp->tte_data); if ((data & TD_FAKE) == 0) { m = PHYS_TO_VM_PAGE(TD_PA(data)); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); if ((data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((data & TD_PV) != 0) { if ((data & TD_W) != 0) vm_page_dirty(m); if ((data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if (TAILQ_EMPTY(&m->md.tte_list)) vm_page_flag_clear(m, PG_WRITEABLE); pm->pm_stats.resident_count--; } pmap_cache_remove(m, va); } TTE_ZERO(tp); if (PMAP_REMOVE_DONE(pm)) return (0); return (1); } /* * Remove the given range of addresses from the specified map. */ void pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) { struct tte *tp; vm_offset_t va; CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", pm->pm_context[curcpu], start, end); if (PMAP_REMOVE_DONE(pm)) return; vm_page_lock_queues(); PMAP_LOCK(pm); if (end - start > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, start, end, pmap_remove_tte); tlb_context_demap(pm); } else { - for (va = start; va < end; va += PAGE_SIZE) { - if ((tp = tsb_tte_lookup(pm, va)) != NULL) { - if (!pmap_remove_tte(pm, NULL, tp, va)) - break; - } - } + for (va = start; va < end; va += PAGE_SIZE) + if ((tp = tsb_tte_lookup(pm, va)) != NULL && + !pmap_remove_tte(pm, NULL, tp, va)) + break; tlb_range_demap(pm, start, end - 1); } PMAP_UNLOCK(pm); vm_page_unlock_queues(); } void pmap_remove_all(vm_page_t m) { struct pmap *pm; struct tte *tpn; struct tte *tp; vm_offset_t va; mtx_assert(&vm_page_queue_mtx, MA_OWNED); for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { tpn = TAILQ_NEXT(tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; pm = TTE_GET_PMAP(tp); va = TTE_GET_VA(tp); PMAP_LOCK(pm); if ((tp->tte_data & TD_WIRED) != 0) pm->pm_stats.wired_count--; if ((tp->tte_data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if ((tp->tte_data & TD_W) != 0) vm_page_dirty(m); tp->tte_data &= ~TD_V; tlb_page_demap(pm, va); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); pm->pm_stats.resident_count--; pmap_cache_remove(m, va); TTE_ZERO(tp); PMAP_UNLOCK(pm); } vm_page_flag_clear(m, PG_WRITEABLE); } int pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, - vm_offset_t va) + vm_offset_t va) { u_long data; vm_page_t m; data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); if ((data & TD_PV) != 0) { m = PHYS_TO_VM_PAGE(TD_PA(data)); if ((data & TD_REF) != 0) vm_page_flag_set(m, PG_REFERENCED); if ((data & TD_W) != 0) vm_page_dirty(m); } return (1); } /* * Set the physical protection on the specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va; struct tte *tp; CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", pm->pm_context[curcpu], sva, eva, prot); if ((prot & VM_PROT_READ) == VM_PROT_NONE) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) return; vm_page_lock_queues(); PMAP_LOCK(pm); if (eva - sva > PMAP_TSB_THRESH) { tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); tlb_context_demap(pm); } else { - for (va = sva; va < eva; va += PAGE_SIZE) { + for (va = sva; va < eva; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(pm, va)) != NULL) pmap_protect_tte(pm, NULL, tp, va); - } tlb_range_demap(pm, sva, eva - 1); } PMAP_UNLOCK(pm); vm_page_unlock_queues(); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. */ void pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, vm_prot_t prot, boolean_t wired) { vm_page_lock_queues(); PMAP_LOCK(pm); pmap_enter_locked(pm, va, m, prot, wired); vm_page_unlock_queues(); PMAP_UNLOCK(pm); } /* * Map the given physical page at the specified virtual address in the * target pmap with the protection requested. If specified the page * will be wired down. * * The page queues and pmap must be locked. */ static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, boolean_t wired) { struct tte *tp; vm_paddr_t pa; u_long data; int i; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_LOCK_ASSERT(pm, MA_OWNED); PMAP_STATS_INC(pmap_nenter); pa = VM_PAGE_TO_PHYS(m); /* * If this is a fake page from the device_pager, but it covers actual * physical memory, convert to the real backing page. */ if ((m->flags & PG_FICTITIOUS) != 0) { for (i = 0; phys_avail[i + 1] != 0; i += 2) { if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { m = PHYS_TO_VM_PAGE(pa); break; } } } CTR6(KTR_PMAP, "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", pm->pm_context[curcpu], m, va, pa, prot, wired); /* * If there is an existing mapping, and the physical address has not * changed, must be protection or wiring change. */ if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { CTR0(KTR_PMAP, "pmap_enter_locked: update"); PMAP_STATS_INC(pmap_nenter_update); /* * Wiring change, just update stats. */ if (wired) { if ((tp->tte_data & TD_WIRED) == 0) { tp->tte_data |= TD_WIRED; pm->pm_stats.wired_count++; } } else { if ((tp->tte_data & TD_WIRED) != 0) { tp->tte_data &= ~TD_WIRED; pm->pm_stats.wired_count--; } } /* * Save the old bits and clear the ones we're interested in. */ data = tp->tte_data; tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); /* * If we're turning off write permissions, sense modify status. */ if ((prot & VM_PROT_WRITE) != 0) { tp->tte_data |= TD_SW; - if (wired) { + if (wired) tp->tte_data |= TD_W; - } vm_page_flag_set(m, PG_WRITEABLE); - } else if ((data & TD_W) != 0) { + } else if ((data & TD_W) != 0) vm_page_dirty(m); - } /* * If we're turning on execute permissions, flush the icache. */ if ((prot & VM_PROT_EXECUTE) != 0) { - if ((data & TD_EXEC) == 0) { + if ((data & TD_EXEC) == 0) icache_page_inval(pa); - } tp->tte_data |= TD_EXEC; } /* * Delete the old mapping. */ tlb_page_demap(pm, TTE_GET_VA(tp)); } else { /* * If there is an existing mapping, but its for a different * phsyical address, delete the old mapping. */ if (tp != NULL) { CTR0(KTR_PMAP, "pmap_enter_locked: replace"); PMAP_STATS_INC(pmap_nenter_replace); pmap_remove_tte(pm, NULL, tp, va); tlb_page_demap(pm, va); } else { CTR0(KTR_PMAP, "pmap_enter_locked: new"); PMAP_STATS_INC(pmap_nenter_new); } /* * Now set up the data and install the new mapping. */ data = TD_V | TD_8K | TD_PA(pa); if (pm == kernel_pmap) data |= TD_P; if ((prot & VM_PROT_WRITE) != 0) { data |= TD_SW; vm_page_flag_set(m, PG_WRITEABLE); } if (prot & VM_PROT_EXECUTE) { data |= TD_EXEC; icache_page_inval(pa); } /* * If its wired update stats. We also don't need reference or * modify tracking for wired mappings, so set the bits now. */ if (wired) { pm->pm_stats.wired_count++; data |= TD_REF | TD_WIRED; if ((prot & VM_PROT_WRITE) != 0) data |= TD_W; } tsb_tte_enter(pm, m, va, TS_8K, data); } } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; psize = atop(end - start); m = m_start; PMAP_LOCK(pm); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pm, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); m = TAILQ_NEXT(m, listq); } PMAP_UNLOCK(pm); } void pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) { PMAP_LOCK(pm); pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); PMAP_UNLOCK(pm); } void pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, - vm_pindex_t pindex, vm_size_t size) + vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * Change the wiring attribute for a map/virtual-address pair. * The mapping must already exist in the pmap. */ void pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) { struct tte *tp; u_long data; PMAP_LOCK(pm); if ((tp = tsb_tte_lookup(pm, va)) != NULL) { if (wired) { data = atomic_set_long(&tp->tte_data, TD_WIRED); if ((data & TD_WIRED) == 0) pm->pm_stats.wired_count++; } else { data = atomic_clear_long(&tp->tte_data, TD_WIRED); if ((data & TD_WIRED) != 0) pm->pm_stats.wired_count--; } } PMAP_UNLOCK(pm); } static int -pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) +pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, + vm_offset_t va) { vm_page_t m; u_long data; if ((tp->tte_data & TD_FAKE) != 0) return (1); if (tsb_tte_lookup(dst_pmap, va) == NULL) { data = tp->tte_data & ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); tsb_tte_enter(dst_pmap, m, va, TS_8K, data); } return (1); } void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct tte *tp; vm_offset_t va; if (dst_addr != src_addr) return; vm_page_lock_queues(); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } if (len > PMAP_TSB_THRESH) { tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, pmap_copy_tte); tlb_context_demap(dst_pmap); } else { - for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { + for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) pmap_copy_tte(src_pmap, dst_pmap, tp, va); - } tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); } vm_page_unlock_queues(); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } void pmap_zero_page(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page: fake page")); PMAP_STATS_INC(pmap_nzero_page); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_area(vm_page_t m, int off, int size) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_area: fake page")); KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); PMAP_STATS_INC(pmap_nzero_page_area); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_area_nc); aszero(ASI_PHYS_USE_EC, pa + off, size); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_area_c); va = TLB_PHYS_TO_DIRECT(pa); bzero((void *)(va + off), size); } else { PMAP_STATS_INC(pmap_nzero_page_area_oc); PMAP_LOCK(kernel_pmap); va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); bzero((void *)(va + off), size); tlb_page_demap(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } void pmap_zero_page_idle(vm_page_t m) { struct tte *tp; vm_offset_t va; vm_paddr_t pa; KASSERT((m->flags & PG_FICTITIOUS) == 0, ("pmap_zero_page_idle: fake page")); PMAP_STATS_INC(pmap_nzero_page_idle); pa = VM_PAGE_TO_PHYS(m); if (m->md.color == -1) { PMAP_STATS_INC(pmap_nzero_page_idle_nc); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } else if (m->md.color == DCACHE_COLOR(pa)) { PMAP_STATS_INC(pmap_nzero_page_idle_c); va = TLB_PHYS_TO_DIRECT(pa); cpu_block_zero((void *)va, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_nzero_page_idle_oc); va = pmap_idle_map + (m->md.color * PAGE_SIZE); tp = tsb_kvtotte(va); tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(va, TS_8K); cpu_block_zero((void *)va, PAGE_SIZE); tlb_page_demap(kernel_pmap, va); } } void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t vdst; vm_offset_t vsrc; vm_paddr_t pdst; vm_paddr_t psrc; struct tte *tp; KASSERT((mdst->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake dst page")); KASSERT((msrc->flags & PG_FICTITIOUS) == 0, ("pmap_copy_page: fake src page")); PMAP_STATS_INC(pmap_ncopy_page); pdst = VM_PAGE_TO_PHYS(mdst); psrc = VM_PAGE_TO_PHYS(msrc); if (msrc->md.color == -1 && mdst->md.color == -1) { PMAP_STATS_INC(pmap_ncopy_page_nc); ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); } else if (msrc->md.color == DCACHE_COLOR(psrc) && mdst->md.color == DCACHE_COLOR(pdst)) { PMAP_STATS_INC(pmap_ncopy_page_c); vdst = TLB_PHYS_TO_DIRECT(pdst); vsrc = TLB_PHYS_TO_DIRECT(psrc); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); } else if (msrc->md.color == -1) { if (mdst->md.color == DCACHE_COLOR(pdst)) { PMAP_STATS_INC(pmap_ncopy_page_dc); vdst = TLB_PHYS_TO_DIRECT(pdst); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_doc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); PMAP_UNLOCK(kernel_pmap); } } else if (mdst->md.color == -1) { if (msrc->md.color == DCACHE_COLOR(psrc)) { PMAP_STATS_INC(pmap_ncopy_page_sc); vsrc = TLB_PHYS_TO_DIRECT(psrc); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); } else { PMAP_STATS_INC(pmap_ncopy_page_soc); PMAP_LOCK(kernel_pmap); vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } else { PMAP_STATS_INC(pmap_ncopy_page_oc); PMAP_LOCK(kernel_pmap); vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); tp = tsb_kvtotte(vdst); tp->tte_data = TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vdst, TS_8K); vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); tp = tsb_kvtotte(vsrc); tp->tte_data = TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; tp->tte_vpn = TV_VPN(vsrc, TS_8K); cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); tlb_page_demap(kernel_pmap, vdst); tlb_page_demap(kernel_pmap, vsrc); PMAP_UNLOCK(kernel_pmap); } } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pm, vm_page_t m) { struct tte *tp; int loops; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); loops = 0; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if (TTE_GET_PMAP(tp) == pm) return (TRUE); if (++loops >= 16) break; } return (FALSE); } /* * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct tte *tp; int count; count = 0; if ((m->flags & PG_FICTITIOUS) != 0) return (count); mtx_assert(&vm_page_queue_mtx, MA_OWNED); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) count++; return (count); } /* * Remove all pages from specified address space, this aids process exit * speeds. This is much faster than pmap_remove n the case of running down * an entire address space. Only works for the current pmap. */ void pmap_remove_pages(pmap_t pm) { } /* * Returns TRUE if the given page has a managed mapping. */ boolean_t pmap_page_is_mapped(vm_page_t m) { struct tte *tp; if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); mtx_assert(&vm_page_queue_mtx, MA_OWNED); - TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { + TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) if ((tp->tte_data & TD_PV) != 0) return (TRUE); - } return (FALSE); } /* * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct tte *tpf; struct tte *tpn; struct tte *tp; u_long data; int count; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (0); count = 0; if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { tpn = TAILQ_NEXT(tp, tte_link); TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_REF); if ((data & TD_REF) != 0 && ++count > 4) break; } while ((tp = tpn) != NULL && tp != tpf); } return (count); } boolean_t pmap_is_modified(vm_page_t m) { struct tte *tp; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return (FALSE); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; if ((tp->tte_data & TD_W) != 0) return (TRUE); } return (FALSE); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is elgible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { return (FALSE); } void pmap_clear_modify(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_W); if ((data & TD_W) != 0) tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } void pmap_clear_reference(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_REF); if ((data & TD_REF) != 0) tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } void pmap_remove_write(vm_page_t m) { struct tte *tp; u_long data; mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || (m->flags & PG_WRITEABLE) == 0) return; TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); if ((data & TD_W) != 0) { vm_page_dirty(m); tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); } } vm_page_flag_clear(m, PG_WRITEABLE); } int pmap_mincore(pmap_t pm, vm_offset_t addr) { /* TODO; */ return (0); } /* * Activate a user pmap. The pmap must be activated before its address space * can be accessed in any way. */ void pmap_activate(struct thread *td) { struct vmspace *vm; struct pmap *pm; int context; vm = td->td_proc->p_vmspace; pm = vmspace_pmap(vm); mtx_lock_spin(&sched_lock); context = PCPU_GET(tlb_ctx); if (context == PCPU_GET(tlb_ctx_max)) { tlb_flush_user(); context = PCPU_GET(tlb_ctx_min); } PCPU_SET(tlb_ctx, context + 1); pm->pm_context[curcpu] = context; pm->pm_active |= PCPU_GET(cpumask); PCPU_SET(pmap, pm); stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & TLB_PCXR_PGSZ_MASK) | context); flush(KERNBASE); mtx_unlock_spin(&sched_lock); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { + } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } Index: user/imp/tbemd/sys/sun4v/include/asi.h =================================================================== --- user/imp/tbemd/sys/sun4v/include/asi.h (revision 203849) +++ user/imp/tbemd/sys/sun4v/include/asi.h (revision 203850) @@ -1,171 +1,178 @@ /*- * Copyright (c) 2006 Kip Macy * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: BSDI: asi.h,v 1.3 1997/08/08 14:31:42 torek * $FreeBSD$ */ #ifndef _MACHINE_ASI_H_ #define _MACHINE_ASI_H_ /* * UltraSPARC Architecture 2005 ASIs */ #define ASI_N 0x04 /* ASI_NUCLEUS */ #define ASI_NL 0x0c /* ASI_NUCLEUS_LITTLE */ #define ASI_AIUP 0x10 /* ASI_AS_IF_USER_PRIMARY */ #define ASI_AIUS 0x11 /* ASI_AS_IF_USER_SECONDARY */ #define ASI_REAL 0x14 #define ASI_REAL_IO 0x15 #define ASI_BLK_AIUP 0x16 /* ASI_BLOCK_AS_IF_USER_PRIMARY */ #define ASI_BLK_AIUS 0x17 /* ASI_BLOCK_AS_IF_USER_SECONDARY */ #define ASI_AIUPL 0x18 /* ASI_AS_IF_USER_PRIMARY_LITTLE */ #define ASI_AIUSL 0x19 /* ASI_AS_IF_USER_SECONDARY_LITTLE */ #define ASI_REAL_L 0x1C /* ASI_REAL_LITTLE */ #define ASI_REAL_IO_L 0x1D /* ASI_REAL_IO_LITTLE */ #define ASI_BLK_AIUPL 0x1E /* ASI_BLOCK_AS_IF_USER_PRIMARY_LITTLE */ #define ASI_BLK_AIUSL 0x1F /* ASI_BLOCK_AS_IF_USER_SECONDARY_LITTLE */ #define ASI_SCRATCHPAD 0x20 #define ASI_MMU_CONTEXTID 0x21 #define ASI_LDTD_AIUP 0x22 /* ASI_LOAD_TWIN_DW_AS_IF_USER_PRIMARY */ #define ASI_LDSTBI_AIUP 0x22 #define ASI_LDTD_AIUS 0x23 /* ASI_LOAD_TWIN_DW_AS_IF_USER_SECONDARY */ #define ASI_LDSTBI_AIUS 0x23 #define ASI_QUEUE 0x25 #define ASI_LDTD_REAL 0x26 /* ASI_LOAD_TWIN_DW_REAL */ #define ASI_STBI_REAL 0x26 #define ASI_LDTD_N 0x27 /* ASI_LOAD_TWIN_DW_NUCLEUS */ #define ASI_LDSTBI_N 0x27 #define ASI_LDTD_AIUPL 0x2A /* ASI_LD_TWIN_DW_AS_IF_USER_PRIMARY_LITTLE */ #define ASI_LDTD_AIUSL 0x2B /* ASI_LD_TWIN_DW_AS_IF_USER_SECONDARY_LITTLE */ #define ASI_LDTD_REAL_L 0x2E /* ASI_LOAD_TWIN_DW_REAL_LITTLE */ #define ASI_LDTD_NL 0x2F /* ASI_LOAD_TWIN_DW_NUCLEUS_LITTLE */ #define ASI_P 0x80 /* ASI_PRIMARY */ #define ASI_S 0x81 /* ASI_SECONDARY */ #define ASI_PNF 0x82 /* ASI_PRIMARY_NO_FAULT */ #define ASI_SNF 0x83 /* ASI_SECONDARY_NO_FAULT */ #define ASI_PL 0x88 /* ASI_PRIMARY_LITTLE */ #define ASI_SL 0x89 /* ASI_SECONDARY_LITTLE */ #define ASI_PNFL 0x8a /* ASI_PRIMARY_NO_FAULT_LITTLE */ #define ASI_SNFL 0x8b /* ASI_SECONDARY_NO_FAULT_LITTLE */ #define ASI_PST8_P 0xc0 #define ASI_PST8_S 0xc1 #define ASI_PST16_P 0xc2 #define ASI_PST16_S 0xc3 #define ASI_PST32_P 0xc4 #define ASI_PST32_S 0xc5 #define ASI_PST8_PL 0xc8 #define ASI_PST8_SL 0xc9 #define ASI_PST16_PL 0xca #define ASI_PST16_SL 0xcb #define ASI_PST32_PL 0xcc #define ASI_PST32_SL 0xcd #define ASI_FL8_P 0xd0 #define ASI_FL8_S 0xd1 #define ASI_FL16_P 0xd2 #define ASI_FL16_S 0xd3 #define ASI_FL8_PL 0xd8 #define ASI_FL8_SL 0xd9 #define ASI_FL16_PL 0xda #define ASI_FL16_SL 0xdb #define ASI_LDTD_P 0xe2 /* ASI_LOAD_TWIN_DW_PRIMARY */ #define ASI_LDSTBI_P 0xe2 #define ASI_LDTD_S 0xe3 /* ASI_LOAD_TWIN_DW_SECONDARY */ #define ASI_LDTD_PL 0xea /* ASI_LOAD_TWIN_DW_PRIMARY_LITTLE */ #define ASI_LDTD_SL 0xeb /* ASI_LOAD_TWIN_DW_SECONDARY_LITTLE */ #define ASI_BLK_P 0xf0 /* ASI_BLOCK_PRIMARY */ #define ASI_BLK_S 0xf1 /* ASI_BLOCK_SECONDARY */ #define ASI_BLK_PL 0xf8 /* ASI_BLOCK_PRIMARY_LITTLE */ #define ASI_BLK_SL 0xf9 /* ASI_BLOCK_SECONDARY_LITTLE */ #define ASI_SCRATCHPAD_0_REG 0x00 #define ASI_SCRATCHPAD_1_REG 0x08 #define ASI_SCRATCHPAD_2_REG 0x10 #define ASI_SCRATCHPAD_3_REG 0x18 #define ASI_SCRATCHPAD_6_REG 0x30 #define ASI_SCRATCHPAD_7_REG 0x38 #define SCRATCH_REG_MMFSA ASI_SCRATCHPAD_0_REG #define SCRATCH_REG_PCPU ASI_SCRATCHPAD_1_REG #define SCRATCH_REG_HASH_KERNEL ASI_SCRATCHPAD_2_REG #define SCRATCH_REG_TSB_KERNEL ASI_SCRATCHPAD_3_REG #define SCRATCH_REG_HASH_USER ASI_SCRATCHPAD_6_REG #define SCRATCH_REG_TSB_USER ASI_SCRATCHPAD_7_REG #define MMU_CID_P 0x08 #define MMU_CID_S 0x10 #define CPU_MONDO_QUEUE_HEAD 0x3c0 #define CPU_MONDO_QUEUE_TAIL 0x3c8 #define DEV_MONDO_QUEUE_HEAD 0x3d0 #define DEV_MONDO_QUEUE_TAIL 0x3d8 #define RESUMABLE_ERROR_QUEUE_HEAD 0x3e0 #define RESUMABLE_ERROR_QUEUE_TAIL 0x3e8 #define NONRESUMABLE_ERROR_QUEUE_HEAD 0x3f0 #define NONRESUMABLE_ERROR_QUEUE_TAIL 0x3f8 #define Q(queue_head) (queue_head >> 4) /* * sparc64 compat for the loader */ #define AA_IMMU_TAR 0x30 #define AA_DMMU_TAR 0x30 #define ASI_UPA_CONFIG_REG 0x4a /* US-I, II */ +#define ASI_FIREPLANE_CONFIG_REG 0x4a /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_CONFIG 0x0 /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_ADDRESS 0x8 /* US-III{,+}, IV{,+} */ +#define AA_FIREPLANE_CONFIG_2 0x10 /* US-IV{,+} */ +#define ASI_JBUS_CONFIG_REG 0x4a /* US-IIIi{,+} */ #define ASI_IMMU 0x50 #define ASI_ITLB_DATA_IN_REG 0x54 #define ASI_ITLB_DATA_ACCESS_REG 0x55 #define ASI_ITLB_TAG_READ_REG 0x56 #define ASI_IMMU_DEMAP 0x57 #define ASI_DMMU 0x58 #define ASI_DTLB_DATA_IN_REG 0x5c #define ASI_DTLB_DATA_ACCESS_REG 0x5d #define ASI_DTLB_TAG_READ_REG 0x5e +#define ASI_INTR_ID 0x63 /* US-IV{,+} */ +#define AA_INTR_ID 0x0 /* US-IV{,+} */ #endif /* !_MACHINE_ASI_H_ */ Index: user/imp/tbemd/sys/sun4v/include/cmt.h =================================================================== --- user/imp/tbemd/sys/sun4v/include/cmt.h (nonexistent) +++ user/imp/tbemd/sys/sun4v/include/cmt.h (revision 203850) @@ -0,0 +1,7 @@ +/*- + * This file is in the public domain. + * + * $FreeBSD$ + */ + +#include Property changes on: user/imp/tbemd/sys/sun4v/include/cmt.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/sun4v/include/fireplane.h =================================================================== --- user/imp/tbemd/sys/sun4v/include/fireplane.h (nonexistent) +++ user/imp/tbemd/sys/sun4v/include/fireplane.h (revision 203850) @@ -0,0 +1,7 @@ +/*- + * This file is in the public domain. + * + * $FreeBSD$ + */ + +#include Property changes on: user/imp/tbemd/sys/sun4v/include/fireplane.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/sun4v/include/jbus.h =================================================================== --- user/imp/tbemd/sys/sun4v/include/jbus.h (nonexistent) +++ user/imp/tbemd/sys/sun4v/include/jbus.h (revision 203850) @@ -0,0 +1,7 @@ +/*- + * This file is in the public domain. + * + * $FreeBSD$ + */ + +#include Property changes on: user/imp/tbemd/sys/sun4v/include/jbus.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/imp/tbemd/sys/ufs/ffs/ffs_alloc.c =================================================================== --- user/imp/tbemd/sys/ufs/ffs/ffs_alloc.c (revision 203849) +++ user/imp/tbemd/sys/ufs/ffs/ffs_alloc.c (revision 203850) @@ -1,2729 +1,2731 @@ /*- * Copyright (c) 2002 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Marshall * Kirk McKusick and Network Associates Laboratories, the Security * Research Division of Network Associates, Inc. under DARPA/SPAWAR * contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS * research program * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)ffs_alloc.c 8.18 (Berkeley) 5/26/95 */ #include __FBSDID("$FreeBSD$"); #include "opt_quota.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef ufs2_daddr_t allocfcn_t(struct inode *ip, u_int cg, ufs2_daddr_t bpref, int size); static ufs2_daddr_t ffs_alloccg(struct inode *, u_int, ufs2_daddr_t, int); static ufs2_daddr_t ffs_alloccgblk(struct inode *, struct buf *, ufs2_daddr_t); #ifdef INVARIANTS static int ffs_checkblk(struct inode *, ufs2_daddr_t, long); #endif static ufs2_daddr_t ffs_clusteralloc(struct inode *, u_int, ufs2_daddr_t, int); static void ffs_clusteracct(struct ufsmount *, struct fs *, struct cg *, ufs1_daddr_t, int); static ino_t ffs_dirpref(struct inode *); static ufs2_daddr_t ffs_fragextend(struct inode *, u_int, ufs2_daddr_t, int, int); static void ffs_fserr(struct fs *, ino_t, char *); static ufs2_daddr_t ffs_hashalloc (struct inode *, u_int, ufs2_daddr_t, int, allocfcn_t *); static ufs2_daddr_t ffs_nodealloccg(struct inode *, u_int, ufs2_daddr_t, int); static ufs1_daddr_t ffs_mapsearch(struct fs *, struct cg *, ufs2_daddr_t, int); static int ffs_reallocblks_ufs1(struct vop_reallocblks_args *); static int ffs_reallocblks_ufs2(struct vop_reallocblks_args *); /* * Allocate a block in the filesystem. * * The size of the requested block is given, which must be some * multiple of fs_fsize and <= fs_bsize. * A preference may be optionally specified. If a preference is given * the following hierarchy is used to allocate a block: * 1) allocate the requested block. * 2) allocate a rotationally optimal block in the same cylinder. * 3) allocate a block in the same cylinder group. * 4) quadradically rehash into other cylinder groups, until an * available block is located. * If no block preference is given the following hierarchy is used * to allocate a block: * 1) allocate a block in the cylinder group that contains the * inode for the file. * 2) quadradically rehash into other cylinder groups, until an * available block is located. */ int ffs_alloc(ip, lbn, bpref, size, flags, cred, bnp) struct inode *ip; ufs2_daddr_t lbn, bpref; int size, flags; struct ucred *cred; ufs2_daddr_t *bnp; { struct fs *fs; struct ufsmount *ump; ufs2_daddr_t bno; u_int cg, reclaimed; static struct timeval lastfail; static int curfail; int64_t delta; #ifdef QUOTA int error; #endif *bnp = 0; fs = ip->i_fs; ump = ip->i_ump; mtx_assert(UFS_MTX(ump), MA_OWNED); #ifdef INVARIANTS if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) { printf("dev = %s, bsize = %ld, size = %d, fs = %s\n", devtoname(ip->i_dev), (long)fs->fs_bsize, size, fs->fs_fsmnt); panic("ffs_alloc: bad size"); } if (cred == NOCRED) panic("ffs_alloc: missing credential"); #endif /* INVARIANTS */ reclaimed = 0; retry: #ifdef QUOTA UFS_UNLOCK(ump); error = chkdq(ip, btodb(size), cred, 0); if (error) return (error); UFS_LOCK(ump); #endif if (size == fs->fs_bsize && fs->fs_cstotal.cs_nbfree == 0) goto nospace; if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE, 0) && freespace(fs, fs->fs_minfree) - numfrags(fs, size) < 0) goto nospace; if (bpref >= fs->fs_size) bpref = 0; if (bpref == 0) cg = ino_to_cg(fs, ip->i_number); else cg = dtog(fs, bpref); bno = ffs_hashalloc(ip, cg, bpref, size, ffs_alloccg); if (bno > 0) { delta = btodb(size); if (ip->i_flag & IN_SPACECOUNTED) { UFS_LOCK(ump); fs->fs_pendingblocks += delta; UFS_UNLOCK(ump); } DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta); if (flags & IO_EXT) ip->i_flag |= IN_CHANGE; else ip->i_flag |= IN_CHANGE | IN_UPDATE; *bnp = bno; return (0); } nospace: #ifdef QUOTA UFS_UNLOCK(ump); /* * Restore user's disk quota because allocation failed. */ (void) chkdq(ip, -btodb(size), cred, FORCE); UFS_LOCK(ump); #endif if (fs->fs_pendingblocks > 0 && reclaimed == 0) { reclaimed = 1; softdep_request_cleanup(fs, ITOV(ip)); goto retry; } UFS_UNLOCK(ump); if (ppsratecheck(&lastfail, &curfail, 1)) { ffs_fserr(fs, ip->i_number, "filesystem full"); uprintf("\n%s: write failed, filesystem is full\n", fs->fs_fsmnt); } return (ENOSPC); } /* * Reallocate a fragment to a bigger size * * The number and size of the old block is given, and a preference * and new size is also specified. The allocator attempts to extend * the original block. Failing that, the regular block allocator is * invoked to get an appropriate block. */ int ffs_realloccg(ip, lbprev, bprev, bpref, osize, nsize, flags, cred, bpp) struct inode *ip; ufs2_daddr_t lbprev; ufs2_daddr_t bprev; ufs2_daddr_t bpref; int osize, nsize, flags; struct ucred *cred; struct buf **bpp; { struct vnode *vp; struct fs *fs; struct buf *bp; struct ufsmount *ump; u_int cg, request, reclaimed; int error; ufs2_daddr_t bno; static struct timeval lastfail; static int curfail; int64_t delta; *bpp = 0; vp = ITOV(ip); fs = ip->i_fs; bp = NULL; ump = ip->i_ump; mtx_assert(UFS_MTX(ump), MA_OWNED); #ifdef INVARIANTS if (vp->v_mount->mnt_kern_flag & MNTK_SUSPENDED) panic("ffs_realloccg: allocation on suspended filesystem"); if ((u_int)osize > fs->fs_bsize || fragoff(fs, osize) != 0 || (u_int)nsize > fs->fs_bsize || fragoff(fs, nsize) != 0) { printf( "dev = %s, bsize = %ld, osize = %d, nsize = %d, fs = %s\n", devtoname(ip->i_dev), (long)fs->fs_bsize, osize, nsize, fs->fs_fsmnt); panic("ffs_realloccg: bad size"); } if (cred == NOCRED) panic("ffs_realloccg: missing credential"); #endif /* INVARIANTS */ reclaimed = 0; retry: if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE, 0) && freespace(fs, fs->fs_minfree) - numfrags(fs, nsize - osize) < 0) { goto nospace; } if (bprev == 0) { printf("dev = %s, bsize = %ld, bprev = %jd, fs = %s\n", devtoname(ip->i_dev), (long)fs->fs_bsize, (intmax_t)bprev, fs->fs_fsmnt); panic("ffs_realloccg: bad bprev"); } UFS_UNLOCK(ump); /* * Allocate the extra space in the buffer. */ error = bread(vp, lbprev, osize, NOCRED, &bp); if (error) { brelse(bp); return (error); } if (bp->b_blkno == bp->b_lblkno) { if (lbprev >= NDADDR) panic("ffs_realloccg: lbprev out of range"); bp->b_blkno = fsbtodb(fs, bprev); } #ifdef QUOTA error = chkdq(ip, btodb(nsize - osize), cred, 0); if (error) { brelse(bp); return (error); } #endif /* * Check for extension in the existing location. */ cg = dtog(fs, bprev); UFS_LOCK(ump); bno = ffs_fragextend(ip, cg, bprev, osize, nsize); if (bno) { if (bp->b_blkno != fsbtodb(fs, bno)) panic("ffs_realloccg: bad blockno"); delta = btodb(nsize - osize); if (ip->i_flag & IN_SPACECOUNTED) { UFS_LOCK(ump); fs->fs_pendingblocks += delta; UFS_UNLOCK(ump); } DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta); if (flags & IO_EXT) ip->i_flag |= IN_CHANGE; else ip->i_flag |= IN_CHANGE | IN_UPDATE; allocbuf(bp, nsize); bp->b_flags |= B_DONE; bzero(bp->b_data + osize, nsize - osize); if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO) vfs_bio_set_valid(bp, osize, nsize - osize); *bpp = bp; return (0); } /* * Allocate a new disk location. */ if (bpref >= fs->fs_size) bpref = 0; switch ((int)fs->fs_optim) { case FS_OPTSPACE: /* * Allocate an exact sized fragment. Although this makes * best use of space, we will waste time relocating it if * the file continues to grow. If the fragmentation is * less than half of the minimum free reserve, we choose * to begin optimizing for time. */ request = nsize; if (fs->fs_minfree <= 5 || fs->fs_cstotal.cs_nffree > (off_t)fs->fs_dsize * fs->fs_minfree / (2 * 100)) break; log(LOG_NOTICE, "%s: optimization changed from SPACE to TIME\n", fs->fs_fsmnt); fs->fs_optim = FS_OPTTIME; break; case FS_OPTTIME: /* * At this point we have discovered a file that is trying to * grow a small fragment to a larger fragment. To save time, * we allocate a full sized block, then free the unused portion. * If the file continues to grow, the `ffs_fragextend' call * above will be able to grow it in place without further * copying. If aberrant programs cause disk fragmentation to * grow within 2% of the free reserve, we choose to begin * optimizing for space. */ request = fs->fs_bsize; if (fs->fs_cstotal.cs_nffree < (off_t)fs->fs_dsize * (fs->fs_minfree - 2) / 100) break; log(LOG_NOTICE, "%s: optimization changed from TIME to SPACE\n", fs->fs_fsmnt); fs->fs_optim = FS_OPTSPACE; break; default: printf("dev = %s, optim = %ld, fs = %s\n", devtoname(ip->i_dev), (long)fs->fs_optim, fs->fs_fsmnt); panic("ffs_realloccg: bad optim"); /* NOTREACHED */ } bno = ffs_hashalloc(ip, cg, bpref, request, ffs_alloccg); if (bno > 0) { bp->b_blkno = fsbtodb(fs, bno); if (!DOINGSOFTDEP(vp)) ffs_blkfree(ump, fs, ip->i_devvp, bprev, (long)osize, ip->i_number); if (nsize < request) ffs_blkfree(ump, fs, ip->i_devvp, bno + numfrags(fs, nsize), (long)(request - nsize), ip->i_number); delta = btodb(nsize - osize); if (ip->i_flag & IN_SPACECOUNTED) { UFS_LOCK(ump); fs->fs_pendingblocks += delta; UFS_UNLOCK(ump); } DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + delta); if (flags & IO_EXT) ip->i_flag |= IN_CHANGE; else ip->i_flag |= IN_CHANGE | IN_UPDATE; allocbuf(bp, nsize); bp->b_flags |= B_DONE; bzero(bp->b_data + osize, nsize - osize); if ((bp->b_flags & (B_MALLOC | B_VMIO)) == B_VMIO) vfs_bio_set_valid(bp, osize, nsize - osize); *bpp = bp; return (0); } #ifdef QUOTA UFS_UNLOCK(ump); /* * Restore user's disk quota because allocation failed. */ (void) chkdq(ip, -btodb(nsize - osize), cred, FORCE); UFS_LOCK(ump); #endif nospace: /* * no space available */ if (fs->fs_pendingblocks > 0 && reclaimed == 0) { reclaimed = 1; softdep_request_cleanup(fs, vp); UFS_UNLOCK(ump); - if (bp) + if (bp) { brelse(bp); + bp = NULL; + } UFS_LOCK(ump); goto retry; } UFS_UNLOCK(ump); if (bp) brelse(bp); if (ppsratecheck(&lastfail, &curfail, 1)) { ffs_fserr(fs, ip->i_number, "filesystem full"); uprintf("\n%s: write failed, filesystem is full\n", fs->fs_fsmnt); } return (ENOSPC); } /* * Reallocate a sequence of blocks into a contiguous sequence of blocks. * * The vnode and an array of buffer pointers for a range of sequential * logical blocks to be made contiguous is given. The allocator attempts * to find a range of sequential blocks starting as close as possible * from the end of the allocation for the logical block immediately * preceding the current range. If successful, the physical block numbers * in the buffer pointers and in the inode are changed to reflect the new * allocation. If unsuccessful, the allocation is left unchanged. The * success in doing the reallocation is returned. Note that the error * return is not reflected back to the user. Rather the previous block * allocation will be used. */ SYSCTL_NODE(_vfs, OID_AUTO, ffs, CTLFLAG_RW, 0, "FFS filesystem"); static int doasyncfree = 1; SYSCTL_INT(_vfs_ffs, OID_AUTO, doasyncfree, CTLFLAG_RW, &doasyncfree, 0, ""); static int doreallocblks = 1; SYSCTL_INT(_vfs_ffs, OID_AUTO, doreallocblks, CTLFLAG_RW, &doreallocblks, 0, ""); #ifdef DEBUG static volatile int prtrealloc = 0; #endif int ffs_reallocblks(ap) struct vop_reallocblks_args /* { struct vnode *a_vp; struct cluster_save *a_buflist; } */ *ap; { if (doreallocblks == 0) return (ENOSPC); if (VTOI(ap->a_vp)->i_ump->um_fstype == UFS1) return (ffs_reallocblks_ufs1(ap)); return (ffs_reallocblks_ufs2(ap)); } static int ffs_reallocblks_ufs1(ap) struct vop_reallocblks_args /* { struct vnode *a_vp; struct cluster_save *a_buflist; } */ *ap; { struct fs *fs; struct inode *ip; struct vnode *vp; struct buf *sbp, *ebp; ufs1_daddr_t *bap, *sbap, *ebap = 0; struct cluster_save *buflist; struct ufsmount *ump; ufs_lbn_t start_lbn, end_lbn; ufs1_daddr_t soff, newblk, blkno; ufs2_daddr_t pref; struct indir start_ap[NIADDR + 1], end_ap[NIADDR + 1], *idp; int i, len, start_lvl, end_lvl, ssize; vp = ap->a_vp; ip = VTOI(vp); fs = ip->i_fs; ump = ip->i_ump; if (fs->fs_contigsumsize <= 0) return (ENOSPC); buflist = ap->a_buflist; len = buflist->bs_nchildren; start_lbn = buflist->bs_children[0]->b_lblkno; end_lbn = start_lbn + len - 1; #ifdef INVARIANTS for (i = 0; i < len; i++) if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 1"); for (i = 1; i < len; i++) if (buflist->bs_children[i]->b_lblkno != start_lbn + i) panic("ffs_reallocblks: non-logical cluster"); blkno = buflist->bs_children[0]->b_blkno; ssize = fsbtodb(fs, fs->fs_frag); for (i = 1; i < len - 1; i++) if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize)) panic("ffs_reallocblks: non-physical cluster %d", i); #endif /* * If the latest allocation is in a new cylinder group, assume that * the filesystem has decided to move and do not force it back to * the previous cylinder group. */ if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) != dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno))) return (ENOSPC); if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) || ufs_getlbns(vp, end_lbn, end_ap, &end_lvl)) return (ENOSPC); /* * Get the starting offset and block map for the first block. */ if (start_lvl == 0) { sbap = &ip->i_din1->di_db[0]; soff = start_lbn; } else { idp = &start_ap[start_lvl - 1]; if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) { brelse(sbp); return (ENOSPC); } sbap = (ufs1_daddr_t *)sbp->b_data; soff = idp->in_off; } /* * If the block range spans two block maps, get the second map. */ if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) { ssize = len; } else { #ifdef INVARIANTS if (start_lvl > 0 && start_ap[start_lvl - 1].in_lbn == idp->in_lbn) panic("ffs_reallocblk: start == end"); #endif ssize = len - (idp->in_off + 1); if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp)) goto fail; ebap = (ufs1_daddr_t *)ebp->b_data; } /* * Find the preferred location for the cluster. */ UFS_LOCK(ump); pref = ffs_blkpref_ufs1(ip, start_lbn, soff, sbap); /* * Search the block map looking for an allocation of the desired size. */ if ((newblk = ffs_hashalloc(ip, dtog(fs, pref), pref, len, ffs_clusteralloc)) == 0) { UFS_UNLOCK(ump); goto fail; } /* * We have found a new contiguous block. * * First we have to replace the old block pointers with the new * block pointers in the inode and indirect blocks associated * with the file. */ #ifdef DEBUG if (prtrealloc) printf("realloc: ino %d, lbns %jd-%jd\n\told:", ip->i_number, (intmax_t)start_lbn, (intmax_t)end_lbn); #endif blkno = newblk; for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) { if (i == ssize) { bap = ebap; soff = -i; } #ifdef INVARIANTS if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 2"); if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap) panic("ffs_reallocblks: alloc mismatch"); #endif #ifdef DEBUG if (prtrealloc) printf(" %d,", *bap); #endif if (DOINGSOFTDEP(vp)) { if (sbap == &ip->i_din1->di_db[0] && i < ssize) softdep_setup_allocdirect(ip, start_lbn + i, blkno, *bap, fs->fs_bsize, fs->fs_bsize, buflist->bs_children[i]); else softdep_setup_allocindir_page(ip, start_lbn + i, i < ssize ? sbp : ebp, soff + i, blkno, *bap, buflist->bs_children[i]); } *bap++ = blkno; } /* * Next we must write out the modified inode and indirect blocks. * For strict correctness, the writes should be synchronous since * the old block values may have been written to disk. In practise * they are almost never written, but if we are concerned about * strict correctness, the `doasyncfree' flag should be set to zero. * * The test on `doasyncfree' should be changed to test a flag * that shows whether the associated buffers and inodes have * been written. The flag should be set when the cluster is * started and cleared whenever the buffer or inode is flushed. * We can then check below to see if it is set, and do the * synchronous write only when it has been cleared. */ if (sbap != &ip->i_din1->di_db[0]) { if (doasyncfree) bdwrite(sbp); else bwrite(sbp); } else { ip->i_flag |= IN_CHANGE | IN_UPDATE; if (!doasyncfree) ffs_update(vp, 1); } if (ssize < len) { if (doasyncfree) bdwrite(ebp); else bwrite(ebp); } /* * Last, free the old blocks and assign the new blocks to the buffers. */ #ifdef DEBUG if (prtrealloc) printf("\n\tnew:"); #endif for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) { if (!DOINGSOFTDEP(vp)) ffs_blkfree(ump, fs, ip->i_devvp, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize, ip->i_number); buflist->bs_children[i]->b_blkno = fsbtodb(fs, blkno); #ifdef INVARIANTS if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 3"); #endif #ifdef DEBUG if (prtrealloc) printf(" %d,", blkno); #endif } #ifdef DEBUG if (prtrealloc) { prtrealloc--; printf("\n"); } #endif return (0); fail: if (ssize < len) brelse(ebp); if (sbap != &ip->i_din1->di_db[0]) brelse(sbp); return (ENOSPC); } static int ffs_reallocblks_ufs2(ap) struct vop_reallocblks_args /* { struct vnode *a_vp; struct cluster_save *a_buflist; } */ *ap; { struct fs *fs; struct inode *ip; struct vnode *vp; struct buf *sbp, *ebp; ufs2_daddr_t *bap, *sbap, *ebap = 0; struct cluster_save *buflist; struct ufsmount *ump; ufs_lbn_t start_lbn, end_lbn; ufs2_daddr_t soff, newblk, blkno, pref; struct indir start_ap[NIADDR + 1], end_ap[NIADDR + 1], *idp; int i, len, start_lvl, end_lvl, ssize; vp = ap->a_vp; ip = VTOI(vp); fs = ip->i_fs; ump = ip->i_ump; if (fs->fs_contigsumsize <= 0) return (ENOSPC); buflist = ap->a_buflist; len = buflist->bs_nchildren; start_lbn = buflist->bs_children[0]->b_lblkno; end_lbn = start_lbn + len - 1; #ifdef INVARIANTS for (i = 0; i < len; i++) if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 1"); for (i = 1; i < len; i++) if (buflist->bs_children[i]->b_lblkno != start_lbn + i) panic("ffs_reallocblks: non-logical cluster"); blkno = buflist->bs_children[0]->b_blkno; ssize = fsbtodb(fs, fs->fs_frag); for (i = 1; i < len - 1; i++) if (buflist->bs_children[i]->b_blkno != blkno + (i * ssize)) panic("ffs_reallocblks: non-physical cluster %d", i); #endif /* * If the latest allocation is in a new cylinder group, assume that * the filesystem has decided to move and do not force it back to * the previous cylinder group. */ if (dtog(fs, dbtofsb(fs, buflist->bs_children[0]->b_blkno)) != dtog(fs, dbtofsb(fs, buflist->bs_children[len - 1]->b_blkno))) return (ENOSPC); if (ufs_getlbns(vp, start_lbn, start_ap, &start_lvl) || ufs_getlbns(vp, end_lbn, end_ap, &end_lvl)) return (ENOSPC); /* * Get the starting offset and block map for the first block. */ if (start_lvl == 0) { sbap = &ip->i_din2->di_db[0]; soff = start_lbn; } else { idp = &start_ap[start_lvl - 1]; if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &sbp)) { brelse(sbp); return (ENOSPC); } sbap = (ufs2_daddr_t *)sbp->b_data; soff = idp->in_off; } /* * If the block range spans two block maps, get the second map. */ if (end_lvl == 0 || (idp = &end_ap[end_lvl - 1])->in_off + 1 >= len) { ssize = len; } else { #ifdef INVARIANTS if (start_lvl > 0 && start_ap[start_lvl - 1].in_lbn == idp->in_lbn) panic("ffs_reallocblk: start == end"); #endif ssize = len - (idp->in_off + 1); if (bread(vp, idp->in_lbn, (int)fs->fs_bsize, NOCRED, &ebp)) goto fail; ebap = (ufs2_daddr_t *)ebp->b_data; } /* * Find the preferred location for the cluster. */ UFS_LOCK(ump); pref = ffs_blkpref_ufs2(ip, start_lbn, soff, sbap); /* * Search the block map looking for an allocation of the desired size. */ if ((newblk = ffs_hashalloc(ip, dtog(fs, pref), pref, len, ffs_clusteralloc)) == 0) { UFS_UNLOCK(ump); goto fail; } /* * We have found a new contiguous block. * * First we have to replace the old block pointers with the new * block pointers in the inode and indirect blocks associated * with the file. */ #ifdef DEBUG if (prtrealloc) printf("realloc: ino %d, lbns %jd-%jd\n\told:", ip->i_number, (intmax_t)start_lbn, (intmax_t)end_lbn); #endif blkno = newblk; for (bap = &sbap[soff], i = 0; i < len; i++, blkno += fs->fs_frag) { if (i == ssize) { bap = ebap; soff = -i; } #ifdef INVARIANTS if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 2"); if (dbtofsb(fs, buflist->bs_children[i]->b_blkno) != *bap) panic("ffs_reallocblks: alloc mismatch"); #endif #ifdef DEBUG if (prtrealloc) printf(" %jd,", (intmax_t)*bap); #endif if (DOINGSOFTDEP(vp)) { if (sbap == &ip->i_din2->di_db[0] && i < ssize) softdep_setup_allocdirect(ip, start_lbn + i, blkno, *bap, fs->fs_bsize, fs->fs_bsize, buflist->bs_children[i]); else softdep_setup_allocindir_page(ip, start_lbn + i, i < ssize ? sbp : ebp, soff + i, blkno, *bap, buflist->bs_children[i]); } *bap++ = blkno; } /* * Next we must write out the modified inode and indirect blocks. * For strict correctness, the writes should be synchronous since * the old block values may have been written to disk. In practise * they are almost never written, but if we are concerned about * strict correctness, the `doasyncfree' flag should be set to zero. * * The test on `doasyncfree' should be changed to test a flag * that shows whether the associated buffers and inodes have * been written. The flag should be set when the cluster is * started and cleared whenever the buffer or inode is flushed. * We can then check below to see if it is set, and do the * synchronous write only when it has been cleared. */ if (sbap != &ip->i_din2->di_db[0]) { if (doasyncfree) bdwrite(sbp); else bwrite(sbp); } else { ip->i_flag |= IN_CHANGE | IN_UPDATE; if (!doasyncfree) ffs_update(vp, 1); } if (ssize < len) { if (doasyncfree) bdwrite(ebp); else bwrite(ebp); } /* * Last, free the old blocks and assign the new blocks to the buffers. */ #ifdef DEBUG if (prtrealloc) printf("\n\tnew:"); #endif for (blkno = newblk, i = 0; i < len; i++, blkno += fs->fs_frag) { if (!DOINGSOFTDEP(vp)) ffs_blkfree(ump, fs, ip->i_devvp, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize, ip->i_number); buflist->bs_children[i]->b_blkno = fsbtodb(fs, blkno); #ifdef INVARIANTS if (!ffs_checkblk(ip, dbtofsb(fs, buflist->bs_children[i]->b_blkno), fs->fs_bsize)) panic("ffs_reallocblks: unallocated block 3"); #endif #ifdef DEBUG if (prtrealloc) printf(" %jd,", (intmax_t)blkno); #endif } #ifdef DEBUG if (prtrealloc) { prtrealloc--; printf("\n"); } #endif return (0); fail: if (ssize < len) brelse(ebp); if (sbap != &ip->i_din2->di_db[0]) brelse(sbp); return (ENOSPC); } /* * Allocate an inode in the filesystem. * * If allocating a directory, use ffs_dirpref to select the inode. * If allocating in a directory, the following hierarchy is followed: * 1) allocate the preferred inode. * 2) allocate an inode in the same cylinder group. * 3) quadradically rehash into other cylinder groups, until an * available inode is located. * If no inode preference is given the following hierarchy is used * to allocate an inode: * 1) allocate an inode in cylinder group 0. * 2) quadradically rehash into other cylinder groups, until an * available inode is located. */ int ffs_valloc(pvp, mode, cred, vpp) struct vnode *pvp; int mode; struct ucred *cred; struct vnode **vpp; { struct inode *pip; struct fs *fs; struct inode *ip; struct timespec ts; struct ufsmount *ump; ino_t ino, ipref; u_int cg; int error, error1; static struct timeval lastfail; static int curfail; *vpp = NULL; pip = VTOI(pvp); fs = pip->i_fs; ump = pip->i_ump; UFS_LOCK(ump); if (fs->fs_cstotal.cs_nifree == 0) goto noinodes; if ((mode & IFMT) == IFDIR) ipref = ffs_dirpref(pip); else ipref = pip->i_number; if (ipref >= fs->fs_ncg * fs->fs_ipg) ipref = 0; cg = ino_to_cg(fs, ipref); /* * Track number of dirs created one after another * in a same cg without intervening by files. */ if ((mode & IFMT) == IFDIR) { if (fs->fs_contigdirs[cg] < 255) fs->fs_contigdirs[cg]++; } else { if (fs->fs_contigdirs[cg] > 0) fs->fs_contigdirs[cg]--; } ino = (ino_t)ffs_hashalloc(pip, cg, ipref, mode, (allocfcn_t *)ffs_nodealloccg); if (ino == 0) goto noinodes; error = ffs_vget(pvp->v_mount, ino, LK_EXCLUSIVE, vpp); if (error) { error1 = ffs_vgetf(pvp->v_mount, ino, LK_EXCLUSIVE, vpp, FFSV_FORCEINSMQ); ffs_vfree(pvp, ino, mode); if (error1 == 0) { ip = VTOI(*vpp); if (ip->i_mode) goto dup_alloc; ip->i_flag |= IN_MODIFIED; vput(*vpp); } return (error); } ip = VTOI(*vpp); if (ip->i_mode) { dup_alloc: printf("mode = 0%o, inum = %lu, fs = %s\n", ip->i_mode, (u_long)ip->i_number, fs->fs_fsmnt); panic("ffs_valloc: dup alloc"); } if (DIP(ip, i_blocks) && (fs->fs_flags & FS_UNCLEAN) == 0) { /* XXX */ printf("free inode %s/%lu had %ld blocks\n", fs->fs_fsmnt, (u_long)ino, (long)DIP(ip, i_blocks)); DIP_SET(ip, i_blocks, 0); } ip->i_flags = 0; DIP_SET(ip, i_flags, 0); /* * Set up a new generation number for this inode. */ if (ip->i_gen == 0 || ++ip->i_gen == 0) ip->i_gen = arc4random() / 2 + 1; DIP_SET(ip, i_gen, ip->i_gen); if (fs->fs_magic == FS_UFS2_MAGIC) { vfs_timestamp(&ts); ip->i_din2->di_birthtime = ts.tv_sec; ip->i_din2->di_birthnsec = ts.tv_nsec; } ip->i_flag = 0; vnode_destroy_vobject(*vpp); (*vpp)->v_type = VNON; if (fs->fs_magic == FS_UFS2_MAGIC) (*vpp)->v_op = &ffs_vnodeops2; else (*vpp)->v_op = &ffs_vnodeops1; return (0); noinodes: UFS_UNLOCK(ump); if (ppsratecheck(&lastfail, &curfail, 1)) { ffs_fserr(fs, pip->i_number, "out of inodes"); uprintf("\n%s: create/symlink failed, no inodes free\n", fs->fs_fsmnt); } return (ENOSPC); } /* * Find a cylinder group to place a directory. * * The policy implemented by this algorithm is to allocate a * directory inode in the same cylinder group as its parent * directory, but also to reserve space for its files inodes * and data. Restrict the number of directories which may be * allocated one after another in the same cylinder group * without intervening allocation of files. * * If we allocate a first level directory then force allocation * in another cylinder group. */ static ino_t ffs_dirpref(pip) struct inode *pip; { struct fs *fs; u_int cg, prefcg, dirsize, cgsize; u_int avgifree, avgbfree, avgndir, curdirsize; u_int minifree, minbfree, maxndir; u_int mincg, minndir; u_int maxcontigdirs; mtx_assert(UFS_MTX(pip->i_ump), MA_OWNED); fs = pip->i_fs; avgifree = fs->fs_cstotal.cs_nifree / fs->fs_ncg; avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg; avgndir = fs->fs_cstotal.cs_ndir / fs->fs_ncg; /* * Force allocation in another cg if creating a first level dir. */ ASSERT_VOP_LOCKED(ITOV(pip), "ffs_dirpref"); if (ITOV(pip)->v_vflag & VV_ROOT) { prefcg = arc4random() % fs->fs_ncg; mincg = prefcg; minndir = fs->fs_ipg; for (cg = prefcg; cg < fs->fs_ncg; cg++) if (fs->fs_cs(fs, cg).cs_ndir < minndir && fs->fs_cs(fs, cg).cs_nifree >= avgifree && fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { mincg = cg; minndir = fs->fs_cs(fs, cg).cs_ndir; } for (cg = 0; cg < prefcg; cg++) if (fs->fs_cs(fs, cg).cs_ndir < minndir && fs->fs_cs(fs, cg).cs_nifree >= avgifree && fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { mincg = cg; minndir = fs->fs_cs(fs, cg).cs_ndir; } return ((ino_t)(fs->fs_ipg * mincg)); } /* * Count various limits which used for * optimal allocation of a directory inode. */ maxndir = min(avgndir + fs->fs_ipg / 16, fs->fs_ipg); minifree = avgifree - avgifree / 4; if (minifree < 1) minifree = 1; minbfree = avgbfree - avgbfree / 4; if (minbfree < 1) minbfree = 1; cgsize = fs->fs_fsize * fs->fs_fpg; dirsize = fs->fs_avgfilesize * fs->fs_avgfpdir; curdirsize = avgndir ? (cgsize - avgbfree * fs->fs_bsize) / avgndir : 0; if (dirsize < curdirsize) dirsize = curdirsize; if (dirsize <= 0) maxcontigdirs = 0; /* dirsize overflowed */ else maxcontigdirs = min((avgbfree * fs->fs_bsize) / dirsize, 255); if (fs->fs_avgfpdir > 0) maxcontigdirs = min(maxcontigdirs, fs->fs_ipg / fs->fs_avgfpdir); if (maxcontigdirs == 0) maxcontigdirs = 1; /* * Limit number of dirs in one cg and reserve space for * regular files, but only if we have no deficit in * inodes or space. */ prefcg = ino_to_cg(fs, pip->i_number); for (cg = prefcg; cg < fs->fs_ncg; cg++) if (fs->fs_cs(fs, cg).cs_ndir < maxndir && fs->fs_cs(fs, cg).cs_nifree >= minifree && fs->fs_cs(fs, cg).cs_nbfree >= minbfree) { if (fs->fs_contigdirs[cg] < maxcontigdirs) return ((ino_t)(fs->fs_ipg * cg)); } for (cg = 0; cg < prefcg; cg++) if (fs->fs_cs(fs, cg).cs_ndir < maxndir && fs->fs_cs(fs, cg).cs_nifree >= minifree && fs->fs_cs(fs, cg).cs_nbfree >= minbfree) { if (fs->fs_contigdirs[cg] < maxcontigdirs) return ((ino_t)(fs->fs_ipg * cg)); } /* * This is a backstop when we have deficit in space. */ for (cg = prefcg; cg < fs->fs_ncg; cg++) if (fs->fs_cs(fs, cg).cs_nifree >= avgifree) return ((ino_t)(fs->fs_ipg * cg)); for (cg = 0; cg < prefcg; cg++) if (fs->fs_cs(fs, cg).cs_nifree >= avgifree) break; return ((ino_t)(fs->fs_ipg * cg)); } /* * Select the desired position for the next block in a file. The file is * logically divided into sections. The first section is composed of the * direct blocks. Each additional section contains fs_maxbpg blocks. * * If no blocks have been allocated in the first section, the policy is to * request a block in the same cylinder group as the inode that describes * the file. If no blocks have been allocated in any other section, the * policy is to place the section in a cylinder group with a greater than * average number of free blocks. An appropriate cylinder group is found * by using a rotor that sweeps the cylinder groups. When a new group of * blocks is needed, the sweep begins in the cylinder group following the * cylinder group from which the previous allocation was made. The sweep * continues until a cylinder group with greater than the average number * of free blocks is found. If the allocation is for the first block in an * indirect block, the information on the previous allocation is unavailable; * here a best guess is made based upon the logical block number being * allocated. * * If a section is already partially allocated, the policy is to * contiguously allocate fs_maxcontig blocks. The end of one of these * contiguous blocks and the beginning of the next is laid out * contiguously if possible. */ ufs2_daddr_t ffs_blkpref_ufs1(ip, lbn, indx, bap) struct inode *ip; ufs_lbn_t lbn; int indx; ufs1_daddr_t *bap; { struct fs *fs; u_int cg; u_int avgbfree, startcg; mtx_assert(UFS_MTX(ip->i_ump), MA_OWNED); fs = ip->i_fs; if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) { if (lbn < NDADDR + NINDIR(fs)) { cg = ino_to_cg(fs, ip->i_number); return (cgbase(fs, cg) + fs->fs_frag); } /* * Find a cylinder with greater than average number of * unused data blocks. */ if (indx == 0 || bap[indx - 1] == 0) startcg = ino_to_cg(fs, ip->i_number) + lbn / fs->fs_maxbpg; else startcg = dtog(fs, bap[indx - 1]) + 1; startcg %= fs->fs_ncg; avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg; for (cg = startcg; cg < fs->fs_ncg; cg++) if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { fs->fs_cgrotor = cg; return (cgbase(fs, cg) + fs->fs_frag); } for (cg = 0; cg <= startcg; cg++) if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { fs->fs_cgrotor = cg; return (cgbase(fs, cg) + fs->fs_frag); } return (0); } /* * We just always try to lay things out contiguously. */ return (bap[indx - 1] + fs->fs_frag); } /* * Same as above, but for UFS2 */ ufs2_daddr_t ffs_blkpref_ufs2(ip, lbn, indx, bap) struct inode *ip; ufs_lbn_t lbn; int indx; ufs2_daddr_t *bap; { struct fs *fs; u_int cg; u_int avgbfree, startcg; mtx_assert(UFS_MTX(ip->i_ump), MA_OWNED); fs = ip->i_fs; if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) { if (lbn < NDADDR + NINDIR(fs)) { cg = ino_to_cg(fs, ip->i_number); return (cgbase(fs, cg) + fs->fs_frag); } /* * Find a cylinder with greater than average number of * unused data blocks. */ if (indx == 0 || bap[indx - 1] == 0) startcg = ino_to_cg(fs, ip->i_number) + lbn / fs->fs_maxbpg; else startcg = dtog(fs, bap[indx - 1]) + 1; startcg %= fs->fs_ncg; avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg; for (cg = startcg; cg < fs->fs_ncg; cg++) if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { fs->fs_cgrotor = cg; return (cgbase(fs, cg) + fs->fs_frag); } for (cg = 0; cg <= startcg; cg++) if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) { fs->fs_cgrotor = cg; return (cgbase(fs, cg) + fs->fs_frag); } return (0); } /* * We just always try to lay things out contiguously. */ return (bap[indx - 1] + fs->fs_frag); } /* * Implement the cylinder overflow algorithm. * * The policy implemented by this algorithm is: * 1) allocate the block in its requested cylinder group. * 2) quadradically rehash on the cylinder group number. * 3) brute force search for a free block. * * Must be called with the UFS lock held. Will release the lock on success * and return with it held on failure. */ /*VARARGS5*/ static ufs2_daddr_t ffs_hashalloc(ip, cg, pref, size, allocator) struct inode *ip; u_int cg; ufs2_daddr_t pref; int size; /* size for data blocks, mode for inodes */ allocfcn_t *allocator; { struct fs *fs; ufs2_daddr_t result; u_int i, icg = cg; mtx_assert(UFS_MTX(ip->i_ump), MA_OWNED); #ifdef INVARIANTS if (ITOV(ip)->v_mount->mnt_kern_flag & MNTK_SUSPENDED) panic("ffs_hashalloc: allocation on suspended filesystem"); #endif fs = ip->i_fs; /* * 1: preferred cylinder group */ result = (*allocator)(ip, cg, pref, size); if (result) return (result); /* * 2: quadratic rehash */ for (i = 1; i < fs->fs_ncg; i *= 2) { cg += i; if (cg >= fs->fs_ncg) cg -= fs->fs_ncg; result = (*allocator)(ip, cg, 0, size); if (result) return (result); } /* * 3: brute force search * Note that we start at i == 2, since 0 was checked initially, * and 1 is always checked in the quadratic rehash. */ cg = (icg + 2) % fs->fs_ncg; for (i = 2; i < fs->fs_ncg; i++) { result = (*allocator)(ip, cg, 0, size); if (result) return (result); cg++; if (cg == fs->fs_ncg) cg = 0; } return (0); } /* * Determine whether a fragment can be extended. * * Check to see if the necessary fragments are available, and * if they are, allocate them. */ static ufs2_daddr_t ffs_fragextend(ip, cg, bprev, osize, nsize) struct inode *ip; u_int cg; ufs2_daddr_t bprev; int osize, nsize; { struct fs *fs; struct cg *cgp; struct buf *bp; struct ufsmount *ump; int nffree; long bno; int frags, bbase; int i, error; u_int8_t *blksfree; ump = ip->i_ump; fs = ip->i_fs; if (fs->fs_cs(fs, cg).cs_nffree < numfrags(fs, nsize - osize)) return (0); frags = numfrags(fs, nsize); bbase = fragnum(fs, bprev); if (bbase > fragnum(fs, (bprev + frags - 1))) { /* cannot extend across a block boundary */ return (0); } UFS_UNLOCK(ump); error = bread(ip->i_devvp, fsbtodb(fs, cgtod(fs, cg)), (int)fs->fs_cgsize, NOCRED, &bp); if (error) goto fail; cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) goto fail; bp->b_xflags |= BX_BKGRDWRITE; cgp->cg_old_time = cgp->cg_time = time_second; bno = dtogd(fs, bprev); blksfree = cg_blksfree(cgp); for (i = numfrags(fs, osize); i < frags; i++) if (isclr(blksfree, bno + i)) goto fail; /* * the current fragment can be extended * deduct the count on fragment being extended into * increase the count on the remaining fragment (if any) * allocate the extended piece */ for (i = frags; i < fs->fs_frag - bbase; i++) if (isclr(blksfree, bno + i)) break; cgp->cg_frsum[i - numfrags(fs, osize)]--; if (i != frags) cgp->cg_frsum[i - frags]++; for (i = numfrags(fs, osize), nffree = 0; i < frags; i++) { clrbit(blksfree, bno + i); cgp->cg_cs.cs_nffree--; nffree++; } UFS_LOCK(ump); fs->fs_cstotal.cs_nffree -= nffree; fs->fs_cs(fs, cg).cs_nffree -= nffree; fs->fs_fmod = 1; ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); if (DOINGSOFTDEP(ITOV(ip))) softdep_setup_blkmapdep(bp, UFSTOVFS(ump), bprev); bdwrite(bp); return (bprev); fail: brelse(bp); UFS_LOCK(ump); return (0); } /* * Determine whether a block can be allocated. * * Check to see if a block of the appropriate size is available, * and if it is, allocate it. */ static ufs2_daddr_t ffs_alloccg(ip, cg, bpref, size) struct inode *ip; u_int cg; ufs2_daddr_t bpref; int size; { struct fs *fs; struct cg *cgp; struct buf *bp; struct ufsmount *ump; ufs1_daddr_t bno; ufs2_daddr_t blkno; int i, allocsiz, error, frags; u_int8_t *blksfree; ump = ip->i_ump; fs = ip->i_fs; if (fs->fs_cs(fs, cg).cs_nbfree == 0 && size == fs->fs_bsize) return (0); UFS_UNLOCK(ump); error = bread(ip->i_devvp, fsbtodb(fs, cgtod(fs, cg)), (int)fs->fs_cgsize, NOCRED, &bp); if (error) goto fail; cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp) || (cgp->cg_cs.cs_nbfree == 0 && size == fs->fs_bsize)) goto fail; bp->b_xflags |= BX_BKGRDWRITE; cgp->cg_old_time = cgp->cg_time = time_second; if (size == fs->fs_bsize) { UFS_LOCK(ump); blkno = ffs_alloccgblk(ip, bp, bpref); ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); bdwrite(bp); return (blkno); } /* * check to see if any fragments are already available * allocsiz is the size which will be allocated, hacking * it down to a smaller size if necessary */ blksfree = cg_blksfree(cgp); frags = numfrags(fs, size); for (allocsiz = frags; allocsiz < fs->fs_frag; allocsiz++) if (cgp->cg_frsum[allocsiz] != 0) break; if (allocsiz == fs->fs_frag) { /* * no fragments were available, so a block will be * allocated, and hacked up */ if (cgp->cg_cs.cs_nbfree == 0) goto fail; UFS_LOCK(ump); blkno = ffs_alloccgblk(ip, bp, bpref); bno = dtogd(fs, blkno); for (i = frags; i < fs->fs_frag; i++) setbit(blksfree, bno + i); i = fs->fs_frag - frags; cgp->cg_cs.cs_nffree += i; fs->fs_cstotal.cs_nffree += i; fs->fs_cs(fs, cg).cs_nffree += i; fs->fs_fmod = 1; cgp->cg_frsum[i]++; ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); bdwrite(bp); return (blkno); } bno = ffs_mapsearch(fs, cgp, bpref, allocsiz); if (bno < 0) goto fail; for (i = 0; i < frags; i++) clrbit(blksfree, bno + i); cgp->cg_cs.cs_nffree -= frags; cgp->cg_frsum[allocsiz]--; if (frags != allocsiz) cgp->cg_frsum[allocsiz - frags]++; UFS_LOCK(ump); fs->fs_cstotal.cs_nffree -= frags; fs->fs_cs(fs, cg).cs_nffree -= frags; fs->fs_fmod = 1; blkno = cgbase(fs, cg) + bno; ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); if (DOINGSOFTDEP(ITOV(ip))) softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno); bdwrite(bp); return (blkno); fail: brelse(bp); UFS_LOCK(ump); return (0); } /* * Allocate a block in a cylinder group. * * This algorithm implements the following policy: * 1) allocate the requested block. * 2) allocate a rotationally optimal block in the same cylinder. * 3) allocate the next available block on the block rotor for the * specified cylinder group. * Note that this routine only allocates fs_bsize blocks; these * blocks may be fragmented by the routine that allocates them. */ static ufs2_daddr_t ffs_alloccgblk(ip, bp, bpref) struct inode *ip; struct buf *bp; ufs2_daddr_t bpref; { struct fs *fs; struct cg *cgp; struct ufsmount *ump; ufs1_daddr_t bno; ufs2_daddr_t blkno; u_int8_t *blksfree; fs = ip->i_fs; ump = ip->i_ump; mtx_assert(UFS_MTX(ump), MA_OWNED); cgp = (struct cg *)bp->b_data; blksfree = cg_blksfree(cgp); if (bpref == 0 || dtog(fs, bpref) != cgp->cg_cgx) { bpref = cgp->cg_rotor; } else { bpref = blknum(fs, bpref); bno = dtogd(fs, bpref); /* * if the requested block is available, use it */ if (ffs_isblock(fs, blksfree, fragstoblks(fs, bno))) goto gotit; } /* * Take the next available block in this cylinder group. */ bno = ffs_mapsearch(fs, cgp, bpref, (int)fs->fs_frag); if (bno < 0) return (0); cgp->cg_rotor = bno; gotit: blkno = fragstoblks(fs, bno); ffs_clrblock(fs, blksfree, (long)blkno); ffs_clusteracct(ump, fs, cgp, blkno, -1); cgp->cg_cs.cs_nbfree--; fs->fs_cstotal.cs_nbfree--; fs->fs_cs(fs, cgp->cg_cgx).cs_nbfree--; fs->fs_fmod = 1; blkno = cgbase(fs, cgp->cg_cgx) + bno; /* XXX Fixme. */ UFS_UNLOCK(ump); if (DOINGSOFTDEP(ITOV(ip))) softdep_setup_blkmapdep(bp, UFSTOVFS(ump), blkno); UFS_LOCK(ump); return (blkno); } /* * Determine whether a cluster can be allocated. * * We do not currently check for optimal rotational layout if there * are multiple choices in the same cylinder group. Instead we just * take the first one that we find following bpref. */ static ufs2_daddr_t ffs_clusteralloc(ip, cg, bpref, len) struct inode *ip; u_int cg; ufs2_daddr_t bpref; int len; { struct fs *fs; struct cg *cgp; struct buf *bp; struct ufsmount *ump; int i, run, bit, map, got; ufs2_daddr_t bno; u_char *mapp; int32_t *lp; u_int8_t *blksfree; fs = ip->i_fs; ump = ip->i_ump; if (fs->fs_maxcluster[cg] < len) return (0); UFS_UNLOCK(ump); if (bread(ip->i_devvp, fsbtodb(fs, cgtod(fs, cg)), (int)fs->fs_cgsize, NOCRED, &bp)) goto fail_lock; cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) goto fail_lock; bp->b_xflags |= BX_BKGRDWRITE; /* * Check to see if a cluster of the needed size (or bigger) is * available in this cylinder group. */ lp = &cg_clustersum(cgp)[len]; for (i = len; i <= fs->fs_contigsumsize; i++) if (*lp++ > 0) break; if (i > fs->fs_contigsumsize) { /* * This is the first time looking for a cluster in this * cylinder group. Update the cluster summary information * to reflect the true maximum sized cluster so that * future cluster allocation requests can avoid reading * the cylinder group map only to find no clusters. */ lp = &cg_clustersum(cgp)[len - 1]; for (i = len - 1; i > 0; i--) if (*lp-- > 0) break; UFS_LOCK(ump); fs->fs_maxcluster[cg] = i; goto fail; } /* * Search the cluster map to find a big enough cluster. * We take the first one that we find, even if it is larger * than we need as we prefer to get one close to the previous * block allocation. We do not search before the current * preference point as we do not want to allocate a block * that is allocated before the previous one (as we will * then have to wait for another pass of the elevator * algorithm before it will be read). We prefer to fail and * be recalled to try an allocation in the next cylinder group. */ if (dtog(fs, bpref) != cg) bpref = 0; else bpref = fragstoblks(fs, dtogd(fs, blknum(fs, bpref))); mapp = &cg_clustersfree(cgp)[bpref / NBBY]; map = *mapp++; bit = 1 << (bpref % NBBY); for (run = 0, got = bpref; got < cgp->cg_nclusterblks; got++) { if ((map & bit) == 0) { run = 0; } else { run++; if (run == len) break; } if ((got & (NBBY - 1)) != (NBBY - 1)) { bit <<= 1; } else { map = *mapp++; bit = 1; } } if (got >= cgp->cg_nclusterblks) goto fail_lock; /* * Allocate the cluster that we have found. */ blksfree = cg_blksfree(cgp); for (i = 1; i <= len; i++) if (!ffs_isblock(fs, blksfree, got - run + i)) panic("ffs_clusteralloc: map mismatch"); bno = cgbase(fs, cg) + blkstofrags(fs, got - run + 1); if (dtog(fs, bno) != cg) panic("ffs_clusteralloc: allocated out of group"); len = blkstofrags(fs, len); UFS_LOCK(ump); for (i = 0; i < len; i += fs->fs_frag) if (ffs_alloccgblk(ip, bp, bno + i) != bno + i) panic("ffs_clusteralloc: lost block"); ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); bdwrite(bp); return (bno); fail_lock: UFS_LOCK(ump); fail: brelse(bp); return (0); } /* * Determine whether an inode can be allocated. * * Check to see if an inode is available, and if it is, * allocate it using the following policy: * 1) allocate the requested inode. * 2) allocate the next available inode after the requested * inode in the specified cylinder group. */ static ufs2_daddr_t ffs_nodealloccg(ip, cg, ipref, mode) struct inode *ip; u_int cg; ufs2_daddr_t ipref; int mode; { struct fs *fs; struct cg *cgp; struct buf *bp, *ibp; struct ufsmount *ump; u_int8_t *inosused; struct ufs2_dinode *dp2; int error, start, len, loc, map, i; fs = ip->i_fs; ump = ip->i_ump; if (fs->fs_cs(fs, cg).cs_nifree == 0) return (0); UFS_UNLOCK(ump); error = bread(ip->i_devvp, fsbtodb(fs, cgtod(fs, cg)), (int)fs->fs_cgsize, NOCRED, &bp); if (error) { brelse(bp); UFS_LOCK(ump); return (0); } cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp) || cgp->cg_cs.cs_nifree == 0) { brelse(bp); UFS_LOCK(ump); return (0); } bp->b_xflags |= BX_BKGRDWRITE; cgp->cg_old_time = cgp->cg_time = time_second; inosused = cg_inosused(cgp); if (ipref) { ipref %= fs->fs_ipg; if (isclr(inosused, ipref)) goto gotit; } start = cgp->cg_irotor / NBBY; len = howmany(fs->fs_ipg - cgp->cg_irotor, NBBY); loc = skpc(0xff, len, &inosused[start]); if (loc == 0) { len = start + 1; start = 0; loc = skpc(0xff, len, &inosused[0]); if (loc == 0) { printf("cg = %d, irotor = %ld, fs = %s\n", cg, (long)cgp->cg_irotor, fs->fs_fsmnt); panic("ffs_nodealloccg: map corrupted"); /* NOTREACHED */ } } i = start + len - loc; map = inosused[i]; ipref = i * NBBY; for (i = 1; i < (1 << NBBY); i <<= 1, ipref++) { if ((map & i) == 0) { cgp->cg_irotor = ipref; goto gotit; } } printf("fs = %s\n", fs->fs_fsmnt); panic("ffs_nodealloccg: block not in map"); /* NOTREACHED */ gotit: /* * Check to see if we need to initialize more inodes. */ ibp = NULL; if (fs->fs_magic == FS_UFS2_MAGIC && ipref + INOPB(fs) > cgp->cg_initediblk && cgp->cg_initediblk < cgp->cg_niblk) { ibp = getblk(ip->i_devvp, fsbtodb(fs, ino_to_fsba(fs, cg * fs->fs_ipg + cgp->cg_initediblk)), (int)fs->fs_bsize, 0, 0, 0); bzero(ibp->b_data, (int)fs->fs_bsize); dp2 = (struct ufs2_dinode *)(ibp->b_data); for (i = 0; i < INOPB(fs); i++) { dp2->di_gen = arc4random() / 2 + 1; dp2++; } cgp->cg_initediblk += INOPB(fs); } UFS_LOCK(ump); ACTIVECLEAR(fs, cg); setbit(inosused, ipref); cgp->cg_cs.cs_nifree--; fs->fs_cstotal.cs_nifree--; fs->fs_cs(fs, cg).cs_nifree--; fs->fs_fmod = 1; if ((mode & IFMT) == IFDIR) { cgp->cg_cs.cs_ndir++; fs->fs_cstotal.cs_ndir++; fs->fs_cs(fs, cg).cs_ndir++; } UFS_UNLOCK(ump); if (DOINGSOFTDEP(ITOV(ip))) softdep_setup_inomapdep(bp, ip, cg * fs->fs_ipg + ipref); bdwrite(bp); if (ibp != NULL) bawrite(ibp); return ((ino_t)(cg * fs->fs_ipg + ipref)); } /* * check if a block is free */ static int ffs_isfreeblock(struct fs *fs, u_char *cp, ufs1_daddr_t h) { switch ((int)fs->fs_frag) { case 8: return (cp[h] == 0); case 4: return ((cp[h >> 1] & (0x0f << ((h & 0x1) << 2))) == 0); case 2: return ((cp[h >> 2] & (0x03 << ((h & 0x3) << 1))) == 0); case 1: return ((cp[h >> 3] & (0x01 << (h & 0x7))) == 0); default: panic("ffs_isfreeblock"); } return (0); } /* * Free a block or fragment. * * The specified block or fragment is placed back in the * free map. If a fragment is deallocated, a possible * block reassembly is checked. */ void ffs_blkfree(ump, fs, devvp, bno, size, inum) struct ufsmount *ump; struct fs *fs; struct vnode *devvp; ufs2_daddr_t bno; long size; ino_t inum; { struct cg *cgp; struct buf *bp; ufs1_daddr_t fragno, cgbno; ufs2_daddr_t cgblkno; int i, blk, frags, bbase; u_int cg; u_int8_t *blksfree; struct cdev *dev; cg = dtog(fs, bno); if (devvp->v_type == VREG) { /* devvp is a snapshot */ dev = VTOI(devvp)->i_devvp->v_rdev; cgblkno = fragstoblks(fs, cgtod(fs, cg)); } else { /* devvp is a normal disk device */ dev = devvp->v_rdev; cgblkno = fsbtodb(fs, cgtod(fs, cg)); ASSERT_VOP_LOCKED(devvp, "ffs_blkfree"); if ((devvp->v_vflag & VV_COPYONWRITE) && ffs_snapblkfree(fs, devvp, bno, size, inum)) return; } #ifdef INVARIANTS if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0 || fragnum(fs, bno) + numfrags(fs, size) > fs->fs_frag) { printf("dev=%s, bno = %jd, bsize = %ld, size = %ld, fs = %s\n", devtoname(dev), (intmax_t)bno, (long)fs->fs_bsize, size, fs->fs_fsmnt); panic("ffs_blkfree: bad size"); } #endif if ((u_int)bno >= fs->fs_size) { printf("bad block %jd, ino %lu\n", (intmax_t)bno, (u_long)inum); ffs_fserr(fs, inum, "bad block"); return; } if (bread(devvp, cgblkno, (int)fs->fs_cgsize, NOCRED, &bp)) { brelse(bp); return; } cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) { brelse(bp); return; } bp->b_xflags |= BX_BKGRDWRITE; cgp->cg_old_time = cgp->cg_time = time_second; cgbno = dtogd(fs, bno); blksfree = cg_blksfree(cgp); UFS_LOCK(ump); if (size == fs->fs_bsize) { fragno = fragstoblks(fs, cgbno); if (!ffs_isfreeblock(fs, blksfree, fragno)) { if (devvp->v_type == VREG) { UFS_UNLOCK(ump); /* devvp is a snapshot */ brelse(bp); return; } printf("dev = %s, block = %jd, fs = %s\n", devtoname(dev), (intmax_t)bno, fs->fs_fsmnt); panic("ffs_blkfree: freeing free block"); } ffs_setblock(fs, blksfree, fragno); ffs_clusteracct(ump, fs, cgp, fragno, 1); cgp->cg_cs.cs_nbfree++; fs->fs_cstotal.cs_nbfree++; fs->fs_cs(fs, cg).cs_nbfree++; } else { bbase = cgbno - fragnum(fs, cgbno); /* * decrement the counts associated with the old frags */ blk = blkmap(fs, blksfree, bbase); ffs_fragacct(fs, blk, cgp->cg_frsum, -1); /* * deallocate the fragment */ frags = numfrags(fs, size); for (i = 0; i < frags; i++) { if (isset(blksfree, cgbno + i)) { printf("dev = %s, block = %jd, fs = %s\n", devtoname(dev), (intmax_t)(bno + i), fs->fs_fsmnt); panic("ffs_blkfree: freeing free frag"); } setbit(blksfree, cgbno + i); } cgp->cg_cs.cs_nffree += i; fs->fs_cstotal.cs_nffree += i; fs->fs_cs(fs, cg).cs_nffree += i; /* * add back in counts associated with the new frags */ blk = blkmap(fs, blksfree, bbase); ffs_fragacct(fs, blk, cgp->cg_frsum, 1); /* * if a complete block has been reassembled, account for it */ fragno = fragstoblks(fs, bbase); if (ffs_isblock(fs, blksfree, fragno)) { cgp->cg_cs.cs_nffree -= fs->fs_frag; fs->fs_cstotal.cs_nffree -= fs->fs_frag; fs->fs_cs(fs, cg).cs_nffree -= fs->fs_frag; ffs_clusteracct(ump, fs, cgp, fragno, 1); cgp->cg_cs.cs_nbfree++; fs->fs_cstotal.cs_nbfree++; fs->fs_cs(fs, cg).cs_nbfree++; } } fs->fs_fmod = 1; ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); bdwrite(bp); } #ifdef INVARIANTS /* * Verify allocation of a block or fragment. Returns true if block or * fragment is allocated, false if it is free. */ static int ffs_checkblk(ip, bno, size) struct inode *ip; ufs2_daddr_t bno; long size; { struct fs *fs; struct cg *cgp; struct buf *bp; ufs1_daddr_t cgbno; int i, error, frags, free; u_int8_t *blksfree; fs = ip->i_fs; if ((u_int)size > fs->fs_bsize || fragoff(fs, size) != 0) { printf("bsize = %ld, size = %ld, fs = %s\n", (long)fs->fs_bsize, size, fs->fs_fsmnt); panic("ffs_checkblk: bad size"); } if ((u_int)bno >= fs->fs_size) panic("ffs_checkblk: bad block %jd", (intmax_t)bno); error = bread(ip->i_devvp, fsbtodb(fs, cgtod(fs, dtog(fs, bno))), (int)fs->fs_cgsize, NOCRED, &bp); if (error) panic("ffs_checkblk: cg bread failed"); cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) panic("ffs_checkblk: cg magic mismatch"); bp->b_xflags |= BX_BKGRDWRITE; blksfree = cg_blksfree(cgp); cgbno = dtogd(fs, bno); if (size == fs->fs_bsize) { free = ffs_isblock(fs, blksfree, fragstoblks(fs, cgbno)); } else { frags = numfrags(fs, size); for (free = 0, i = 0; i < frags; i++) if (isset(blksfree, cgbno + i)) free++; if (free != 0 && free != frags) panic("ffs_checkblk: partially free fragment"); } brelse(bp); return (!free); } #endif /* INVARIANTS */ /* * Free an inode. */ int ffs_vfree(pvp, ino, mode) struct vnode *pvp; ino_t ino; int mode; { struct inode *ip; if (DOINGSOFTDEP(pvp)) { softdep_freefile(pvp, ino, mode); return (0); } ip = VTOI(pvp); return (ffs_freefile(ip->i_ump, ip->i_fs, ip->i_devvp, ino, mode)); } /* * Do the actual free operation. * The specified inode is placed back in the free map. */ int ffs_freefile(ump, fs, devvp, ino, mode) struct ufsmount *ump; struct fs *fs; struct vnode *devvp; ino_t ino; int mode; { struct cg *cgp; struct buf *bp; ufs2_daddr_t cgbno; int error; u_int cg; u_int8_t *inosused; struct cdev *dev; cg = ino_to_cg(fs, ino); if (devvp->v_type == VREG) { /* devvp is a snapshot */ dev = VTOI(devvp)->i_devvp->v_rdev; cgbno = fragstoblks(fs, cgtod(fs, cg)); } else { /* devvp is a normal disk device */ dev = devvp->v_rdev; cgbno = fsbtodb(fs, cgtod(fs, cg)); } if (ino >= fs->fs_ipg * fs->fs_ncg) panic("ffs_freefile: range: dev = %s, ino = %lu, fs = %s", devtoname(dev), (u_long)ino, fs->fs_fsmnt); if ((error = bread(devvp, cgbno, (int)fs->fs_cgsize, NOCRED, &bp))) { brelse(bp); return (error); } cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) { brelse(bp); return (0); } bp->b_xflags |= BX_BKGRDWRITE; cgp->cg_old_time = cgp->cg_time = time_second; inosused = cg_inosused(cgp); ino %= fs->fs_ipg; if (isclr(inosused, ino)) { printf("dev = %s, ino = %u, fs = %s\n", devtoname(dev), ino + cg * fs->fs_ipg, fs->fs_fsmnt); if (fs->fs_ronly == 0) panic("ffs_freefile: freeing free inode"); } clrbit(inosused, ino); if (ino < cgp->cg_irotor) cgp->cg_irotor = ino; cgp->cg_cs.cs_nifree++; UFS_LOCK(ump); fs->fs_cstotal.cs_nifree++; fs->fs_cs(fs, cg).cs_nifree++; if ((mode & IFMT) == IFDIR) { cgp->cg_cs.cs_ndir--; fs->fs_cstotal.cs_ndir--; fs->fs_cs(fs, cg).cs_ndir--; } fs->fs_fmod = 1; ACTIVECLEAR(fs, cg); UFS_UNLOCK(ump); bdwrite(bp); return (0); } /* * Check to see if a file is free. */ int ffs_checkfreefile(fs, devvp, ino) struct fs *fs; struct vnode *devvp; ino_t ino; { struct cg *cgp; struct buf *bp; ufs2_daddr_t cgbno; int ret; u_int cg; u_int8_t *inosused; cg = ino_to_cg(fs, ino); if (devvp->v_type == VREG) { /* devvp is a snapshot */ cgbno = fragstoblks(fs, cgtod(fs, cg)); } else { /* devvp is a normal disk device */ cgbno = fsbtodb(fs, cgtod(fs, cg)); } if (ino >= fs->fs_ipg * fs->fs_ncg) return (1); if (bread(devvp, cgbno, (int)fs->fs_cgsize, NOCRED, &bp)) { brelse(bp); return (1); } cgp = (struct cg *)bp->b_data; if (!cg_chkmagic(cgp)) { brelse(bp); return (1); } inosused = cg_inosused(cgp); ino %= fs->fs_ipg; ret = isclr(inosused, ino); brelse(bp); return (ret); } /* * Find a block of the specified size in the specified cylinder group. * * It is a panic if a request is made to find a block if none are * available. */ static ufs1_daddr_t ffs_mapsearch(fs, cgp, bpref, allocsiz) struct fs *fs; struct cg *cgp; ufs2_daddr_t bpref; int allocsiz; { ufs1_daddr_t bno; int start, len, loc, i; int blk, field, subfield, pos; u_int8_t *blksfree; /* * find the fragment by searching through the free block * map for an appropriate bit pattern */ if (bpref) start = dtogd(fs, bpref) / NBBY; else start = cgp->cg_frotor / NBBY; blksfree = cg_blksfree(cgp); len = howmany(fs->fs_fpg, NBBY) - start; loc = scanc((u_int)len, (u_char *)&blksfree[start], fragtbl[fs->fs_frag], (u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY)))); if (loc == 0) { len = start + 1; start = 0; loc = scanc((u_int)len, (u_char *)&blksfree[0], fragtbl[fs->fs_frag], (u_char)(1 << (allocsiz - 1 + (fs->fs_frag % NBBY)))); if (loc == 0) { printf("start = %d, len = %d, fs = %s\n", start, len, fs->fs_fsmnt); panic("ffs_alloccg: map corrupted"); /* NOTREACHED */ } } bno = (start + len - loc) * NBBY; cgp->cg_frotor = bno; /* * found the byte in the map * sift through the bits to find the selected frag */ for (i = bno + NBBY; bno < i; bno += fs->fs_frag) { blk = blkmap(fs, blksfree, bno); blk <<= 1; field = around[allocsiz]; subfield = inside[allocsiz]; for (pos = 0; pos <= fs->fs_frag - allocsiz; pos++) { if ((blk & field) == subfield) return (bno + pos); field <<= 1; subfield <<= 1; } } printf("bno = %lu, fs = %s\n", (u_long)bno, fs->fs_fsmnt); panic("ffs_alloccg: block not in map"); return (-1); } /* * Update the cluster map because of an allocation or free. * * Cnt == 1 means free; cnt == -1 means allocating. */ void ffs_clusteracct(ump, fs, cgp, blkno, cnt) struct ufsmount *ump; struct fs *fs; struct cg *cgp; ufs1_daddr_t blkno; int cnt; { int32_t *sump; int32_t *lp; u_char *freemapp, *mapp; int i, start, end, forw, back, map, bit; mtx_assert(UFS_MTX(ump), MA_OWNED); if (fs->fs_contigsumsize <= 0) return; freemapp = cg_clustersfree(cgp); sump = cg_clustersum(cgp); /* * Allocate or clear the actual block. */ if (cnt > 0) setbit(freemapp, blkno); else clrbit(freemapp, blkno); /* * Find the size of the cluster going forward. */ start = blkno + 1; end = start + fs->fs_contigsumsize; if (end >= cgp->cg_nclusterblks) end = cgp->cg_nclusterblks; mapp = &freemapp[start / NBBY]; map = *mapp++; bit = 1 << (start % NBBY); for (i = start; i < end; i++) { if ((map & bit) == 0) break; if ((i & (NBBY - 1)) != (NBBY - 1)) { bit <<= 1; } else { map = *mapp++; bit = 1; } } forw = i - start; /* * Find the size of the cluster going backward. */ start = blkno - 1; end = start - fs->fs_contigsumsize; if (end < 0) end = -1; mapp = &freemapp[start / NBBY]; map = *mapp--; bit = 1 << (start % NBBY); for (i = start; i > end; i--) { if ((map & bit) == 0) break; if ((i & (NBBY - 1)) != 0) { bit >>= 1; } else { map = *mapp--; bit = 1 << (NBBY - 1); } } back = start - i; /* * Account for old cluster and the possibly new forward and * back clusters. */ i = back + forw + 1; if (i > fs->fs_contigsumsize) i = fs->fs_contigsumsize; sump[i] += cnt; if (back > 0) sump[back] -= cnt; if (forw > 0) sump[forw] -= cnt; /* * Update cluster summary information. */ lp = &sump[fs->fs_contigsumsize]; for (i = fs->fs_contigsumsize; i > 0; i--) if (*lp-- > 0) break; fs->fs_maxcluster[cgp->cg_cgx] = i; } /* * Fserr prints the name of a filesystem with an error diagnostic. * * The form of the error message is: * fs: error message */ static void ffs_fserr(fs, inum, cp) struct fs *fs; ino_t inum; char *cp; { struct thread *td = curthread; /* XXX */ struct proc *p = td->td_proc; log(LOG_ERR, "pid %d (%s), uid %d inumber %d on %s: %s\n", p->p_pid, p->p_comm, td->td_ucred->cr_uid, inum, fs->fs_fsmnt, cp); } /* * This function provides the capability for the fsck program to * update an active filesystem. Fourteen operations are provided: * * adjrefcnt(inode, amt) - adjusts the reference count on the * specified inode by the specified amount. Under normal * operation the count should always go down. Decrementing * the count to zero will cause the inode to be freed. * adjblkcnt(inode, amt) - adjust the number of blocks used to * by the specifed amount. * adjndir, adjbfree, adjifree, adjffree, adjnumclusters(amt) - * adjust the superblock summary. * freedirs(inode, count) - directory inodes [inode..inode + count - 1] * are marked as free. Inodes should never have to be marked * as in use. * freefiles(inode, count) - file inodes [inode..inode + count - 1] * are marked as free. Inodes should never have to be marked * as in use. * freeblks(blockno, size) - blocks [blockno..blockno + size - 1] * are marked as free. Blocks should never have to be marked * as in use. * setflags(flags, set/clear) - the fs_flags field has the specified * flags set (second parameter +1) or cleared (second parameter -1). * setcwd(dirinode) - set the current directory to dirinode in the * filesystem associated with the snapshot. * setdotdot(oldvalue, newvalue) - Verify that the inode number for ".." * in the current directory is oldvalue then change it to newvalue. * unlink(nameptr, oldvalue) - Verify that the inode number associated * with nameptr in the current directory is oldvalue then unlink it. */ static int sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_vfs_ffs, FFS_ADJ_REFCNT, adjrefcnt, CTLFLAG_WR|CTLTYPE_STRUCT, 0, 0, sysctl_ffs_fsck, "S,fsck", "Adjust Inode Reference Count"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_BLKCNT, adjblkcnt, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust Inode Used Blocks Count"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NDIR, adjndir, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust number of directories"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NBFREE, adjnbfree, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust number of free blocks"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NIFREE, adjnifree, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust number of free inodes"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NFFREE, adjnffree, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust number of free frags"); static SYSCTL_NODE(_vfs_ffs, FFS_ADJ_NUMCLUSTERS, adjnumclusters, CTLFLAG_WR, sysctl_ffs_fsck, "Adjust number of free clusters"); static SYSCTL_NODE(_vfs_ffs, FFS_DIR_FREE, freedirs, CTLFLAG_WR, sysctl_ffs_fsck, "Free Range of Directory Inodes"); static SYSCTL_NODE(_vfs_ffs, FFS_FILE_FREE, freefiles, CTLFLAG_WR, sysctl_ffs_fsck, "Free Range of File Inodes"); static SYSCTL_NODE(_vfs_ffs, FFS_BLK_FREE, freeblks, CTLFLAG_WR, sysctl_ffs_fsck, "Free Range of Blocks"); static SYSCTL_NODE(_vfs_ffs, FFS_SET_FLAGS, setflags, CTLFLAG_WR, sysctl_ffs_fsck, "Change Filesystem Flags"); static SYSCTL_NODE(_vfs_ffs, FFS_SET_CWD, setcwd, CTLFLAG_WR, sysctl_ffs_fsck, "Set Current Working Directory"); static SYSCTL_NODE(_vfs_ffs, FFS_SET_DOTDOT, setdotdot, CTLFLAG_WR, sysctl_ffs_fsck, "Change Value of .. Entry"); static SYSCTL_NODE(_vfs_ffs, FFS_UNLINK, unlink, CTLFLAG_WR, sysctl_ffs_fsck, "Unlink a Duplicate Name"); #ifdef DEBUG static int fsckcmds = 0; SYSCTL_INT(_debug, OID_AUTO, fsckcmds, CTLFLAG_RW, &fsckcmds, 0, ""); #endif /* DEBUG */ static int sysctl_ffs_fsck(SYSCTL_HANDLER_ARGS) { struct thread *td = curthread; struct fsck_cmd cmd; struct ufsmount *ump; struct vnode *vp, *vpold, *dvp, *fdvp; struct inode *ip, *dp; struct mount *mp; struct fs *fs; ufs2_daddr_t blkno; long blkcnt, blksize; struct filedesc *fdp; struct file *fp; int vfslocked, filetype, error; if (req->newlen > sizeof cmd) return (EBADRPC); if ((error = SYSCTL_IN(req, &cmd, sizeof cmd)) != 0) return (error); if (cmd.version != FFS_CMD_VERSION) return (ERPCMISMATCH); if ((error = getvnode(curproc->p_fd, cmd.handle, &fp)) != 0) return (error); vp = fp->f_data; if (vp->v_type != VREG && vp->v_type != VDIR) { fdrop(fp, td); return (EINVAL); } vn_start_write(vp, &mp, V_WAIT); if (mp == 0 || strncmp(mp->mnt_stat.f_fstypename, "ufs", MFSNAMELEN)) { vn_finished_write(mp); fdrop(fp, td); return (EINVAL); } if (mp->mnt_flag & MNT_RDONLY) { vn_finished_write(mp); fdrop(fp, td); return (EROFS); } ump = VFSTOUFS(mp); fs = ump->um_fs; filetype = IFREG; switch (oidp->oid_number) { case FFS_SET_FLAGS: #ifdef DEBUG if (fsckcmds) printf("%s: %s flags\n", mp->mnt_stat.f_mntonname, cmd.size > 0 ? "set" : "clear"); #endif /* DEBUG */ if (cmd.size > 0) fs->fs_flags |= (long)cmd.value; else fs->fs_flags &= ~(long)cmd.value; break; case FFS_ADJ_REFCNT: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust inode %jd count by %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value, (intmax_t)cmd.size); } #endif /* DEBUG */ if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp))) break; ip = VTOI(vp); ip->i_nlink += cmd.size; DIP_SET(ip, i_nlink, ip->i_nlink); ip->i_effnlink += cmd.size; ip->i_flag |= IN_CHANGE; if (DOINGSOFTDEP(vp)) softdep_change_linkcnt(ip); vput(vp); break; case FFS_ADJ_BLKCNT: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust inode %jd block count by %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value, (intmax_t)cmd.size); } #endif /* DEBUG */ if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &vp))) break; ip = VTOI(vp); if (ip->i_flag & IN_SPACECOUNTED) { UFS_LOCK(ump); fs->fs_pendingblocks += cmd.size; UFS_UNLOCK(ump); } DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + cmd.size); ip->i_flag |= IN_CHANGE; vput(vp); break; case FFS_DIR_FREE: filetype = IFDIR; /* fall through */ case FFS_FILE_FREE: #ifdef DEBUG if (fsckcmds) { if (cmd.size == 1) printf("%s: free %s inode %d\n", mp->mnt_stat.f_mntonname, filetype == IFDIR ? "directory" : "file", (ino_t)cmd.value); else printf("%s: free %s inodes %d-%d\n", mp->mnt_stat.f_mntonname, filetype == IFDIR ? "directory" : "file", (ino_t)cmd.value, (ino_t)(cmd.value + cmd.size - 1)); } #endif /* DEBUG */ while (cmd.size > 0) { if ((error = ffs_freefile(ump, fs, ump->um_devvp, cmd.value, filetype))) break; cmd.size -= 1; cmd.value += 1; } break; case FFS_BLK_FREE: #ifdef DEBUG if (fsckcmds) { if (cmd.size == 1) printf("%s: free block %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); else printf("%s: free blocks %jd-%jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value, (intmax_t)cmd.value + cmd.size - 1); } #endif /* DEBUG */ blkno = cmd.value; blkcnt = cmd.size; blksize = fs->fs_frag - (blkno % fs->fs_frag); while (blkcnt > 0) { if (blksize > blkcnt) blksize = blkcnt; ffs_blkfree(ump, fs, ump->um_devvp, blkno, blksize * fs->fs_fsize, ROOTINO); blkno += blksize; blkcnt -= blksize; blksize = fs->fs_frag; } break; /* * Adjust superblock summaries. fsck(8) is expected to * submit deltas when necessary. */ case FFS_ADJ_NDIR: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust number of directories by %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ fs->fs_cstotal.cs_ndir += cmd.value; break; case FFS_ADJ_NBFREE: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust number of free blocks by %+jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ fs->fs_cstotal.cs_nbfree += cmd.value; break; case FFS_ADJ_NIFREE: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust number of free inodes by %+jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ fs->fs_cstotal.cs_nifree += cmd.value; break; case FFS_ADJ_NFFREE: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust number of free frags by %+jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ fs->fs_cstotal.cs_nffree += cmd.value; break; case FFS_ADJ_NUMCLUSTERS: #ifdef DEBUG if (fsckcmds) { printf("%s: adjust number of free clusters by %+jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ fs->fs_cstotal.cs_numclusters += cmd.value; break; case FFS_SET_CWD: #ifdef DEBUG if (fsckcmds) { printf("%s: set current directory to inode %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value); } #endif /* DEBUG */ if ((error = ffs_vget(mp, (ino_t)cmd.value, LK_SHARED, &vp))) break; vfslocked = VFS_LOCK_GIANT(vp->v_mount); AUDIT_ARG_VNODE1(vp); if ((error = change_dir(vp, td)) != 0) { vput(vp); VFS_UNLOCK_GIANT(vfslocked); break; } VOP_UNLOCK(vp, 0); VFS_UNLOCK_GIANT(vfslocked); fdp = td->td_proc->p_fd; FILEDESC_XLOCK(fdp); vpold = fdp->fd_cdir; fdp->fd_cdir = vp; FILEDESC_XUNLOCK(fdp); vfslocked = VFS_LOCK_GIANT(vpold->v_mount); vrele(vpold); VFS_UNLOCK_GIANT(vfslocked); break; case FFS_SET_DOTDOT: #ifdef DEBUG if (fsckcmds) { printf("%s: change .. in cwd from %jd to %jd\n", mp->mnt_stat.f_mntonname, (intmax_t)cmd.value, (intmax_t)cmd.size); } #endif /* DEBUG */ /* * First we have to get and lock the parent directory * to which ".." points. */ error = ffs_vget(mp, (ino_t)cmd.value, LK_EXCLUSIVE, &fdvp); if (error) break; /* * Now we get and lock the child directory containing "..". */ FILEDESC_SLOCK(td->td_proc->p_fd); dvp = td->td_proc->p_fd->fd_cdir; FILEDESC_SUNLOCK(td->td_proc->p_fd); if ((error = vget(dvp, LK_EXCLUSIVE, td)) != 0) { vput(fdvp); break; } dp = VTOI(dvp); dp->i_offset = 12; /* XXX mastertemplate.dot_reclen */ error = ufs_dirrewrite(dp, VTOI(fdvp), (ino_t)cmd.size, DT_DIR, 0); cache_purge(fdvp); cache_purge(dvp); vput(dvp); vput(fdvp); break; case FFS_UNLINK: #ifdef DEBUG if (fsckcmds) { char buf[32]; if (copyinstr((char *)(intptr_t)cmd.value, buf,32,NULL)) strncpy(buf, "Name_too_long", 32); printf("%s: unlink %s (inode %jd)\n", mp->mnt_stat.f_mntonname, buf, (intmax_t)cmd.size); } #endif /* DEBUG */ /* * kern_unlinkat will do its own start/finish writes and * they do not nest, so drop ours here. Setting mp == NULL * indicates that vn_finished_write is not needed down below. */ vn_finished_write(mp); mp = NULL; error = kern_unlinkat(td, AT_FDCWD, (char *)(intptr_t)cmd.value, UIO_USERSPACE, (ino_t)cmd.size); break; default: #ifdef DEBUG if (fsckcmds) { printf("Invalid request %d from fsck\n", oidp->oid_number); } #endif /* DEBUG */ error = EINVAL; break; } fdrop(fp, td); vn_finished_write(mp); return (error); } Index: user/imp/tbemd =================================================================== --- user/imp/tbemd (revision 203849) +++ user/imp/tbemd (revision 203850) Property changes on: user/imp/tbemd ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r203813-203844