Index: head/sys/dev/ciss/ciss.c =================================================================== --- head/sys/dev/ciss/ciss.c (revision 298930) +++ head/sys/dev/ciss/ciss.c (revision 298931) @@ -1,4730 +1,4730 @@ /*- * Copyright (c) 2001 Michael Smith * Copyright (c) 2004 Paul Saab * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * Common Interface for SCSI-3 Support driver. * * CISS claims to provide a common interface between a generic SCSI * transport and an intelligent host adapter. * * This driver supports CISS as defined in the document "CISS Command * Interface for SCSI-3 Support Open Specification", Version 1.04, * Valence Number 1, dated 20001127, produced by Compaq Computer * Corporation. This document appears to be a hastily and somewhat * arbitrarlily cut-down version of a larger (and probably even more * chaotic and inconsistent) Compaq internal document. Various * details were also gleaned from Compaq's "cciss" driver for Linux. * * We provide a shim layer between the CISS interface and CAM, * offloading most of the queueing and being-a-disk chores onto CAM. * Entry to the driver is via the PCI bus attachment (ciss_probe, * ciss_attach, etc) and via the CAM interface (ciss_cam_action, * ciss_cam_poll). The Compaq CISS adapters are, however, poor SCSI * citizens and we have to fake up some responses to get reasonable * behaviour out of them. In addition, the CISS command set is by no * means adequate to support the functionality of a RAID controller, * and thus the supported Compaq adapters utilise portions of the * control protocol from earlier Compaq adapter families. * * Note that we only support the "simple" transport layer over PCI. * This interface (ab)uses the I2O register set (specifically the post * queues) to exchange commands with the adapter. Other interfaces * are available, but we aren't supposed to know about them, and it is * dubious whether they would provide major performance improvements * except under extreme load. * * Currently the only supported CISS adapters are the Compaq Smart * Array 5* series (5300, 5i, 532). Even with only three adapters, * Compaq still manage to have interface variations. * * * Thanks must go to Fred Harris and Darryl DeVinney at Compaq, as * well as Paul Saab at Yahoo! for their assistance in making this * driver happen. * * More thanks must go to John Cagle at HP for the countless hours * spent making this driver "work" with the MSA* series storage * enclosures. Without his help (and nagging), this driver could not * be used with these enclosures. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(CISS_MALLOC_CLASS, "ciss_data", "ciss internal data buffers"); /* pci interface */ static int ciss_lookup(device_t dev); static int ciss_probe(device_t dev); static int ciss_attach(device_t dev); static int ciss_detach(device_t dev); static int ciss_shutdown(device_t dev); /* (de)initialisation functions, control wrappers */ static int ciss_init_pci(struct ciss_softc *sc); static int ciss_setup_msix(struct ciss_softc *sc); static int ciss_init_perf(struct ciss_softc *sc); static int ciss_wait_adapter(struct ciss_softc *sc); static int ciss_flush_adapter(struct ciss_softc *sc); static int ciss_init_requests(struct ciss_softc *sc); static void ciss_command_map_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error); static int ciss_identify_adapter(struct ciss_softc *sc); static int ciss_init_logical(struct ciss_softc *sc); static int ciss_init_physical(struct ciss_softc *sc); static int ciss_filter_physical(struct ciss_softc *sc, struct ciss_lun_report *cll); static int ciss_identify_logical(struct ciss_softc *sc, struct ciss_ldrive *ld); static int ciss_get_ldrive_status(struct ciss_softc *sc, struct ciss_ldrive *ld); static int ciss_update_config(struct ciss_softc *sc); static int ciss_accept_media(struct ciss_softc *sc, struct ciss_ldrive *ld); static void ciss_init_sysctl(struct ciss_softc *sc); static void ciss_soft_reset(struct ciss_softc *sc); static void ciss_free(struct ciss_softc *sc); static void ciss_spawn_notify_thread(struct ciss_softc *sc); static void ciss_kill_notify_thread(struct ciss_softc *sc); /* request submission/completion */ static int ciss_start(struct ciss_request *cr); static void ciss_done(struct ciss_softc *sc, cr_qhead_t *qh); static void ciss_perf_done(struct ciss_softc *sc, cr_qhead_t *qh); static void ciss_intr(void *arg); static void ciss_perf_intr(void *arg); static void ciss_perf_msi_intr(void *arg); static void ciss_complete(struct ciss_softc *sc, cr_qhead_t *qh); static int _ciss_report_request(struct ciss_request *cr, int *command_status, int *scsi_status, const char *func); static int ciss_synch_request(struct ciss_request *cr, int timeout); static int ciss_poll_request(struct ciss_request *cr, int timeout); static int ciss_wait_request(struct ciss_request *cr, int timeout); #if 0 static int ciss_abort_request(struct ciss_request *cr); #endif /* request queueing */ static int ciss_get_request(struct ciss_softc *sc, struct ciss_request **crp); static void ciss_preen_command(struct ciss_request *cr); static void ciss_release_request(struct ciss_request *cr); /* request helpers */ static int ciss_get_bmic_request(struct ciss_softc *sc, struct ciss_request **crp, int opcode, void **bufp, size_t bufsize); static int ciss_user_command(struct ciss_softc *sc, IOCTL_Command_struct *ioc); /* DMA map/unmap */ static int ciss_map_request(struct ciss_request *cr); static void ciss_request_map_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error); static void ciss_unmap_request(struct ciss_request *cr); /* CAM interface */ static int ciss_cam_init(struct ciss_softc *sc); static void ciss_cam_rescan_target(struct ciss_softc *sc, int bus, int target); static void ciss_cam_action(struct cam_sim *sim, union ccb *ccb); static int ciss_cam_action_io(struct cam_sim *sim, struct ccb_scsiio *csio); static int ciss_cam_emulate(struct ciss_softc *sc, struct ccb_scsiio *csio); static void ciss_cam_poll(struct cam_sim *sim); static void ciss_cam_complete(struct ciss_request *cr); static void ciss_cam_complete_fixup(struct ciss_softc *sc, struct ccb_scsiio *csio); static int ciss_name_device(struct ciss_softc *sc, int bus, int target); /* periodic status monitoring */ static void ciss_periodic(void *arg); static void ciss_nop_complete(struct ciss_request *cr); static void ciss_disable_adapter(struct ciss_softc *sc); static void ciss_notify_event(struct ciss_softc *sc); static void ciss_notify_complete(struct ciss_request *cr); static int ciss_notify_abort(struct ciss_softc *sc); static int ciss_notify_abort_bmic(struct ciss_softc *sc); static void ciss_notify_hotplug(struct ciss_softc *sc, struct ciss_notify *cn); static void ciss_notify_logical(struct ciss_softc *sc, struct ciss_notify *cn); static void ciss_notify_physical(struct ciss_softc *sc, struct ciss_notify *cn); /* debugging output */ static void ciss_print_request(struct ciss_request *cr); static void ciss_print_ldrive(struct ciss_softc *sc, struct ciss_ldrive *ld); static const char *ciss_name_ldrive_status(int status); static int ciss_decode_ldrive_status(int status); static const char *ciss_name_ldrive_org(int org); static const char *ciss_name_command_status(int status); /* * PCI bus interface. */ static device_method_t ciss_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ciss_probe), DEVMETHOD(device_attach, ciss_attach), DEVMETHOD(device_detach, ciss_detach), DEVMETHOD(device_shutdown, ciss_shutdown), { 0, 0 } }; static driver_t ciss_pci_driver = { "ciss", ciss_methods, sizeof(struct ciss_softc) }; static devclass_t ciss_devclass; DRIVER_MODULE(ciss, pci, ciss_pci_driver, ciss_devclass, 0, 0); MODULE_DEPEND(ciss, cam, 1, 1, 1); MODULE_DEPEND(ciss, pci, 1, 1, 1); /* * Control device interface. */ static d_open_t ciss_open; static d_close_t ciss_close; static d_ioctl_t ciss_ioctl; static struct cdevsw ciss_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_open = ciss_open, .d_close = ciss_close, .d_ioctl = ciss_ioctl, .d_name = "ciss", }; /* * This tunable can be set at boot time and controls whether physical devices * that are marked hidden by the firmware should be exposed anyways. */ static unsigned int ciss_expose_hidden_physical = 0; TUNABLE_INT("hw.ciss.expose_hidden_physical", &ciss_expose_hidden_physical); static unsigned int ciss_nop_message_heartbeat = 0; TUNABLE_INT("hw.ciss.nop_message_heartbeat", &ciss_nop_message_heartbeat); /* * This tunable can force a particular transport to be used: * <= 0 : use default * 1 : force simple * 2 : force performant */ static int ciss_force_transport = 0; TUNABLE_INT("hw.ciss.force_transport", &ciss_force_transport); /* * This tunable can force a particular interrupt delivery method to be used: * <= 0 : use default * 1 : force INTx * 2 : force MSIX */ static int ciss_force_interrupt = 0; TUNABLE_INT("hw.ciss.force_interrupt", &ciss_force_interrupt); /************************************************************************ * CISS adapters amazingly don't have a defined programming interface * value. (One could say some very despairing things about PCI and * people just not getting the general idea.) So we are forced to * stick with matching against subvendor/subdevice, and thus have to * be updated for every new CISS adapter that appears. */ #define CISS_BOARD_UNKNWON 0 #define CISS_BOARD_SA5 1 #define CISS_BOARD_SA5B 2 #define CISS_BOARD_NOMSI (1<<4) #define CISS_BOARD_SIMPLE (1<<5) static struct { u_int16_t subvendor; u_int16_t subdevice; int flags; char *desc; } ciss_vendor_data[] = { { 0x0e11, 0x4070, CISS_BOARD_SA5|CISS_BOARD_NOMSI|CISS_BOARD_SIMPLE, "Compaq Smart Array 5300" }, { 0x0e11, 0x4080, CISS_BOARD_SA5B|CISS_BOARD_NOMSI, "Compaq Smart Array 5i" }, { 0x0e11, 0x4082, CISS_BOARD_SA5B|CISS_BOARD_NOMSI, "Compaq Smart Array 532" }, { 0x0e11, 0x4083, CISS_BOARD_SA5B|CISS_BOARD_NOMSI, "HP Smart Array 5312" }, { 0x0e11, 0x4091, CISS_BOARD_SA5, "HP Smart Array 6i" }, { 0x0e11, 0x409A, CISS_BOARD_SA5, "HP Smart Array 641" }, { 0x0e11, 0x409B, CISS_BOARD_SA5, "HP Smart Array 642" }, { 0x0e11, 0x409C, CISS_BOARD_SA5, "HP Smart Array 6400" }, { 0x0e11, 0x409D, CISS_BOARD_SA5, "HP Smart Array 6400 EM" }, { 0x103C, 0x3211, CISS_BOARD_SA5, "HP Smart Array E200i" }, { 0x103C, 0x3212, CISS_BOARD_SA5, "HP Smart Array E200" }, { 0x103C, 0x3213, CISS_BOARD_SA5, "HP Smart Array E200i" }, { 0x103C, 0x3214, CISS_BOARD_SA5, "HP Smart Array E200i" }, { 0x103C, 0x3215, CISS_BOARD_SA5, "HP Smart Array E200i" }, { 0x103C, 0x3220, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3222, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3223, CISS_BOARD_SA5, "HP Smart Array P800" }, { 0x103C, 0x3225, CISS_BOARD_SA5, "HP Smart Array P600" }, { 0x103C, 0x3230, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3231, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3232, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3233, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3234, CISS_BOARD_SA5, "HP Smart Array P400" }, { 0x103C, 0x3235, CISS_BOARD_SA5, "HP Smart Array P400i" }, { 0x103C, 0x3236, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3237, CISS_BOARD_SA5, "HP Smart Array E500" }, { 0x103C, 0x3238, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3239, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x323A, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x323B, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x323C, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x323D, CISS_BOARD_SA5, "HP Smart Array P700m" }, { 0x103C, 0x3241, CISS_BOARD_SA5, "HP Smart Array P212" }, { 0x103C, 0x3243, CISS_BOARD_SA5, "HP Smart Array P410" }, { 0x103C, 0x3245, CISS_BOARD_SA5, "HP Smart Array P410i" }, { 0x103C, 0x3247, CISS_BOARD_SA5, "HP Smart Array P411" }, { 0x103C, 0x3249, CISS_BOARD_SA5, "HP Smart Array P812" }, { 0x103C, 0x324A, CISS_BOARD_SA5, "HP Smart Array P712m" }, { 0x103C, 0x324B, CISS_BOARD_SA5, "HP Smart Array" }, { 0x103C, 0x3350, CISS_BOARD_SA5, "HP Smart Array P222" }, { 0x103C, 0x3351, CISS_BOARD_SA5, "HP Smart Array P420" }, { 0x103C, 0x3352, CISS_BOARD_SA5, "HP Smart Array P421" }, { 0x103C, 0x3353, CISS_BOARD_SA5, "HP Smart Array P822" }, { 0x103C, 0x3354, CISS_BOARD_SA5, "HP Smart Array P420i" }, { 0x103C, 0x3355, CISS_BOARD_SA5, "HP Smart Array P220i" }, { 0x103C, 0x3356, CISS_BOARD_SA5, "HP Smart Array P721m" }, { 0x103C, 0x1920, CISS_BOARD_SA5, "HP Smart Array P430i" }, { 0x103C, 0x1921, CISS_BOARD_SA5, "HP Smart Array P830i" }, { 0x103C, 0x1922, CISS_BOARD_SA5, "HP Smart Array P430" }, { 0x103C, 0x1923, CISS_BOARD_SA5, "HP Smart Array P431" }, { 0x103C, 0x1924, CISS_BOARD_SA5, "HP Smart Array P830" }, { 0x103C, 0x1926, CISS_BOARD_SA5, "HP Smart Array P731m" }, { 0x103C, 0x1928, CISS_BOARD_SA5, "HP Smart Array P230i" }, { 0x103C, 0x1929, CISS_BOARD_SA5, "HP Smart Array P530" }, { 0x103C, 0x192A, CISS_BOARD_SA5, "HP Smart Array P531" }, { 0x103C, 0x21BD, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21BE, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21BF, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C0, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C2, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C3, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C5, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C6, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C7, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21C8, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21CA, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21CB, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21CC, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21CD, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0x103C, 0x21CE, CISS_BOARD_SA5, "HP Smart Array TBD" }, { 0, 0, 0, NULL } }; /************************************************************************ * Find a match for the device in our list of known adapters. */ static int ciss_lookup(device_t dev) { int i; for (i = 0; ciss_vendor_data[i].desc != NULL; i++) if ((pci_get_subvendor(dev) == ciss_vendor_data[i].subvendor) && (pci_get_subdevice(dev) == ciss_vendor_data[i].subdevice)) { return(i); } return(-1); } /************************************************************************ * Match a known CISS adapter. */ static int ciss_probe(device_t dev) { int i; i = ciss_lookup(dev); if (i != -1) { device_set_desc(dev, ciss_vendor_data[i].desc); return(BUS_PROBE_DEFAULT); } return(ENOENT); } /************************************************************************ * Attach the driver to this adapter. */ static int ciss_attach(device_t dev) { struct ciss_softc *sc; int error; debug_called(1); #ifdef CISS_DEBUG /* print structure/union sizes */ debug_struct(ciss_command); debug_struct(ciss_header); debug_union(ciss_device_address); debug_struct(ciss_cdb); debug_struct(ciss_report_cdb); debug_struct(ciss_notify_cdb); debug_struct(ciss_notify); debug_struct(ciss_message_cdb); debug_struct(ciss_error_info_pointer); debug_struct(ciss_error_info); debug_struct(ciss_sg_entry); debug_struct(ciss_config_table); debug_struct(ciss_bmic_cdb); debug_struct(ciss_bmic_id_ldrive); debug_struct(ciss_bmic_id_lstatus); debug_struct(ciss_bmic_id_table); debug_struct(ciss_bmic_id_pdrive); debug_struct(ciss_bmic_blink_pdrive); debug_struct(ciss_bmic_flush_cache); debug_const(CISS_MAX_REQUESTS); debug_const(CISS_MAX_LOGICAL); debug_const(CISS_INTERRUPT_COALESCE_DELAY); debug_const(CISS_INTERRUPT_COALESCE_COUNT); debug_const(CISS_COMMAND_ALLOC_SIZE); debug_const(CISS_COMMAND_SG_LENGTH); debug_type(cciss_pci_info_struct); debug_type(cciss_coalint_struct); debug_type(cciss_coalint_struct); debug_type(NodeName_type); debug_type(NodeName_type); debug_type(Heartbeat_type); debug_type(BusTypes_type); debug_type(FirmwareVer_type); debug_type(DriverVer_type); debug_type(IOCTL_Command_struct); #endif sc = device_get_softc(dev); sc->ciss_dev = dev; mtx_init(&sc->ciss_mtx, "cissmtx", NULL, MTX_DEF); callout_init_mtx(&sc->ciss_periodic, &sc->ciss_mtx, 0); /* * Do PCI-specific init. */ if ((error = ciss_init_pci(sc)) != 0) goto out; /* * Initialise driver queues. */ ciss_initq_free(sc); ciss_initq_notify(sc); /* - * Initalize device sysctls. + * Initialize device sysctls. */ ciss_init_sysctl(sc); /* * Initialise command/request pool. */ if ((error = ciss_init_requests(sc)) != 0) goto out; /* * Get adapter information. */ if ((error = ciss_identify_adapter(sc)) != 0) goto out; /* * Find all the physical devices. */ if ((error = ciss_init_physical(sc)) != 0) goto out; /* * Build our private table of logical devices. */ if ((error = ciss_init_logical(sc)) != 0) goto out; /* * Enable interrupts so that the CAM scan can complete. */ CISS_TL_SIMPLE_ENABLE_INTERRUPTS(sc); /* * Initialise the CAM interface. */ if ((error = ciss_cam_init(sc)) != 0) goto out; /* * Start the heartbeat routine and event chain. */ ciss_periodic(sc); /* * Create the control device. */ sc->ciss_dev_t = make_dev(&ciss_cdevsw, device_get_unit(sc->ciss_dev), UID_ROOT, GID_OPERATOR, S_IRUSR | S_IWUSR, "ciss%d", device_get_unit(sc->ciss_dev)); sc->ciss_dev_t->si_drv1 = sc; /* * The adapter is running; synchronous commands can now sleep * waiting for an interrupt to signal completion. */ sc->ciss_flags |= CISS_FLAG_RUNNING; ciss_spawn_notify_thread(sc); error = 0; out: if (error != 0) { /* ciss_free() expects the mutex to be held */ mtx_lock(&sc->ciss_mtx); ciss_free(sc); } return(error); } /************************************************************************ * Detach the driver from this adapter. */ static int ciss_detach(device_t dev) { struct ciss_softc *sc = device_get_softc(dev); debug_called(1); mtx_lock(&sc->ciss_mtx); if (sc->ciss_flags & CISS_FLAG_CONTROL_OPEN) { mtx_unlock(&sc->ciss_mtx); return (EBUSY); } /* flush adapter cache */ ciss_flush_adapter(sc); /* release all resources. The mutex is released and freed here too. */ ciss_free(sc); return(0); } /************************************************************************ * Prepare adapter for system shutdown. */ static int ciss_shutdown(device_t dev) { struct ciss_softc *sc = device_get_softc(dev); debug_called(1); mtx_lock(&sc->ciss_mtx); /* flush adapter cache */ ciss_flush_adapter(sc); if (sc->ciss_soft_reset) ciss_soft_reset(sc); mtx_unlock(&sc->ciss_mtx); return(0); } static void ciss_init_sysctl(struct ciss_softc *sc) { SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->ciss_dev), SYSCTL_CHILDREN(device_get_sysctl_tree(sc->ciss_dev)), OID_AUTO, "soft_reset", CTLFLAG_RW, &sc->ciss_soft_reset, 0, ""); } /************************************************************************ * Perform PCI-specific attachment actions. */ static int ciss_init_pci(struct ciss_softc *sc) { uintptr_t cbase, csize, cofs; uint32_t method, supported_methods; int error, sqmask, i; void *intr; debug_called(1); /* * Work out adapter type. */ i = ciss_lookup(sc->ciss_dev); if (i < 0) { ciss_printf(sc, "unknown adapter type\n"); return (ENXIO); } if (ciss_vendor_data[i].flags & CISS_BOARD_SA5) { sqmask = CISS_TL_SIMPLE_INTR_OPQ_SA5; } else if (ciss_vendor_data[i].flags & CISS_BOARD_SA5B) { sqmask = CISS_TL_SIMPLE_INTR_OPQ_SA5B; } else { /* * XXX Big hammer, masks/unmasks all possible interrupts. This should * work on all hardware variants. Need to add code to handle the - * "controller crashed" interupt bit that this unmasks. + * "controller crashed" interrupt bit that this unmasks. */ sqmask = ~0; } /* * Allocate register window first (we need this to find the config * struct). */ error = ENXIO; sc->ciss_regs_rid = CISS_TL_SIMPLE_BAR_REGS; if ((sc->ciss_regs_resource = bus_alloc_resource_any(sc->ciss_dev, SYS_RES_MEMORY, &sc->ciss_regs_rid, RF_ACTIVE)) == NULL) { ciss_printf(sc, "can't allocate register window\n"); return(ENXIO); } sc->ciss_regs_bhandle = rman_get_bushandle(sc->ciss_regs_resource); sc->ciss_regs_btag = rman_get_bustag(sc->ciss_regs_resource); /* * Find the BAR holding the config structure. If it's not the one * we already mapped for registers, map it too. */ sc->ciss_cfg_rid = CISS_TL_SIMPLE_READ(sc, CISS_TL_SIMPLE_CFG_BAR) & 0xffff; if (sc->ciss_cfg_rid != sc->ciss_regs_rid) { if ((sc->ciss_cfg_resource = bus_alloc_resource_any(sc->ciss_dev, SYS_RES_MEMORY, &sc->ciss_cfg_rid, RF_ACTIVE)) == NULL) { ciss_printf(sc, "can't allocate config window\n"); return(ENXIO); } cbase = (uintptr_t)rman_get_virtual(sc->ciss_cfg_resource); csize = rman_get_end(sc->ciss_cfg_resource) - rman_get_start(sc->ciss_cfg_resource) + 1; } else { cbase = (uintptr_t)rman_get_virtual(sc->ciss_regs_resource); csize = rman_get_end(sc->ciss_regs_resource) - rman_get_start(sc->ciss_regs_resource) + 1; } cofs = CISS_TL_SIMPLE_READ(sc, CISS_TL_SIMPLE_CFG_OFF); /* * Use the base/size/offset values we just calculated to * sanity-check the config structure. If it's OK, point to it. */ if ((cofs + sizeof(struct ciss_config_table)) > csize) { ciss_printf(sc, "config table outside window\n"); return(ENXIO); } sc->ciss_cfg = (struct ciss_config_table *)(cbase + cofs); debug(1, "config struct at %p", sc->ciss_cfg); /* * Calculate the number of request structures/commands we are * going to provide for this adapter. */ sc->ciss_max_requests = min(CISS_MAX_REQUESTS, sc->ciss_cfg->max_outstanding_commands); /* * Validate the config structure. If we supported other transport * methods, we could select amongst them at this point in time. */ if (strncmp(sc->ciss_cfg->signature, "CISS", 4)) { ciss_printf(sc, "config signature mismatch (got '%c%c%c%c')\n", sc->ciss_cfg->signature[0], sc->ciss_cfg->signature[1], sc->ciss_cfg->signature[2], sc->ciss_cfg->signature[3]); return(ENXIO); } /* * Select the mode of operation, prefer Performant. */ if (!(sc->ciss_cfg->supported_methods & (CISS_TRANSPORT_METHOD_SIMPLE | CISS_TRANSPORT_METHOD_PERF))) { ciss_printf(sc, "No supported transport layers: 0x%x\n", sc->ciss_cfg->supported_methods); } switch (ciss_force_transport) { case 1: supported_methods = CISS_TRANSPORT_METHOD_SIMPLE; break; case 2: supported_methods = CISS_TRANSPORT_METHOD_PERF; break; default: /* * Override the capabilities of the BOARD and specify SIMPLE * MODE */ if (ciss_vendor_data[i].flags & CISS_BOARD_SIMPLE) supported_methods = CISS_TRANSPORT_METHOD_SIMPLE; else supported_methods = sc->ciss_cfg->supported_methods; break; } setup: if ((supported_methods & CISS_TRANSPORT_METHOD_PERF) != 0) { method = CISS_TRANSPORT_METHOD_PERF; sc->ciss_perf = (struct ciss_perf_config *)(cbase + cofs + sc->ciss_cfg->transport_offset); if (ciss_init_perf(sc)) { supported_methods &= ~method; goto setup; } } else if (supported_methods & CISS_TRANSPORT_METHOD_SIMPLE) { method = CISS_TRANSPORT_METHOD_SIMPLE; } else { ciss_printf(sc, "No supported transport methods: 0x%x\n", sc->ciss_cfg->supported_methods); return(ENXIO); } /* * Tell it we're using the low 4GB of RAM. Set the default interrupt * coalescing options. */ sc->ciss_cfg->requested_method = method; sc->ciss_cfg->command_physlimit = 0; sc->ciss_cfg->interrupt_coalesce_delay = CISS_INTERRUPT_COALESCE_DELAY; sc->ciss_cfg->interrupt_coalesce_count = CISS_INTERRUPT_COALESCE_COUNT; #ifdef __i386__ sc->ciss_cfg->host_driver |= CISS_DRIVER_SCSI_PREFETCH; #endif if (ciss_update_config(sc)) { ciss_printf(sc, "adapter refuses to accept config update (IDBR 0x%x)\n", CISS_TL_SIMPLE_READ(sc, CISS_TL_SIMPLE_IDBR)); return(ENXIO); } if ((sc->ciss_cfg->active_method & method) == 0) { supported_methods &= ~method; if (supported_methods == 0) { ciss_printf(sc, "adapter refuses to go into available transports " "mode (0x%x, 0x%x)\n", supported_methods, sc->ciss_cfg->active_method); return(ENXIO); } else goto setup; } /* * Wait for the adapter to come ready. */ if ((error = ciss_wait_adapter(sc)) != 0) return(error); /* Prepare to possibly use MSIX and/or PERFORMANT interrupts. Normal * interrupts have a rid of 0, this will be overridden if MSIX is used. */ sc->ciss_irq_rid[0] = 0; if (method == CISS_TRANSPORT_METHOD_PERF) { ciss_printf(sc, "PERFORMANT Transport\n"); if ((ciss_force_interrupt != 1) && (ciss_setup_msix(sc) == 0)) { intr = ciss_perf_msi_intr; } else { intr = ciss_perf_intr; } /* XXX The docs say that the 0x01 bit is only for SAS controllers. * Unfortunately, there is no good way to know if this is a SAS * controller. Hopefully enabling this bit universally will work OK. * It seems to work fine for SA6i controllers. */ sc->ciss_interrupt_mask = CISS_TL_PERF_INTR_OPQ | CISS_TL_PERF_INTR_MSI; } else { ciss_printf(sc, "SIMPLE Transport\n"); /* MSIX doesn't seem to work in SIMPLE mode, only enable if it forced */ if (ciss_force_interrupt == 2) /* If this fails, we automatically revert to INTx */ ciss_setup_msix(sc); sc->ciss_perf = NULL; intr = ciss_intr; sc->ciss_interrupt_mask = sqmask; } /* * Turn off interrupts before we go routing anything. */ CISS_TL_SIMPLE_DISABLE_INTERRUPTS(sc); /* * Allocate and set up our interrupt. */ if ((sc->ciss_irq_resource = bus_alloc_resource_any(sc->ciss_dev, SYS_RES_IRQ, &sc->ciss_irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { ciss_printf(sc, "can't allocate interrupt\n"); return(ENXIO); } if (bus_setup_intr(sc->ciss_dev, sc->ciss_irq_resource, INTR_TYPE_CAM|INTR_MPSAFE, NULL, intr, sc, &sc->ciss_intr)) { ciss_printf(sc, "can't set up interrupt\n"); return(ENXIO); } /* * Allocate the parent bus DMA tag appropriate for our PCI * interface. * * Note that "simple" adapters can only address within a 32-bit * span. */ if (bus_dma_tag_create(bus_get_dma_tag(sc->ciss_dev),/* PCI parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->ciss_parent_dmat)) { ciss_printf(sc, "can't allocate parent DMA tag\n"); return(ENOMEM); } /* * Create DMA tag for mapping buffers into adapter-addressable * space. */ if (bus_dma_tag_create(sc->ciss_parent_dmat, /* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ (CISS_MAX_SG_ELEMENTS - 1) * PAGE_SIZE, /* maxsize */ CISS_MAX_SG_ELEMENTS, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ busdma_lock_mutex, &sc->ciss_mtx, /* lockfunc, lockarg */ &sc->ciss_buffer_dmat)) { ciss_printf(sc, "can't allocate buffer DMA tag\n"); return(ENOMEM); } return(0); } /************************************************************************ * Setup MSI/MSIX operation (Performant only) * Four interrupts are available, but we only use 1 right now. If MSI-X * isn't avaialble, try using MSI instead. */ static int ciss_setup_msix(struct ciss_softc *sc) { int val, i; /* Weed out devices that don't actually support MSI */ i = ciss_lookup(sc->ciss_dev); if (ciss_vendor_data[i].flags & CISS_BOARD_NOMSI) return (EINVAL); /* * Only need to use the minimum number of MSI vectors, as the driver * doesn't support directed MSIX interrupts. */ val = pci_msix_count(sc->ciss_dev); if (val < CISS_MSI_COUNT) { val = pci_msi_count(sc->ciss_dev); device_printf(sc->ciss_dev, "got %d MSI messages]\n", val); if (val < CISS_MSI_COUNT) return (EINVAL); } val = MIN(val, CISS_MSI_COUNT); if (pci_alloc_msix(sc->ciss_dev, &val) != 0) { if (pci_alloc_msi(sc->ciss_dev, &val) != 0) return (EINVAL); } sc->ciss_msi = val; if (bootverbose) ciss_printf(sc, "Using %d MSIX interrupt%s\n", val, (val != 1) ? "s" : ""); for (i = 0; i < val; i++) sc->ciss_irq_rid[i] = i + 1; return (0); } /************************************************************************ * Setup the Performant structures. */ static int ciss_init_perf(struct ciss_softc *sc) { struct ciss_perf_config *pc = sc->ciss_perf; int reply_size; /* * Create the DMA tag for the reply queue. */ reply_size = sizeof(uint64_t) * sc->ciss_max_requests; if (bus_dma_tag_create(sc->ciss_parent_dmat, /* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ reply_size, 1, /* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->ciss_reply_dmat)) { ciss_printf(sc, "can't allocate reply DMA tag\n"); return(ENOMEM); } /* * Allocate memory and make it available for DMA. */ if (bus_dmamem_alloc(sc->ciss_reply_dmat, (void **)&sc->ciss_reply, BUS_DMA_NOWAIT, &sc->ciss_reply_map)) { ciss_printf(sc, "can't allocate reply memory\n"); return(ENOMEM); } bus_dmamap_load(sc->ciss_reply_dmat, sc->ciss_reply_map, sc->ciss_reply, reply_size, ciss_command_map_helper, &sc->ciss_reply_phys, 0); bzero(sc->ciss_reply, reply_size); sc->ciss_cycle = 0x1; sc->ciss_rqidx = 0; /* * Preload the fetch table with common command sizes. This allows the * hardware to not waste bus cycles for typical i/o commands, but also not * tax the driver to be too exact in choosing sizes. The table is optimized * for page-aligned i/o's, but since most i/o comes from the various pagers, * it's a reasonable assumption to make. */ pc->fetch_count[CISS_SG_FETCH_NONE] = (sizeof(struct ciss_command) + 15) / 16; pc->fetch_count[CISS_SG_FETCH_1] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 1 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_2] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 2 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_4] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 4 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_8] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 8 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_16] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 16 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_32] = (sizeof(struct ciss_command) + sizeof(struct ciss_sg_entry) * 32 + 15) / 16; pc->fetch_count[CISS_SG_FETCH_MAX] = (CISS_COMMAND_ALLOC_SIZE + 15) / 16; pc->rq_size = sc->ciss_max_requests; /* XXX less than the card supports? */ pc->rq_count = 1; /* XXX Hardcode for a single queue */ pc->rq_bank_hi = 0; pc->rq_bank_lo = 0; pc->rq[0].rq_addr_hi = 0x0; pc->rq[0].rq_addr_lo = sc->ciss_reply_phys; return(0); } /************************************************************************ * Wait for the adapter to come ready. */ static int ciss_wait_adapter(struct ciss_softc *sc) { int i; debug_called(1); /* * Wait for the adapter to come ready. */ if (!(sc->ciss_cfg->active_method & CISS_TRANSPORT_METHOD_READY)) { ciss_printf(sc, "waiting for adapter to come ready...\n"); for (i = 0; !(sc->ciss_cfg->active_method & CISS_TRANSPORT_METHOD_READY); i++) { DELAY(1000000); /* one second */ if (i > 30) { ciss_printf(sc, "timed out waiting for adapter to come ready\n"); return(EIO); } } } return(0); } /************************************************************************ * Flush the adapter cache. */ static int ciss_flush_adapter(struct ciss_softc *sc) { struct ciss_request *cr; struct ciss_bmic_flush_cache *cbfc; int error, command_status; debug_called(1); cr = NULL; cbfc = NULL; /* * Build a BMIC request to flush the cache. We don't disable * it, as we may be going to do more I/O (eg. we are emulating * the Synchronise Cache command). */ if ((cbfc = malloc(sizeof(*cbfc), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO)) == NULL) { error = ENOMEM; goto out; } if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_FLUSH_CACHE, (void **)&cbfc, sizeof(*cbfc))) != 0) goto out; /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC FLUSH_CACHE command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: break; default: ciss_printf(sc, "error flushing cache (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } out: if (cbfc != NULL) free(cbfc, CISS_MALLOC_CLASS); if (cr != NULL) ciss_release_request(cr); return(error); } static void ciss_soft_reset(struct ciss_softc *sc) { struct ciss_request *cr = NULL; struct ciss_command *cc; int i, error = 0; for (i = 0; i < sc->ciss_max_logical_bus; i++) { /* only reset proxy controllers */ if (sc->ciss_controllers[i].physical.bus == 0) continue; if ((error = ciss_get_request(sc, &cr)) != 0) break; if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_SOFT_RESET, NULL, 0)) != 0) break; cc = cr->cr_cc; cc->header.address = sc->ciss_controllers[i]; if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) break; ciss_release_request(cr); } if (error) ciss_printf(sc, "error resetting controller (%d)\n", error); if (cr != NULL) ciss_release_request(cr); } /************************************************************************ * Allocate memory for the adapter command structures, initialise * the request structures. * * Note that the entire set of commands are allocated in a single * contiguous slab. */ static int ciss_init_requests(struct ciss_softc *sc) { struct ciss_request *cr; int i; debug_called(1); if (bootverbose) ciss_printf(sc, "using %d of %d available commands\n", sc->ciss_max_requests, sc->ciss_cfg->max_outstanding_commands); /* * Create the DMA tag for commands. */ if (bus_dma_tag_create(sc->ciss_parent_dmat, /* parent */ 32, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ CISS_COMMAND_ALLOC_SIZE * sc->ciss_max_requests, 1, /* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->ciss_command_dmat)) { ciss_printf(sc, "can't allocate command DMA tag\n"); return(ENOMEM); } /* * Allocate memory and make it available for DMA. */ if (bus_dmamem_alloc(sc->ciss_command_dmat, (void **)&sc->ciss_command, BUS_DMA_NOWAIT, &sc->ciss_command_map)) { ciss_printf(sc, "can't allocate command memory\n"); return(ENOMEM); } bus_dmamap_load(sc->ciss_command_dmat, sc->ciss_command_map,sc->ciss_command, CISS_COMMAND_ALLOC_SIZE * sc->ciss_max_requests, ciss_command_map_helper, &sc->ciss_command_phys, 0); bzero(sc->ciss_command, CISS_COMMAND_ALLOC_SIZE * sc->ciss_max_requests); /* * Set up the request and command structures, push requests onto * the free queue. */ for (i = 1; i < sc->ciss_max_requests; i++) { cr = &sc->ciss_request[i]; cr->cr_sc = sc; cr->cr_tag = i; cr->cr_cc = (struct ciss_command *)((uintptr_t)sc->ciss_command + CISS_COMMAND_ALLOC_SIZE * i); cr->cr_ccphys = sc->ciss_command_phys + CISS_COMMAND_ALLOC_SIZE * i; bus_dmamap_create(sc->ciss_buffer_dmat, 0, &cr->cr_datamap); ciss_enqueue_free(cr); } return(0); } static void ciss_command_map_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error) { uint32_t *addr; addr = arg; *addr = segs[0].ds_addr; } /************************************************************************ * Identify the adapter, print some information about it. */ static int ciss_identify_adapter(struct ciss_softc *sc) { struct ciss_request *cr; int error, command_status; debug_called(1); cr = NULL; /* * Get a request, allocate storage for the adapter data. */ if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_ID_CTLR, (void **)&sc->ciss_id, sizeof(*sc->ciss_id))) != 0) goto out; /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC ID_CTLR command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: /* buffer right size */ break; case CISS_CMD_STATUS_DATA_UNDERRUN: case CISS_CMD_STATUS_DATA_OVERRUN: ciss_printf(sc, "data over/underrun reading adapter information\n"); default: ciss_printf(sc, "error reading adapter information (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } /* sanity-check reply */ if (!(sc->ciss_id->controller_flags & CONTROLLER_FLAGS_BIG_MAP_SUPPORT)) { ciss_printf(sc, "adapter does not support BIG_MAP\n"); error = ENXIO; goto out; } #if 0 /* XXX later revisions may not need this */ sc->ciss_flags |= CISS_FLAG_FAKE_SYNCH; #endif /* XXX only really required for old 5300 adapters? */ sc->ciss_flags |= CISS_FLAG_BMIC_ABORT; /* * Earlier controller specs do not contain these config * entries, so assume that a 0 means its old and assign * these values to the defaults that were established * when this driver was developed for them */ if (sc->ciss_cfg->max_logical_supported == 0) sc->ciss_cfg->max_logical_supported = CISS_MAX_LOGICAL; if (sc->ciss_cfg->max_physical_supported == 0) sc->ciss_cfg->max_physical_supported = CISS_MAX_PHYSICAL; /* print information */ if (bootverbose) { ciss_printf(sc, " %d logical drive%s configured\n", sc->ciss_id->configured_logical_drives, (sc->ciss_id->configured_logical_drives == 1) ? "" : "s"); ciss_printf(sc, " firmware %4.4s\n", sc->ciss_id->running_firmware_revision); ciss_printf(sc, " %d SCSI channels\n", sc->ciss_id->scsi_chip_count); ciss_printf(sc, " signature '%.4s'\n", sc->ciss_cfg->signature); ciss_printf(sc, " valence %d\n", sc->ciss_cfg->valence); ciss_printf(sc, " supported I/O methods 0x%b\n", sc->ciss_cfg->supported_methods, "\20\1READY\2simple\3performant\4MEMQ\n"); ciss_printf(sc, " active I/O method 0x%b\n", sc->ciss_cfg->active_method, "\20\2simple\3performant\4MEMQ\n"); ciss_printf(sc, " 4G page base 0x%08x\n", sc->ciss_cfg->command_physlimit); ciss_printf(sc, " interrupt coalesce delay %dus\n", sc->ciss_cfg->interrupt_coalesce_delay); ciss_printf(sc, " interrupt coalesce count %d\n", sc->ciss_cfg->interrupt_coalesce_count); ciss_printf(sc, " max outstanding commands %d\n", sc->ciss_cfg->max_outstanding_commands); ciss_printf(sc, " bus types 0x%b\n", sc->ciss_cfg->bus_types, "\20\1ultra2\2ultra3\10fibre1\11fibre2\n"); ciss_printf(sc, " server name '%.16s'\n", sc->ciss_cfg->server_name); ciss_printf(sc, " heartbeat 0x%x\n", sc->ciss_cfg->heartbeat); ciss_printf(sc, " max logical logical volumes: %d\n", sc->ciss_cfg->max_logical_supported); ciss_printf(sc, " max physical disks supported: %d\n", sc->ciss_cfg->max_physical_supported); ciss_printf(sc, " max physical disks per logical volume: %d\n", sc->ciss_cfg->max_physical_per_logical); ciss_printf(sc, " JBOD Support is %s\n", (sc->ciss_id->uiYetMoreControllerFlags & YMORE_CONTROLLER_FLAGS_JBOD_SUPPORTED) ? "Available" : "Unavailable"); ciss_printf(sc, " JBOD Mode is %s\n", (sc->ciss_id->PowerUPNvramFlags & PWR_UP_FLAG_JBOD_ENABLED) ? "Enabled" : "Disabled"); } out: if (error) { if (sc->ciss_id != NULL) { free(sc->ciss_id, CISS_MALLOC_CLASS); sc->ciss_id = NULL; } } if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Helper routine for generating a list of logical and physical luns. */ static struct ciss_lun_report * ciss_report_luns(struct ciss_softc *sc, int opcode, int nunits) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_report_cdb *crc; struct ciss_lun_report *cll; int command_status; int report_size; int error = 0; debug_called(1); cr = NULL; cll = NULL; /* * Get a request, allocate storage for the address list. */ if ((error = ciss_get_request(sc, &cr)) != 0) goto out; report_size = sizeof(*cll) + nunits * sizeof(union ciss_device_address); if ((cll = malloc(report_size, CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO)) == NULL) { ciss_printf(sc, "can't allocate memory for lun report\n"); error = ENOMEM; goto out; } /* * Build the Report Logical/Physical LUNs command. */ cc = cr->cr_cc; cr->cr_data = cll; cr->cr_length = report_size; cr->cr_flags = CISS_REQ_DATAIN; cc->header.address.physical.mode = CISS_HDR_ADDRESS_MODE_PERIPHERAL; cc->header.address.physical.bus = 0; cc->header.address.physical.target = 0; cc->cdb.cdb_length = sizeof(*crc); cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_READ; cc->cdb.timeout = 30; /* XXX better suggestions? */ crc = (struct ciss_report_cdb *)&(cc->cdb.cdb[0]); bzero(crc, sizeof(*crc)); crc->opcode = opcode; crc->length = htonl(report_size); /* big-endian field */ cll->list_size = htonl(report_size - sizeof(*cll)); /* big-endian field */ /* * Submit the request and wait for it to complete. (timeout * here should be much greater than above) */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending %d LUN command (%d)\n", opcode, error); goto out; } /* * Check response. Note that data over/underrun is OK. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: /* buffer right size */ case CISS_CMD_STATUS_DATA_UNDERRUN: /* buffer too large, not bad */ break; case CISS_CMD_STATUS_DATA_OVERRUN: ciss_printf(sc, "WARNING: more units than driver limit (%d)\n", sc->ciss_cfg->max_logical_supported); break; default: ciss_printf(sc, "error detecting logical drive configuration (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } ciss_release_request(cr); cr = NULL; out: if (cr != NULL) ciss_release_request(cr); if (error && cll != NULL) { free(cll, CISS_MALLOC_CLASS); cll = NULL; } return(cll); } /************************************************************************ * Find logical drives on the adapter. */ static int ciss_init_logical(struct ciss_softc *sc) { struct ciss_lun_report *cll; int error = 0, i, j; int ndrives; debug_called(1); cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_LOGICAL_LUNS, sc->ciss_cfg->max_logical_supported); if (cll == NULL) { error = ENXIO; goto out; } /* sanity-check reply */ ndrives = (ntohl(cll->list_size) / sizeof(union ciss_device_address)); if ((ndrives < 0) || (ndrives > sc->ciss_cfg->max_logical_supported)) { ciss_printf(sc, "adapter claims to report absurd number of logical drives (%d > %d)\n", ndrives, sc->ciss_cfg->max_logical_supported); error = ENXIO; goto out; } /* * Save logical drive information. */ if (bootverbose) { ciss_printf(sc, "%d logical drive%s\n", ndrives, (ndrives > 1 || ndrives == 0) ? "s" : ""); } sc->ciss_logical = malloc(sc->ciss_max_logical_bus * sizeof(struct ciss_ldrive *), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_logical == NULL) { error = ENXIO; goto out; } for (i = 0; i < sc->ciss_max_logical_bus; i++) { sc->ciss_logical[i] = malloc(sc->ciss_cfg->max_logical_supported * sizeof(struct ciss_ldrive), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_logical[i] == NULL) { error = ENXIO; goto out; } for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) sc->ciss_logical[i][j].cl_status = CISS_LD_NONEXISTENT; } for (i = 0; i < sc->ciss_cfg->max_logical_supported; i++) { if (i < ndrives) { struct ciss_ldrive *ld; int bus, target; bus = CISS_LUN_TO_BUS(cll->lun[i].logical.lun); target = CISS_LUN_TO_TARGET(cll->lun[i].logical.lun); ld = &sc->ciss_logical[bus][target]; ld->cl_address = cll->lun[i]; ld->cl_controller = &sc->ciss_controllers[bus]; if (ciss_identify_logical(sc, ld) != 0) continue; /* * If the drive has had media exchanged, we should bring it online. */ if (ld->cl_lstatus->media_exchanged) ciss_accept_media(sc, ld); } } out: if (cll != NULL) free(cll, CISS_MALLOC_CLASS); return(error); } static int ciss_init_physical(struct ciss_softc *sc) { struct ciss_lun_report *cll; int error = 0, i; int nphys; int bus, target; debug_called(1); bus = 0; target = 0; cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_PHYSICAL_LUNS, sc->ciss_cfg->max_physical_supported); if (cll == NULL) { error = ENXIO; goto out; } nphys = (ntohl(cll->list_size) / sizeof(union ciss_device_address)); if (bootverbose) { ciss_printf(sc, "%d physical device%s\n", nphys, (nphys > 1 || nphys == 0) ? "s" : ""); } /* * Figure out the bus mapping. * Logical buses include both the local logical bus for local arrays and * proxy buses for remote arrays. Physical buses are numbered by the * controller and represent physical buses that hold physical devices. * We shift these bus numbers so that everything fits into a single flat * numbering space for CAM. Logical buses occupy the first 32 CAM bus * numbers, and the physical bus numbers are shifted to be above that. * This results in the various driver arrays being indexed as follows: * * ciss_controllers[] - indexed by logical bus * ciss_cam_sim[] - indexed by both logical and physical, with physical * being shifted by 32. * ciss_logical[][] - indexed by logical bus * ciss_physical[][] - indexed by physical bus * * XXX This is getting more and more hackish. CISS really doesn't play * well with a standard SCSI model; devices are addressed via magic * cookies, not via b/t/l addresses. Since there is no way to store * the cookie in the CAM device object, we have to keep these lookup * tables handy so that the devices can be found quickly at the cost * of wasting memory and having a convoluted lookup scheme. This * driver should probably be converted to block interface. */ /* * If the L2 and L3 SCSI addresses are 0, this signifies a proxy * controller. A proxy controller is another physical controller * behind the primary PCI controller. We need to know about this * so that BMIC commands can be properly targeted. There can be * proxy controllers attached to a single PCI controller, so * find the highest numbered one so the array can be properly * sized. */ sc->ciss_max_logical_bus = 1; for (i = 0; i < nphys; i++) { if (cll->lun[i].physical.extra_address == 0) { bus = cll->lun[i].physical.bus; sc->ciss_max_logical_bus = max(sc->ciss_max_logical_bus, bus) + 1; } else { bus = CISS_EXTRA_BUS2(cll->lun[i].physical.extra_address); sc->ciss_max_physical_bus = max(sc->ciss_max_physical_bus, bus); } } sc->ciss_controllers = malloc(sc->ciss_max_logical_bus * sizeof (union ciss_device_address), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_controllers == NULL) { ciss_printf(sc, "Could not allocate memory for controller map\n"); error = ENOMEM; goto out; } /* setup a map of controller addresses */ for (i = 0; i < nphys; i++) { if (cll->lun[i].physical.extra_address == 0) { sc->ciss_controllers[cll->lun[i].physical.bus] = cll->lun[i]; } } sc->ciss_physical = malloc(sc->ciss_max_physical_bus * sizeof(struct ciss_pdrive *), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_physical == NULL) { ciss_printf(sc, "Could not allocate memory for physical device map\n"); error = ENOMEM; goto out; } for (i = 0; i < sc->ciss_max_physical_bus; i++) { sc->ciss_physical[i] = malloc(sizeof(struct ciss_pdrive) * CISS_MAX_PHYSTGT, CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_physical[i] == NULL) { ciss_printf(sc, "Could not allocate memory for target map\n"); error = ENOMEM; goto out; } } ciss_filter_physical(sc, cll); out: if (cll != NULL) free(cll, CISS_MALLOC_CLASS); return(error); } static int ciss_filter_physical(struct ciss_softc *sc, struct ciss_lun_report *cll) { u_int32_t ea; int i, nphys; int bus, target; nphys = (ntohl(cll->list_size) / sizeof(union ciss_device_address)); for (i = 0; i < nphys; i++) { if (cll->lun[i].physical.extra_address == 0) continue; /* * Filter out devices that we don't want. Level 3 LUNs could * probably be supported, but the docs don't give enough of a * hint to know how. * * The mode field of the physical address is likely set to have * hard disks masked out. Honor it unless the user has overridden * us with the tunable. We also munge the inquiry data for these * disks so that they only show up as passthrough devices. Keeping * them visible in this fashion is useful for doing things like * flashing firmware. */ ea = cll->lun[i].physical.extra_address; if ((CISS_EXTRA_BUS3(ea) != 0) || (CISS_EXTRA_TARGET3(ea) != 0) || (CISS_EXTRA_MODE2(ea) == 0x3)) continue; if ((ciss_expose_hidden_physical == 0) && (cll->lun[i].physical.mode == CISS_HDR_ADDRESS_MODE_MASK_PERIPHERAL)) continue; /* * Note: CISS firmware numbers physical busses starting at '1', not * '0'. This numbering is internal to the firmware and is only * used as a hint here. */ bus = CISS_EXTRA_BUS2(ea) - 1; target = CISS_EXTRA_TARGET2(ea); sc->ciss_physical[bus][target].cp_address = cll->lun[i]; sc->ciss_physical[bus][target].cp_online = 1; } return (0); } static int ciss_inquiry_logical(struct ciss_softc *sc, struct ciss_ldrive *ld) { struct ciss_request *cr; struct ciss_command *cc; struct scsi_inquiry *inq; int error; int command_status; cr = NULL; bzero(&ld->cl_geometry, sizeof(ld->cl_geometry)); if ((error = ciss_get_request(sc, &cr)) != 0) goto out; cc = cr->cr_cc; cr->cr_data = &ld->cl_geometry; cr->cr_length = sizeof(ld->cl_geometry); cr->cr_flags = CISS_REQ_DATAIN; cc->header.address = ld->cl_address; cc->cdb.cdb_length = 6; cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_READ; cc->cdb.timeout = 30; inq = (struct scsi_inquiry *)&(cc->cdb.cdb[0]); inq->opcode = INQUIRY; inq->byte2 = SI_EVPD; inq->page_code = CISS_VPD_LOGICAL_DRIVE_GEOMETRY; scsi_ulto2b(sizeof(ld->cl_geometry), inq->length); if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error getting geometry (%d)\n", error); goto out; } ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: case CISS_CMD_STATUS_DATA_UNDERRUN: break; case CISS_CMD_STATUS_DATA_OVERRUN: ciss_printf(sc, "WARNING: Data overrun\n"); break; default: ciss_printf(sc, "Error detecting logical drive geometry (%s)\n", ciss_name_command_status(command_status)); break; } out: if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Identify a logical drive, initialise state related to it. */ static int ciss_identify_logical(struct ciss_softc *sc, struct ciss_ldrive *ld) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_bmic_cdb *cbc; int error, command_status; debug_called(1); cr = NULL; /* * Build a BMIC request to fetch the drive ID. */ if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_ID_LDRIVE, (void **)&ld->cl_ldrive, sizeof(*ld->cl_ldrive))) != 0) goto out; cc = cr->cr_cc; cc->header.address = *ld->cl_controller; /* target controller */ cbc = (struct ciss_bmic_cdb *)&(cc->cdb.cdb[0]); cbc->log_drive = CISS_LUN_TO_TARGET(ld->cl_address.logical.lun); /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC LDRIVE command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: /* buffer right size */ break; case CISS_CMD_STATUS_DATA_UNDERRUN: case CISS_CMD_STATUS_DATA_OVERRUN: ciss_printf(sc, "data over/underrun reading logical drive ID\n"); default: ciss_printf(sc, "error reading logical drive ID (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } ciss_release_request(cr); cr = NULL; /* * Build a CISS BMIC command to get the logical drive status. */ if ((error = ciss_get_ldrive_status(sc, ld)) != 0) goto out; /* * Get the logical drive geometry. */ if ((error = ciss_inquiry_logical(sc, ld)) != 0) goto out; /* * Print the drive's basic characteristics. */ if (bootverbose) { ciss_printf(sc, "logical drive (b%dt%d): %s, %dMB ", CISS_LUN_TO_BUS(ld->cl_address.logical.lun), CISS_LUN_TO_TARGET(ld->cl_address.logical.lun), ciss_name_ldrive_org(ld->cl_ldrive->fault_tolerance), ((ld->cl_ldrive->blocks_available / (1024 * 1024)) * ld->cl_ldrive->block_size)); ciss_print_ldrive(sc, ld); } out: if (error != 0) { /* make the drive not-exist */ ld->cl_status = CISS_LD_NONEXISTENT; if (ld->cl_ldrive != NULL) { free(ld->cl_ldrive, CISS_MALLOC_CLASS); ld->cl_ldrive = NULL; } if (ld->cl_lstatus != NULL) { free(ld->cl_lstatus, CISS_MALLOC_CLASS); ld->cl_lstatus = NULL; } } if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Get status for a logical drive. * * XXX should we also do this in response to Test Unit Ready? */ static int ciss_get_ldrive_status(struct ciss_softc *sc, struct ciss_ldrive *ld) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_bmic_cdb *cbc; int error, command_status; /* * Build a CISS BMIC command to get the logical drive status. */ if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_ID_LSTATUS, (void **)&ld->cl_lstatus, sizeof(*ld->cl_lstatus))) != 0) goto out; cc = cr->cr_cc; cc->header.address = *ld->cl_controller; /* target controller */ cbc = (struct ciss_bmic_cdb *)&(cc->cdb.cdb[0]); cbc->log_drive = CISS_LUN_TO_TARGET(ld->cl_address.logical.lun); /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC LSTATUS command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: /* buffer right size */ break; case CISS_CMD_STATUS_DATA_UNDERRUN: case CISS_CMD_STATUS_DATA_OVERRUN: ciss_printf(sc, "data over/underrun reading logical drive status\n"); default: ciss_printf(sc, "error reading logical drive status (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } /* * Set the drive's summary status based on the returned status. * * XXX testing shows that a failed JBOD drive comes back at next * boot in "queued for expansion" mode. WTF? */ ld->cl_status = ciss_decode_ldrive_status(ld->cl_lstatus->status); out: if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Notify the adapter of a config update. */ static int ciss_update_config(struct ciss_softc *sc) { int i; debug_called(1); CISS_TL_SIMPLE_WRITE(sc, CISS_TL_SIMPLE_IDBR, CISS_TL_SIMPLE_IDBR_CFG_TABLE); for (i = 0; i < 1000; i++) { if (!(CISS_TL_SIMPLE_READ(sc, CISS_TL_SIMPLE_IDBR) & CISS_TL_SIMPLE_IDBR_CFG_TABLE)) { return(0); } DELAY(1000); } return(1); } /************************************************************************ * Accept new media into a logical drive. * * XXX The drive has previously been offline; it would be good if we * could make sure it's not open right now. */ static int ciss_accept_media(struct ciss_softc *sc, struct ciss_ldrive *ld) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_bmic_cdb *cbc; int command_status; int error = 0, ldrive; ldrive = CISS_LUN_TO_TARGET(ld->cl_address.logical.lun); debug(0, "bringing logical drive %d back online", ldrive); /* * Build a CISS BMIC command to bring the drive back online. */ if ((error = ciss_get_bmic_request(sc, &cr, CISS_BMIC_ACCEPT_MEDIA, NULL, 0)) != 0) goto out; cc = cr->cr_cc; cc->header.address = *ld->cl_controller; /* target controller */ cbc = (struct ciss_bmic_cdb *)&(cc->cdb.cdb[0]); cbc->log_drive = ldrive; /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC ACCEPT MEDIA command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: /* all OK */ /* we should get a logical drive status changed event here */ break; default: ciss_printf(cr->cr_sc, "error accepting media into failed logical drive (%s)\n", ciss_name_command_status(command_status)); break; } out: if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Release adapter resources. */ static void ciss_free(struct ciss_softc *sc) { struct ciss_request *cr; int i, j; debug_called(1); /* we're going away */ sc->ciss_flags |= CISS_FLAG_ABORTING; /* terminate the periodic heartbeat routine */ callout_stop(&sc->ciss_periodic); /* cancel the Event Notify chain */ ciss_notify_abort(sc); ciss_kill_notify_thread(sc); /* disconnect from CAM */ if (sc->ciss_cam_sim) { for (i = 0; i < sc->ciss_max_logical_bus; i++) { if (sc->ciss_cam_sim[i]) { xpt_bus_deregister(cam_sim_path(sc->ciss_cam_sim[i])); cam_sim_free(sc->ciss_cam_sim[i], 0); } } for (i = CISS_PHYSICAL_BASE; i < sc->ciss_max_physical_bus + CISS_PHYSICAL_BASE; i++) { if (sc->ciss_cam_sim[i]) { xpt_bus_deregister(cam_sim_path(sc->ciss_cam_sim[i])); cam_sim_free(sc->ciss_cam_sim[i], 0); } } free(sc->ciss_cam_sim, CISS_MALLOC_CLASS); } if (sc->ciss_cam_devq) cam_simq_free(sc->ciss_cam_devq); /* remove the control device */ mtx_unlock(&sc->ciss_mtx); if (sc->ciss_dev_t != NULL) destroy_dev(sc->ciss_dev_t); /* Final cleanup of the callout. */ callout_drain(&sc->ciss_periodic); mtx_destroy(&sc->ciss_mtx); /* free the controller data */ if (sc->ciss_id != NULL) free(sc->ciss_id, CISS_MALLOC_CLASS); /* release I/O resources */ if (sc->ciss_regs_resource != NULL) bus_release_resource(sc->ciss_dev, SYS_RES_MEMORY, sc->ciss_regs_rid, sc->ciss_regs_resource); if (sc->ciss_cfg_resource != NULL) bus_release_resource(sc->ciss_dev, SYS_RES_MEMORY, sc->ciss_cfg_rid, sc->ciss_cfg_resource); if (sc->ciss_intr != NULL) bus_teardown_intr(sc->ciss_dev, sc->ciss_irq_resource, sc->ciss_intr); if (sc->ciss_irq_resource != NULL) bus_release_resource(sc->ciss_dev, SYS_RES_IRQ, sc->ciss_irq_rid[0], sc->ciss_irq_resource); if (sc->ciss_msi) pci_release_msi(sc->ciss_dev); while ((cr = ciss_dequeue_free(sc)) != NULL) bus_dmamap_destroy(sc->ciss_buffer_dmat, cr->cr_datamap); if (sc->ciss_buffer_dmat) bus_dma_tag_destroy(sc->ciss_buffer_dmat); /* destroy command memory and DMA tag */ if (sc->ciss_command != NULL) { bus_dmamap_unload(sc->ciss_command_dmat, sc->ciss_command_map); bus_dmamem_free(sc->ciss_command_dmat, sc->ciss_command, sc->ciss_command_map); } if (sc->ciss_command_dmat) bus_dma_tag_destroy(sc->ciss_command_dmat); if (sc->ciss_reply) { bus_dmamap_unload(sc->ciss_reply_dmat, sc->ciss_reply_map); bus_dmamem_free(sc->ciss_reply_dmat, sc->ciss_reply, sc->ciss_reply_map); } if (sc->ciss_reply_dmat) bus_dma_tag_destroy(sc->ciss_reply_dmat); /* destroy DMA tags */ if (sc->ciss_parent_dmat) bus_dma_tag_destroy(sc->ciss_parent_dmat); if (sc->ciss_logical) { for (i = 0; i < sc->ciss_max_logical_bus; i++) { for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) { if (sc->ciss_logical[i][j].cl_ldrive) free(sc->ciss_logical[i][j].cl_ldrive, CISS_MALLOC_CLASS); if (sc->ciss_logical[i][j].cl_lstatus) free(sc->ciss_logical[i][j].cl_lstatus, CISS_MALLOC_CLASS); } free(sc->ciss_logical[i], CISS_MALLOC_CLASS); } free(sc->ciss_logical, CISS_MALLOC_CLASS); } if (sc->ciss_physical) { for (i = 0; i < sc->ciss_max_physical_bus; i++) free(sc->ciss_physical[i], CISS_MALLOC_CLASS); free(sc->ciss_physical, CISS_MALLOC_CLASS); } if (sc->ciss_controllers) free(sc->ciss_controllers, CISS_MALLOC_CLASS); } /************************************************************************ * Give a command to the adapter. * * Note that this uses the simple transport layer directly. If we * want to add support for other layers, we'll need a switch of some * sort. * * Note that the simple transport layer has no way of refusing a * command; we only have as many request structures as the adapter * supports commands, so we don't have to check (this presumes that * the adapter can handle commands as fast as we throw them at it). */ static int ciss_start(struct ciss_request *cr) { struct ciss_command *cc; /* XXX debugging only */ int error; cc = cr->cr_cc; debug(2, "post command %d tag %d ", cr->cr_tag, cc->header.host_tag); /* * Map the request's data. */ if ((error = ciss_map_request(cr))) return(error); #if 0 ciss_print_request(cr); #endif return(0); } /************************************************************************ * Fetch completed request(s) from the adapter, queue them for * completion handling. * * Note that this uses the simple transport layer directly. If we * want to add support for other layers, we'll need a switch of some * sort. * * Note that the simple transport mechanism does not require any * reentrancy protection; the OPQ read is atomic. If there is a * chance of a race with something else that might move the request * off the busy list, then we will have to lock against that * (eg. timeouts, etc.) */ static void ciss_done(struct ciss_softc *sc, cr_qhead_t *qh) { struct ciss_request *cr; struct ciss_command *cc; u_int32_t tag, index; debug_called(3); /* * Loop quickly taking requests from the adapter and moving them * to the completed queue. */ for (;;) { tag = CISS_TL_SIMPLE_FETCH_CMD(sc); if (tag == CISS_TL_SIMPLE_OPQ_EMPTY) break; index = tag >> 2; debug(2, "completed command %d%s", index, (tag & CISS_HDR_HOST_TAG_ERROR) ? " with error" : ""); if (index >= sc->ciss_max_requests) { ciss_printf(sc, "completed invalid request %d (0x%x)\n", index, tag); continue; } cr = &(sc->ciss_request[index]); cc = cr->cr_cc; cc->header.host_tag = tag; /* not updated by adapter */ ciss_enqueue_complete(cr, qh); } } static void ciss_perf_done(struct ciss_softc *sc, cr_qhead_t *qh) { struct ciss_request *cr; struct ciss_command *cc; u_int32_t tag, index; debug_called(3); /* * Loop quickly taking requests from the adapter and moving them * to the completed queue. */ for (;;) { tag = sc->ciss_reply[sc->ciss_rqidx]; if ((tag & CISS_CYCLE_MASK) != sc->ciss_cycle) break; index = tag >> 2; debug(2, "completed command %d%s\n", index, (tag & CISS_HDR_HOST_TAG_ERROR) ? " with error" : ""); if (index < sc->ciss_max_requests) { cr = &(sc->ciss_request[index]); cc = cr->cr_cc; cc->header.host_tag = tag; /* not updated by adapter */ ciss_enqueue_complete(cr, qh); } else { ciss_printf(sc, "completed invalid request %d (0x%x)\n", index, tag); } if (++sc->ciss_rqidx == sc->ciss_max_requests) { sc->ciss_rqidx = 0; sc->ciss_cycle ^= 1; } } } /************************************************************************ * Take an interrupt from the adapter. */ static void ciss_intr(void *arg) { cr_qhead_t qh; struct ciss_softc *sc = (struct ciss_softc *)arg; /* * The only interrupt we recognise indicates that there are * entries in the outbound post queue. */ STAILQ_INIT(&qh); ciss_done(sc, &qh); mtx_lock(&sc->ciss_mtx); ciss_complete(sc, &qh); mtx_unlock(&sc->ciss_mtx); } static void ciss_perf_intr(void *arg) { struct ciss_softc *sc = (struct ciss_softc *)arg; /* Clear the interrupt and flush the bridges. Docs say that the flush * needs to be done twice, which doesn't seem right. */ CISS_TL_PERF_CLEAR_INT(sc); CISS_TL_PERF_FLUSH_INT(sc); ciss_perf_msi_intr(sc); } static void ciss_perf_msi_intr(void *arg) { cr_qhead_t qh; struct ciss_softc *sc = (struct ciss_softc *)arg; STAILQ_INIT(&qh); ciss_perf_done(sc, &qh); mtx_lock(&sc->ciss_mtx); ciss_complete(sc, &qh); mtx_unlock(&sc->ciss_mtx); } /************************************************************************ * Process completed requests. * * Requests can be completed in three fashions: * * - by invoking a callback function (cr_complete is non-null) * - by waking up a sleeper (cr_flags has CISS_REQ_SLEEP set) * - by clearing the CISS_REQ_POLL flag in interrupt/timeout context */ static void ciss_complete(struct ciss_softc *sc, cr_qhead_t *qh) { struct ciss_request *cr; debug_called(2); /* * Loop taking requests off the completed queue and performing * completion processing on them. */ for (;;) { if ((cr = ciss_dequeue_complete(sc, qh)) == NULL) break; ciss_unmap_request(cr); if ((cr->cr_flags & CISS_REQ_BUSY) == 0) ciss_printf(sc, "WARNING: completing non-busy request\n"); cr->cr_flags &= ~CISS_REQ_BUSY; /* * If the request has a callback, invoke it. */ if (cr->cr_complete != NULL) { cr->cr_complete(cr); continue; } /* * If someone is sleeping on this request, wake them up. */ if (cr->cr_flags & CISS_REQ_SLEEP) { cr->cr_flags &= ~CISS_REQ_SLEEP; wakeup(cr); continue; } /* * If someone is polling this request for completion, signal. */ if (cr->cr_flags & CISS_REQ_POLL) { cr->cr_flags &= ~CISS_REQ_POLL; continue; } /* * Give up and throw the request back on the free queue. This * should never happen; resources will probably be lost. */ ciss_printf(sc, "WARNING: completed command with no submitter\n"); ciss_enqueue_free(cr); } } /************************************************************************ * Report on the completion status of a request, and pass back SCSI * and command status values. */ static int _ciss_report_request(struct ciss_request *cr, int *command_status, int *scsi_status, const char *func) { struct ciss_command *cc; struct ciss_error_info *ce; debug_called(2); cc = cr->cr_cc; ce = (struct ciss_error_info *)&(cc->sg[0]); /* * We don't consider data under/overrun an error for the Report * Logical/Physical LUNs commands. */ if ((cc->header.host_tag & CISS_HDR_HOST_TAG_ERROR) && ((ce->command_status == CISS_CMD_STATUS_DATA_OVERRUN) || (ce->command_status == CISS_CMD_STATUS_DATA_UNDERRUN)) && ((cc->cdb.cdb[0] == CISS_OPCODE_REPORT_LOGICAL_LUNS) || (cc->cdb.cdb[0] == CISS_OPCODE_REPORT_PHYSICAL_LUNS) || (cc->cdb.cdb[0] == INQUIRY))) { cc->header.host_tag &= ~CISS_HDR_HOST_TAG_ERROR; debug(2, "ignoring irrelevant under/overrun error"); } /* * Check the command's error bit, if clear, there's no status and * everything is OK. */ if (!(cc->header.host_tag & CISS_HDR_HOST_TAG_ERROR)) { if (scsi_status != NULL) *scsi_status = SCSI_STATUS_OK; if (command_status != NULL) *command_status = CISS_CMD_STATUS_SUCCESS; return(0); } else { if (command_status != NULL) *command_status = ce->command_status; if (scsi_status != NULL) { if (ce->command_status == CISS_CMD_STATUS_TARGET_STATUS) { *scsi_status = ce->scsi_status; } else { *scsi_status = -1; } } if (bootverbose) ciss_printf(cr->cr_sc, "command status 0x%x (%s) scsi status 0x%x\n", ce->command_status, ciss_name_command_status(ce->command_status), ce->scsi_status); if (ce->command_status == CISS_CMD_STATUS_INVALID_COMMAND) { ciss_printf(cr->cr_sc, "invalid command, offense size %d at %d, value 0x%x, function %s\n", ce->additional_error_info.invalid_command.offense_size, ce->additional_error_info.invalid_command.offense_offset, ce->additional_error_info.invalid_command.offense_value, func); } } #if 0 ciss_print_request(cr); #endif return(1); } /************************************************************************ * Issue a request and don't return until it's completed. * * Depending on adapter status, we may poll or sleep waiting for * completion. */ static int ciss_synch_request(struct ciss_request *cr, int timeout) { if (cr->cr_sc->ciss_flags & CISS_FLAG_RUNNING) { return(ciss_wait_request(cr, timeout)); } else { return(ciss_poll_request(cr, timeout)); } } /************************************************************************ * Issue a request and poll for completion. * * Timeout in milliseconds. */ static int ciss_poll_request(struct ciss_request *cr, int timeout) { cr_qhead_t qh; struct ciss_softc *sc; int error; debug_called(2); STAILQ_INIT(&qh); sc = cr->cr_sc; cr->cr_flags |= CISS_REQ_POLL; if ((error = ciss_start(cr)) != 0) return(error); do { if (sc->ciss_perf) ciss_perf_done(sc, &qh); else ciss_done(sc, &qh); ciss_complete(sc, &qh); if (!(cr->cr_flags & CISS_REQ_POLL)) return(0); DELAY(1000); } while (timeout-- >= 0); return(EWOULDBLOCK); } /************************************************************************ * Issue a request and sleep waiting for completion. * * Timeout in milliseconds. Note that a spurious wakeup will reset * the timeout. */ static int ciss_wait_request(struct ciss_request *cr, int timeout) { int error; debug_called(2); cr->cr_flags |= CISS_REQ_SLEEP; if ((error = ciss_start(cr)) != 0) return(error); while ((cr->cr_flags & CISS_REQ_SLEEP) && (error != EWOULDBLOCK)) { error = msleep_sbt(cr, &cr->cr_sc->ciss_mtx, PRIBIO, "cissREQ", SBT_1MS * timeout, 0, 0); } return(error); } #if 0 /************************************************************************ * Abort a request. Note that a potential exists here to race the * request being completed; the caller must deal with this. */ static int ciss_abort_request(struct ciss_request *ar) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_message_cdb *cmc; int error; debug_called(1); /* get a request */ if ((error = ciss_get_request(ar->cr_sc, &cr)) != 0) return(error); /* build the abort command */ cc = cr->cr_cc; cc->header.address.mode.mode = CISS_HDR_ADDRESS_MODE_PERIPHERAL; /* addressing? */ cc->header.address.physical.target = 0; cc->header.address.physical.bus = 0; cc->cdb.cdb_length = sizeof(*cmc); cc->cdb.type = CISS_CDB_TYPE_MESSAGE; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_NONE; cc->cdb.timeout = 30; cmc = (struct ciss_message_cdb *)&(cc->cdb.cdb[0]); cmc->opcode = CISS_OPCODE_MESSAGE_ABORT; cmc->type = CISS_MESSAGE_ABORT_TASK; cmc->abort_tag = ar->cr_tag; /* endianness?? */ /* * Send the request and wait for a response. If we believe we * aborted the request OK, clear the flag that indicates it's * running. */ error = ciss_synch_request(cr, 35 * 1000); if (!error) error = ciss_report_request(cr, NULL, NULL); ciss_release_request(cr); return(error); } #endif /************************************************************************ * Fetch and initialise a request */ static int ciss_get_request(struct ciss_softc *sc, struct ciss_request **crp) { struct ciss_request *cr; debug_called(2); /* * Get a request and clean it up. */ if ((cr = ciss_dequeue_free(sc)) == NULL) return(ENOMEM); cr->cr_data = NULL; cr->cr_flags = 0; cr->cr_complete = NULL; cr->cr_private = NULL; cr->cr_sg_tag = CISS_SG_MAX; /* Backstop to prevent accidents */ ciss_preen_command(cr); *crp = cr; return(0); } static void ciss_preen_command(struct ciss_request *cr) { struct ciss_command *cc; u_int32_t cmdphys; /* * Clean up the command structure. * * Note that we set up the error_info structure here, since the * length can be overwritten by any command. */ cc = cr->cr_cc; cc->header.sg_in_list = 0; /* kinda inefficient this way */ cc->header.sg_total = 0; cc->header.host_tag = cr->cr_tag << 2; cc->header.host_tag_zeroes = 0; bzero(&(cc->sg[0]), CISS_COMMAND_ALLOC_SIZE - sizeof(struct ciss_command)); cmdphys = cr->cr_ccphys; cc->error_info.error_info_address = cmdphys + sizeof(struct ciss_command); cc->error_info.error_info_length = CISS_COMMAND_ALLOC_SIZE - sizeof(struct ciss_command); } /************************************************************************ * Release a request to the free list. */ static void ciss_release_request(struct ciss_request *cr) { struct ciss_softc *sc; debug_called(2); sc = cr->cr_sc; /* release the request to the free queue */ ciss_requeue_free(cr); } /************************************************************************ * Allocate a request that will be used to send a BMIC command. Do some * of the common setup here to avoid duplicating it everywhere else. */ static int ciss_get_bmic_request(struct ciss_softc *sc, struct ciss_request **crp, int opcode, void **bufp, size_t bufsize) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_bmic_cdb *cbc; void *buf; int error; int dataout; debug_called(2); cr = NULL; buf = NULL; /* * Get a request. */ if ((error = ciss_get_request(sc, &cr)) != 0) goto out; /* * Allocate data storage if requested, determine the data direction. */ dataout = 0; if ((bufsize > 0) && (bufp != NULL)) { if (*bufp == NULL) { if ((buf = malloc(bufsize, CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO)) == NULL) { error = ENOMEM; goto out; } } else { buf = *bufp; dataout = 1; /* we are given a buffer, so we are writing */ } } /* * Build a CISS BMIC command to get the logical drive ID. */ cr->cr_data = buf; cr->cr_length = bufsize; if (!dataout) cr->cr_flags = CISS_REQ_DATAIN; cc = cr->cr_cc; cc->header.address.physical.mode = CISS_HDR_ADDRESS_MODE_PERIPHERAL; cc->header.address.physical.bus = 0; cc->header.address.physical.target = 0; cc->cdb.cdb_length = sizeof(*cbc); cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = dataout ? CISS_CDB_DIRECTION_WRITE : CISS_CDB_DIRECTION_READ; cc->cdb.timeout = 0; cbc = (struct ciss_bmic_cdb *)&(cc->cdb.cdb[0]); bzero(cbc, sizeof(*cbc)); cbc->opcode = dataout ? CISS_ARRAY_CONTROLLER_WRITE : CISS_ARRAY_CONTROLLER_READ; cbc->bmic_opcode = opcode; cbc->size = htons((u_int16_t)bufsize); out: if (error) { if (cr != NULL) ciss_release_request(cr); } else { *crp = cr; if ((bufp != NULL) && (*bufp == NULL) && (buf != NULL)) *bufp = buf; } return(error); } /************************************************************************ * Handle a command passed in from userspace. */ static int ciss_user_command(struct ciss_softc *sc, IOCTL_Command_struct *ioc) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_error_info *ce; int error = 0; debug_called(1); cr = NULL; /* * Get a request. */ while (ciss_get_request(sc, &cr) != 0) msleep(sc, &sc->ciss_mtx, PPAUSE, "cissREQ", hz); cc = cr->cr_cc; /* * Allocate an in-kernel databuffer if required, copy in user data. */ mtx_unlock(&sc->ciss_mtx); cr->cr_length = ioc->buf_size; if (ioc->buf_size > 0) { if ((cr->cr_data = malloc(ioc->buf_size, CISS_MALLOC_CLASS, M_NOWAIT)) == NULL) { error = ENOMEM; goto out_unlocked; } if ((error = copyin(ioc->buf, cr->cr_data, ioc->buf_size))) { debug(0, "copyin: bad data buffer %p/%d", ioc->buf, ioc->buf_size); goto out_unlocked; } } /* * Build the request based on the user command. */ bcopy(&ioc->LUN_info, &cc->header.address, sizeof(cc->header.address)); bcopy(&ioc->Request, &cc->cdb, sizeof(cc->cdb)); /* XXX anything else to populate here? */ mtx_lock(&sc->ciss_mtx); /* * Run the command. */ if ((error = ciss_synch_request(cr, 60 * 1000))) { debug(0, "request failed - %d", error); goto out; } /* * Check to see if the command succeeded. */ ce = (struct ciss_error_info *)&(cc->sg[0]); if ((cc->header.host_tag & CISS_HDR_HOST_TAG_ERROR) == 0) bzero(ce, sizeof(*ce)); /* * Copy the results back to the user. */ bcopy(ce, &ioc->error_info, sizeof(*ce)); mtx_unlock(&sc->ciss_mtx); if ((ioc->buf_size > 0) && (error = copyout(cr->cr_data, ioc->buf, ioc->buf_size))) { debug(0, "copyout: bad data buffer %p/%d", ioc->buf, ioc->buf_size); goto out_unlocked; } /* done OK */ error = 0; out_unlocked: mtx_lock(&sc->ciss_mtx); out: if ((cr != NULL) && (cr->cr_data != NULL)) free(cr->cr_data, CISS_MALLOC_CLASS); if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Map a request into bus-visible space, initialise the scatter/gather * list. */ static int ciss_map_request(struct ciss_request *cr) { struct ciss_softc *sc; int error = 0; debug_called(2); sc = cr->cr_sc; /* check that mapping is necessary */ if (cr->cr_flags & CISS_REQ_MAPPED) return(0); cr->cr_flags |= CISS_REQ_MAPPED; bus_dmamap_sync(sc->ciss_command_dmat, sc->ciss_command_map, BUS_DMASYNC_PREWRITE); if (cr->cr_data != NULL) { if (cr->cr_flags & CISS_REQ_CCB) error = bus_dmamap_load_ccb(sc->ciss_buffer_dmat, cr->cr_datamap, cr->cr_data, ciss_request_map_helper, cr, 0); else error = bus_dmamap_load(sc->ciss_buffer_dmat, cr->cr_datamap, cr->cr_data, cr->cr_length, ciss_request_map_helper, cr, 0); if (error != 0) return (error); } else { /* * Post the command to the adapter. */ cr->cr_sg_tag = CISS_SG_NONE; cr->cr_flags |= CISS_REQ_BUSY; if (sc->ciss_perf) CISS_TL_PERF_POST_CMD(sc, cr); else CISS_TL_SIMPLE_POST_CMD(sc, cr->cr_ccphys); } return(0); } static void ciss_request_map_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error) { struct ciss_command *cc; struct ciss_request *cr; struct ciss_softc *sc; int i; debug_called(2); cr = (struct ciss_request *)arg; sc = cr->cr_sc; cc = cr->cr_cc; for (i = 0; i < nseg; i++) { cc->sg[i].address = segs[i].ds_addr; cc->sg[i].length = segs[i].ds_len; cc->sg[i].extension = 0; } /* we leave the s/g table entirely within the command */ cc->header.sg_in_list = nseg; cc->header.sg_total = nseg; if (cr->cr_flags & CISS_REQ_DATAIN) bus_dmamap_sync(sc->ciss_buffer_dmat, cr->cr_datamap, BUS_DMASYNC_PREREAD); if (cr->cr_flags & CISS_REQ_DATAOUT) bus_dmamap_sync(sc->ciss_buffer_dmat, cr->cr_datamap, BUS_DMASYNC_PREWRITE); if (nseg == 0) cr->cr_sg_tag = CISS_SG_NONE; else if (nseg == 1) cr->cr_sg_tag = CISS_SG_1; else if (nseg == 2) cr->cr_sg_tag = CISS_SG_2; else if (nseg <= 4) cr->cr_sg_tag = CISS_SG_4; else if (nseg <= 8) cr->cr_sg_tag = CISS_SG_8; else if (nseg <= 16) cr->cr_sg_tag = CISS_SG_16; else if (nseg <= 32) cr->cr_sg_tag = CISS_SG_32; else cr->cr_sg_tag = CISS_SG_MAX; /* * Post the command to the adapter. */ cr->cr_flags |= CISS_REQ_BUSY; if (sc->ciss_perf) CISS_TL_PERF_POST_CMD(sc, cr); else CISS_TL_SIMPLE_POST_CMD(sc, cr->cr_ccphys); } /************************************************************************ * Unmap a request from bus-visible space. */ static void ciss_unmap_request(struct ciss_request *cr) { struct ciss_softc *sc; debug_called(2); sc = cr->cr_sc; /* check that unmapping is necessary */ if ((cr->cr_flags & CISS_REQ_MAPPED) == 0) return; bus_dmamap_sync(sc->ciss_command_dmat, sc->ciss_command_map, BUS_DMASYNC_POSTWRITE); if (cr->cr_data == NULL) goto out; if (cr->cr_flags & CISS_REQ_DATAIN) bus_dmamap_sync(sc->ciss_buffer_dmat, cr->cr_datamap, BUS_DMASYNC_POSTREAD); if (cr->cr_flags & CISS_REQ_DATAOUT) bus_dmamap_sync(sc->ciss_buffer_dmat, cr->cr_datamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->ciss_buffer_dmat, cr->cr_datamap); out: cr->cr_flags &= ~CISS_REQ_MAPPED; } /************************************************************************ * Attach the driver to CAM. * * We put all the logical drives on a single SCSI bus. */ static int ciss_cam_init(struct ciss_softc *sc) { int i, maxbus; debug_called(1); /* * Allocate a devq. We can reuse this for the masked physical * devices if we decide to export these as well. */ if ((sc->ciss_cam_devq = cam_simq_alloc(sc->ciss_max_requests - 2)) == NULL) { ciss_printf(sc, "can't allocate CAM SIM queue\n"); return(ENOMEM); } /* * Create a SIM. * * This naturally wastes a bit of memory. The alternative is to allocate * and register each bus as it is found, and then track them on a linked * list. Unfortunately, the driver has a few places where it needs to * look up the SIM based solely on bus number, and it's unclear whether * a list traversal would work for these situations. */ maxbus = max(sc->ciss_max_logical_bus, sc->ciss_max_physical_bus + CISS_PHYSICAL_BASE); sc->ciss_cam_sim = malloc(maxbus * sizeof(struct cam_sim*), CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO); if (sc->ciss_cam_sim == NULL) { ciss_printf(sc, "can't allocate memory for controller SIM\n"); return(ENOMEM); } for (i = 0; i < sc->ciss_max_logical_bus; i++) { if ((sc->ciss_cam_sim[i] = cam_sim_alloc(ciss_cam_action, ciss_cam_poll, "ciss", sc, device_get_unit(sc->ciss_dev), &sc->ciss_mtx, 2, sc->ciss_max_requests - 2, sc->ciss_cam_devq)) == NULL) { ciss_printf(sc, "can't allocate CAM SIM for controller %d\n", i); return(ENOMEM); } /* * Register bus with this SIM. */ mtx_lock(&sc->ciss_mtx); if (i == 0 || sc->ciss_controllers[i].physical.bus != 0) { if (xpt_bus_register(sc->ciss_cam_sim[i], sc->ciss_dev, i) != 0) { ciss_printf(sc, "can't register SCSI bus %d\n", i); mtx_unlock(&sc->ciss_mtx); return (ENXIO); } } mtx_unlock(&sc->ciss_mtx); } for (i = CISS_PHYSICAL_BASE; i < sc->ciss_max_physical_bus + CISS_PHYSICAL_BASE; i++) { if ((sc->ciss_cam_sim[i] = cam_sim_alloc(ciss_cam_action, ciss_cam_poll, "ciss", sc, device_get_unit(sc->ciss_dev), &sc->ciss_mtx, 1, sc->ciss_max_requests - 2, sc->ciss_cam_devq)) == NULL) { ciss_printf(sc, "can't allocate CAM SIM for controller %d\n", i); return (ENOMEM); } mtx_lock(&sc->ciss_mtx); if (xpt_bus_register(sc->ciss_cam_sim[i], sc->ciss_dev, i) != 0) { ciss_printf(sc, "can't register SCSI bus %d\n", i); mtx_unlock(&sc->ciss_mtx); return (ENXIO); } mtx_unlock(&sc->ciss_mtx); } return(0); } /************************************************************************ * Initiate a rescan of the 'logical devices' SIM */ static void ciss_cam_rescan_target(struct ciss_softc *sc, int bus, int target) { union ccb *ccb; debug_called(1); if ((ccb = xpt_alloc_ccb_nowait()) == NULL) { ciss_printf(sc, "rescan failed (can't allocate CCB)\n"); return; } if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->ciss_cam_sim[bus]), target, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { ciss_printf(sc, "rescan failed (can't create path)\n"); xpt_free_ccb(ccb); return; } xpt_rescan(ccb); /* scan is now in progress */ } /************************************************************************ * Handle requests coming from CAM */ static void ciss_cam_action(struct cam_sim *sim, union ccb *ccb) { struct ciss_softc *sc; struct ccb_scsiio *csio; int bus, target; int physical; sc = cam_sim_softc(sim); bus = cam_sim_bus(sim); csio = (struct ccb_scsiio *)&ccb->csio; target = csio->ccb_h.target_id; physical = CISS_IS_PHYSICAL(bus); switch (ccb->ccb_h.func_code) { /* perform SCSI I/O */ case XPT_SCSI_IO: if (!ciss_cam_action_io(sim, csio)) return; break; /* perform geometry calculations */ case XPT_CALC_GEOMETRY: { struct ccb_calc_geometry *ccg = &ccb->ccg; struct ciss_ldrive *ld; debug(1, "XPT_CALC_GEOMETRY %d:%d:%d", cam_sim_bus(sim), ccb->ccb_h.target_id, ccb->ccb_h.target_lun); ld = NULL; if (!physical) ld = &sc->ciss_logical[bus][target]; /* * Use the cached geometry settings unless the fault tolerance * is invalid. */ if (physical || ld->cl_geometry.fault_tolerance == 0xFF) { u_int32_t secs_per_cylinder; ccg->heads = 255; ccg->secs_per_track = 32; secs_per_cylinder = ccg->heads * ccg->secs_per_track; ccg->cylinders = ccg->volume_size / secs_per_cylinder; } else { ccg->heads = ld->cl_geometry.heads; ccg->secs_per_track = ld->cl_geometry.sectors; ccg->cylinders = ntohs(ld->cl_geometry.cylinders); } ccb->ccb_h.status = CAM_REQ_CMP; break; } /* handle path attribute inquiry */ case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; int sg_length; debug(1, "XPT_PATH_INQ %d:%d:%d", cam_sim_bus(sim), ccb->ccb_h.target_id, ccb->ccb_h.target_lun); cpi->version_num = 1; cpi->hba_inquiry = PI_TAG_ABLE; /* XXX is this correct? */ cpi->target_sprt = 0; cpi->hba_misc = 0; cpi->max_target = sc->ciss_cfg->max_logical_supported; cpi->max_lun = 0; /* 'logical drive' channel only */ cpi->initiator_id = sc->ciss_cfg->max_logical_supported; strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strncpy(cpi->hba_vid, "msmith@freebsd.org", HBA_IDLEN); strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->base_transfer_speed = 132 * 1024; /* XXX what to set this to? */ cpi->transport = XPORT_SPI; cpi->transport_version = 2; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_2; if (sc->ciss_cfg->max_sg_length == 0) { sg_length = 17; } else { /* XXX Fix for ZMR cards that advertise max_sg_length == 32 * Confusing bit here. max_sg_length is usually a power of 2. We always * need to subtract 1 to account for partial pages. Then we need to * align on a valid PAGE_SIZE so we round down to the nearest power of 2. * Add 1 so we can then subtract it out in the assignment to maxio. * The reason for all these shenanigans is to create a maxio value that * creates IO operations to volumes that yield consistent operations * with good performance. */ sg_length = sc->ciss_cfg->max_sg_length - 1; sg_length = (1 << (fls(sg_length) - 1)) + 1; } cpi->maxio = (min(CISS_MAX_SG_ELEMENTS, sg_length) - 1) * PAGE_SIZE; ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts = &ccb->cts; int bus, target; struct ccb_trans_settings_spi *spi = &cts->xport_specific.spi; struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi; bus = cam_sim_bus(sim); target = cts->ccb_h.target_id; debug(1, "XPT_GET_TRAN_SETTINGS %d:%d", bus, target); /* disconnect always OK */ cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_2; cts->transport = XPORT_SPI; cts->transport_version = 2; spi->valid = CTS_SPI_VALID_DISC; spi->flags = CTS_SPI_FLAGS_DISC_ENB; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; cts->ccb_h.status = CAM_REQ_CMP; break; } default: /* we can't do this */ debug(1, "unspported func_code = 0x%x", ccb->ccb_h.func_code); ccb->ccb_h.status = CAM_REQ_INVALID; break; } xpt_done(ccb); } /************************************************************************ * Handle a CAM SCSI I/O request. */ static int ciss_cam_action_io(struct cam_sim *sim, struct ccb_scsiio *csio) { struct ciss_softc *sc; int bus, target; struct ciss_request *cr; struct ciss_command *cc; int error; sc = cam_sim_softc(sim); bus = cam_sim_bus(sim); target = csio->ccb_h.target_id; debug(2, "XPT_SCSI_IO %d:%d:%d", bus, target, csio->ccb_h.target_lun); /* check that the CDB pointer is not to a physical address */ if ((csio->ccb_h.flags & CAM_CDB_POINTER) && (csio->ccb_h.flags & CAM_CDB_PHYS)) { debug(3, " CDB pointer is to physical address"); csio->ccb_h.status = CAM_REQ_CMP_ERR; } /* abandon aborted ccbs or those that have failed validation */ if ((csio->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG) { debug(3, "abandoning CCB due to abort/validation failure"); return(EINVAL); } /* handle emulation of some SCSI commands ourself */ if (ciss_cam_emulate(sc, csio)) return(0); /* * Get a request to manage this command. If we can't, return the * ccb, freeze the queue and flag so that we unfreeze it when a * request completes. */ if ((error = ciss_get_request(sc, &cr)) != 0) { xpt_freeze_simq(sim, 1); sc->ciss_flags |= CISS_FLAG_BUSY; csio->ccb_h.status |= CAM_REQUEUE_REQ; return(error); } /* * Build the command. */ cc = cr->cr_cc; cr->cr_data = csio; cr->cr_length = csio->dxfer_len; cr->cr_complete = ciss_cam_complete; cr->cr_private = csio; /* * Target the right logical volume. */ if (CISS_IS_PHYSICAL(bus)) cc->header.address = sc->ciss_physical[CISS_CAM_TO_PBUS(bus)][target].cp_address; else cc->header.address = sc->ciss_logical[bus][target].cl_address; cc->cdb.cdb_length = csio->cdb_len; cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; /* XXX ordered tags? */ if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { cr->cr_flags = CISS_REQ_DATAOUT | CISS_REQ_CCB; cc->cdb.direction = CISS_CDB_DIRECTION_WRITE; } else if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { cr->cr_flags = CISS_REQ_DATAIN | CISS_REQ_CCB; cc->cdb.direction = CISS_CDB_DIRECTION_READ; } else { cr->cr_data = NULL; cr->cr_flags = 0; cc->cdb.direction = CISS_CDB_DIRECTION_NONE; } cc->cdb.timeout = (csio->ccb_h.timeout / 1000) + 1; if (csio->ccb_h.flags & CAM_CDB_POINTER) { bcopy(csio->cdb_io.cdb_ptr, &cc->cdb.cdb[0], csio->cdb_len); } else { bcopy(csio->cdb_io.cdb_bytes, &cc->cdb.cdb[0], csio->cdb_len); } /* * Submit the request to the adapter. * * Note that this may fail if we're unable to map the request (and * if we ever learn a transport layer other than simple, may fail * if the adapter rejects the command). */ if ((error = ciss_start(cr)) != 0) { xpt_freeze_simq(sim, 1); csio->ccb_h.status |= CAM_RELEASE_SIMQ; if (error == EINPROGRESS) { error = 0; } else { csio->ccb_h.status |= CAM_REQUEUE_REQ; ciss_release_request(cr); } return(error); } return(0); } /************************************************************************ * Emulate SCSI commands the adapter doesn't handle as we might like. */ static int ciss_cam_emulate(struct ciss_softc *sc, struct ccb_scsiio *csio) { int bus, target; u_int8_t opcode; target = csio->ccb_h.target_id; bus = cam_sim_bus(xpt_path_sim(csio->ccb_h.path)); opcode = (csio->ccb_h.flags & CAM_CDB_POINTER) ? *(u_int8_t *)csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes[0]; if (CISS_IS_PHYSICAL(bus)) { if (sc->ciss_physical[CISS_CAM_TO_PBUS(bus)][target].cp_online != 1) { csio->ccb_h.status |= CAM_SEL_TIMEOUT; xpt_done((union ccb *)csio); return(1); } else return(0); } /* * Handle requests for volumes that don't exist or are not online. * A selection timeout is slightly better than an illegal request. * Other errors might be better. */ if (sc->ciss_logical[bus][target].cl_status != CISS_LD_ONLINE) { csio->ccb_h.status |= CAM_SEL_TIMEOUT; xpt_done((union ccb *)csio); return(1); } /* if we have to fake Synchronise Cache */ if (sc->ciss_flags & CISS_FLAG_FAKE_SYNCH) { /* * If this is a Synchronise Cache command, typically issued when * a device is closed, flush the adapter and complete now. */ if (((csio->ccb_h.flags & CAM_CDB_POINTER) ? *(u_int8_t *)csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes[0]) == SYNCHRONIZE_CACHE) { ciss_flush_adapter(sc); csio->ccb_h.status |= CAM_REQ_CMP; xpt_done((union ccb *)csio); return(1); } } /* * A CISS target can only ever have one lun per target. REPORT_LUNS requires * at least one LUN field to be pre created for us, so snag it and fill in * the least significant byte indicating 1 LUN here. Emulate the command * return to shut up warning on console of a CDB error. swb */ if (opcode == REPORT_LUNS && csio->dxfer_len > 0) { csio->data_ptr[3] = 8; csio->ccb_h.status |= CAM_REQ_CMP; xpt_done((union ccb *)csio); return(1); } return(0); } /************************************************************************ * Check for possibly-completed commands. */ static void ciss_cam_poll(struct cam_sim *sim) { cr_qhead_t qh; struct ciss_softc *sc = cam_sim_softc(sim); debug_called(2); STAILQ_INIT(&qh); if (sc->ciss_perf) ciss_perf_done(sc, &qh); else ciss_done(sc, &qh); ciss_complete(sc, &qh); } /************************************************************************ * Handle completion of a command - pass results back through the CCB */ static void ciss_cam_complete(struct ciss_request *cr) { struct ciss_softc *sc; struct ciss_command *cc; struct ciss_error_info *ce; struct ccb_scsiio *csio; int scsi_status; int command_status; debug_called(2); sc = cr->cr_sc; cc = cr->cr_cc; ce = (struct ciss_error_info *)&(cc->sg[0]); csio = (struct ccb_scsiio *)cr->cr_private; /* * Extract status values from request. */ ciss_report_request(cr, &command_status, &scsi_status); csio->scsi_status = scsi_status; /* * Handle specific SCSI status values. */ switch(scsi_status) { /* no status due to adapter error */ case -1: debug(0, "adapter error"); csio->ccb_h.status |= CAM_REQ_CMP_ERR; break; /* no status due to command completed OK */ case SCSI_STATUS_OK: /* CISS_SCSI_STATUS_GOOD */ debug(2, "SCSI_STATUS_OK"); csio->ccb_h.status |= CAM_REQ_CMP; break; /* check condition, sense data included */ case SCSI_STATUS_CHECK_COND: /* CISS_SCSI_STATUS_CHECK_CONDITION */ debug(0, "SCSI_STATUS_CHECK_COND sense size %d resid %d\n", ce->sense_length, ce->residual_count); bzero(&csio->sense_data, SSD_FULL_SIZE); bcopy(&ce->sense_info[0], &csio->sense_data, ce->sense_length); if (csio->sense_len > ce->sense_length) csio->sense_resid = csio->sense_len - ce->sense_length; else csio->sense_resid = 0; csio->resid = ce->residual_count; csio->ccb_h.status |= CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID; #ifdef CISS_DEBUG { struct scsi_sense_data *sns = (struct scsi_sense_data *)&ce->sense_info[0]; debug(0, "sense key %x", scsi_get_sense_key(sns, csio->sense_len - csio->sense_resid, /*show_errors*/ 1)); } #endif break; case SCSI_STATUS_BUSY: /* CISS_SCSI_STATUS_BUSY */ debug(0, "SCSI_STATUS_BUSY"); csio->ccb_h.status |= CAM_SCSI_BUSY; break; default: debug(0, "unknown status 0x%x", csio->scsi_status); csio->ccb_h.status |= CAM_REQ_CMP_ERR; break; } /* handle post-command fixup */ ciss_cam_complete_fixup(sc, csio); ciss_release_request(cr); if (sc->ciss_flags & CISS_FLAG_BUSY) { sc->ciss_flags &= ~CISS_FLAG_BUSY; if (csio->ccb_h.status & CAM_RELEASE_SIMQ) xpt_release_simq(xpt_path_sim(csio->ccb_h.path), 0); else csio->ccb_h.status |= CAM_RELEASE_SIMQ; } xpt_done((union ccb *)csio); } /******************************************************************************** * Fix up the result of some commands here. */ static void ciss_cam_complete_fixup(struct ciss_softc *sc, struct ccb_scsiio *csio) { struct scsi_inquiry_data *inq; struct ciss_ldrive *cl; uint8_t *cdb; int bus, target; cdb = (csio->ccb_h.flags & CAM_CDB_POINTER) ? (uint8_t *)csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes; if (cdb[0] == INQUIRY && (cdb[1] & SI_EVPD) == 0 && (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN && csio->dxfer_len >= SHORT_INQUIRY_LENGTH) { inq = (struct scsi_inquiry_data *)csio->data_ptr; target = csio->ccb_h.target_id; bus = cam_sim_bus(xpt_path_sim(csio->ccb_h.path)); /* * If the controller is in JBOD mode, there are no logical volumes. * Let the disks be probed and dealt with via CAM. Else, mask off * the physical disks and setup the parts of the inq structure for * the logical volume. swb */ if( !(sc->ciss_id->PowerUPNvramFlags & PWR_UP_FLAG_JBOD_ENABLED)){ if (CISS_IS_PHYSICAL(bus)) { if (SID_TYPE(inq) == T_DIRECT) inq->device = (inq->device & 0xe0) | T_NODEVICE; return; } cl = &sc->ciss_logical[bus][target]; padstr(inq->vendor, "HP", SID_VENDOR_SIZE); padstr(inq->product, ciss_name_ldrive_org(cl->cl_ldrive->fault_tolerance), SID_PRODUCT_SIZE); padstr(inq->revision, ciss_name_ldrive_status(cl->cl_lstatus->status), SID_REVISION_SIZE); } } } /******************************************************************************** * Name the device at (target) * * XXX is this strictly correct? */ static int ciss_name_device(struct ciss_softc *sc, int bus, int target) { struct cam_periph *periph; struct cam_path *path; int status; if (CISS_IS_PHYSICAL(bus)) return (0); status = xpt_create_path(&path, NULL, cam_sim_path(sc->ciss_cam_sim[bus]), target, 0); if (status == CAM_REQ_CMP) { xpt_path_lock(path); periph = cam_periph_find(path, NULL); xpt_path_unlock(path); xpt_free_path(path); if (periph != NULL) { sprintf(sc->ciss_logical[bus][target].cl_name, "%s%d", periph->periph_name, periph->unit_number); return(0); } } sc->ciss_logical[bus][target].cl_name[0] = 0; return(ENOENT); } /************************************************************************ * Periodic status monitoring. */ static void ciss_periodic(void *arg) { struct ciss_softc *sc; struct ciss_request *cr = NULL; struct ciss_command *cc = NULL; int error = 0; debug_called(1); sc = (struct ciss_softc *)arg; /* * Check the adapter heartbeat. */ if (sc->ciss_cfg->heartbeat == sc->ciss_heartbeat) { sc->ciss_heart_attack++; debug(0, "adapter heart attack in progress 0x%x/%d", sc->ciss_heartbeat, sc->ciss_heart_attack); if (sc->ciss_heart_attack == 3) { ciss_printf(sc, "ADAPTER HEARTBEAT FAILED\n"); ciss_disable_adapter(sc); return; } } else { sc->ciss_heartbeat = sc->ciss_cfg->heartbeat; sc->ciss_heart_attack = 0; debug(3, "new heartbeat 0x%x", sc->ciss_heartbeat); } /* * Send the NOP message and wait for a response. */ if (ciss_nop_message_heartbeat != 0 && (error = ciss_get_request(sc, &cr)) == 0) { cc = cr->cr_cc; cr->cr_complete = ciss_nop_complete; cc->cdb.cdb_length = 1; cc->cdb.type = CISS_CDB_TYPE_MESSAGE; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_WRITE; cc->cdb.timeout = 0; cc->cdb.cdb[0] = CISS_OPCODE_MESSAGE_NOP; if ((error = ciss_start(cr)) != 0) { ciss_printf(sc, "SENDING NOP MESSAGE FAILED\n"); } } /* * If the notify event request has died for some reason, or has * not started yet, restart it. */ if (!(sc->ciss_flags & CISS_FLAG_NOTIFY_OK)) { debug(0, "(re)starting Event Notify chain"); ciss_notify_event(sc); } /* * Reschedule. */ callout_reset(&sc->ciss_periodic, CISS_HEARTBEAT_RATE * hz, ciss_periodic, sc); } static void ciss_nop_complete(struct ciss_request *cr) { struct ciss_softc *sc; static int first_time = 1; sc = cr->cr_sc; if (ciss_report_request(cr, NULL, NULL) != 0) { if (first_time == 1) { first_time = 0; ciss_printf(sc, "SENDING NOP MESSAGE FAILED (not logging anymore)\n"); } } ciss_release_request(cr); } /************************************************************************ * Disable the adapter. * * The all requests in completed queue is failed with hardware error. * This will cause failover in a multipath configuration. */ static void ciss_disable_adapter(struct ciss_softc *sc) { cr_qhead_t qh; struct ciss_request *cr; struct ciss_command *cc; struct ciss_error_info *ce; int i; CISS_TL_SIMPLE_DISABLE_INTERRUPTS(sc); pci_disable_busmaster(sc->ciss_dev); sc->ciss_flags &= ~CISS_FLAG_RUNNING; for (i = 1; i < sc->ciss_max_requests; i++) { cr = &sc->ciss_request[i]; if ((cr->cr_flags & CISS_REQ_BUSY) == 0) continue; cc = cr->cr_cc; ce = (struct ciss_error_info *)&(cc->sg[0]); ce->command_status = CISS_CMD_STATUS_HARDWARE_ERROR; ciss_enqueue_complete(cr, &qh); } for (;;) { if ((cr = ciss_dequeue_complete(sc, &qh)) == NULL) break; /* * If the request has a callback, invoke it. */ if (cr->cr_complete != NULL) { cr->cr_complete(cr); continue; } /* * If someone is sleeping on this request, wake them up. */ if (cr->cr_flags & CISS_REQ_SLEEP) { cr->cr_flags &= ~CISS_REQ_SLEEP; wakeup(cr); continue; } } } /************************************************************************ * Request a notification response from the adapter. * * If (cr) is NULL, this is the first request of the adapter, so * reset the adapter's message pointer and start with the oldest * message available. */ static void ciss_notify_event(struct ciss_softc *sc) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_notify_cdb *cnc; int error; debug_called(1); cr = sc->ciss_periodic_notify; /* get a request if we don't already have one */ if (cr == NULL) { if ((error = ciss_get_request(sc, &cr)) != 0) { debug(0, "can't get notify event request"); goto out; } sc->ciss_periodic_notify = cr; cr->cr_complete = ciss_notify_complete; debug(1, "acquired request %d", cr->cr_tag); } /* * Get a databuffer if we don't already have one, note that the * adapter command wants a larger buffer than the actual * structure. */ if (cr->cr_data == NULL) { if ((cr->cr_data = malloc(CISS_NOTIFY_DATA_SIZE, CISS_MALLOC_CLASS, M_NOWAIT)) == NULL) { debug(0, "can't get notify event request buffer"); error = ENOMEM; goto out; } cr->cr_length = CISS_NOTIFY_DATA_SIZE; } /* re-setup the request's command (since we never release it) XXX overkill*/ ciss_preen_command(cr); /* (re)build the notify event command */ cc = cr->cr_cc; cc->header.address.physical.mode = CISS_HDR_ADDRESS_MODE_PERIPHERAL; cc->header.address.physical.bus = 0; cc->header.address.physical.target = 0; cc->cdb.cdb_length = sizeof(*cnc); cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_READ; cc->cdb.timeout = 0; /* no timeout, we hope */ cnc = (struct ciss_notify_cdb *)&(cc->cdb.cdb[0]); bzero(cr->cr_data, CISS_NOTIFY_DATA_SIZE); cnc->opcode = CISS_OPCODE_READ; cnc->command = CISS_COMMAND_NOTIFY_ON_EVENT; cnc->timeout = 0; /* no timeout, we hope */ cnc->synchronous = 0; cnc->ordered = 0; cnc->seek_to_oldest = 0; if ((sc->ciss_flags & CISS_FLAG_RUNNING) == 0) cnc->new_only = 1; else cnc->new_only = 0; cnc->length = htonl(CISS_NOTIFY_DATA_SIZE); /* submit the request */ error = ciss_start(cr); out: if (error) { if (cr != NULL) { if (cr->cr_data != NULL) free(cr->cr_data, CISS_MALLOC_CLASS); ciss_release_request(cr); } sc->ciss_periodic_notify = NULL; debug(0, "can't submit notify event request"); sc->ciss_flags &= ~CISS_FLAG_NOTIFY_OK; } else { debug(1, "notify event submitted"); sc->ciss_flags |= CISS_FLAG_NOTIFY_OK; } } static void ciss_notify_complete(struct ciss_request *cr) { struct ciss_command *cc; struct ciss_notify *cn; struct ciss_softc *sc; int scsi_status; int command_status; debug_called(1); cc = cr->cr_cc; cn = (struct ciss_notify *)cr->cr_data; sc = cr->cr_sc; /* * Report request results, decode status. */ ciss_report_request(cr, &command_status, &scsi_status); /* * Abort the chain on a fatal error. * * XXX which of these are actually errors? */ if ((command_status != CISS_CMD_STATUS_SUCCESS) && (command_status != CISS_CMD_STATUS_TARGET_STATUS) && (command_status != CISS_CMD_STATUS_TIMEOUT)) { /* XXX timeout? */ ciss_printf(sc, "fatal error in Notify Event request (%s)\n", ciss_name_command_status(command_status)); ciss_release_request(cr); sc->ciss_flags &= ~CISS_FLAG_NOTIFY_OK; return; } /* * If the adapter gave us a text message, print it. */ if (cn->message[0] != 0) ciss_printf(sc, "*** %.80s\n", cn->message); debug(0, "notify event class %d subclass %d detail %d", cn->class, cn->subclass, cn->detail); /* * If the response indicates that the notifier has been aborted, * release the notifier command. */ if ((cn->class == CISS_NOTIFY_NOTIFIER) && (cn->subclass == CISS_NOTIFY_NOTIFIER_STATUS) && (cn->detail == 1)) { debug(0, "notifier exiting"); sc->ciss_flags &= ~CISS_FLAG_NOTIFY_OK; ciss_release_request(cr); sc->ciss_periodic_notify = NULL; wakeup(&sc->ciss_periodic_notify); } else { /* Handle notify events in a kernel thread */ ciss_enqueue_notify(cr); sc->ciss_periodic_notify = NULL; wakeup(&sc->ciss_periodic_notify); wakeup(&sc->ciss_notify); } /* * Send a new notify event command, if we're not aborting. */ if (!(sc->ciss_flags & CISS_FLAG_ABORTING)) { ciss_notify_event(sc); } } /************************************************************************ * Abort the Notify Event chain. * * Note that we can't just abort the command in progress; we have to * explicitly issue an Abort Notify Event command in order for the * adapter to clean up correctly. * * If we are called with CISS_FLAG_ABORTING set in the adapter softc, * the chain will not restart itself. */ static int ciss_notify_abort(struct ciss_softc *sc) { struct ciss_request *cr; struct ciss_command *cc; struct ciss_notify_cdb *cnc; int error, command_status, scsi_status; debug_called(1); cr = NULL; error = 0; /* verify that there's an outstanding command */ if (!(sc->ciss_flags & CISS_FLAG_NOTIFY_OK)) goto out; /* get a command to issue the abort with */ if ((error = ciss_get_request(sc, &cr))) goto out; /* get a buffer for the result */ if ((cr->cr_data = malloc(CISS_NOTIFY_DATA_SIZE, CISS_MALLOC_CLASS, M_NOWAIT)) == NULL) { debug(0, "can't get notify event request buffer"); error = ENOMEM; goto out; } cr->cr_length = CISS_NOTIFY_DATA_SIZE; /* build the CDB */ cc = cr->cr_cc; cc->header.address.physical.mode = CISS_HDR_ADDRESS_MODE_PERIPHERAL; cc->header.address.physical.bus = 0; cc->header.address.physical.target = 0; cc->cdb.cdb_length = sizeof(*cnc); cc->cdb.type = CISS_CDB_TYPE_COMMAND; cc->cdb.attribute = CISS_CDB_ATTRIBUTE_SIMPLE; cc->cdb.direction = CISS_CDB_DIRECTION_READ; cc->cdb.timeout = 0; /* no timeout, we hope */ cnc = (struct ciss_notify_cdb *)&(cc->cdb.cdb[0]); bzero(cnc, sizeof(*cnc)); cnc->opcode = CISS_OPCODE_WRITE; cnc->command = CISS_COMMAND_ABORT_NOTIFY; cnc->length = htonl(CISS_NOTIFY_DATA_SIZE); ciss_print_request(cr); /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "Abort Notify Event command failed (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, &scsi_status); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: break; case CISS_CMD_STATUS_INVALID_COMMAND: /* * Some older adapters don't support the CISS version of this * command. Fall back to using the BMIC version. */ error = ciss_notify_abort_bmic(sc); if (error != 0) goto out; break; case CISS_CMD_STATUS_TARGET_STATUS: /* * This can happen if the adapter thinks there wasn't an outstanding * Notify Event command but we did. We clean up here. */ if (scsi_status == CISS_SCSI_STATUS_CHECK_CONDITION) { if (sc->ciss_periodic_notify != NULL) ciss_release_request(sc->ciss_periodic_notify); error = 0; goto out; } /* FALLTHROUGH */ default: ciss_printf(sc, "Abort Notify Event command failed (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } /* * Sleep waiting for the notifier command to complete. Note * that if it doesn't, we may end up in a bad situation, since * the adapter may deliver it later. Also note that the adapter * requires the Notify Event command to be cancelled in order to * maintain internal bookkeeping. */ while (sc->ciss_periodic_notify != NULL) { error = msleep(&sc->ciss_periodic_notify, &sc->ciss_mtx, PRIBIO, "cissNEA", hz * 5); if (error == EWOULDBLOCK) { ciss_printf(sc, "Notify Event command failed to abort, adapter may wedge.\n"); break; } } out: /* release the cancel request */ if (cr != NULL) { if (cr->cr_data != NULL) free(cr->cr_data, CISS_MALLOC_CLASS); ciss_release_request(cr); } if (error == 0) sc->ciss_flags &= ~CISS_FLAG_NOTIFY_OK; return(error); } /************************************************************************ * Abort the Notify Event chain using a BMIC command. */ static int ciss_notify_abort_bmic(struct ciss_softc *sc) { struct ciss_request *cr; int error, command_status; debug_called(1); cr = NULL; error = 0; /* verify that there's an outstanding command */ if (!(sc->ciss_flags & CISS_FLAG_NOTIFY_OK)) goto out; /* * Build a BMIC command to cancel the Notify on Event command. * * Note that we are sending a CISS opcode here. Odd. */ if ((error = ciss_get_bmic_request(sc, &cr, CISS_COMMAND_ABORT_NOTIFY, NULL, 0)) != 0) goto out; /* * Submit the request and wait for it to complete. */ if ((error = ciss_synch_request(cr, 60 * 1000)) != 0) { ciss_printf(sc, "error sending BMIC Cancel Notify on Event command (%d)\n", error); goto out; } /* * Check response. */ ciss_report_request(cr, &command_status, NULL); switch(command_status) { case CISS_CMD_STATUS_SUCCESS: break; default: ciss_printf(sc, "error cancelling Notify on Event (%s)\n", ciss_name_command_status(command_status)); error = EIO; goto out; } out: if (cr != NULL) ciss_release_request(cr); return(error); } /************************************************************************ * Handle rescanning all the logical volumes when a notify event * causes the drives to come online or offline. */ static void ciss_notify_rescan_logical(struct ciss_softc *sc) { struct ciss_lun_report *cll; struct ciss_ldrive *ld; int i, j, ndrives; /* * We must rescan all logical volumes to get the right logical * drive address. */ cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_LOGICAL_LUNS, sc->ciss_cfg->max_logical_supported); if (cll == NULL) return; ndrives = (ntohl(cll->list_size) / sizeof(union ciss_device_address)); /* * Delete any of the drives which were destroyed by the * firmware. */ for (i = 0; i < sc->ciss_max_logical_bus; i++) { for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) { ld = &sc->ciss_logical[i][j]; if (ld->cl_update == 0) continue; if (ld->cl_status != CISS_LD_ONLINE) { ciss_cam_rescan_target(sc, i, j); ld->cl_update = 0; if (ld->cl_ldrive) free(ld->cl_ldrive, CISS_MALLOC_CLASS); if (ld->cl_lstatus) free(ld->cl_lstatus, CISS_MALLOC_CLASS); ld->cl_ldrive = NULL; ld->cl_lstatus = NULL; } } } /* * Scan for new drives. */ for (i = 0; i < ndrives; i++) { int bus, target; bus = CISS_LUN_TO_BUS(cll->lun[i].logical.lun); target = CISS_LUN_TO_TARGET(cll->lun[i].logical.lun); ld = &sc->ciss_logical[bus][target]; if (ld->cl_update == 0) continue; ld->cl_update = 0; ld->cl_address = cll->lun[i]; ld->cl_controller = &sc->ciss_controllers[bus]; if (ciss_identify_logical(sc, ld) == 0) { ciss_cam_rescan_target(sc, bus, target); } } free(cll, CISS_MALLOC_CLASS); } /************************************************************************ * Handle a notify event relating to the status of a logical drive. * * XXX need to be able to defer some of these to properly handle * calling the "ID Physical drive" command, unless the 'extended' * drive IDs are always in BIG_MAP format. */ static void ciss_notify_logical(struct ciss_softc *sc, struct ciss_notify *cn) { struct ciss_ldrive *ld; int ostatus, bus, target; debug_called(2); bus = cn->device.physical.bus; target = cn->data.logical_status.logical_drive; ld = &sc->ciss_logical[bus][target]; switch (cn->subclass) { case CISS_NOTIFY_LOGICAL_STATUS: switch (cn->detail) { case 0: ciss_name_device(sc, bus, target); ciss_printf(sc, "logical drive %d (%s) changed status %s->%s, spare status 0x%b\n", cn->data.logical_status.logical_drive, ld->cl_name, ciss_name_ldrive_status(cn->data.logical_status.previous_state), ciss_name_ldrive_status(cn->data.logical_status.new_state), cn->data.logical_status.spare_state, "\20\1configured\2rebuilding\3failed\4in use\5available\n"); /* * Update our idea of the drive's status. */ ostatus = ciss_decode_ldrive_status(cn->data.logical_status.previous_state); ld->cl_status = ciss_decode_ldrive_status(cn->data.logical_status.new_state); if (ld->cl_lstatus != NULL) ld->cl_lstatus->status = cn->data.logical_status.new_state; /* * Have CAM rescan the drive if its status has changed. */ if (ostatus != ld->cl_status) { ld->cl_update = 1; ciss_notify_rescan_logical(sc); } break; case 1: /* logical drive has recognised new media, needs Accept Media Exchange */ ciss_name_device(sc, bus, target); ciss_printf(sc, "logical drive %d (%s) media exchanged, ready to go online\n", cn->data.logical_status.logical_drive, ld->cl_name); ciss_accept_media(sc, ld); ld->cl_update = 1; ld->cl_status = ciss_decode_ldrive_status(cn->data.logical_status.new_state); ciss_notify_rescan_logical(sc); break; case 2: case 3: ciss_printf(sc, "rebuild of logical drive %d (%s) failed due to %s error\n", cn->data.rebuild_aborted.logical_drive, ld->cl_name, (cn->detail == 2) ? "read" : "write"); break; } break; case CISS_NOTIFY_LOGICAL_ERROR: if (cn->detail == 0) { ciss_printf(sc, "FATAL I/O ERROR on logical drive %d (%s), SCSI port %d ID %d\n", cn->data.io_error.logical_drive, ld->cl_name, cn->data.io_error.failure_bus, cn->data.io_error.failure_drive); /* XXX should we take the drive down at this point, or will we be told? */ } break; case CISS_NOTIFY_LOGICAL_SURFACE: if (cn->detail == 0) ciss_printf(sc, "logical drive %d (%s) completed consistency initialisation\n", cn->data.consistency_completed.logical_drive, ld->cl_name); break; } } /************************************************************************ * Handle a notify event relating to the status of a physical drive. */ static void ciss_notify_physical(struct ciss_softc *sc, struct ciss_notify *cn) { } /************************************************************************ * Handle a notify event relating to the status of a physical drive. */ static void ciss_notify_hotplug(struct ciss_softc *sc, struct ciss_notify *cn) { struct ciss_lun_report *cll = NULL; int bus, target; switch (cn->subclass) { case CISS_NOTIFY_HOTPLUG_PHYSICAL: case CISS_NOTIFY_HOTPLUG_NONDISK: bus = CISS_BIG_MAP_BUS(sc, cn->data.drive.big_physical_drive_number); target = CISS_BIG_MAP_TARGET(sc, cn->data.drive.big_physical_drive_number); if (cn->detail == 0) { /* * Mark the device offline so that it'll start producing selection * timeouts to the upper layer. */ if ((bus >= 0) && (target >= 0)) sc->ciss_physical[bus][target].cp_online = 0; } else { /* * Rescan the physical lun list for new items */ cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_PHYSICAL_LUNS, sc->ciss_cfg->max_physical_supported); if (cll == NULL) { ciss_printf(sc, "Warning, cannot get physical lun list\n"); break; } ciss_filter_physical(sc, cll); } break; default: ciss_printf(sc, "Unknown hotplug event %d\n", cn->subclass); return; } if (cll != NULL) free(cll, CISS_MALLOC_CLASS); } /************************************************************************ * Handle deferred processing of notify events. Notify events may need * sleep which is unsafe during an interrupt. */ static void ciss_notify_thread(void *arg) { struct ciss_softc *sc; struct ciss_request *cr; struct ciss_notify *cn; sc = (struct ciss_softc *)arg; mtx_lock(&sc->ciss_mtx); for (;;) { if (STAILQ_EMPTY(&sc->ciss_notify) != 0 && (sc->ciss_flags & CISS_FLAG_THREAD_SHUT) == 0) { msleep(&sc->ciss_notify, &sc->ciss_mtx, PUSER, "idle", 0); } if (sc->ciss_flags & CISS_FLAG_THREAD_SHUT) break; cr = ciss_dequeue_notify(sc); if (cr == NULL) panic("cr null"); cn = (struct ciss_notify *)cr->cr_data; switch (cn->class) { case CISS_NOTIFY_HOTPLUG: ciss_notify_hotplug(sc, cn); break; case CISS_NOTIFY_LOGICAL: ciss_notify_logical(sc, cn); break; case CISS_NOTIFY_PHYSICAL: ciss_notify_physical(sc, cn); break; } ciss_release_request(cr); } sc->ciss_notify_thread = NULL; wakeup(&sc->ciss_notify_thread); mtx_unlock(&sc->ciss_mtx); kproc_exit(0); } /************************************************************************ * Start the notification kernel thread. */ static void ciss_spawn_notify_thread(struct ciss_softc *sc) { if (kproc_create((void(*)(void *))ciss_notify_thread, sc, &sc->ciss_notify_thread, 0, 0, "ciss_notify%d", device_get_unit(sc->ciss_dev))) panic("Could not create notify thread\n"); } /************************************************************************ * Kill the notification kernel thread. */ static void ciss_kill_notify_thread(struct ciss_softc *sc) { if (sc->ciss_notify_thread == NULL) return; sc->ciss_flags |= CISS_FLAG_THREAD_SHUT; wakeup(&sc->ciss_notify); msleep(&sc->ciss_notify_thread, &sc->ciss_mtx, PUSER, "thtrm", 0); } /************************************************************************ * Print a request. */ static void ciss_print_request(struct ciss_request *cr) { struct ciss_softc *sc; struct ciss_command *cc; int i; sc = cr->cr_sc; cc = cr->cr_cc; ciss_printf(sc, "REQUEST @ %p\n", cr); ciss_printf(sc, " data %p/%d tag %d flags %b\n", cr->cr_data, cr->cr_length, cr->cr_tag, cr->cr_flags, "\20\1mapped\2sleep\3poll\4dataout\5datain\n"); ciss_printf(sc, " sg list/total %d/%d host tag 0x%x\n", cc->header.sg_in_list, cc->header.sg_total, cc->header.host_tag); switch(cc->header.address.mode.mode) { case CISS_HDR_ADDRESS_MODE_PERIPHERAL: case CISS_HDR_ADDRESS_MODE_MASK_PERIPHERAL: ciss_printf(sc, " physical bus %d target %d\n", cc->header.address.physical.bus, cc->header.address.physical.target); break; case CISS_HDR_ADDRESS_MODE_LOGICAL: ciss_printf(sc, " logical unit %d\n", cc->header.address.logical.lun); break; } ciss_printf(sc, " %s cdb length %d type %s attribute %s\n", (cc->cdb.direction == CISS_CDB_DIRECTION_NONE) ? "no-I/O" : (cc->cdb.direction == CISS_CDB_DIRECTION_READ) ? "READ" : (cc->cdb.direction == CISS_CDB_DIRECTION_WRITE) ? "WRITE" : "??", cc->cdb.cdb_length, (cc->cdb.type == CISS_CDB_TYPE_COMMAND) ? "command" : (cc->cdb.type == CISS_CDB_TYPE_MESSAGE) ? "message" : "??", (cc->cdb.attribute == CISS_CDB_ATTRIBUTE_UNTAGGED) ? "untagged" : (cc->cdb.attribute == CISS_CDB_ATTRIBUTE_SIMPLE) ? "simple" : (cc->cdb.attribute == CISS_CDB_ATTRIBUTE_HEAD_OF_QUEUE) ? "head-of-queue" : (cc->cdb.attribute == CISS_CDB_ATTRIBUTE_ORDERED) ? "ordered" : (cc->cdb.attribute == CISS_CDB_ATTRIBUTE_AUTO_CONTINGENT) ? "auto-contingent" : "??"); ciss_printf(sc, " %*D\n", cc->cdb.cdb_length, &cc->cdb.cdb[0], " "); if (cc->header.host_tag & CISS_HDR_HOST_TAG_ERROR) { /* XXX print error info */ } else { /* since we don't use chained s/g, don't support it here */ for (i = 0; i < cc->header.sg_in_list; i++) { if ((i % 4) == 0) ciss_printf(sc, " "); printf("0x%08x/%d ", (u_int32_t)cc->sg[i].address, cc->sg[i].length); if ((((i + 1) % 4) == 0) || (i == (cc->header.sg_in_list - 1))) printf("\n"); } } } /************************************************************************ * Print information about the status of a logical drive. */ static void ciss_print_ldrive(struct ciss_softc *sc, struct ciss_ldrive *ld) { int bus, target, i; if (ld->cl_lstatus == NULL) { printf("does not exist\n"); return; } /* print drive status */ switch(ld->cl_lstatus->status) { case CISS_LSTATUS_OK: printf("online\n"); break; case CISS_LSTATUS_INTERIM_RECOVERY: printf("in interim recovery mode\n"); break; case CISS_LSTATUS_READY_RECOVERY: printf("ready to begin recovery\n"); break; case CISS_LSTATUS_RECOVERING: bus = CISS_BIG_MAP_BUS(sc, ld->cl_lstatus->drive_rebuilding); target = CISS_BIG_MAP_BUS(sc, ld->cl_lstatus->drive_rebuilding); printf("being recovered, working on physical drive %d.%d, %u blocks remaining\n", bus, target, ld->cl_lstatus->blocks_to_recover); break; case CISS_LSTATUS_EXPANDING: printf("being expanded, %u blocks remaining\n", ld->cl_lstatus->blocks_to_recover); break; case CISS_LSTATUS_QUEUED_FOR_EXPANSION: printf("queued for expansion\n"); break; case CISS_LSTATUS_FAILED: printf("queued for expansion\n"); break; case CISS_LSTATUS_WRONG_PDRIVE: printf("wrong physical drive inserted\n"); break; case CISS_LSTATUS_MISSING_PDRIVE: printf("missing a needed physical drive\n"); break; case CISS_LSTATUS_BECOMING_READY: printf("becoming ready\n"); break; } /* print failed physical drives */ for (i = 0; i < CISS_BIG_MAP_ENTRIES / 8; i++) { bus = CISS_BIG_MAP_BUS(sc, ld->cl_lstatus->drive_failure_map[i]); target = CISS_BIG_MAP_TARGET(sc, ld->cl_lstatus->drive_failure_map[i]); if (bus == -1) continue; ciss_printf(sc, "physical drive %d:%d (%x) failed\n", bus, target, ld->cl_lstatus->drive_failure_map[i]); } } #ifdef CISS_DEBUG #include "opt_ddb.h" #ifdef DDB #include /************************************************************************ * Print information about the controller/driver. */ static void ciss_print_adapter(struct ciss_softc *sc) { int i, j; ciss_printf(sc, "ADAPTER:\n"); for (i = 0; i < CISSQ_COUNT; i++) { ciss_printf(sc, "%s %d/%d\n", i == 0 ? "free" : i == 1 ? "busy" : "complete", sc->ciss_qstat[i].q_length, sc->ciss_qstat[i].q_max); } ciss_printf(sc, "max_requests %d\n", sc->ciss_max_requests); ciss_printf(sc, "flags %b\n", sc->ciss_flags, "\20\1notify_ok\2control_open\3aborting\4running\21fake_synch\22bmic_abort\n"); for (i = 0; i < sc->ciss_max_logical_bus; i++) { for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) { ciss_printf(sc, "LOGICAL DRIVE %d: ", i); ciss_print_ldrive(sc, &sc->ciss_logical[i][j]); } } /* XXX Should physical drives be printed out here? */ for (i = 1; i < sc->ciss_max_requests; i++) ciss_print_request(sc->ciss_request + i); } /* DDB hook */ DB_COMMAND(ciss_prt, db_ciss_prt) { struct ciss_softc *sc; devclass_t dc; int maxciss, i; dc = devclass_find("ciss"); if ( dc == NULL ) { printf("%s: can't find devclass!\n", __func__); return; } maxciss = devclass_get_maxunit(dc); for (i = 0; i < maxciss; i++) { sc = devclass_get_softc(dc, i); ciss_print_adapter(sc); } } #endif #endif /************************************************************************ * Return a name for a logical drive status value. */ static const char * ciss_name_ldrive_status(int status) { switch (status) { case CISS_LSTATUS_OK: return("OK"); case CISS_LSTATUS_FAILED: return("failed"); case CISS_LSTATUS_NOT_CONFIGURED: return("not configured"); case CISS_LSTATUS_INTERIM_RECOVERY: return("interim recovery"); case CISS_LSTATUS_READY_RECOVERY: return("ready for recovery"); case CISS_LSTATUS_RECOVERING: return("recovering"); case CISS_LSTATUS_WRONG_PDRIVE: return("wrong physical drive inserted"); case CISS_LSTATUS_MISSING_PDRIVE: return("missing physical drive"); case CISS_LSTATUS_EXPANDING: return("expanding"); case CISS_LSTATUS_BECOMING_READY: return("becoming ready"); case CISS_LSTATUS_QUEUED_FOR_EXPANSION: return("queued for expansion"); } return("unknown status"); } /************************************************************************ * Return an online/offline/nonexistent value for a logical drive * status value. */ static int ciss_decode_ldrive_status(int status) { switch(status) { case CISS_LSTATUS_NOT_CONFIGURED: return(CISS_LD_NONEXISTENT); case CISS_LSTATUS_OK: case CISS_LSTATUS_INTERIM_RECOVERY: case CISS_LSTATUS_READY_RECOVERY: case CISS_LSTATUS_RECOVERING: case CISS_LSTATUS_EXPANDING: case CISS_LSTATUS_QUEUED_FOR_EXPANSION: return(CISS_LD_ONLINE); case CISS_LSTATUS_FAILED: case CISS_LSTATUS_WRONG_PDRIVE: case CISS_LSTATUS_MISSING_PDRIVE: case CISS_LSTATUS_BECOMING_READY: default: return(CISS_LD_OFFLINE); } } /************************************************************************ * Return a name for a logical drive's organisation. */ static const char * ciss_name_ldrive_org(int org) { switch(org) { case CISS_LDRIVE_RAID0: return("RAID 0"); case CISS_LDRIVE_RAID1: return("RAID 1(1+0)"); case CISS_LDRIVE_RAID4: return("RAID 4"); case CISS_LDRIVE_RAID5: return("RAID 5"); case CISS_LDRIVE_RAID51: return("RAID 5+1"); case CISS_LDRIVE_RAIDADG: return("RAID ADG"); } - return("unkown"); + return("unknown"); } /************************************************************************ * Return a name for a command status value. */ static const char * ciss_name_command_status(int status) { switch(status) { case CISS_CMD_STATUS_SUCCESS: return("success"); case CISS_CMD_STATUS_TARGET_STATUS: return("target status"); case CISS_CMD_STATUS_DATA_UNDERRUN: return("data underrun"); case CISS_CMD_STATUS_DATA_OVERRUN: return("data overrun"); case CISS_CMD_STATUS_INVALID_COMMAND: return("invalid command"); case CISS_CMD_STATUS_PROTOCOL_ERROR: return("protocol error"); case CISS_CMD_STATUS_HARDWARE_ERROR: return("hardware error"); case CISS_CMD_STATUS_CONNECTION_LOST: return("connection lost"); case CISS_CMD_STATUS_ABORTED: return("aborted"); case CISS_CMD_STATUS_ABORT_FAILED: return("abort failed"); case CISS_CMD_STATUS_UNSOLICITED_ABORT: return("unsolicited abort"); case CISS_CMD_STATUS_TIMEOUT: return("timeout"); case CISS_CMD_STATUS_UNABORTABLE: return("unabortable"); } return("unknown status"); } /************************************************************************ * Handle an open on the control device. */ static int ciss_open(struct cdev *dev, int flags, int fmt, struct thread *p) { struct ciss_softc *sc; debug_called(1); sc = (struct ciss_softc *)dev->si_drv1; /* we might want to veto if someone already has us open */ mtx_lock(&sc->ciss_mtx); sc->ciss_flags |= CISS_FLAG_CONTROL_OPEN; mtx_unlock(&sc->ciss_mtx); return(0); } /************************************************************************ * Handle the last close on the control device. */ static int ciss_close(struct cdev *dev, int flags, int fmt, struct thread *p) { struct ciss_softc *sc; debug_called(1); sc = (struct ciss_softc *)dev->si_drv1; mtx_lock(&sc->ciss_mtx); sc->ciss_flags &= ~CISS_FLAG_CONTROL_OPEN; mtx_unlock(&sc->ciss_mtx); return (0); } /******************************************************************************** * Handle adapter-specific control operations. * * Note that the API here is compatible with the Linux driver, in order to * simplify the porting of Compaq's userland tools. */ static int ciss_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *p) { struct ciss_softc *sc; IOCTL_Command_struct *ioc = (IOCTL_Command_struct *)addr; #ifdef __amd64__ IOCTL_Command_struct32 *ioc32 = (IOCTL_Command_struct32 *)addr; IOCTL_Command_struct ioc_swab; #endif int error; debug_called(1); sc = (struct ciss_softc *)dev->si_drv1; error = 0; mtx_lock(&sc->ciss_mtx); switch(cmd) { case CCISS_GETQSTATS: { union ciss_statrequest *cr = (union ciss_statrequest *)addr; switch (cr->cs_item) { case CISSQ_FREE: case CISSQ_NOTIFY: bcopy(&sc->ciss_qstat[cr->cs_item], &cr->cs_qstat, sizeof(struct ciss_qstat)); break; default: error = ENOIOCTL; break; } break; } case CCISS_GETPCIINFO: { cciss_pci_info_struct *pis = (cciss_pci_info_struct *)addr; pis->bus = pci_get_bus(sc->ciss_dev); pis->dev_fn = pci_get_slot(sc->ciss_dev); pis->board_id = (pci_get_subvendor(sc->ciss_dev) << 16) | pci_get_subdevice(sc->ciss_dev); break; } case CCISS_GETINTINFO: { cciss_coalint_struct *cis = (cciss_coalint_struct *)addr; cis->delay = sc->ciss_cfg->interrupt_coalesce_delay; cis->count = sc->ciss_cfg->interrupt_coalesce_count; break; } case CCISS_SETINTINFO: { cciss_coalint_struct *cis = (cciss_coalint_struct *)addr; if ((cis->delay == 0) && (cis->count == 0)) { error = EINVAL; break; } /* * XXX apparently this is only safe if the controller is idle, * we should suspend it before doing this. */ sc->ciss_cfg->interrupt_coalesce_delay = cis->delay; sc->ciss_cfg->interrupt_coalesce_count = cis->count; if (ciss_update_config(sc)) error = EIO; /* XXX resume the controller here */ break; } case CCISS_GETNODENAME: bcopy(sc->ciss_cfg->server_name, (NodeName_type *)addr, sizeof(NodeName_type)); break; case CCISS_SETNODENAME: bcopy((NodeName_type *)addr, sc->ciss_cfg->server_name, sizeof(NodeName_type)); if (ciss_update_config(sc)) error = EIO; break; case CCISS_GETHEARTBEAT: *(Heartbeat_type *)addr = sc->ciss_cfg->heartbeat; break; case CCISS_GETBUSTYPES: *(BusTypes_type *)addr = sc->ciss_cfg->bus_types; break; case CCISS_GETFIRMVER: bcopy(sc->ciss_id->running_firmware_revision, (FirmwareVer_type *)addr, sizeof(FirmwareVer_type)); break; case CCISS_GETDRIVERVER: *(DriverVer_type *)addr = CISS_DRIVER_VERSION; break; case CCISS_REVALIDVOLS: /* * This is a bit ugly; to do it "right" we really need * to find any disks that have changed, kick CAM off them, * then rescan only these disks. It'd be nice if they * a) told us which disk(s) they were going to play with, * and b) which ones had arrived. 8( */ break; #ifdef __amd64__ case CCISS_PASSTHRU32: ioc_swab.LUN_info = ioc32->LUN_info; ioc_swab.Request = ioc32->Request; ioc_swab.error_info = ioc32->error_info; ioc_swab.buf_size = ioc32->buf_size; ioc_swab.buf = (u_int8_t *)(uintptr_t)ioc32->buf; ioc = &ioc_swab; /* FALLTHROUGH */ #endif case CCISS_PASSTHRU: error = ciss_user_command(sc, ioc); break; default: debug(0, "unknown ioctl 0x%lx", cmd); debug(1, "CCISS_GETPCIINFO: 0x%lx", CCISS_GETPCIINFO); debug(1, "CCISS_GETINTINFO: 0x%lx", CCISS_GETINTINFO); debug(1, "CCISS_SETINTINFO: 0x%lx", CCISS_SETINTINFO); debug(1, "CCISS_GETNODENAME: 0x%lx", CCISS_GETNODENAME); debug(1, "CCISS_SETNODENAME: 0x%lx", CCISS_SETNODENAME); debug(1, "CCISS_GETHEARTBEAT: 0x%lx", CCISS_GETHEARTBEAT); debug(1, "CCISS_GETBUSTYPES: 0x%lx", CCISS_GETBUSTYPES); debug(1, "CCISS_GETFIRMVER: 0x%lx", CCISS_GETFIRMVER); debug(1, "CCISS_GETDRIVERVER: 0x%lx", CCISS_GETDRIVERVER); debug(1, "CCISS_REVALIDVOLS: 0x%lx", CCISS_REVALIDVOLS); debug(1, "CCISS_PASSTHRU: 0x%lx", CCISS_PASSTHRU); error = ENOIOCTL; break; } mtx_unlock(&sc->ciss_mtx); return(error); } Index: head/sys/dev/drm2/i915/i915_gem.c =================================================================== --- head/sys/dev/drm2/i915/i915_gem.c (revision 298930) +++ head/sys/dev/drm2/i915/i915_gem.c (revision 298931) @@ -1,4798 +1,4798 @@ /* * Copyright © 2008 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Eric Anholt * * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, unsigned alignment, bool map_and_fenceable, bool nonblocking); static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file); static void i915_gem_write_fence(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj); static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, struct drm_i915_fence_reg *fence, bool enable); static void i915_gem_inactive_shrink(void *); static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); static void i915_gem_shrink_all(struct drm_i915_private *dev_priv); static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, off_t start, off_t end); static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh); MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); long i915_gem_wired_pages_cnt; static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) { if (obj->tiling_mode) i915_gem_release_mmap(obj); /* As we do not have an associated fence register, we will force * a tiling change if we ever need to acquire one. */ obj->fence_dirty = false; obj->fence_reg = I915_FENCE_REG_NONE; } /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) { dev_priv->mm.object_count++; dev_priv->mm.object_memory += size; } static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) { dev_priv->mm.object_count--; dev_priv->mm.object_memory -= size; } static int i915_gem_wait_for_error(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct completion *x = &dev_priv->error_completion; int ret; if (!atomic_read(&dev_priv->mm.wedged)) return 0; /* * Only wait 10 seconds for the gpu reset to complete to avoid hanging * userspace. If it takes that long something really bad is going on and * we should simply try to bail out and fail as gracefully as possible. */ ret = wait_for_completion_interruptible_timeout(x, 10*HZ); if (ret == 0) { DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); return -EIO; } else if (ret < 0) { return ret; } if (atomic_read(&dev_priv->mm.wedged)) { /* GPU is hung, bump the completion count to account for * the token we just consumed so that we never hit zero and * end up waiting upon a subsequent completion event that * will never happen. */ mtx_lock(&x->lock); x->done++; mtx_unlock(&x->lock); } return 0; } int i915_mutex_lock_interruptible(struct drm_device *dev) { int ret; ret = i915_gem_wait_for_error(dev); if (ret) return ret; /* * interruptible shall it be. might indeed be if dev_lock is * changed to sx */ ret = sx_xlock_sig(&dev->dev_struct_lock); if (ret) return -EINTR; WARN_ON(i915_verify_lists(dev)); return 0; } static inline bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) { return obj->gtt_space && !obj->active; } int i915_gem_init_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_init *args = data; if (drm_core_check_feature(dev, DRIVER_MODESET)) return -ENODEV; if (args->gtt_start >= args->gtt_end || (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) return -EINVAL; /* GEM with user mode setting was never supported on ilk and later. */ if (INTEL_INFO(dev)->gen >= 5) return -ENODEV; /* * XXXKIB. The second-time initialization should be guarded * against. */ DRM_LOCK(dev); i915_gem_init_global_gtt(dev, args->gtt_start, args->gtt_end, args->gtt_end); DRM_UNLOCK(dev); return 0; } int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_get_aperture *args = data; struct drm_i915_gem_object *obj; size_t pinned; pinned = 0; DRM_LOCK(dev); list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) if (obj->pin_count) pinned += obj->gtt_space->size; DRM_UNLOCK(dev); args->aper_size = dev_priv->mm.gtt_total; args->aper_available_size = args->aper_size - pinned; return 0; } static int i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, uint32_t *handle_p) { struct drm_i915_gem_object *obj; int ret; u32 handle; size = roundup(size, PAGE_SIZE); if (size == 0) return -EINVAL; /* Allocate the new object */ obj = i915_gem_alloc_object(dev, size); if (obj == NULL) return -ENOMEM; ret = drm_gem_handle_create(file, &obj->base, &handle); if (ret) { drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev->dev_private, obj->base.size); free(obj, DRM_I915_GEM); return ret; } /* drop reference from allocate - handle holds it now */ drm_gem_object_unreference(&obj->base); CTR2(KTR_DRM, "object_create %p %x", obj, size); *handle_p = handle; return 0; } int i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args) { /* have to work out size/pitch and return them */ args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); args->size = args->pitch * args->height; return i915_gem_create(file, dev, args->size, &args->handle); } int i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, uint32_t handle) { return drm_gem_handle_delete(file, handle); } /** * Creates a new mm object and returns a handle to it. */ int i915_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_create *args = data; return i915_gem_create(file, dev, args->size, &args->handle); } static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = obj->base.dev->dev_private; return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && obj->tiling_mode != I915_TILING_NONE; } static inline int __copy_to_user_swizzled(char __user *cpu_vaddr, const char *gpu_vaddr, int gpu_offset, int length) { int ret, cpu_offset = 0; while (length > 0) { int cacheline_end = roundup2(gpu_offset + 1, 64); int this_length = min(cacheline_end - gpu_offset, length); int swizzled_gpu_offset = gpu_offset ^ 64; ret = __copy_to_user(cpu_vaddr + cpu_offset, gpu_vaddr + swizzled_gpu_offset, this_length); if (ret) return ret + length; cpu_offset += this_length; gpu_offset += this_length; length -= this_length; } return 0; } static inline int __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, const char __user *cpu_vaddr, int length) { int ret, cpu_offset = 0; while (length > 0) { int cacheline_end = roundup2(gpu_offset + 1, 64); int this_length = min(cacheline_end - gpu_offset, length); int swizzled_gpu_offset = gpu_offset ^ 64; ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, cpu_vaddr + cpu_offset, this_length); if (ret) return ret + length; cpu_offset += this_length; gpu_offset += this_length; length -= this_length; } return 0; } /* Per-page copy function for the shmem pread fastpath. * Flushes invalid cachelines before reading the target if * needs_clflush is set. */ static int shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush) { char *vaddr; struct sf_buf *sf; int ret; if (unlikely(page_do_bit17_swizzling)) return -EINVAL; sched_pin(); sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); if (sf == NULL) { sched_unpin(); return (-EFAULT); } vaddr = (char *)sf_buf_kva(sf); if (needs_clflush) drm_clflush_virt_range(vaddr + shmem_page_offset, page_length); ret = __copy_to_user_inatomic(user_data, vaddr + shmem_page_offset, page_length); sf_buf_free(sf); sched_unpin(); return ret ? -EFAULT : 0; } static void shmem_clflush_swizzled_range(char *addr, unsigned long length, bool swizzled) { if (unlikely(swizzled)) { unsigned long start = (unsigned long) addr; unsigned long end = (unsigned long) addr + length; /* For swizzling simply ensure that we always flush both * channels. Lame, but simple and it works. Swizzled * pwrite/pread is far from a hotpath - current userspace * doesn't use it at all. */ start = round_down(start, 128); end = round_up(end, 128); drm_clflush_virt_range((void *)start, end - start); } else { drm_clflush_virt_range(addr, length); } } /* Only difference to the fast-path function is that this can handle bit17 * and uses non-atomic copy and kmap functions. */ static int shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush) { char *vaddr; struct sf_buf *sf; int ret; sf = sf_buf_alloc(page, 0); vaddr = (char *)sf_buf_kva(sf); if (needs_clflush) shmem_clflush_swizzled_range(vaddr + shmem_page_offset, page_length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) ret = __copy_to_user_swizzled(user_data, vaddr, shmem_page_offset, page_length); else ret = __copy_to_user(user_data, vaddr + shmem_page_offset, page_length); sf_buf_free(sf); return ret ? - EFAULT : 0; } static int i915_gem_shmem_pread(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pread *args, struct drm_file *file) { char __user *user_data; ssize_t remain; off_t offset; int shmem_page_offset, page_length, ret = 0; int obj_do_bit17_swizzling, page_do_bit17_swizzling; int hit_slowpath = 0; int prefaulted = 0; int needs_clflush = 0; user_data = to_user_ptr(args->data_ptr); remain = args->size; obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { /* If we're not in the cpu read domain, set ourself into the gtt * read domain and manually flush cachelines (if required). This * optimizes for the case when the gpu will dirty the data * anyway again before the next pread happens. */ if (obj->cache_level == I915_CACHE_NONE) needs_clflush = 1; if (obj->gtt_space) { ret = i915_gem_object_set_to_gtt_domain(obj, false); if (ret) return ret; } } ret = i915_gem_object_get_pages(obj); if (ret) return ret; i915_gem_object_pin_pages(obj); offset = args->offset; VM_OBJECT_WLOCK(obj->base.vm_obj); for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, OFF_TO_IDX(offset));; page = vm_page_next(page)) { VM_OBJECT_WUNLOCK(obj->base.vm_obj); if (remain <= 0) break; /* Operation in this page * * shmem_page_offset = offset within page in shmem file * page_length = bytes to copy for this page */ shmem_page_offset = offset_in_page(offset); page_length = remain; if ((shmem_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - shmem_page_offset; page_do_bit17_swizzling = obj_do_bit17_swizzling && (page_to_phys(page) & (1 << 17)) != 0; ret = shmem_pread_fast(page, shmem_page_offset, page_length, user_data, page_do_bit17_swizzling, needs_clflush); if (ret == 0) goto next_page; hit_slowpath = 1; DRM_UNLOCK(dev); if (!prefaulted) { ret = fault_in_multipages_writeable(user_data, remain); /* Userspace is tricking us, but we've already clobbered * its pages with the prefault and promised to write the * data up to the first fault. Hence ignore any errors * and just continue. */ (void)ret; prefaulted = 1; } ret = shmem_pread_slow(page, shmem_page_offset, page_length, user_data, page_do_bit17_swizzling, needs_clflush); DRM_LOCK(dev); next_page: vm_page_reference(page); if (ret) goto out; remain -= page_length; user_data += page_length; offset += page_length; VM_OBJECT_WLOCK(obj->base.vm_obj); } out: i915_gem_object_unpin_pages(obj); if (hit_slowpath) { /* Fixup: Kill any reinstated backing storage pages */ if (obj->madv == __I915_MADV_PURGED) i915_gem_object_truncate(obj); } return ret; } /** * Reads data from the object referenced by handle. * * On error, the contents of *data are undefined. */ int i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pread *args = data; struct drm_i915_gem_object *obj; int ret = 0; if (args->size == 0) return 0; if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE)) return -EFAULT; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } /* Bounds check source. */ if (args->offset > obj->base.size || args->size > obj->base.size - args->offset) { ret = -EINVAL; goto out; } #ifdef FREEBSD_WIP /* prime objects have no backing filp to GEM pread/pwrite * pages from. */ if (!obj->base.filp) { ret = -EINVAL; goto out; } #endif /* FREEBSD_WIP */ CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size); ret = i915_gem_shmem_pread(dev, obj, args, file); out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } /* This is the fast write path which cannot handle * page faults in the source data */ static inline int fast_user_write(vm_paddr_t mapping_addr, off_t page_base, int page_offset, char __user *user_data, int length) { void __iomem *vaddr_atomic; void *vaddr; unsigned long unwritten; vaddr_atomic = pmap_mapdev_attr(mapping_addr + page_base, length, PAT_WRITE_COMBINING); /* We can use the cpu mem copy function because this is X86. */ vaddr = (char __force*)vaddr_atomic + page_offset; unwritten = __copy_from_user_inatomic_nocache(vaddr, user_data, length); pmap_unmapdev((vm_offset_t)vaddr_atomic, length); return unwritten; } /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. */ static int i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file) { drm_i915_private_t *dev_priv = dev->dev_private; ssize_t remain; off_t offset, page_base; char __user *user_data; int page_offset, page_length, ret; ret = i915_gem_object_pin(obj, 0, true, true); if (ret) goto out; ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) goto out_unpin; ret = i915_gem_object_put_fence(obj); if (ret) goto out_unpin; user_data = to_user_ptr(args->data_ptr); remain = args->size; offset = obj->gtt_offset + args->offset; while (remain > 0) { /* Operation in this page * * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ page_base = offset & ~PAGE_MASK; page_offset = offset_in_page(offset); page_length = remain; if ((page_offset + remain) > PAGE_SIZE) page_length = PAGE_SIZE - page_offset; /* If we get a fault while copying data, then (presumably) our * source page isn't available. Return the error and we'll * retry in the slow path. */ if (fast_user_write(dev_priv->mm.gtt_base_addr, page_base, page_offset, user_data, page_length)) { ret = -EFAULT; goto out_unpin; } remain -= page_length; user_data += page_length; offset += page_length; } out_unpin: i915_gem_object_unpin(obj); out: return ret; } /* Per-page copy function for the shmem pwrite fastpath. * Flushes invalid cachelines before writing to the target if * needs_clflush_before is set and flushes out any written cachelines after * writing if needs_clflush is set. */ static int shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush_before, bool needs_clflush_after) { char *vaddr; struct sf_buf *sf; int ret; if (unlikely(page_do_bit17_swizzling)) return -EINVAL; sched_pin(); sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE); if (sf == NULL) { sched_unpin(); return (-EFAULT); } vaddr = (char *)sf_buf_kva(sf); if (needs_clflush_before) drm_clflush_virt_range(vaddr + shmem_page_offset, page_length); ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, user_data, page_length); if (needs_clflush_after) drm_clflush_virt_range(vaddr + shmem_page_offset, page_length); sf_buf_free(sf); sched_unpin(); return ret ? -EFAULT : 0; } /* Only difference to the fast-path function is that this can handle bit17 * and uses non-atomic copy and kmap functions. */ static int shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush_before, bool needs_clflush_after) { char *vaddr; struct sf_buf *sf; int ret; sf = sf_buf_alloc(page, 0); vaddr = (char *)sf_buf_kva(sf); if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) shmem_clflush_swizzled_range(vaddr + shmem_page_offset, page_length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, user_data, page_length); else ret = __copy_from_user(vaddr + shmem_page_offset, user_data, page_length); if (needs_clflush_after) shmem_clflush_swizzled_range(vaddr + shmem_page_offset, page_length, page_do_bit17_swizzling); sf_buf_free(sf); return ret ? -EFAULT : 0; } static int i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file) { ssize_t remain; off_t offset; char __user *user_data; int shmem_page_offset, page_length, ret = 0; int obj_do_bit17_swizzling, page_do_bit17_swizzling; int hit_slowpath = 0; int needs_clflush_after = 0; int needs_clflush_before = 0; user_data = to_user_ptr(args->data_ptr); remain = args->size; obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { /* If we're not in the cpu write domain, set ourself into the gtt * write domain and manually flush cachelines (if required). This * optimizes for the case when the gpu will use the data * right away and we therefore have to clflush anyway. */ if (obj->cache_level == I915_CACHE_NONE) needs_clflush_after = 1; if (obj->gtt_space) { ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) return ret; } } /* Same trick applies for invalidate partially written cachelines before * writing. */ if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) && obj->cache_level == I915_CACHE_NONE) needs_clflush_before = 1; ret = i915_gem_object_get_pages(obj); if (ret) return ret; i915_gem_object_pin_pages(obj); offset = args->offset; obj->dirty = 1; VM_OBJECT_WLOCK(obj->base.vm_obj); for (vm_page_t page = vm_page_find_least(obj->base.vm_obj, OFF_TO_IDX(offset));; page = vm_page_next(page)) { VM_OBJECT_WUNLOCK(obj->base.vm_obj); int partial_cacheline_write; if (remain <= 0) break; /* Operation in this page * * shmem_page_offset = offset within page in shmem file * page_length = bytes to copy for this page */ shmem_page_offset = offset_in_page(offset); page_length = remain; if ((shmem_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - shmem_page_offset; /* If we don't overwrite a cacheline completely we need to be * careful to have up-to-date data by first clflushing. Don't * overcomplicate things and flush the entire patch. */ partial_cacheline_write = needs_clflush_before && ((shmem_page_offset | page_length) & (cpu_clflush_line_size - 1)); page_do_bit17_swizzling = obj_do_bit17_swizzling && (page_to_phys(page) & (1 << 17)) != 0; ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, user_data, page_do_bit17_swizzling, partial_cacheline_write, needs_clflush_after); if (ret == 0) goto next_page; hit_slowpath = 1; DRM_UNLOCK(dev); ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, user_data, page_do_bit17_swizzling, partial_cacheline_write, needs_clflush_after); DRM_LOCK(dev); next_page: vm_page_dirty(page); vm_page_reference(page); if (ret) goto out; remain -= page_length; user_data += page_length; offset += page_length; VM_OBJECT_WLOCK(obj->base.vm_obj); } out: i915_gem_object_unpin_pages(obj); if (hit_slowpath) { /* Fixup: Kill any reinstated backing storage pages */ if (obj->madv == __I915_MADV_PURGED) i915_gem_object_truncate(obj); /* and flush dirty cachelines in case the object isn't in the cpu write * domain anymore. */ if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { i915_gem_clflush_object(obj); i915_gem_chipset_flush(dev); } } if (needs_clflush_after) i915_gem_chipset_flush(dev); return ret; } /** * Writes data to the object referenced by handle. * * On error, the contents of the buffer that were to be modified are undefined. */ int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pwrite *args = data; struct drm_i915_gem_object *obj; int ret; if (args->size == 0) return 0; if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ)) return -EFAULT; ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), args->size); if (ret) return -EFAULT; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } /* Bounds check destination. */ if (args->offset > obj->base.size || args->size > obj->base.size - args->offset) { ret = -EINVAL; goto out; } #ifdef FREEBSD_WIP /* prime objects have no backing filp to GEM pread/pwrite * pages from. */ if (!obj->base.filp) { ret = -EINVAL; goto out; } #endif /* FREEBSD_WIP */ CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size); ret = -EFAULT; /* We can only do the GTT pwrite on untiled buffers, as otherwise * it would end up going through the fenced access, and we'll get * different detiling behavior between reading and writing. * pread/pwrite currently are reading and writing from the CPU * perspective, requiring manual detiling by the client. */ if (obj->phys_obj) { ret = i915_gem_phys_pwrite(dev, obj, args, file); goto out; } if (obj->cache_level == I915_CACHE_NONE && obj->tiling_mode == I915_TILING_NONE && obj->base.write_domain != I915_GEM_DOMAIN_CPU) { ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); /* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between * textures). Fallback to the shmem path in that case. */ } if (ret == -EFAULT || ret == -ENOSPC) ret = i915_gem_shmem_pwrite(dev, obj, args, file); out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } int i915_gem_check_wedge(struct drm_i915_private *dev_priv, bool interruptible) { if (atomic_read(&dev_priv->mm.wedged)) { struct completion *x = &dev_priv->error_completion; bool recovery_complete; /* Give the error handler a chance to run. */ mtx_lock(&x->lock); recovery_complete = x->done > 0; mtx_unlock(&x->lock); /* Non-interruptible callers can't handle -EAGAIN, hence return * -EIO unconditionally for these. */ if (!interruptible) return -EIO; /* Recovery complete, but still wedged means reset failure. */ if (recovery_complete) return -EIO; return -EAGAIN; } return 0; } /* * Compare seqno against outstanding lazy request. Emit a request if they are * equal. */ static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) { int ret; DRM_LOCK_ASSERT(ring->dev); ret = 0; if (seqno == ring->outstanding_lazy_request) ret = i915_add_request(ring, NULL, NULL); return ret; } /** * __wait_seqno - wait until execution of seqno has finished * @ring: the ring expected to report seqno * @seqno: duh! * @interruptible: do an interruptible wait (normally yes) * @timeout: in - how long to wait (NULL forever); out - how much time remaining * * Returns 0 if the seqno was found within the alloted time. Else returns the * errno with remaining time filled in timeout argument. */ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, bool interruptible, struct timespec *timeout) { drm_i915_private_t *dev_priv = ring->dev->dev_private; struct timespec before, now, wait_time={1,0}; sbintime_t timeout_sbt; long end; bool wait_forever = true; int ret, flags; if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) return 0; CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno); if (timeout != NULL) { wait_time = *timeout; wait_forever = false; } timeout_sbt = tstosbt(wait_time); if (WARN_ON(!ring->irq_get(ring))) return -ENODEV; /* Record current time in case interrupted by signal, or wedged * */ getrawmonotonic(&before); #define EXIT_COND \ (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ atomic_read(&dev_priv->mm.wedged)) flags = interruptible ? PCATCH : 0; mtx_lock(&dev_priv->irq_lock); do { if (EXIT_COND) { end = 1; } else { ret = -msleep_sbt(&ring->irq_queue, &dev_priv->irq_lock, flags, "915gwr", timeout_sbt, 0, 0); /* * NOTE Linux<->FreeBSD: Convert msleep_sbt() return * value to something close to wait_event*_timeout() * functions used on Linux. * * >0 -> condition is true (end = time remaining) * =0 -> sleep timed out * <0 -> error (interrupted) * * We fake the remaining time by returning 1. We * compute a proper value later. */ if (EXIT_COND) /* We fake a remaining time of 1 tick. */ end = 1; else if (ret == -EINTR || ret == -ERESTART) /* Interrupted. */ end = -ERESTARTSYS; else /* Timeout. */ end = 0; } ret = i915_gem_check_wedge(dev_priv, interruptible); if (ret) end = ret; } while (end == 0 && wait_forever); mtx_unlock(&dev_priv->irq_lock); getrawmonotonic(&now); ring->irq_put(ring); CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, end); #undef EXIT_COND if (timeout) { timespecsub(&now, &before); timespecsub(timeout, &now); } switch (end) { case -EIO: case -EAGAIN: /* Wedged */ case -ERESTARTSYS: /* Signal */ case -ETIMEDOUT: /* Timeout */ return (int)end; case 0: /* Timeout */ return -ETIMEDOUT; default: /* Completed */ WARN_ON(end < 0); /* We're not aware of other errors */ return 0; } } /** * Waits for a sequence number to be signaled, and cleans up the * request and object lists appropriately for that event. */ int i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno) { struct drm_device *dev = ring->dev; struct drm_i915_private *dev_priv = dev->dev_private; bool interruptible = dev_priv->mm.interruptible; int ret; DRM_LOCK_ASSERT(dev); BUG_ON(seqno == 0); ret = i915_gem_check_wedge(dev_priv, interruptible); if (ret) return ret; ret = i915_gem_check_olr(ring, seqno); if (ret) return ret; return __wait_seqno(ring, seqno, interruptible, NULL); } /** * Ensures that all rendering to the object has completed and the object is * safe to unbind from the GTT or access from the CPU. */ static __must_check int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool readonly) { struct intel_ring_buffer *ring = obj->ring; u32 seqno; int ret; seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; if (seqno == 0) return 0; ret = i915_wait_seqno(ring, seqno); if (ret) return ret; i915_gem_retire_requests_ring(ring); /* Manually manage the write flush as we may have not yet * retired the buffer. */ if (obj->last_write_seqno && i915_seqno_passed(seqno, obj->last_write_seqno)) { obj->last_write_seqno = 0; obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; } return 0; } /* A nonblocking variant of the above wait. This is a highly dangerous routine * as the object state may change during this call. */ static __must_check int i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, bool readonly) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_ring_buffer *ring = obj->ring; u32 seqno; int ret; DRM_LOCK_ASSERT(dev); BUG_ON(!dev_priv->mm.interruptible); seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno; if (seqno == 0) return 0; ret = i915_gem_check_wedge(dev_priv, true); if (ret) return ret; ret = i915_gem_check_olr(ring, seqno); if (ret) return ret; DRM_UNLOCK(dev); ret = __wait_seqno(ring, seqno, true, NULL); DRM_LOCK(dev); i915_gem_retire_requests_ring(ring); /* Manually manage the write flush as we may have not yet * retired the buffer. */ if (ret == 0 && obj->last_write_seqno && i915_seqno_passed(seqno, obj->last_write_seqno)) { obj->last_write_seqno = 0; obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; } return ret; } /** * Called when user space prepares to use an object with the CPU, either * through the mmap ioctl's mapping or a GTT mapping. */ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_set_domain *args = data; struct drm_i915_gem_object *obj; uint32_t read_domains = args->read_domains; uint32_t write_domain = args->write_domain; int ret; /* Only handle setting domains to types used by the CPU. */ if (write_domain & I915_GEM_GPU_DOMAINS) return -EINVAL; if (read_domains & I915_GEM_GPU_DOMAINS) return -EINVAL; /* Having something in the write domain implies it's in the read * domain, and only that read domain. Enforce that in the request. */ if (write_domain != 0 && read_domains != write_domain) return -EINVAL; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } /* Try to flush the object off the GPU without holding the lock. * We will repeat the flush holding the lock in the normal manner * to catch cases where we are gazumped. */ ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain); if (ret) goto unref; if (read_domains & I915_GEM_DOMAIN_GTT) { ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); /* Silently promote "you're not bound, there was nothing to do" * to success, since the client was just asking us to * make sure everything was done. */ if (ret == -EINVAL) ret = 0; } else { ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); } unref: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } /** * Called when user space has done writes to this buffer */ int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_sw_finish *args = data; struct drm_i915_gem_object *obj; int ret = 0; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } /* Pinned buffers may be scanout, so flush the cache */ if (obj->pin_count) i915_gem_object_flush_cpu_write_domain(obj); drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } /** * Maps the contents of an object, returning the address it is mapped * into. * * While the mapping holds a reference on the contents of the object, it doesn't * imply a ref on the object itself. */ int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_mmap *args = data; struct drm_gem_object *obj; struct proc *p; vm_map_t map; vm_offset_t addr; vm_size_t size; int error, rv; obj = drm_gem_object_lookup(dev, file, args->handle); if (obj == NULL) return -ENOENT; #ifdef FREEBSD_WIP /* prime objects have no backing filp to GEM mmap * pages from. */ if (!obj->filp) { drm_gem_object_unreference_unlocked(obj); return -EINVAL; } #endif /* FREEBSD_WIP */ error = 0; if (args->size == 0) goto out; p = curproc; map = &p->p_vmspace->vm_map; size = round_page(args->size); PROC_LOCK(p); if (map->size + size > lim_cur_proc(p, RLIMIT_VMEM)) { PROC_UNLOCK(p); error = -ENOMEM; goto out; } PROC_UNLOCK(p); addr = 0; vm_object_reference(obj->vm_obj); rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 0, VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); if (rv != KERN_SUCCESS) { vm_object_deallocate(obj->vm_obj); error = -vm_mmap_to_errno(rv); } else { args->addr_ptr = (uint64_t)addr; } out: drm_gem_object_unreference_unlocked(obj); return (error); } static int i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { /* * NOTE Linux<->FreeBSD: drm_gem_mmap_single() takes care of * calling drm_gem_object_reference(). That's why we don't * do this here. i915_gem_pager_dtor(), below, will call * drm_gem_object_unreference(). * * On Linux, drm_gem_vm_open() references the object because * it's called the mapping is copied. drm_gem_vm_open() is not * called when the mapping is created. So the possible sequences * are: * 1. drm_gem_mmap(): ref++ * 2. drm_gem_vm_close(): ref-- * * 1. drm_gem_mmap(): ref++ * 2. drm_gem_vm_open(): ref++ (for the copied vma) * 3. drm_gem_vm_close(): ref-- (for the copied vma) * 4. drm_gem_vm_close(): ref-- (for the initial vma) * * On FreeBSD, i915_gem_pager_ctor() is called once during the * creation of the mapping. No callback is called when the * mapping is shared during a fork(). i915_gem_pager_dtor() is * called when the last reference to the mapping is dropped. So * the only sequence is: * 1. drm_gem_mmap_single(): ref++ * 2. i915_gem_pager_ctor(): * 3. i915_gem_pager_dtor(): ref-- */ *color = 0; /* XXXKIB */ return (0); } /** * i915_gem_fault - fault a page into the GTT * vma: VMA in question * vmf: fault info * * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped * from userspace. The fault handler takes care of binding the object to * the GTT (if needed), allocating and programming a fence register (again, * only if needed based on whether the old reg is still valid or the object * is tiled) and inserting a new PTE into the faulting process. * * Note that the faulting process may involve evicting existing objects * from the GTT and/or fence registers to make room. So performance may * suffer if the GTT working set is large or there are few fence registers * left. */ int i915_intr_pf; static int i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) { struct drm_gem_object *gem_obj = vm_obj->handle; struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; vm_page_t page; int ret = 0; #ifdef FREEBSD_WIP bool write = (prot & VM_PROT_WRITE) != 0; #else bool write = true; #endif /* FREEBSD_WIP */ bool pinned; vm_object_pip_add(vm_obj, 1); /* * Remove the placeholder page inserted by vm_fault() from the * object before dropping the object lock. If * i915_gem_release_mmap() is active in parallel on this gem * object, then it owns the drm device sx and might find the * placeholder already. Then, since the page is busy, * i915_gem_release_mmap() sleeps waiting for the busy state * of the page cleared. We will be unable to acquire drm * device lock until i915_gem_release_mmap() is able to make a * progress. */ if (*mres != NULL) { vm_page_lock(*mres); vm_page_remove(*mres); vm_page_unlock(*mres); } VM_OBJECT_WUNLOCK(vm_obj); retry: ret = 0; pinned = 0; page = NULL; if (i915_intr_pf) { ret = i915_mutex_lock_interruptible(dev); if (ret != 0) goto out; } else DRM_LOCK(dev); /* * Since the object lock was dropped, other thread might have * faulted on the same GTT address and instantiated the * mapping for the page. Recheck. */ VM_OBJECT_WLOCK(vm_obj); page = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); if (page != NULL) { if (vm_page_busied(page)) { DRM_UNLOCK(dev); vm_page_lock(page); VM_OBJECT_WUNLOCK(vm_obj); vm_page_busy_sleep(page, "915pee"); goto retry; } goto have_page; } else VM_OBJECT_WUNLOCK(vm_obj); /* Now bind it into the GTT if needed */ ret = i915_gem_object_pin(obj, 0, true, false); if (ret) goto unlock; pinned = 1; ret = i915_gem_object_set_to_gtt_domain(obj, write); if (ret) goto unpin; ret = i915_gem_object_get_fence(obj); if (ret) goto unpin; obj->fault_mappable = true; VM_OBJECT_WLOCK(vm_obj); page = PHYS_TO_VM_PAGE(dev_priv->mm.gtt_base_addr + obj->gtt_offset + offset); KASSERT((page->flags & PG_FICTITIOUS) != 0, ("physical address %#jx not fictitious", (uintmax_t)(dev_priv->mm.gtt_base_addr + obj->gtt_offset + offset))); if (page == NULL) { VM_OBJECT_WUNLOCK(vm_obj); ret = -EFAULT; goto unpin; } KASSERT((page->flags & PG_FICTITIOUS) != 0, ("not fictitious %p", page)); KASSERT(page->wire_count == 1, ("wire_count not 1 %p", page)); if (vm_page_busied(page)) { i915_gem_object_unpin(obj); DRM_UNLOCK(dev); vm_page_lock(page); VM_OBJECT_WUNLOCK(vm_obj); vm_page_busy_sleep(page, "915pbs"); goto retry; } if (vm_page_insert(page, vm_obj, OFF_TO_IDX(offset))) { i915_gem_object_unpin(obj); DRM_UNLOCK(dev); VM_OBJECT_WUNLOCK(vm_obj); VM_WAIT; goto retry; } page->valid = VM_PAGE_BITS_ALL; have_page: vm_page_xbusy(page); CTR4(KTR_DRM, "fault %p %jx %x phys %x", gem_obj, offset, prot, page->phys_addr); if (pinned) { /* * We may have not pinned the object if the page was * found by the call to vm_page_lookup() */ i915_gem_object_unpin(obj); } DRM_UNLOCK(dev); if (*mres != NULL) { - KASSERT(*mres != page, ("loosing %p %p", *mres, page)); + KASSERT(*mres != page, ("losing %p %p", *mres, page)); vm_page_lock(*mres); vm_page_free(*mres); vm_page_unlock(*mres); } *mres = page; vm_object_pip_wakeup(vm_obj); return (VM_PAGER_OK); unpin: i915_gem_object_unpin(obj); unlock: DRM_UNLOCK(dev); out: KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); CTR4(KTR_DRM, "fault_fail %p %jx %x err %d", gem_obj, offset, prot, -ret); if (ret == -ERESTARTSYS) { /* * NOTE Linux<->FreeBSD: Convert Linux' -ERESTARTSYS to * the more common -EINTR, so the page fault is retried. */ ret = -EINTR; } if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { kern_yield(PRI_USER); goto retry; } VM_OBJECT_WLOCK(vm_obj); vm_object_pip_wakeup(vm_obj); return (VM_PAGER_ERROR); } static void i915_gem_pager_dtor(void *handle) { struct drm_gem_object *obj = handle; struct drm_device *dev = obj->dev; DRM_LOCK(dev); drm_gem_object_unreference(obj); DRM_UNLOCK(dev); } struct cdev_pager_ops i915_gem_pager_ops = { .cdev_pg_fault = i915_gem_pager_fault, .cdev_pg_ctor = i915_gem_pager_ctor, .cdev_pg_dtor = i915_gem_pager_dtor }; /** * i915_gem_release_mmap - remove physical page mappings * @obj: obj in question * * Preserve the reservation of the mmapping with the DRM core code, but * relinquish ownership of the pages back to the system. * * It is vital that we remove the page mapping if we have mapped a tiled * object through the GTT and then lose the fence register due to * resource pressure. Similarly if the object has been moved out of the * aperture, than pages mapped into userspace must be revoked. Removing the * mapping will then trigger a page fault on the next user access, allowing * fixup by i915_gem_fault(). */ void i915_gem_release_mmap(struct drm_i915_gem_object *obj) { vm_object_t devobj; vm_page_t page; int i, page_count; if (!obj->fault_mappable) return; CTR3(KTR_DRM, "release_mmap %p %x %x", obj, obj->gtt_offset, OFF_TO_IDX(obj->base.size)); devobj = cdev_pager_lookup(obj); if (devobj != NULL) { page_count = OFF_TO_IDX(obj->base.size); VM_OBJECT_WLOCK(devobj); retry: for (i = 0; i < page_count; i++) { page = vm_page_lookup(devobj, i); if (page == NULL) continue; if (vm_page_sleep_if_busy(page, "915unm")) goto retry; cdev_pager_free_page(devobj, page); } VM_OBJECT_WUNLOCK(devobj); vm_object_deallocate(devobj); } obj->fault_mappable = false; } static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) { uint32_t gtt_size; if (INTEL_INFO(dev)->gen >= 4 || tiling_mode == I915_TILING_NONE) return size; /* Previous chips need a power-of-two fence region when tiling */ if (INTEL_INFO(dev)->gen == 3) gtt_size = 1024*1024; else gtt_size = 512*1024; while (gtt_size < size) gtt_size <<= 1; return gtt_size; } /** * i915_gem_get_gtt_alignment - return required GTT alignment for an object * @obj: object to check * * Return the required GTT alignment for an object, taking into account * potential fence register mapping. */ static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode) { /* * Minimum alignment is 4k (GTT page size), but might be greater * if a fence register is needed for the object. */ if (INTEL_INFO(dev)->gen >= 4 || tiling_mode == I915_TILING_NONE) return 4096; /* * Previous chips need to be aligned to the size of the smallest * fence register that can contain the object. */ return i915_gem_get_gtt_size(dev, size, tiling_mode); } /** * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an * unfenced object * @dev: the device * @size: size of the object * @tiling_mode: tiling mode of the object * * Return the required GTT alignment for an object, only taking into account * unfenced tiled surface requirements. */ uint32_t i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, int tiling_mode) { /* * Minimum alignment is 4k (GTT page size) for sane hw. */ if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || tiling_mode == I915_TILING_NONE) return 4096; /* Previous hardware however needs to be aligned to a power-of-two * tile height. The simplest method for determining this is to reuse * the power-of-tile object size. */ return i915_gem_get_gtt_size(dev, size, tiling_mode); } static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; int ret; if (obj->base.on_map) return 0; dev_priv->mm.shrinker_no_lock_stealing = true; ret = drm_gem_create_mmap_offset(&obj->base); if (ret != -ENOSPC) goto out; /* Badly fragmented mmap space? The only way we can recover * space is by destroying unwanted objects. We can't randomly release * mmap_offsets as userspace expects them to be persistent for the * lifetime of the objects. The closest we can is to release the * offsets on purgeable objects by truncating it and marking it purged, * which prevents userspace from ever using that object again. */ i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT); ret = drm_gem_create_mmap_offset(&obj->base); if (ret != -ENOSPC) goto out; i915_gem_shrink_all(dev_priv); ret = drm_gem_create_mmap_offset(&obj->base); out: dev_priv->mm.shrinker_no_lock_stealing = false; return ret; } static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) { if (!obj->base.on_map) return; drm_gem_free_mmap_offset(&obj->base); } int i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, uint32_t handle, uint64_t *offset) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->base.size > dev_priv->mm.gtt_mappable_end) { ret = -E2BIG; goto out; } if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to mmap a purgeable buffer\n"); ret = -EINVAL; goto out; } ret = i915_gem_object_create_mmap_offset(obj); if (ret) goto out; *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | DRM_GEM_MAPPING_KEY; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } /** * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing * @dev: DRM device * @data: GTT mapping ioctl data * @file: GEM object info * * Simply returns the fake offset to userspace so it can mmap it. * The mmap call will end up in drm_gem_mmap(), which will set things * up so we can get faults in the handler above. * * The fault handler will take care of binding the object into the GTT * (since it may have been evicted to make room for something), allocating * a fence register, and mapping the appropriate aperture address into * userspace. */ int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_mmap_gtt *args = data; return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); } /* Immediately discard the backing storage */ static void i915_gem_object_truncate(struct drm_i915_gem_object *obj) { vm_object_t vm_obj; vm_obj = obj->base.vm_obj; VM_OBJECT_WLOCK(vm_obj); vm_object_page_remove(vm_obj, 0, 0, false); VM_OBJECT_WUNLOCK(vm_obj); i915_gem_object_free_mmap_offset(obj); obj->madv = __I915_MADV_PURGED; } static inline int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) { return obj->madv == I915_MADV_DONTNEED; } static void i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj, vm_pindex_t si, vm_pindex_t ei) { vm_object_t vm_obj; vm_page_t page; vm_pindex_t i; vm_obj = obj->base.vm_obj; VM_OBJECT_ASSERT_LOCKED(vm_obj); for (i = si, page = vm_page_lookup(vm_obj, i); i < ei; page = vm_page_next(page), i++) { KASSERT(page->pindex == i, ("pindex %jx %jx", (uintmax_t)page->pindex, (uintmax_t)i)); vm_page_lock(page); vm_page_unwire(page, PQ_INACTIVE); if (page->wire_count == 0) atomic_add_long(&i915_gem_wired_pages_cnt, -1); vm_page_unlock(page); } } #define GEM_PARANOID_CHECK_GTT 0 #if GEM_PARANOID_CHECK_GTT static void i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, int page_count) { struct drm_i915_private *dev_priv; vm_paddr_t pa; unsigned long start, end; u_int i; int j; dev_priv = dev->dev_private; start = OFF_TO_IDX(dev_priv->mm.gtt_start); end = OFF_TO_IDX(dev_priv->mm.gtt_end); for (i = start; i < end; i++) { pa = intel_gtt_read_pte_paddr(i); for (j = 0; j < page_count; j++) { if (pa == VM_PAGE_TO_PHYS(ma[j])) { panic("Page %p in GTT pte index %d pte %x", ma[i], i, intel_gtt_read_pte(i)); } } } } #endif static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) { int page_count = obj->base.size / PAGE_SIZE; int ret, i; BUG_ON(obj->madv == __I915_MADV_PURGED); ret = i915_gem_object_set_to_cpu_domain(obj, true); if (ret) { /* In the event of a disaster, abandon all caches and * hope for the best. */ WARN_ON(ret != -EIO); i915_gem_clflush_object(obj); obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; } if (i915_gem_object_needs_bit17_swizzle(obj)) i915_gem_object_save_bit_17_swizzle(obj); if (obj->madv == I915_MADV_DONTNEED) obj->dirty = 0; VM_OBJECT_WLOCK(obj->base.vm_obj); #if GEM_PARANOID_CHECK_GTT i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); #endif for (i = 0; i < page_count; i++) { vm_page_t page = obj->pages[i]; if (obj->dirty) vm_page_dirty(page); if (obj->madv == I915_MADV_WILLNEED) vm_page_reference(page); vm_page_lock(page); vm_page_unwire(obj->pages[i], PQ_ACTIVE); vm_page_unlock(page); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); obj->dirty = 0; free(obj->pages, DRM_I915_GEM); obj->pages = NULL; } static int i915_gem_object_put_pages(struct drm_i915_gem_object *obj) { const struct drm_i915_gem_object_ops *ops = obj->ops; if (obj->pages == NULL) return 0; BUG_ON(obj->gtt_space); if (obj->pages_pin_count) return -EBUSY; /* ->put_pages might need to allocate memory for the bit17 swizzle * array, hence protect them from being reaped by removing them from gtt * lists early. */ list_del(&obj->gtt_list); ops->put_pages(obj); obj->pages = NULL; if (i915_gem_object_is_purgeable(obj)) i915_gem_object_truncate(obj); return 0; } static long __i915_gem_shrink(struct drm_i915_private *dev_priv, long target, bool purgeable_only) { struct drm_i915_gem_object *obj, *next; long count = 0; list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list) { if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && i915_gem_object_put_pages(obj) == 0) { count += obj->base.size >> PAGE_SHIFT; if (target != -1 && count >= target) return count; } } list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, mm_list) { if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) && i915_gem_object_unbind(obj) == 0 && i915_gem_object_put_pages(obj) == 0) { count += obj->base.size >> PAGE_SHIFT; if (target != -1 && count >= target) return count; } } return count; } static long i915_gem_purge(struct drm_i915_private *dev_priv, long target) { return __i915_gem_shrink(dev_priv, target, true); } static void i915_gem_shrink_all(struct drm_i915_private *dev_priv) { struct drm_i915_gem_object *obj, *next; i915_gem_evict_everything(dev_priv->dev); list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list) i915_gem_object_put_pages(obj); } static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj, off_t start, off_t end) { vm_object_t vm_obj; vm_page_t page; vm_pindex_t si, ei, i; bool need_swizzle, fresh; need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0; vm_obj = obj->base.vm_obj; si = OFF_TO_IDX(trunc_page(start)); ei = OFF_TO_IDX(round_page(end)); VM_OBJECT_WLOCK(vm_obj); for (i = si; i < ei; i++) { page = i915_gem_wire_page(vm_obj, i, &fresh); if (page == NULL) goto failed; if (need_swizzle && fresh) i915_gem_object_do_bit_17_swizzle_page(obj, page); } VM_OBJECT_WUNLOCK(vm_obj); return (0); failed: i915_gem_object_put_pages_range_locked(obj, si, i); VM_OBJECT_WUNLOCK(vm_obj); return (-EIO); } static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) { vm_object_t vm_obj; vm_page_t page; vm_pindex_t i, page_count; int res; /* Assert that the object is not currently in any GPU domain. As it * wasn't in the GTT, there shouldn't be any way it could have been in * a GPU cache */ BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); KASSERT(obj->pages == NULL, ("Obj already has pages")); page_count = OFF_TO_IDX(obj->base.size); obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK); res = i915_gem_object_get_pages_range(obj, 0, obj->base.size); if (res != 0) { free(obj->pages, DRM_I915_GEM); obj->pages = NULL; return (res); } vm_obj = obj->base.vm_obj; VM_OBJECT_WLOCK(vm_obj); for (i = 0, page = vm_page_lookup(vm_obj, 0); i < page_count; i++, page = vm_page_next(page)) { KASSERT(page->pindex == i, ("pindex %jx %jx", (uintmax_t)page->pindex, (uintmax_t)i)); obj->pages[i] = page; } VM_OBJECT_WUNLOCK(vm_obj); return (0); } /* Ensure that the associated pages are gathered from the backing storage * and pinned into our object. i915_gem_object_get_pages() may be called * multiple times before they are released by a single call to * i915_gem_object_put_pages() - once the pages are no longer referenced * either as a result of memory pressure (reaping pages under the shrinker) * or as the object is itself released. */ int i915_gem_object_get_pages(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; const struct drm_i915_gem_object_ops *ops = obj->ops; int ret; if (obj->pages) return 0; BUG_ON(obj->pages_pin_count); ret = ops->get_pages(obj); if (ret) return ret; list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); return 0; } void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, struct intel_ring_buffer *ring) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 seqno = intel_ring_get_seqno(ring); BUG_ON(ring == NULL); obj->ring = ring; /* Add a reference if we're newly entering the active list. */ if (!obj->active) { drm_gem_object_reference(&obj->base); obj->active = 1; } /* Move from whatever list we were on to the tail of execution. */ list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); list_move_tail(&obj->ring_list, &ring->active_list); obj->last_read_seqno = seqno; if (obj->fenced_gpu_access) { obj->last_fenced_seqno = seqno; /* Bump MRU to take account of the delayed flush */ if (obj->fence_reg != I915_FENCE_REG_NONE) { struct drm_i915_fence_reg *reg; reg = &dev_priv->fence_regs[obj->fence_reg]; list_move_tail(®->lru_list, &dev_priv->mm.fence_list); } } } static void i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); BUG_ON(!obj->active); list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); list_del_init(&obj->ring_list); obj->ring = NULL; obj->last_read_seqno = 0; obj->last_write_seqno = 0; obj->base.write_domain = 0; obj->last_fenced_seqno = 0; obj->fenced_gpu_access = false; obj->active = 0; drm_gem_object_unreference(&obj->base); WARN_ON(i915_verify_lists(dev)); } static int i915_gem_handle_seqno_wrap(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; int ret, i, j; /* The hardware uses various monotonic 32-bit counters, if we * detect that they will wraparound we need to idle the GPU * and reset those counters. */ ret = 0; for_each_ring(ring, dev_priv, i) { for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) ret |= ring->sync_seqno[j] != 0; } if (ret == 0) return ret; ret = i915_gpu_idle(dev); if (ret) return ret; i915_gem_retire_requests(dev); for_each_ring(ring, dev_priv, i) { for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++) ring->sync_seqno[j] = 0; } return 0; } int i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) { struct drm_i915_private *dev_priv = dev->dev_private; /* reserve 0 for non-seqno */ if (dev_priv->next_seqno == 0) { int ret = i915_gem_handle_seqno_wrap(dev); if (ret) return ret; dev_priv->next_seqno = 1; } *seqno = dev_priv->next_seqno++; return 0; } int i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, u32 *out_seqno) { drm_i915_private_t *dev_priv = ring->dev->dev_private; struct drm_i915_gem_request *request; u32 request_ring_position; int was_empty; int ret; /* * Emit any outstanding flushes - execbuf can fail to emit the flush * after having emitted the batchbuffer command. Hence we need to fix * things up similar to emitting the lazy request. The difference here * is that the flush _must_ happen before the next request, no matter * what. */ ret = intel_ring_flush_all_caches(ring); if (ret) return ret; request = malloc(sizeof(*request), DRM_I915_GEM, M_NOWAIT); if (request == NULL) return -ENOMEM; /* Record the position of the start of the request so that * should we detect the updated seqno part-way through the * GPU processing the request, we never over-estimate the * position of the head. */ request_ring_position = intel_ring_get_tail(ring); ret = ring->add_request(ring); if (ret) { free(request, DRM_I915_GEM); return ret; } request->seqno = intel_ring_get_seqno(ring); request->ring = ring; request->tail = request_ring_position; request->emitted_jiffies = jiffies; was_empty = list_empty(&ring->request_list); list_add_tail(&request->list, &ring->request_list); request->file_priv = NULL; if (file) { struct drm_i915_file_private *file_priv = file->driver_priv; mtx_lock(&file_priv->mm.lock); request->file_priv = file_priv; list_add_tail(&request->client_list, &file_priv->mm.request_list); mtx_unlock(&file_priv->mm.lock); } CTR2(KTR_DRM, "request_add %s %d", ring->name, request->seqno); ring->outstanding_lazy_request = 0; if (!dev_priv->mm.suspended) { if (i915_enable_hangcheck) { callout_schedule(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD); } if (was_empty) { taskqueue_enqueue_timeout(dev_priv->wq, &dev_priv->mm.retire_work, hz); intel_mark_busy(dev_priv->dev); } } if (out_seqno) *out_seqno = request->seqno; return 0; } static inline void i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) { struct drm_i915_file_private *file_priv = request->file_priv; if (!file_priv) return; mtx_lock(&file_priv->mm.lock); if (request->file_priv) { list_del(&request->client_list); request->file_priv = NULL; } mtx_unlock(&file_priv->mm.lock); } static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, struct intel_ring_buffer *ring) { if (ring->dev != NULL) DRM_LOCK_ASSERT(ring->dev); while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; request = list_first_entry(&ring->request_list, struct drm_i915_gem_request, list); list_del(&request->list); i915_gem_request_remove_from_client(request); free(request, DRM_I915_GEM); } while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&ring->active_list, struct drm_i915_gem_object, ring_list); i915_gem_object_move_to_inactive(obj); } } static void i915_gem_reset_fences(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int i; for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; i915_gem_write_fence(dev, i, NULL); if (reg->obj) i915_gem_object_fence_lost(reg->obj); reg->pin_count = 0; reg->obj = NULL; INIT_LIST_HEAD(®->lru_list); } INIT_LIST_HEAD(&dev_priv->mm.fence_list); } void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; struct intel_ring_buffer *ring; int i; for_each_ring(ring, dev_priv, i) i915_gem_reset_ring_lists(dev_priv, ring); /* Move everything out of the GPU domains to ensure we do any * necessary invalidation upon reuse. */ list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; } /* The fence registers are invalidated so clear them out */ i915_gem_reset_fences(dev); } /** * This function clears the request list as sequence numbers are passed. */ void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) { uint32_t seqno; if (list_empty(&ring->request_list)) return; WARN_ON(i915_verify_lists(ring->dev)); seqno = ring->get_seqno(ring, true); CTR2(KTR_DRM, "retire_request_ring %s %d", ring->name, seqno); while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; request = list_first_entry(&ring->request_list, struct drm_i915_gem_request, list); if (!i915_seqno_passed(seqno, request->seqno)) break; CTR2(KTR_DRM, "retire_request_seqno_passed %s %d", ring->name, seqno); /* We know the GPU must have read the request to have * sent us the seqno + interrupt, so use the position * of tail of the request to update the last known position * of the GPU head. */ ring->last_retired_head = request->tail; list_del(&request->list); i915_gem_request_remove_from_client(request); free(request, DRM_I915_GEM); } /* Move any buffers on the active list that are no longer referenced * by the ringbuffer to the flushing/inactive lists as appropriate. */ while (!list_empty(&ring->active_list)) { struct drm_i915_gem_object *obj; obj = list_first_entry(&ring->active_list, struct drm_i915_gem_object, ring_list); if (!i915_seqno_passed(seqno, obj->last_read_seqno)) break; i915_gem_object_move_to_inactive(obj); } if (unlikely(ring->trace_irq_seqno && i915_seqno_passed(seqno, ring->trace_irq_seqno))) { ring->irq_put(ring); ring->trace_irq_seqno = 0; } WARN_ON(i915_verify_lists(ring->dev)); } void i915_gem_retire_requests(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; int i; for_each_ring(ring, dev_priv, i) i915_gem_retire_requests_ring(ring); } static void i915_gem_retire_work_handler(void *arg, int pending) { drm_i915_private_t *dev_priv; struct drm_device *dev; struct intel_ring_buffer *ring; bool idle; int i; dev_priv = arg; dev = dev_priv->dev; /* Come back later if the device is busy... */ if (!sx_try_xlock(&dev->dev_struct_lock)) { taskqueue_enqueue_timeout(dev_priv->wq, &dev_priv->mm.retire_work, hz); return; } CTR0(KTR_DRM, "retire_task"); i915_gem_retire_requests(dev); /* Send a periodic flush down the ring so we don't hold onto GEM * objects indefinitely. */ idle = true; for_each_ring(ring, dev_priv, i) { if (ring->gpu_caches_dirty) i915_add_request(ring, NULL, NULL); idle &= list_empty(&ring->request_list); } if (!dev_priv->mm.suspended && !idle) taskqueue_enqueue_timeout(dev_priv->wq, &dev_priv->mm.retire_work, hz); if (idle) intel_mark_idle(dev); DRM_UNLOCK(dev); } /** * Ensures that an object will eventually get non-busy by flushing any required * write domains, emitting any outstanding lazy request and retiring and * completed requests. */ static int i915_gem_object_flush_active(struct drm_i915_gem_object *obj) { int ret; if (obj->active) { ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); if (ret) return ret; i915_gem_retire_requests_ring(obj->ring); } return 0; } /** * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT * @DRM_IOCTL_ARGS: standard ioctl arguments * * Returns 0 if successful, else an error is returned with the remaining time in * the timeout parameter. * -ETIME: object is still busy after timeout * -ERESTARTSYS: signal interrupted the wait * -ENONENT: object doesn't exist * Also possible, but rare: * -EAGAIN: GPU wedged * -ENOMEM: damn * -ENODEV: Internal IRQ fail * -E?: The add request failed * * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any * non-zero timeout parameter the wait ioctl will wait for the given number of * nanoseconds on an object becoming unbusy. Since the wait itself does so * without holding struct_mutex the object may become re-busied before this * function completes. A similar but shorter * race condition exists in the busy * ioctl */ int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_wait *args = data; struct drm_i915_gem_object *obj; struct intel_ring_buffer *ring = NULL; struct timespec timeout_stack, *timeout = NULL; u32 seqno = 0; int ret = 0; if (args->timeout_ns >= 0) { timeout_stack.tv_sec = args->timeout_ns / 1000000; timeout_stack.tv_nsec = args->timeout_ns % 1000000; timeout = &timeout_stack; } ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle)); if (&obj->base == NULL) { DRM_UNLOCK(dev); return -ENOENT; } /* Need to make sure the object gets inactive eventually. */ ret = i915_gem_object_flush_active(obj); if (ret) goto out; if (obj->active) { seqno = obj->last_read_seqno; ring = obj->ring; } if (seqno == 0) goto out; /* Do this after OLR check to make sure we make forward progress polling * on this IOCTL with a 0 timeout (like busy ioctl) */ if (!args->timeout_ns) { ret = -ETIMEDOUT; goto out; } drm_gem_object_unreference(&obj->base); DRM_UNLOCK(dev); ret = __wait_seqno(ring, seqno, true, timeout); if (timeout) { args->timeout_ns = timeout->tv_sec * 1000000 + timeout->tv_nsec; } return ret; out: drm_gem_object_unreference(&obj->base); DRM_UNLOCK(dev); return ret; } /** * i915_gem_object_sync - sync an object to a ring. * * @obj: object which may be in use on another ring. * @to: ring we wish to use the object on. May be NULL. * * This code is meant to abstract object synchronization with the GPU. * Calling with NULL implies synchronizing the object with the CPU * rather than a particular GPU ring. * * Returns 0 if successful, else propagates up the lower layer error. */ int i915_gem_object_sync(struct drm_i915_gem_object *obj, struct intel_ring_buffer *to) { struct intel_ring_buffer *from = obj->ring; u32 seqno; int ret, idx; if (from == NULL || to == from) return 0; if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) return i915_gem_object_wait_rendering(obj, false); idx = intel_ring_sync_index(from, to); seqno = obj->last_read_seqno; if (seqno <= from->sync_seqno[idx]) return 0; ret = i915_gem_check_olr(obj->ring, seqno); if (ret) return ret; ret = to->sync_to(to, from, seqno); if (!ret) /* We use last_read_seqno because sync_to() * might have just caused seqno wrap under * the radar. */ from->sync_seqno[idx] = obj->last_read_seqno; return ret; } static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) { u32 old_write_domain, old_read_domains; /* Act a barrier for all accesses through the GTT */ mb(); /* Force a pagefault for domain tracking on next user access */ i915_gem_release_mmap(obj); if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) return; old_read_domains = obj->base.read_domains; old_write_domain = obj->base.write_domain; obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; CTR3(KTR_DRM, "object_change_domain finish gtt %p %x %x", obj, old_read_domains, old_write_domain); } /** * Unbinds an object from the GTT aperture. */ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = obj->base.dev->dev_private; int ret = 0; if (obj->gtt_space == NULL) return 0; if (obj->pin_count) return -EBUSY; BUG_ON(obj->pages == NULL); ret = i915_gem_object_finish_gpu(obj); if (ret) return ret; /* Continue on if we fail due to EIO, the GPU is hung so we * should be safe and we need to cleanup or else we might * cause memory corruption through use-after-free. */ i915_gem_object_finish_gtt(obj); /* release the fence reg _after_ flushing */ ret = i915_gem_object_put_fence(obj); if (ret) return ret; if (obj->has_global_gtt_mapping) i915_gem_gtt_unbind_object(obj); if (obj->has_aliasing_ppgtt_mapping) { i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); obj->has_aliasing_ppgtt_mapping = 0; } i915_gem_gtt_finish_object(obj); list_del(&obj->mm_list); list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); /* Avoid an unnecessary call to unbind on rebind. */ obj->map_and_fenceable = true; drm_mm_put_block(obj->gtt_space); obj->gtt_space = NULL; obj->gtt_offset = 0; return 0; } int i915_gpu_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; int ret, i; /* Flush everything onto the inactive list. */ for_each_ring(ring, dev_priv, i) { ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); if (ret) return ret; ret = intel_ring_idle(ring); if (ret) return ret; } return 0; } static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; uint64_t val; if (obj) { u32 size = obj->gtt_space->size; val = (uint64_t)((obj->gtt_offset + size - 4096) & 0xfffff000) << 32; val |= obj->gtt_offset & 0xfffff000; val |= (uint64_t)((obj->stride / 128) - 1) << SANDYBRIDGE_FENCE_PITCH_SHIFT; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; } else val = 0; I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); } static void i965_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; uint64_t val; if (obj) { u32 size = obj->gtt_space->size; val = (uint64_t)((obj->gtt_offset + size - 4096) & 0xfffff000) << 32; val |= obj->gtt_offset & 0xfffff000; val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; } else val = 0; I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); POSTING_READ(FENCE_REG_965_0 + reg * 8); } static void i915_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; u32 val; if (obj) { u32 size = obj->gtt_space->size; int pitch_val; int tile_width; WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || (size & -size) != size || (obj->gtt_offset & (size - 1)), "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", obj->gtt_offset, obj->map_and_fenceable, size); if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) tile_width = 128; else tile_width = 512; /* Note: pitch better be a power of two tile widths */ pitch_val = obj->stride / tile_width; pitch_val = ffs(pitch_val) - 1; val = obj->gtt_offset; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; val |= I915_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; } else val = 0; if (reg < 8) reg = FENCE_REG_830_0 + reg * 4; else reg = FENCE_REG_945_8 + (reg - 8) * 4; I915_WRITE(reg, val); POSTING_READ(reg); } static void i830_write_fence_reg(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; uint32_t val; if (obj) { u32 size = obj->gtt_space->size; uint32_t pitch_val; WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || (size & -size) != size || (obj->gtt_offset & (size - 1)), "object 0x%08x not 512K or pot-size 0x%08x aligned\n", obj->gtt_offset, size); pitch_val = obj->stride / 128; pitch_val = ffs(pitch_val) - 1; val = obj->gtt_offset; if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; val |= I830_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; } else val = 0; I915_WRITE(FENCE_REG_830_0 + reg * 4, val); POSTING_READ(FENCE_REG_830_0 + reg * 4); } static void i915_gem_write_fence(struct drm_device *dev, int reg, struct drm_i915_gem_object *obj) { switch (INTEL_INFO(dev)->gen) { case 7: case 6: sandybridge_write_fence_reg(dev, reg, obj); break; case 5: case 4: i965_write_fence_reg(dev, reg, obj); break; case 3: i915_write_fence_reg(dev, reg, obj); break; case 2: i830_write_fence_reg(dev, reg, obj); break; default: break; } } static inline int fence_number(struct drm_i915_private *dev_priv, struct drm_i915_fence_reg *fence) { return fence - dev_priv->fence_regs; } static void i915_gem_write_fence__ipi(void *data) { wbinvd(); } static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, struct drm_i915_fence_reg *fence, bool enable) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int fence_reg = fence_number(dev_priv, fence); /* In order to fully serialize access to the fenced region and * the update to the fence register we need to take extreme * measures on SNB+. In theory, the write to the fence register * flushes all memory transactions before, and coupled with the * mb() placed around the register write we serialise all memory * operations with respect to the changes in the tiler. Yet, on * SNB+ we need to take a step further and emit an explicit wbinvd() * on each processor in order to manually flush all memory * transactions before updating the fence register. */ if (HAS_LLC(obj->base.dev)) on_each_cpu(i915_gem_write_fence__ipi, NULL, 1); i915_gem_write_fence(dev, fence_reg, enable ? obj : NULL); if (enable) { obj->fence_reg = fence_reg; fence->obj = obj; list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); } else { obj->fence_reg = I915_FENCE_REG_NONE; fence->obj = NULL; list_del_init(&fence->lru_list); } } static int i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) { if (obj->last_fenced_seqno) { int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); if (ret) return ret; obj->last_fenced_seqno = 0; } /* Ensure that all CPU reads are completed before installing a fence * and all writes before removing the fence. */ if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) mb(); obj->fenced_gpu_access = false; return 0; } int i915_gem_object_put_fence(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; int ret; ret = i915_gem_object_flush_fence(obj); if (ret) return ret; if (obj->fence_reg == I915_FENCE_REG_NONE) return 0; i915_gem_object_update_fence(obj, &dev_priv->fence_regs[obj->fence_reg], false); i915_gem_object_fence_lost(obj); return 0; } static struct drm_i915_fence_reg * i915_find_fence_reg(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_fence_reg *reg, *avail; int i; /* First try to find a free reg */ avail = NULL; for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { reg = &dev_priv->fence_regs[i]; if (!reg->obj) return reg; if (!reg->pin_count) avail = reg; } if (avail == NULL) return NULL; /* None available, try to steal one or wait for a user to finish */ list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { if (reg->pin_count) continue; return reg; } return NULL; } /** * i915_gem_object_get_fence - set up fencing for an object * @obj: object to map through a fence reg * * When mapping objects through the GTT, userspace wants to be able to write * to them without having to worry about swizzling if the object is tiled. * This function walks the fence regs looking for a free one for @obj, * stealing one if it can't find any. * * It then sets up the reg based on the object's properties: address, pitch * and tiling format. * * For an untiled surface, this removes any existing fence. */ int i915_gem_object_get_fence(struct drm_i915_gem_object *obj) { struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; bool enable = obj->tiling_mode != I915_TILING_NONE; struct drm_i915_fence_reg *reg; int ret; /* Have we updated the tiling parameters upon the object and so * will need to serialise the write to the associated fence register? */ if (obj->fence_dirty) { ret = i915_gem_object_flush_fence(obj); if (ret) return ret; } /* Just update our place in the LRU if our fence is getting reused. */ if (obj->fence_reg != I915_FENCE_REG_NONE) { reg = &dev_priv->fence_regs[obj->fence_reg]; if (!obj->fence_dirty) { list_move_tail(®->lru_list, &dev_priv->mm.fence_list); return 0; } } else if (enable) { reg = i915_find_fence_reg(dev); if (reg == NULL) return -EDEADLK; if (reg->obj) { struct drm_i915_gem_object *old = reg->obj; ret = i915_gem_object_flush_fence(old); if (ret) return ret; i915_gem_object_fence_lost(old); } } else return 0; i915_gem_object_update_fence(obj, reg, enable); obj->fence_dirty = false; return 0; } static bool i915_gem_valid_gtt_space(struct drm_device *dev, struct drm_mm_node *gtt_space, unsigned long cache_level) { struct drm_mm_node *other; /* On non-LLC machines we have to be careful when putting differing * types of snoopable memory together to avoid the prefetcher - * crossing memory domains and dieing. + * crossing memory domains and dying. */ if (HAS_LLC(dev)) return true; if (gtt_space == NULL) return true; if (list_empty(>t_space->node_list)) return true; other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); if (other->allocated && !other->hole_follows && other->color != cache_level) return false; other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) return false; return true; } static void i915_gem_verify_gtt(struct drm_device *dev) { #if WATCH_GTT struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; int err = 0; list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { if (obj->gtt_space == NULL) { DRM_ERROR("object found on GTT list with no space reserved\n"); err++; continue; } if (obj->cache_level != obj->gtt_space->color) { DRM_ERROR("object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n", obj->gtt_space->start, obj->gtt_space->start + obj->gtt_space->size, obj->cache_level, obj->gtt_space->color); err++; continue; } if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, obj->cache_level)) { DRM_ERROR("invalid GTT space found at [%08lx, %08lx] - color=%x\n", obj->gtt_space->start, obj->gtt_space->start + obj->gtt_space->size, obj->cache_level); err++; continue; } } WARN_ON(err); #endif } /** * Finds free space in the GTT aperture and binds the object there. */ static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, unsigned alignment, bool map_and_fenceable, bool nonblocking) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; struct drm_mm_node *node; u32 size, fence_size, fence_alignment, unfenced_alignment; bool mappable, fenceable; int ret; if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to bind a purgeable object\n"); return -EINVAL; } fence_size = i915_gem_get_gtt_size(dev, obj->base.size, obj->tiling_mode); fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, obj->tiling_mode); unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, obj->base.size, obj->tiling_mode); if (alignment == 0) alignment = map_and_fenceable ? fence_alignment : unfenced_alignment; if (map_and_fenceable && alignment & (fence_alignment - 1)) { DRM_ERROR("Invalid object alignment requested %u\n", alignment); return -EINVAL; } size = map_and_fenceable ? fence_size : obj->base.size; /* If the object is bigger than the entire aperture, reject it early * before evicting everything in a vain attempt to find space. */ if (obj->base.size > (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { DRM_ERROR("Attempting to bind an object larger than the aperture\n"); return -E2BIG; } ret = i915_gem_object_get_pages(obj); if (ret) return ret; i915_gem_object_pin_pages(obj); node = malloc(sizeof(*node), DRM_MEM_MM, M_NOWAIT | M_ZERO); if (node == NULL) { i915_gem_object_unpin_pages(obj); return -ENOMEM; } search_free: if (map_and_fenceable) ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node, size, alignment, obj->cache_level, 0, dev_priv->mm.gtt_mappable_end); else ret = drm_mm_insert_node_generic(&dev_priv->mm.gtt_space, node, size, alignment, obj->cache_level); if (ret) { ret = i915_gem_evict_something(dev, size, alignment, obj->cache_level, map_and_fenceable, nonblocking); if (ret == 0) goto search_free; i915_gem_object_unpin_pages(obj); free(node, DRM_MEM_MM); return ret; } if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) { i915_gem_object_unpin_pages(obj); drm_mm_put_block(node); return -EINVAL; } ret = i915_gem_gtt_prepare_object(obj); if (ret) { i915_gem_object_unpin_pages(obj); drm_mm_put_block(node); return ret; } list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list); list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); obj->gtt_space = node; obj->gtt_offset = node->start; fenceable = node->size == fence_size && (node->start & (fence_alignment - 1)) == 0; mappable = obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; obj->map_and_fenceable = mappable && fenceable; i915_gem_object_unpin_pages(obj); CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset, obj->base.size, map_and_fenceable); i915_gem_verify_gtt(dev); return 0; } void i915_gem_clflush_object(struct drm_i915_gem_object *obj) { /* If we don't have a page list set up, then we're not pinned * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ if (obj->pages == NULL) return; /* If the GPU is snooping the contents of the CPU cache, * we do not need to manually clear the CPU cache lines. However, * the caches are only snooped when the render cache is * flushed/invalidated. As we always have to emit invalidations * and flushes when moving into and out of the RENDER domain, correct * snooping behaviour occurs naturally as the result of our domain * tracking. */ if (obj->cache_level != I915_CACHE_NONE) return; CTR1(KTR_DRM, "object_clflush %p", obj); drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); } /** Flushes the GTT write domain for the object if it's dirty. */ static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) { uint32_t old_write_domain; if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) return; /* No actual flushing is required for the GTT write domain. Writes * to it immediately go to main memory as far as we know, so there's * no chipset flush. It also doesn't land in render cache. * * However, we do have to enforce the order so that all writes through * the GTT land before any writes to the device, such as updates to * the GATT itself. */ wmb(); old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj, obj->base.read_domains, old_write_domain); } /** Flushes the CPU write domain for the object if it's dirty. */ static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) { uint32_t old_write_domain; if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) return; i915_gem_clflush_object(obj); i915_gem_chipset_flush(obj->base.dev); old_write_domain = obj->base.write_domain; obj->base.write_domain = 0; CTR3(KTR_DRM, "object_change_domain flush_cpu_write %p %x %x", obj, obj->base.read_domains, old_write_domain); } /** * Moves a single object to the GTT read, and possibly write domain. * * This function returns when the move is complete, including waiting on * flushes to occur. */ int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { drm_i915_private_t *dev_priv = obj->base.dev->dev_private; uint32_t old_write_domain, old_read_domains; int ret; /* Not valid to be called on unbound objects. */ if (obj->gtt_space == NULL) return -EINVAL; if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) return 0; ret = i915_gem_object_wait_rendering(obj, !write); if (ret) return ret; i915_gem_object_flush_cpu_write_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; obj->dirty = 1; } CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj, old_read_domains, old_write_domain); /* And bump the LRU for this access */ if (i915_gem_object_is_inactive(obj)) list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); return 0; } int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, enum i915_cache_level cache_level) { struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; int ret; if (obj->cache_level == cache_level) return 0; if (obj->pin_count) { DRM_DEBUG("can not change the cache level of pinned objects\n"); return -EBUSY; } if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { ret = i915_gem_object_unbind(obj); if (ret) return ret; } if (obj->gtt_space) { ret = i915_gem_object_finish_gpu(obj); if (ret) return ret; i915_gem_object_finish_gtt(obj); /* Before SandyBridge, you could not use tiling or fence * registers with snooped memory, so relinquish any fences * currently pointing to our region in the aperture. */ if (INTEL_INFO(dev)->gen < 6) { ret = i915_gem_object_put_fence(obj); if (ret) return ret; } if (obj->has_global_gtt_mapping) i915_gem_gtt_bind_object(obj, cache_level); if (obj->has_aliasing_ppgtt_mapping) i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, obj, cache_level); obj->gtt_space->color = cache_level; } if (cache_level == I915_CACHE_NONE) { u32 old_read_domains, old_write_domain; /* If we're coming from LLC cached, then we haven't * actually been tracking whether the data is in the * CPU cache or not, since we only allow one bit set * in obj->write_domain and have been skipping the clflushes. * Just set it to the CPU cache for now. */ WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU); WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU); old_read_domains = obj->base.read_domains; old_write_domain = obj->base.write_domain; obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x", obj, old_read_domains, old_write_domain); } obj->cache_level = cache_level; i915_gem_verify_gtt(dev); return 0; } int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } args->caching = obj->cache_level != I915_CACHE_NONE; drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; int ret; switch (args->caching) { case I915_CACHING_NONE: level = I915_CACHE_NONE; break; case I915_CACHING_CACHED: level = I915_CACHE_LLC; break; default: return -EINVAL; } ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } ret = i915_gem_object_set_cache_level(obj, level); drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } static bool is_pin_display(struct drm_i915_gem_object *obj) { /* There are 3 sources that pin objects: * 1. The display engine (scanouts, sprites, cursors); * 2. Reservations for execbuffer; * 3. The user. * * We can ignore reservations as we hold the struct_mutex and * are only called outside of the reservation path. The user * can only increment pin_count once, and so if after * subtracting the potential reference by the user, any pin_count * remains, it must be due to another use by the display engine. */ return obj->pin_count - !!obj->user_pin_count; } /* * Prepare buffer for display plane (scanout, cursors, etc). * Can be called from an uninterruptible phase (modesetting) and allows * any flushes to be pipelined (for pageflips). */ int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 alignment, struct intel_ring_buffer *pipelined) { u32 old_read_domains, old_write_domain; int ret; if (pipelined != obj->ring) { ret = i915_gem_object_sync(obj, pipelined); if (ret) return ret; } /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */ obj->pin_display = true; /* The display engine is not coherent with the LLC cache on gen6. As * a result, we make sure that the pinning that is about to occur is * done with uncached PTEs. This is lowest common denominator for all * chipsets. * * However for gen6+, we could do better by using the GFDT bit instead * of uncaching, which would allow us to flush all the LLC-cached data * with that bit in the PTE to main memory with just one PIPE_CONTROL. */ ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); if (ret) goto err_unpin_display; /* As the user may map the buffer once pinned in the display plane * (e.g. libkms for the bootup splash), we have to ensure that we * always use map_and_fenceable for all scanout buffers. */ ret = i915_gem_object_pin(obj, alignment, true, false); if (ret) goto err_unpin_display; i915_gem_object_flush_cpu_write_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ obj->base.write_domain = 0; obj->base.read_domains |= I915_GEM_DOMAIN_GTT; CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x", obj, old_read_domains, old_write_domain); return 0; err_unpin_display: obj->pin_display = is_pin_display(obj); return ret; } void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj) { i915_gem_object_unpin(obj); obj->pin_display = is_pin_display(obj); } int i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) { int ret; if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) return 0; ret = i915_gem_object_wait_rendering(obj, false); if (ret) return ret; /* Ensure that we invalidate the GPU's caches and TLBs. */ obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; return 0; } /** * Moves a single object to the CPU read, and possibly write domain. * * This function returns when the move is complete, including waiting on * flushes to occur. */ int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) { uint32_t old_write_domain, old_read_domains; int ret; if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) return 0; ret = i915_gem_object_wait_rendering(obj, !write); if (ret) return ret; i915_gem_object_flush_gtt_write_domain(obj); old_write_domain = obj->base.write_domain; old_read_domains = obj->base.read_domains; /* Flush the CPU cache if it's still invalid. */ if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { i915_gem_clflush_object(obj); obj->base.read_domains |= I915_GEM_DOMAIN_CPU; } /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); /* If we're writing through the CPU, then the GPU read domains will * need to be invalidated at next use. */ if (write) { obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj, old_read_domains, old_write_domain); return 0; } /* Throttle our rendering by waiting until the ring has completed our requests * emitted over 20 msec ago. * * Note that if we were to use the current jiffies each time around the loop, * we wouldn't escape the function with any frames outstanding if the time to * render a frame was over 20ms. * * This should get us reasonable parallelism between CPU and GPU but also * relatively low latency when blocking on a particular request to finish. */ static int i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - msecs_to_jiffies(20); struct drm_i915_gem_request *request; struct intel_ring_buffer *ring = NULL; u32 seqno = 0; int ret; if (atomic_read(&dev_priv->mm.wedged)) return -EIO; mtx_lock(&file_priv->mm.lock); list_for_each_entry(request, &file_priv->mm.request_list, client_list) { if (time_after_eq(request->emitted_jiffies, recent_enough)) break; ring = request->ring; seqno = request->seqno; } mtx_unlock(&file_priv->mm.lock); if (seqno == 0) return 0; ret = __wait_seqno(ring, seqno, true, NULL); if (ret == 0) taskqueue_enqueue_timeout(dev_priv->wq, &dev_priv->mm.retire_work, 0); return ret; } int i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, bool map_and_fenceable, bool nonblocking) { int ret; if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) return -EBUSY; if (obj->gtt_space != NULL) { if ((alignment && obj->gtt_offset & (alignment - 1)) || (map_and_fenceable && !obj->map_and_fenceable)) { WARN(obj->pin_count, "bo is already pinned with incorrect alignment:" " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," " obj->map_and_fenceable=%d\n", obj->gtt_offset, alignment, map_and_fenceable, obj->map_and_fenceable); ret = i915_gem_object_unbind(obj); if (ret) return ret; } } if (obj->gtt_space == NULL) { struct drm_i915_private *dev_priv = obj->base.dev->dev_private; ret = i915_gem_object_bind_to_gtt(obj, alignment, map_and_fenceable, nonblocking); if (ret) return ret; if (!dev_priv->mm.aliasing_ppgtt) i915_gem_gtt_bind_object(obj, obj->cache_level); } if (!obj->has_global_gtt_mapping && map_and_fenceable) i915_gem_gtt_bind_object(obj, obj->cache_level); obj->pin_count++; obj->pin_mappable |= map_and_fenceable; return 0; } void i915_gem_object_unpin(struct drm_i915_gem_object *obj) { BUG_ON(obj->pin_count == 0); BUG_ON(obj->gtt_space == NULL); if (--obj->pin_count == 0) obj->pin_mappable = false; } int i915_gem_pin_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pin *args = data; struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to pin a purgeable buffer\n"); ret = -EINVAL; goto out; } if (obj->pin_filp != NULL && obj->pin_filp != file) { DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } if (obj->user_pin_count == 0) { ret = i915_gem_object_pin(obj, args->alignment, true, false); if (ret) goto out; } obj->user_pin_count++; obj->pin_filp = file; /* XXX - flush the CPU caches for pinned objects * as the X server doesn't manage domains yet */ i915_gem_object_flush_cpu_write_domain(obj); args->offset = obj->gtt_offset; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } int i915_gem_unpin_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pin *args = data; struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->pin_filp != file) { DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } obj->user_pin_count--; if (obj->user_pin_count == 0) { obj->pin_filp = NULL; i915_gem_object_unpin(obj); } out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } int i915_gem_busy_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } /* Count all active objects as busy, even if they are currently not used * by the gpu. Users of this interface expect objects to eventually * become non-busy without any further actions, therefore emit any * necessary flushes here. */ ret = i915_gem_object_flush_active(obj); args->busy = obj->active; if (obj->ring) { BUILD_BUG_ON(I915_NUM_RINGS > 16); args->busy |= intel_ring_flag(obj->ring) << 16; } drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } int i915_gem_throttle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { return i915_gem_ring_throttle(dev, file_priv); } int i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_i915_gem_madvise *args = data; struct drm_i915_gem_object *obj; int ret; switch (args->madv) { case I915_MADV_DONTNEED: case I915_MADV_WILLNEED: break; default: return -EINVAL; } ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); if (&obj->base == NULL) { ret = -ENOENT; goto unlock; } if (obj->pin_count) { ret = -EINVAL; goto out; } if (obj->madv != __I915_MADV_PURGED) obj->madv = args->madv; /* if the object is no longer attached, discard its backing storage */ if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) i915_gem_object_truncate(obj); args->retained = obj->madv != __I915_MADV_PURGED; out: drm_gem_object_unreference(&obj->base); unlock: DRM_UNLOCK(dev); return ret; } void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops) { INIT_LIST_HEAD(&obj->mm_list); INIT_LIST_HEAD(&obj->gtt_list); INIT_LIST_HEAD(&obj->ring_list); INIT_LIST_HEAD(&obj->exec_list); obj->ops = ops; obj->fence_reg = I915_FENCE_REG_NONE; obj->madv = I915_MADV_WILLNEED; /* Avoid an unnecessary call to unbind on the first bind. */ obj->map_and_fenceable = true; i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); } static const struct drm_i915_gem_object_ops i915_gem_object_ops = { .get_pages = i915_gem_object_get_pages_gtt, .put_pages = i915_gem_object_put_pages_gtt, }; struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, size_t size) { struct drm_i915_gem_object *obj; obj = malloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); if (obj == NULL) return NULL; if (drm_gem_object_init(dev, &obj->base, size) != 0) { free(obj, DRM_I915_GEM); return NULL; } #ifdef FREEBSD_WIP mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { /* 965gm cannot relocate objects above 4GiB. */ mask &= ~__GFP_HIGHMEM; mask |= __GFP_DMA32; } mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; mapping_set_gfp_mask(mapping, mask); #endif /* FREEBSD_WIP */ i915_gem_object_init(obj, &i915_gem_object_ops); obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; if (HAS_LLC(dev)) { /* On some devices, we can have the GPU use the LLC (the CPU * cache) for about a 10% performance improvement * compared to uncached. Graphics requests other than * display scanout are coherent with the CPU in * accessing this cache. This means in this mode we * don't need to clflush on the CPU side, and on the * GPU side we only need to flush internal caches to * get data visible to the CPU. * * However, we maintain the display planes as UC, and so * need to rebind when first used as such. */ obj->cache_level = I915_CACHE_LLC; } else obj->cache_level = I915_CACHE_NONE; return obj; } int i915_gem_init_object(struct drm_gem_object *obj) { printf("i915_gem_init_object called\n"); return 0; } void i915_gem_free_object(struct drm_gem_object *gem_obj) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; CTR1(KTR_DRM, "object_destroy_tail %p", obj); if (obj->phys_obj) i915_gem_detach_phys_object(dev, obj); obj->pin_count = 0; if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { bool was_interruptible; was_interruptible = dev_priv->mm.interruptible; dev_priv->mm.interruptible = false; WARN_ON(i915_gem_object_unbind(obj)); dev_priv->mm.interruptible = was_interruptible; } obj->pages_pin_count = 0; i915_gem_object_put_pages(obj); i915_gem_object_free_mmap_offset(obj); BUG_ON(obj->pages); #ifdef FREEBSD_WIP if (obj->base.import_attach) drm_prime_gem_destroy(&obj->base, NULL); #endif /* FREEBSD_WIP */ drm_gem_object_release(&obj->base); i915_gem_info_remove_obj(dev_priv, obj->base.size); free(obj->bit_17, DRM_I915_GEM); free(obj, DRM_I915_GEM); } int i915_gem_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; DRM_LOCK(dev); if (dev_priv->mm.suspended) { DRM_UNLOCK(dev); return 0; } ret = i915_gpu_idle(dev); if (ret) { DRM_UNLOCK(dev); return ret; } i915_gem_retire_requests(dev); /* Under UMS, be paranoid and evict. */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) i915_gem_evict_everything(dev); i915_gem_reset_fences(dev); /* Hack! Don't let anybody do execbuf while we don't control the chip. * We need to replace this with a semaphore, or something. * And not confound mm.suspended! */ dev_priv->mm.suspended = 1; callout_stop(&dev_priv->hangcheck_timer); i915_kernel_lost_context(dev); i915_gem_cleanup_ringbuffer(dev); DRM_UNLOCK(dev); /* Cancel the retire work handler, which should be idle now. */ taskqueue_cancel_timeout(dev_priv->wq, &dev_priv->mm.retire_work, NULL); return 0; } void i915_gem_l3_remap(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; u32 misccpctl; int i; if (!HAS_L3_GPU_CACHE(dev)) return; if (!dev_priv->l3_parity.remap_info) return; misccpctl = I915_READ(GEN7_MISCCPCTL); I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); POSTING_READ(GEN7_MISCCPCTL); for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { u32 remap = I915_READ(GEN7_L3LOG_BASE + i); if (remap && remap != dev_priv->l3_parity.remap_info[i/4]) DRM_DEBUG("0x%x was already programmed to %x\n", GEN7_L3LOG_BASE + i, remap); if (remap && !dev_priv->l3_parity.remap_info[i/4]) DRM_DEBUG_DRIVER("Clearing remapped register\n"); I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]); } /* Make sure all the writes land before disabling dop clock gating */ POSTING_READ(GEN7_L3LOG_BASE); I915_WRITE(GEN7_MISCCPCTL, misccpctl); } void i915_gem_init_swizzling(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; if (INTEL_INFO(dev)->gen < 5 || dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) return; I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | DISP_TILE_SURFACE_SWIZZLING); if (IS_GEN5(dev)) return; I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); if (IS_GEN6(dev)) I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); else I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); } static bool intel_enable_blt(struct drm_device *dev) { if (!HAS_BLT(dev)) return false; /* The blitter was dysfunctional on early prototypes */ if (IS_GEN6(dev) && pci_get_revid(dev->dev) < 8) { DRM_INFO("BLT not supported on this pre-production hardware;" " graphics performance will be degraded.\n"); return false; } return true; } int i915_gem_init_hw(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; #ifdef FREEBSD_WIP if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) return -EIO; #endif /* FREEBSD_WIP */ if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); i915_gem_l3_remap(dev); i915_gem_init_swizzling(dev); ret = intel_init_render_ring_buffer(dev); if (ret) return ret; if (HAS_BSD(dev)) { ret = intel_init_bsd_ring_buffer(dev); if (ret) goto cleanup_render_ring; } if (intel_enable_blt(dev)) { ret = intel_init_blt_ring_buffer(dev); if (ret) goto cleanup_bsd_ring; } dev_priv->next_seqno = 1; /* * XXX: There was some w/a described somewhere suggesting loading * contexts before PPGTT. */ i915_gem_context_init(dev); i915_gem_init_ppgtt(dev); return 0; cleanup_bsd_ring: intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); cleanup_render_ring: intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); return ret; } static bool intel_enable_ppgtt(struct drm_device *dev) { if (i915_enable_ppgtt >= 0) return i915_enable_ppgtt; #ifdef CONFIG_INTEL_IOMMU /* Disable ppgtt on SNB if VT-d is on. */ if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) return false; #endif return true; } int i915_gem_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; unsigned long gtt_size, mappable_size; int ret; gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; DRM_LOCK(dev); if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { /* PPGTT pdes are stolen from global gtt ptes, so shrink the * aperture accordingly when using aliasing ppgtt. */ gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); ret = i915_gem_init_aliasing_ppgtt(dev); if (ret) { DRM_UNLOCK(dev); return ret; } } else { /* Let GEM Manage all of the aperture. * * However, leave one page at the end still bound to the scratch * page. There are a number of places where the hardware * apparently prefetches past the end of the object, and we've * seen multiple hangs with the GPU head pointer stuck in a * batchbuffer bound at the last page of the aperture. One page * should be enough to keep any prefetching inside of the * aperture. */ i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size); } ret = i915_gem_init_hw(dev); DRM_UNLOCK(dev); if (ret) { i915_gem_cleanup_aliasing_ppgtt(dev); return ret; } /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->dri1.allow_batchbuffer = 1; return 0; } void i915_gem_cleanup_ringbuffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_ring_buffer *ring; int i; for_each_ring(ring, dev_priv, i) intel_cleanup_ring_buffer(ring); } int i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; if (atomic_read(&dev_priv->mm.wedged)) { DRM_ERROR("Reenabling wedged hardware, good luck\n"); atomic_set(&dev_priv->mm.wedged, 0); } DRM_LOCK(dev); dev_priv->mm.suspended = 0; ret = i915_gem_init_hw(dev); if (ret != 0) { DRM_UNLOCK(dev); return ret; } BUG_ON(!list_empty(&dev_priv->mm.active_list)); DRM_UNLOCK(dev); ret = drm_irq_install(dev); if (ret) goto cleanup_ringbuffer; return 0; cleanup_ringbuffer: DRM_LOCK(dev); i915_gem_cleanup_ringbuffer(dev); dev_priv->mm.suspended = 1; DRM_UNLOCK(dev); return ret; } int i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; drm_irq_uninstall(dev); return i915_gem_idle(dev); } void i915_gem_lastclose(struct drm_device *dev) { int ret; if (drm_core_check_feature(dev, DRIVER_MODESET)) return; ret = i915_gem_idle(dev); if (ret) DRM_ERROR("failed to idle hardware: %d\n", ret); } static void init_ring_lists(struct intel_ring_buffer *ring) { INIT_LIST_HEAD(&ring->active_list); INIT_LIST_HEAD(&ring->request_list); } void i915_gem_load(struct drm_device *dev) { int i; drm_i915_private_t *dev_priv = dev->dev_private; INIT_LIST_HEAD(&dev_priv->mm.active_list); INIT_LIST_HEAD(&dev_priv->mm.inactive_list); INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); for (i = 0; i < I915_NUM_RINGS; i++) init_ring_lists(&dev_priv->ring[i]); for (i = 0; i < I915_MAX_NUM_FENCES; i++) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); TIMEOUT_TASK_INIT(dev_priv->wq, &dev_priv->mm.retire_work, 0, i915_gem_retire_work_handler, dev_priv); init_completion(&dev_priv->error_completion); /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ if (IS_GEN3(dev)) { I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); } dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; /* Old X drivers will take 0-2 for front, back, depth buffers */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->fence_reg_start = 3; if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) dev_priv->num_fence_regs = 16; else dev_priv->num_fence_regs = 8; /* Initialize fence registers to zero */ i915_gem_reset_fences(dev); i915_gem_detect_bit_6_swizzle(dev); DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue); dev_priv->mm.interruptible = true; dev_priv->mm.inactive_shrinker = EVENTHANDLER_REGISTER(vm_lowmem, i915_gem_inactive_shrink, dev, EVENTHANDLER_PRI_ANY); } /* * Create a physically contiguous memory object for this object * e.g. for cursor + overlay regs */ static int i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_phys_object *phys_obj; int ret; if (dev_priv->mm.phys_objs[id - 1] || !size) return 0; phys_obj = malloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, M_WAITOK | M_ZERO); if (!phys_obj) return -ENOMEM; phys_obj->id = id; phys_obj->handle = drm_pci_alloc(dev, size, align, BUS_SPACE_MAXADDR); if (!phys_obj->handle) { ret = -ENOMEM; goto kfree_obj; } #ifdef CONFIG_X86 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, size / PAGE_SIZE, PAT_WRITE_COMBINING); #endif dev_priv->mm.phys_objs[id - 1] = phys_obj; return 0; kfree_obj: free(phys_obj, DRM_I915_GEM); return ret; } static void i915_gem_free_phys_object(struct drm_device *dev, int id) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_phys_object *phys_obj; if (!dev_priv->mm.phys_objs[id - 1]) return; phys_obj = dev_priv->mm.phys_objs[id - 1]; if (phys_obj->cur_obj) { i915_gem_detach_phys_object(dev, phys_obj->cur_obj); } #ifdef FREEBSD_WIP #ifdef CONFIG_X86 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE); #endif #endif /* FREEBSD_WIP */ drm_pci_free(dev, phys_obj->handle); free(phys_obj, DRM_I915_GEM); dev_priv->mm.phys_objs[id - 1] = NULL; } void i915_gem_free_all_phys_object(struct drm_device *dev) { int i; for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) i915_gem_free_phys_object(dev, i); } void i915_gem_detach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj) { struct sf_buf *sf; char *vaddr; char *dst; int i; int page_count; if (!obj->phys_obj) return; vaddr = obj->phys_obj->handle->vaddr; page_count = obj->base.size / PAGE_SIZE; VM_OBJECT_WLOCK(obj->base.vm_obj); for (i = 0; i < page_count; i++) { vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); if (page == NULL) continue; /* XXX */ VM_OBJECT_WUNLOCK(obj->base.vm_obj); sf = sf_buf_alloc(page, 0); if (sf != NULL) { dst = (char *)sf_buf_kva(sf); memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); sf_buf_free(sf); } drm_clflush_pages(&page, 1); VM_OBJECT_WLOCK(obj->base.vm_obj); vm_page_reference(page); vm_page_lock(page); vm_page_dirty(page); vm_page_unwire(page, PQ_INACTIVE); vm_page_unlock(page); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); i915_gem_chipset_flush(dev); obj->phys_obj->cur_obj = NULL; obj->phys_obj = NULL; } int i915_gem_attach_phys_object(struct drm_device *dev, struct drm_i915_gem_object *obj, int id, int align) { drm_i915_private_t *dev_priv = dev->dev_private; struct sf_buf *sf; char *dst, *src; int ret = 0; int page_count; int i; if (id > I915_MAX_PHYS_OBJECT) return -EINVAL; if (obj->phys_obj) { if (obj->phys_obj->id == id) return 0; i915_gem_detach_phys_object(dev, obj); } /* create a new object */ if (!dev_priv->mm.phys_objs[id - 1]) { ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); if (ret) { DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->base.size); return ret; } } /* bind to the object */ obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; obj->phys_obj->cur_obj = obj; page_count = obj->base.size / PAGE_SIZE; VM_OBJECT_WLOCK(obj->base.vm_obj); for (i = 0; i < page_count; i++) { vm_page_t page = i915_gem_wire_page(obj->base.vm_obj, i, NULL); if (page == NULL) { ret = -EIO; break; } VM_OBJECT_WUNLOCK(obj->base.vm_obj); sf = sf_buf_alloc(page, 0); src = (char *)sf_buf_kva(sf); dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); memcpy(dst, src, PAGE_SIZE); sf_buf_free(sf); VM_OBJECT_WLOCK(obj->base.vm_obj); vm_page_reference(page); vm_page_lock(page); vm_page_unwire(page, PQ_INACTIVE); vm_page_unlock(page); atomic_add_long(&i915_gem_wired_pages_cnt, -1); } VM_OBJECT_WUNLOCK(obj->base.vm_obj); return ret; } static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file_priv) { void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset; char __user *user_data = to_user_ptr(args->data_ptr); if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { unsigned long unwritten; /* The physical object once assigned is fixed for the lifetime * of the obj, so we can safely drop the lock and continue * to access vaddr. */ DRM_UNLOCK(dev); unwritten = copy_from_user(vaddr, user_data, args->size); DRM_LOCK(dev); if (unwritten) return -EFAULT; } i915_gem_chipset_flush(dev); return 0; } void i915_gem_release(struct drm_device *dev, struct drm_file *file) { struct drm_i915_file_private *file_priv = file->driver_priv; /* Clean up our request list when the client is going away, so that * later retire_requests won't dereference our soon-to-be-gone * file_priv. */ mtx_lock(&file_priv->mm.lock); while (!list_empty(&file_priv->mm.request_list)) { struct drm_i915_gem_request *request; request = list_first_entry(&file_priv->mm.request_list, struct drm_i915_gem_request, client_list); list_del(&request->client_list); request->file_priv = NULL; } mtx_unlock(&file_priv->mm.lock); } static void i915_gem_inactive_shrink(void *arg) { struct drm_device *dev = arg; struct drm_i915_private *dev_priv = dev->dev_private; int pass1, pass2; if (!sx_try_xlock(&dev->dev_struct_lock)) { return; } CTR0(KTR_DRM, "gem_lowmem"); pass1 = i915_gem_purge(dev_priv, -1); pass2 = __i915_gem_shrink(dev_priv, -1, false); if (pass2 <= pass1 / 100) i915_gem_shrink_all(dev_priv); DRM_UNLOCK(dev); } static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh) { vm_page_t page; int rv; VM_OBJECT_ASSERT_WLOCKED(object); page = vm_page_grab(object, pindex, VM_ALLOC_NORMAL); if (page->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(object, pindex, NULL, NULL)) { rv = vm_pager_get_pages(object, &page, 1, NULL, NULL); if (rv != VM_PAGER_OK) { vm_page_lock(page); vm_page_free(page); vm_page_unlock(page); return (NULL); } if (fresh != NULL) *fresh = true; } else { pmap_zero_page(page); page->valid = VM_PAGE_BITS_ALL; page->dirty = 0; if (fresh != NULL) *fresh = false; } } else if (fresh != NULL) { *fresh = false; } vm_page_lock(page); vm_page_wire(page); vm_page_unlock(page); vm_page_xunbusy(page); atomic_add_long(&i915_gem_wired_pages_cnt, 1); return (page); } Index: head/sys/dev/drm2/i915/intel_dp.c =================================================================== --- head/sys/dev/drm2/i915/intel_dp.c (revision 298930) +++ head/sys/dev/drm2/i915/intel_dp.c (revision 298931) @@ -1,2924 +1,2924 @@ /* * Copyright © 2008 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * * Authors: * Keith Packard * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #define DP_LINK_CHECK_TIMEOUT (10 * 1000) /** * is_edp - is the given port attached to an eDP panel (either CPU or PCH) * @intel_dp: DP struct * * If a CPU or PCH DP output is attached to an eDP panel, this function * will return true, and false otherwise. */ static bool is_edp(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); return intel_dig_port->base.type == INTEL_OUTPUT_EDP; } /** * is_pch_edp - is the port on the PCH and attached to an eDP panel? * @intel_dp: DP struct * * Returns true if the given DP struct corresponds to a PCH DP port attached * to an eDP panel, false otherwise. Helpful for determining whether we * may need FDI resources for a given DP output or not. */ static bool is_pch_edp(struct intel_dp *intel_dp) { return intel_dp->is_pch_edp; } /** * is_cpu_edp - is the port on the CPU and attached to an eDP panel? * @intel_dp: DP struct * * Returns true if the given DP struct corresponds to a CPU eDP port. */ static bool is_cpu_edp(struct intel_dp *intel_dp) { return is_edp(intel_dp) && !is_pch_edp(intel_dp); } static struct drm_device *intel_dp_to_dev(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); return intel_dig_port->base.base.dev; } static struct intel_dp *intel_attached_dp(struct drm_connector *connector) { return enc_to_intel_dp(&intel_attached_encoder(connector)->base); } /** * intel_encoder_is_pch_edp - is the given encoder a PCH attached eDP? * @encoder: DRM encoder * * Return true if @encoder corresponds to a PCH attached eDP panel. Needed * by intel_display.c. */ bool intel_encoder_is_pch_edp(struct drm_encoder *encoder) { struct intel_dp *intel_dp; if (!encoder) return false; intel_dp = enc_to_intel_dp(encoder); return is_pch_edp(intel_dp); } static void intel_dp_link_down(struct intel_dp *intel_dp); void intel_edp_link_config(struct intel_encoder *intel_encoder, int *lane_num, int *link_bw) { struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); *lane_num = intel_dp->lane_count; *link_bw = drm_dp_bw_code_to_link_rate(intel_dp->link_bw); } int intel_edp_target_clock(struct intel_encoder *intel_encoder, struct drm_display_mode *mode) { struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); struct intel_connector *intel_connector = intel_dp->attached_connector; if (intel_connector->panel.fixed_mode) return intel_connector->panel.fixed_mode->clock; else return mode->clock; } static int intel_dp_max_link_bw(struct intel_dp *intel_dp) { int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE]; switch (max_link_bw) { case DP_LINK_BW_1_62: case DP_LINK_BW_2_7: break; default: max_link_bw = DP_LINK_BW_1_62; break; } return max_link_bw; } static int intel_dp_link_clock(uint8_t link_bw) { if (link_bw == DP_LINK_BW_2_7) return 270000; else return 162000; } /* * The units on the numbers in the next two are... bizarre. Examples will * make it clearer; this one parallels an example in the eDP spec. * * intel_dp_max_data_rate for one lane of 2.7GHz evaluates as: * * 270000 * 1 * 8 / 10 == 216000 * * The actual data capacity of that configuration is 2.16Gbit/s, so the * units are decakilobits. ->clock in a drm_display_mode is in kilohertz - * or equivalently, kilopixels per second - so for 1680x1050R it'd be * 119000. At 18bpp that's 2142000 kilobits per second. * * Thus the strange-looking division by 10 in intel_dp_link_required, to * get the result in decakilobits instead of kilobits. */ static int intel_dp_link_required(int pixel_clock, int bpp) { return (pixel_clock * bpp + 9) / 10; } static int intel_dp_max_data_rate(int max_link_clock, int max_lanes) { return (max_link_clock * max_lanes * 8) / 10; } static bool intel_dp_adjust_dithering(struct intel_dp *intel_dp, struct drm_display_mode *mode, bool adjust_mode) { int max_link_clock = intel_dp_link_clock(intel_dp_max_link_bw(intel_dp)); int max_lanes = drm_dp_max_lane_count(intel_dp->dpcd); int max_rate, mode_rate; mode_rate = intel_dp_link_required(mode->clock, 24); max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes); if (mode_rate > max_rate) { mode_rate = intel_dp_link_required(mode->clock, 18); if (mode_rate > max_rate) return false; if (adjust_mode) mode->private_flags |= INTEL_MODE_DP_FORCE_6BPC; return true; } return true; } static int intel_dp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; if (is_edp(intel_dp) && fixed_mode) { if (mode->hdisplay > fixed_mode->hdisplay) return MODE_PANEL; if (mode->vdisplay > fixed_mode->vdisplay) return MODE_PANEL; } if (!intel_dp_adjust_dithering(intel_dp, mode, false)) return MODE_CLOCK_HIGH; if (mode->clock < 10000) return MODE_CLOCK_LOW; if (mode->flags & DRM_MODE_FLAG_DBLCLK) return MODE_H_ILLEGAL; return MODE_OK; } static uint32_t pack_aux(uint8_t *src, int src_bytes) { int i; uint32_t v = 0; if (src_bytes > 4) src_bytes = 4; for (i = 0; i < src_bytes; i++) v |= ((uint32_t) src[i]) << ((3-i) * 8); return v; } static void unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes) { int i; if (dst_bytes > 4) dst_bytes = 4; for (i = 0; i < dst_bytes; i++) dst[i] = src >> ((3-i) * 8); } /* hrawclock is 1/4 the FSB frequency */ static int intel_hrawclk(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; uint32_t clkcfg; /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */ if (IS_VALLEYVIEW(dev)) return 200; clkcfg = I915_READ(CLKCFG); switch (clkcfg & CLKCFG_FSB_MASK) { case CLKCFG_FSB_400: return 100; case CLKCFG_FSB_533: return 133; case CLKCFG_FSB_667: return 166; case CLKCFG_FSB_800: return 200; case CLKCFG_FSB_1067: return 266; case CLKCFG_FSB_1333: return 333; /* these two are just a guess; one of them might be right */ case CLKCFG_FSB_1600: case CLKCFG_FSB_1600_ALT: return 400; default: return 133; } } static bool ironlake_edp_have_panel_power(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; return (I915_READ(PCH_PP_STATUS) & PP_ON) != 0; } static bool ironlake_edp_have_panel_vdd(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; return (I915_READ(PCH_PP_CONTROL) & EDP_FORCE_VDD) != 0; } static void intel_dp_check_edp(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; if (!is_edp(intel_dp)) return; if (!ironlake_edp_have_panel_power(intel_dp) && !ironlake_edp_have_panel_vdd(intel_dp)) { WARN(1, "eDP powered off while attempting aux channel communication.\n"); DRM_DEBUG_KMS("Status 0x%08x Control 0x%08x\n", I915_READ(PCH_PP_STATUS), I915_READ(PCH_PP_CONTROL)); } } static int intel_dp_aux_ch(struct intel_dp *intel_dp, uint8_t *send, int send_bytes, uint8_t *recv, int recv_size) { uint32_t output_reg = intel_dp->output_reg; struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; uint32_t ch_ctl = output_reg + 0x10; uint32_t ch_data = ch_ctl + 4; int i; int recv_bytes; uint32_t status; uint32_t aux_clock_divider; int try, precharge; if (IS_HASWELL(dev)) { switch (intel_dig_port->port) { case PORT_A: ch_ctl = DPA_AUX_CH_CTL; ch_data = DPA_AUX_CH_DATA1; break; case PORT_B: ch_ctl = PCH_DPB_AUX_CH_CTL; ch_data = PCH_DPB_AUX_CH_DATA1; break; case PORT_C: ch_ctl = PCH_DPC_AUX_CH_CTL; ch_data = PCH_DPC_AUX_CH_DATA1; break; case PORT_D: ch_ctl = PCH_DPD_AUX_CH_CTL; ch_data = PCH_DPD_AUX_CH_DATA1; break; default: BUG(); } } intel_dp_check_edp(intel_dp); /* The clock divider is based off the hrawclk, * and would like to run at 2MHz. So, take the * hrawclk value and divide by 2 and use that * * Note that PCH attached eDP panels should use a 125MHz input * clock divider. */ if (is_cpu_edp(intel_dp)) { if (IS_HASWELL(dev)) aux_clock_divider = intel_ddi_get_cdclk_freq(dev_priv) >> 1; else if (IS_VALLEYVIEW(dev)) aux_clock_divider = 100; else if (IS_GEN6(dev) || IS_GEN7(dev)) aux_clock_divider = 200; /* SNB & IVB eDP input clock at 400Mhz */ else aux_clock_divider = 225; /* eDP input clock at 450Mhz */ } else if (HAS_PCH_SPLIT(dev)) aux_clock_divider = DIV_ROUND_UP(intel_pch_rawclk(dev), 2); else aux_clock_divider = intel_hrawclk(dev) / 2; if (IS_GEN6(dev)) precharge = 3; else precharge = 5; /* Try to wait for any previous AUX channel activity */ for (try = 0; try < 3; try++) { status = I915_READ(ch_ctl); if ((status & DP_AUX_CH_CTL_SEND_BUSY) == 0) break; DRM_MSLEEP(1); } if (try == 3) { WARN(1, "dp_aux_ch not started status 0x%08x\n", I915_READ(ch_ctl)); return -EBUSY; } /* Must try at least 3 times according to DP spec */ for (try = 0; try < 5; try++) { /* Load the send data into the aux channel data registers */ for (i = 0; i < send_bytes; i += 4) I915_WRITE(ch_data + i, pack_aux(send + i, send_bytes - i)); /* Send the command and wait for it to complete */ I915_WRITE(ch_ctl, DP_AUX_CH_CTL_SEND_BUSY | DP_AUX_CH_CTL_TIME_OUT_400us | (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) | (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) | (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT) | DP_AUX_CH_CTL_DONE | DP_AUX_CH_CTL_TIME_OUT_ERROR | DP_AUX_CH_CTL_RECEIVE_ERROR); for (;;) { status = I915_READ(ch_ctl); if ((status & DP_AUX_CH_CTL_SEND_BUSY) == 0) break; udelay(100); } /* Clear done status and any errors */ I915_WRITE(ch_ctl, status | DP_AUX_CH_CTL_DONE | DP_AUX_CH_CTL_TIME_OUT_ERROR | DP_AUX_CH_CTL_RECEIVE_ERROR); if (status & (DP_AUX_CH_CTL_TIME_OUT_ERROR | DP_AUX_CH_CTL_RECEIVE_ERROR)) continue; if (status & DP_AUX_CH_CTL_DONE) break; } if ((status & DP_AUX_CH_CTL_DONE) == 0) { DRM_ERROR("dp_aux_ch not done status 0x%08x\n", status); return -EBUSY; } /* Check for timeout or receive error. * Timeouts occur when the sink is not connected */ if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) { DRM_ERROR("dp_aux_ch receive error status 0x%08x\n", status); return -EIO; } /* Timeouts occur when the device isn't connected, so they're * "normal" -- don't fill the kernel log with these */ if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR) { DRM_DEBUG_KMS("dp_aux_ch timeout status 0x%08x\n", status); return -ETIMEDOUT; } /* Unload any bytes sent back from the other side */ recv_bytes = ((status & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >> DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT); if (recv_bytes > recv_size) recv_bytes = recv_size; for (i = 0; i < recv_bytes; i += 4) unpack_aux(I915_READ(ch_data + i), recv + i, recv_bytes - i); return recv_bytes; } /* Write data to the aux channel in native mode */ static int intel_dp_aux_native_write(struct intel_dp *intel_dp, uint16_t address, uint8_t *send, int send_bytes) { int ret; uint8_t msg[20]; int msg_bytes; uint8_t ack; intel_dp_check_edp(intel_dp); if (send_bytes > 16) return -1; msg[0] = AUX_NATIVE_WRITE << 4; msg[1] = address >> 8; msg[2] = address & 0xff; msg[3] = send_bytes - 1; memcpy(&msg[4], send, send_bytes); msg_bytes = send_bytes + 4; for (;;) { ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, &ack, 1); if (ret < 0) return ret; if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK) break; else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER) udelay(100); else return -EIO; } return send_bytes; } /* Write a single byte to the aux channel in native mode */ static int intel_dp_aux_native_write_1(struct intel_dp *intel_dp, uint16_t address, uint8_t byte) { return intel_dp_aux_native_write(intel_dp, address, &byte, 1); } /* read bytes from a native aux channel */ static int intel_dp_aux_native_read(struct intel_dp *intel_dp, uint16_t address, uint8_t *recv, int recv_bytes) { uint8_t msg[4]; int msg_bytes; uint8_t reply[20]; int reply_bytes; uint8_t ack; int ret; intel_dp_check_edp(intel_dp); msg[0] = AUX_NATIVE_READ << 4; msg[1] = address >> 8; msg[2] = address & 0xff; msg[3] = recv_bytes - 1; msg_bytes = 4; reply_bytes = recv_bytes + 1; for (;;) { ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, reply, reply_bytes); if (ret == 0) return -EPROTO; if (ret < 0) return ret; ack = reply[0]; if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK) { memcpy(recv, reply + 1, ret - 1); return ret - 1; } else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER) udelay(100); else return -EIO; } } static int intel_dp_i2c_aux_ch(device_t adapter, int mode, uint8_t write_byte, uint8_t *read_byte) { struct iic_dp_aux_data *algo_data = device_get_softc(adapter); struct intel_dp *intel_dp = algo_data->priv; uint16_t address = algo_data->address; uint8_t msg[5]; uint8_t reply[2]; unsigned retry; int msg_bytes; int reply_bytes; int ret; intel_dp_check_edp(intel_dp); /* Set up the command byte */ if (mode & MODE_I2C_READ) msg[0] = AUX_I2C_READ << 4; else msg[0] = AUX_I2C_WRITE << 4; if (!(mode & MODE_I2C_STOP)) msg[0] |= AUX_I2C_MOT << 4; msg[1] = address >> 8; msg[2] = address; switch (mode) { case MODE_I2C_WRITE: msg[3] = 0; msg[4] = write_byte; msg_bytes = 5; reply_bytes = 1; break; case MODE_I2C_READ: msg[3] = 0; msg_bytes = 4; reply_bytes = 2; break; default: msg_bytes = 3; reply_bytes = 1; break; } for (retry = 0; retry < 5; retry++) { ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, reply, reply_bytes); if (ret < 0) { DRM_DEBUG_KMS("aux_ch failed %d\n", ret); return ret; } switch (reply[0] & AUX_NATIVE_REPLY_MASK) { case AUX_NATIVE_REPLY_ACK: /* I2C-over-AUX Reply field is only valid * when paired with AUX ACK. */ break; case AUX_NATIVE_REPLY_NACK: DRM_DEBUG_KMS("aux_ch native nack\n"); return -EREMOTEIO; case AUX_NATIVE_REPLY_DEFER: udelay(100); continue; default: DRM_ERROR("aux_ch invalid native reply 0x%02x\n", reply[0]); return -EREMOTEIO; } switch (reply[0] & AUX_I2C_REPLY_MASK) { case AUX_I2C_REPLY_ACK: if (mode == MODE_I2C_READ) { *read_byte = reply[1]; } return reply_bytes - 1; case AUX_I2C_REPLY_NACK: DRM_DEBUG_KMS("aux_i2c nack\n"); return -EREMOTEIO; case AUX_I2C_REPLY_DEFER: DRM_DEBUG_KMS("aux_i2c defer\n"); udelay(100); break; default: DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]); return -EREMOTEIO; } } DRM_ERROR("too many retries, giving up\n"); return -EREMOTEIO; } static int intel_dp_i2c_init(struct intel_dp *intel_dp, struct intel_connector *intel_connector, const char *name) { int ret; DRM_DEBUG_KMS("i2c_init %s\n", name); ironlake_edp_panel_vdd_on(intel_dp); ret = iic_dp_aux_add_bus(intel_connector->base.dev->dev, name, intel_dp_i2c_aux_ch, intel_dp, &intel_dp->dp_iic_bus, &intel_dp->adapter); ironlake_edp_panel_vdd_off(intel_dp, false); return ret; } bool intel_dp_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = encoder->dev; struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_connector *intel_connector = intel_dp->attached_connector; int lane_count, clock; int max_lane_count = drm_dp_max_lane_count(intel_dp->dpcd); int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0; int bpp, mode_rate; static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 }; if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) { intel_fixed_panel_mode(intel_connector->panel.fixed_mode, adjusted_mode); intel_pch_panel_fitting(dev, intel_connector->panel.fitting_mode, mode, adjusted_mode); } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) return false; DRM_DEBUG_KMS("DP link computation with max lane count %i " "max bw %02x pixel clock %iKHz\n", max_lane_count, bws[max_clock], adjusted_mode->clock); if (!intel_dp_adjust_dithering(intel_dp, adjusted_mode, true)) return false; bpp = adjusted_mode->private_flags & INTEL_MODE_DP_FORCE_6BPC ? 18 : 24; mode_rate = intel_dp_link_required(adjusted_mode->clock, bpp); for (clock = 0; clock <= max_clock; clock++) { for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) { int link_avail = intel_dp_max_data_rate(intel_dp_link_clock(bws[clock]), lane_count); if (mode_rate <= link_avail) { intel_dp->link_bw = bws[clock]; intel_dp->lane_count = lane_count; adjusted_mode->clock = intel_dp_link_clock(intel_dp->link_bw); DRM_DEBUG_KMS("DP link bw %02x lane " "count %d clock %d bpp %d\n", intel_dp->link_bw, intel_dp->lane_count, adjusted_mode->clock, bpp); DRM_DEBUG_KMS("DP link bw required %i available %i\n", mode_rate, link_avail); return true; } } } return false; } struct intel_dp_m_n { uint32_t tu; uint32_t gmch_m; uint32_t gmch_n; uint32_t link_m; uint32_t link_n; }; static void intel_reduce_ratio(uint32_t *num, uint32_t *den) { while (*num > 0xffffff || *den > 0xffffff) { *num >>= 1; *den >>= 1; } } static void intel_dp_compute_m_n(int bpp, int nlanes, int pixel_clock, int link_clock, struct intel_dp_m_n *m_n) { m_n->tu = 64; m_n->gmch_m = (pixel_clock * bpp) >> 3; m_n->gmch_n = link_clock * nlanes; intel_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n); m_n->link_m = pixel_clock; m_n->link_n = link_clock; intel_reduce_ratio(&m_n->link_m, &m_n->link_n); } void intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = crtc->dev; struct intel_encoder *intel_encoder; struct intel_dp *intel_dp; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int lane_count = 4; struct intel_dp_m_n m_n; int pipe = intel_crtc->pipe; enum transcoder cpu_transcoder = intel_crtc->cpu_transcoder; int target_clock; /* * Find the lane count in the intel_encoder private */ for_each_encoder_on_crtc(dev, crtc, intel_encoder) { intel_dp = enc_to_intel_dp(&intel_encoder->base); if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || intel_encoder->type == INTEL_OUTPUT_EDP) { lane_count = intel_dp->lane_count; break; } } target_clock = mode->clock; for_each_encoder_on_crtc(dev, crtc, intel_encoder) { if (intel_encoder->type == INTEL_OUTPUT_EDP) { target_clock = intel_edp_target_clock(intel_encoder, mode); break; } } /* * Compute the GMCH and Link ratios. The '3' here is * the number of bytes_per_pixel post-LUT, which we always * set up for 8-bits of R/G/B, or 3 bytes total. */ intel_dp_compute_m_n(intel_crtc->bpp, lane_count, target_clock, adjusted_mode->clock, &m_n); if (IS_HASWELL(dev)) { I915_WRITE(PIPE_DATA_M1(cpu_transcoder), TU_SIZE(m_n.tu) | m_n.gmch_m); I915_WRITE(PIPE_DATA_N1(cpu_transcoder), m_n.gmch_n); I915_WRITE(PIPE_LINK_M1(cpu_transcoder), m_n.link_m); I915_WRITE(PIPE_LINK_N1(cpu_transcoder), m_n.link_n); } else if (HAS_PCH_SPLIT(dev)) { I915_WRITE(TRANSDATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m); I915_WRITE(TRANSDATA_N1(pipe), m_n.gmch_n); I915_WRITE(TRANSDPLINK_M1(pipe), m_n.link_m); I915_WRITE(TRANSDPLINK_N1(pipe), m_n.link_n); } else if (IS_VALLEYVIEW(dev)) { I915_WRITE(PIPE_DATA_M1(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m); I915_WRITE(PIPE_DATA_N1(pipe), m_n.gmch_n); I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m); I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n); } else { I915_WRITE(PIPE_GMCH_DATA_M(pipe), TU_SIZE(m_n.tu) | m_n.gmch_m); I915_WRITE(PIPE_GMCH_DATA_N(pipe), m_n.gmch_n); I915_WRITE(PIPE_DP_LINK_M(pipe), m_n.link_m); I915_WRITE(PIPE_DP_LINK_N(pipe), m_n.link_n); } } void intel_dp_init_link_config(struct intel_dp *intel_dp) { memset(intel_dp->link_configuration, 0, DP_LINK_CONFIGURATION_SIZE); intel_dp->link_configuration[0] = intel_dp->link_bw; intel_dp->link_configuration[1] = intel_dp->lane_count; intel_dp->link_configuration[8] = DP_SET_ANSI_8B10B; /* * Check for DPCD version > 1.1 and enhanced framing support */ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && (intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP)) { intel_dp->link_configuration[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN; } } static void intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode) { struct drm_device *dev = encoder->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct drm_crtc *crtc = encoder->crtc; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); /* * There are four kinds of DP registers: * * IBX PCH * SNB CPU * IVB CPU * CPT PCH * * IBX PCH and CPU are the same for almost everything, * except that the CPU DP PLL is configured in this * register * * CPT PCH is quite different, having many bits moved * to the TRANS_DP_CTL register instead. That * configuration happens (oddly) in ironlake_pch_enable */ /* Preserve the BIOS-computed detected bit. This is * supposed to be read-only. */ intel_dp->DP = I915_READ(intel_dp->output_reg) & DP_DETECTED; /* Handle DP bits in common between all three register formats */ intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0; switch (intel_dp->lane_count) { case 1: intel_dp->DP |= DP_PORT_WIDTH_1; break; case 2: intel_dp->DP |= DP_PORT_WIDTH_2; break; case 4: intel_dp->DP |= DP_PORT_WIDTH_4; break; } if (intel_dp->has_audio) { DRM_DEBUG_DRIVER("Enabling DP audio on pipe %c\n", pipe_name(intel_crtc->pipe)); intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE; intel_write_eld(encoder, adjusted_mode); } intel_dp_init_link_config(intel_dp); /* Split out the IBX/CPU vs CPT settings */ if (is_cpu_edp(intel_dp) && IS_GEN7(dev) && !IS_VALLEYVIEW(dev)) { if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) intel_dp->DP |= DP_SYNC_HS_HIGH; if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) intel_dp->DP |= DP_SYNC_VS_HIGH; intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; if (intel_dp->link_configuration[1] & DP_LANE_COUNT_ENHANCED_FRAME_EN) intel_dp->DP |= DP_ENHANCED_FRAMING; intel_dp->DP |= intel_crtc->pipe << 29; /* don't miss out required setting for eDP */ if (adjusted_mode->clock < 200000) intel_dp->DP |= DP_PLL_FREQ_160MHZ; else intel_dp->DP |= DP_PLL_FREQ_270MHZ; } else if (!HAS_PCH_CPT(dev) || is_cpu_edp(intel_dp)) { intel_dp->DP |= intel_dp->color_range; if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) intel_dp->DP |= DP_SYNC_HS_HIGH; if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) intel_dp->DP |= DP_SYNC_VS_HIGH; intel_dp->DP |= DP_LINK_TRAIN_OFF; if (intel_dp->link_configuration[1] & DP_LANE_COUNT_ENHANCED_FRAME_EN) intel_dp->DP |= DP_ENHANCED_FRAMING; if (intel_crtc->pipe == 1) intel_dp->DP |= DP_PIPEB_SELECT; if (is_cpu_edp(intel_dp)) { /* don't miss out required setting for eDP */ if (adjusted_mode->clock < 200000) intel_dp->DP |= DP_PLL_FREQ_160MHZ; else intel_dp->DP |= DP_PLL_FREQ_270MHZ; } } else { intel_dp->DP |= DP_LINK_TRAIN_OFF_CPT; } } #define IDLE_ON_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) #define IDLE_ON_VALUE (PP_ON | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_ON_IDLE) #define IDLE_OFF_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | 0 | PP_SEQUENCE_STATE_MASK) #define IDLE_OFF_VALUE (0 | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) #define IDLE_CYCLE_MASK (PP_ON | 0 | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK) #define IDLE_CYCLE_VALUE (0 | 0 | PP_SEQUENCE_NONE | 0 | PP_SEQUENCE_STATE_OFF_IDLE) static void ironlake_wait_panel_status(struct intel_dp *intel_dp, u32 mask, u32 value) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; DRM_DEBUG_KMS("mask %08x value %08x status %08x control %08x\n", mask, value, I915_READ(PCH_PP_STATUS), I915_READ(PCH_PP_CONTROL)); if (_intel_wait_for(dev, (I915_READ(PCH_PP_STATUS) & mask) == value, 5000, 10, "915iwp")) { DRM_ERROR("Panel status timeout: status %08x control %08x\n", I915_READ(PCH_PP_STATUS), I915_READ(PCH_PP_CONTROL)); } } static void ironlake_wait_panel_on(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power on\n"); ironlake_wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE); } static void ironlake_wait_panel_off(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power off time\n"); ironlake_wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE); } static void ironlake_wait_panel_power_cycle(struct intel_dp *intel_dp) { DRM_DEBUG_KMS("Wait for panel power cycle\n"); ironlake_wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE); } /* Read the current pp_control value, unlocking the register if it * is locked */ static u32 ironlake_get_pp_control(struct drm_i915_private *dev_priv) { u32 control = I915_READ(PCH_PP_CONTROL); control &= ~PANEL_UNLOCK_MASK; control |= PANEL_UNLOCK_REGS; return control; } void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; if (!is_edp(intel_dp)) return; DRM_DEBUG_KMS("Turn eDP VDD on\n"); WARN(intel_dp->want_panel_vdd, "eDP VDD already requested on\n"); intel_dp->want_panel_vdd = true; if (ironlake_edp_have_panel_vdd(intel_dp)) { DRM_DEBUG_KMS("eDP VDD already on\n"); return; } if (!ironlake_edp_have_panel_power(intel_dp)) ironlake_wait_panel_power_cycle(intel_dp); pp = ironlake_get_pp_control(dev_priv); pp |= EDP_FORCE_VDD; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); DRM_DEBUG_KMS("PCH_PP_STATUS: 0x%08x PCH_PP_CONTROL: 0x%08x\n", I915_READ(PCH_PP_STATUS), I915_READ(PCH_PP_CONTROL)); /* * If the panel wasn't on, delay before accessing aux channel */ if (!ironlake_edp_have_panel_power(intel_dp)) { DRM_DEBUG_KMS("eDP was not running\n"); DRM_MSLEEP(intel_dp->panel_power_up_delay); } } static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; if (!intel_dp->want_panel_vdd && ironlake_edp_have_panel_vdd(intel_dp)) { pp = ironlake_get_pp_control(dev_priv); pp &= ~EDP_FORCE_VDD; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); /* Make sure sequencer is idle before allowing subsequent activity */ DRM_DEBUG_KMS("PCH_PP_STATUS: 0x%08x PCH_PP_CONTROL: 0x%08x\n", I915_READ(PCH_PP_STATUS), I915_READ(PCH_PP_CONTROL)); DRM_MSLEEP(intel_dp->panel_power_down_delay); } } static void ironlake_panel_vdd_work(void *arg, int pending __unused) { struct intel_dp *intel_dp = arg; struct drm_device *dev = intel_dp_to_dev(intel_dp); sx_xlock(&dev->mode_config.mutex); ironlake_panel_vdd_off_sync(intel_dp); sx_xunlock(&dev->mode_config.mutex); } void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync) { if (!is_edp(intel_dp)) return; DRM_DEBUG_KMS("Turn eDP VDD off %d\n", intel_dp->want_panel_vdd); WARN(!intel_dp->want_panel_vdd, "eDP VDD not forced on"); intel_dp->want_panel_vdd = false; if (sync) { ironlake_panel_vdd_off_sync(intel_dp); } else { /* * Queue the timer to fire a long * time from now (relative to the power down delay) * to keep the panel power up across a sequence of operations */ struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; taskqueue_enqueue_timeout(dev_priv->wq, &intel_dp->panel_vdd_work, msecs_to_jiffies(intel_dp->panel_power_cycle_delay * 5)); } } void ironlake_edp_panel_on(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; if (!is_edp(intel_dp)) return; DRM_DEBUG_KMS("Turn eDP power on\n"); if (ironlake_edp_have_panel_power(intel_dp)) { DRM_DEBUG_KMS("eDP power already on\n"); return; } ironlake_wait_panel_power_cycle(intel_dp); pp = ironlake_get_pp_control(dev_priv); if (IS_GEN5(dev)) { /* ILK workaround: disable reset around power sequence */ pp &= ~PANEL_POWER_RESET; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); } pp |= POWER_TARGET_ON; if (!IS_GEN5(dev)) pp |= PANEL_POWER_RESET; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); ironlake_wait_panel_on(intel_dp); if (IS_GEN5(dev)) { pp |= PANEL_POWER_RESET; /* restore panel reset bit */ I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); } } void ironlake_edp_panel_off(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; if (!is_edp(intel_dp)) return; DRM_DEBUG_KMS("Turn eDP power off\n"); WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n"); pp = ironlake_get_pp_control(dev_priv); /* We need to switch off panel power _and_ force vdd, for otherwise some * panels get very unhappy and cease to work. */ pp &= ~(POWER_TARGET_ON | EDP_FORCE_VDD | PANEL_POWER_RESET | EDP_BLC_ENABLE); I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); intel_dp->want_panel_vdd = false; ironlake_wait_panel_off(intel_dp); } void ironlake_edp_backlight_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int pipe = to_intel_crtc(intel_dig_port->base.base.crtc)->pipe; u32 pp; if (!is_edp(intel_dp)) return; DRM_DEBUG_KMS("\n"); /* * If we enable the backlight right away following a panel power * on, we may see slight flicker as the panel syncs with the eDP * link. So delay a bit to make sure the image is solid before * allowing it to appear. */ DRM_MSLEEP(intel_dp->backlight_on_delay); pp = ironlake_get_pp_control(dev_priv); pp |= EDP_BLC_ENABLE; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); intel_panel_enable_backlight(dev, pipe); } void ironlake_edp_backlight_off(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; u32 pp; if (!is_edp(intel_dp)) return; intel_panel_disable_backlight(dev); DRM_DEBUG_KMS("\n"); pp = ironlake_get_pp_control(dev_priv); pp &= ~EDP_BLC_ENABLE; I915_WRITE(PCH_PP_CONTROL, pp); POSTING_READ(PCH_PP_CONTROL); DRM_MSLEEP(intel_dp->backlight_off_delay); } static void ironlake_edp_pll_on(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_crtc *crtc = intel_dig_port->base.base.crtc; struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 dpa_ctl; assert_pipe_disabled(dev_priv, to_intel_crtc(crtc)->pipe); DRM_DEBUG_KMS("\n"); dpa_ctl = I915_READ(DP_A); WARN(dpa_ctl & DP_PLL_ENABLE, "dp pll on, should be off\n"); WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n"); /* We don't adjust intel_dp->DP while tearing down the link, to * facilitate link retraining (e.g. after hotplug). Hence clear all * enable bits here to ensure that we don't enable too much. */ intel_dp->DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE); intel_dp->DP |= DP_PLL_ENABLE; I915_WRITE(DP_A, intel_dp->DP); POSTING_READ(DP_A); udelay(200); } static void ironlake_edp_pll_off(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_crtc *crtc = intel_dig_port->base.base.crtc; struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 dpa_ctl; assert_pipe_disabled(dev_priv, to_intel_crtc(crtc)->pipe); dpa_ctl = I915_READ(DP_A); WARN((dpa_ctl & DP_PLL_ENABLE) == 0, "dp pll off, should be on\n"); WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n"); /* We can't rely on the value tracked for the DP register in * intel_dp->DP because link_down must not change that (otherwise link * re-training will fail. */ dpa_ctl &= ~DP_PLL_ENABLE; I915_WRITE(DP_A, dpa_ctl); POSTING_READ(DP_A); udelay(200); } /* If the sink supports it, try to set the power state appropriately */ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) { int ret, i; /* Should have a valid DPCD by this point */ if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) return; if (mode != DRM_MODE_DPMS_ON) { ret = intel_dp_aux_native_write_1(intel_dp, DP_SET_POWER, DP_SET_POWER_D3); if (ret != 1) DRM_DEBUG_DRIVER("failed to write sink power state\n"); } else { /* * When turning on, we need to retry for 1ms to give the sink * time to wake up. */ for (i = 0; i < 3; i++) { ret = intel_dp_aux_native_write_1(intel_dp, DP_SET_POWER, DP_SET_POWER_D0); if (ret == 1) break; DRM_MSLEEP(1); } } } static bool intel_dp_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; u32 tmp = I915_READ(intel_dp->output_reg); if (!(tmp & DP_PORT_EN)) return false; if (is_cpu_edp(intel_dp) && IS_GEN7(dev)) { *pipe = PORT_TO_PIPE_CPT(tmp); } else if (!HAS_PCH_CPT(dev) || is_cpu_edp(intel_dp)) { *pipe = PORT_TO_PIPE(tmp); } else { u32 trans_sel; u32 trans_dp; int i; switch (intel_dp->output_reg) { case PCH_DP_B: trans_sel = TRANS_DP_PORT_SEL_B; break; case PCH_DP_C: trans_sel = TRANS_DP_PORT_SEL_C; break; case PCH_DP_D: trans_sel = TRANS_DP_PORT_SEL_D; break; default: return true; } for_each_pipe(i) { trans_dp = I915_READ(TRANS_DP_CTL(i)); if ((trans_dp & TRANS_DP_PORT_SEL_MASK) == trans_sel) { *pipe = i; return true; } } DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n", intel_dp->output_reg); } return true; } static void intel_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); /* Make sure the panel is off before trying to change the mode. But also * ensure that we have vdd while we switch off the panel. */ ironlake_edp_panel_vdd_on(intel_dp); ironlake_edp_backlight_off(intel_dp); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); ironlake_edp_panel_off(intel_dp); /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */ if (!is_cpu_edp(intel_dp)) intel_dp_link_down(intel_dp); } static void intel_post_disable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); if (is_cpu_edp(intel_dp)) { intel_dp_link_down(intel_dp); ironlake_edp_pll_off(intel_dp); } } static void intel_enable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); struct drm_device *dev = encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; uint32_t dp_reg = I915_READ(intel_dp->output_reg); if (WARN_ON(dp_reg & DP_PORT_EN)) return; ironlake_edp_panel_vdd_on(intel_dp); intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); intel_dp_start_link_train(intel_dp); ironlake_edp_panel_on(intel_dp); ironlake_edp_panel_vdd_off(intel_dp, true); intel_dp_complete_link_train(intel_dp); ironlake_edp_backlight_on(intel_dp); } static void intel_pre_enable_dp(struct intel_encoder *encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base); if (is_cpu_edp(intel_dp)) ironlake_edp_pll_on(intel_dp); } /* * Native read with retry for link status and receiver capability reads for * cases where the sink may still be asleep. */ static bool intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address, uint8_t *recv, int recv_bytes) { int ret, i; /* * Sinks are *supposed* to come up within 1ms from an off state, * but we're also supposed to retry 3 times per the spec. */ for (i = 0; i < 3; i++) { ret = intel_dp_aux_native_read(intel_dp, address, recv, recv_bytes); if (ret == recv_bytes) return true; DRM_MSLEEP(1); } return false; } /* * Fetch AUX CH registers 0x202 - 0x207 which contain * link status information */ static bool intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]) { return intel_dp_aux_native_read_retry(intel_dp, DP_LANE0_1_STATUS, link_status, DP_LINK_STATUS_SIZE); } #if 0 static char *voltage_names[] = { "0.4V", "0.6V", "0.8V", "1.2V" }; static char *pre_emph_names[] = { "0dB", "3.5dB", "6dB", "9.5dB" }; static char *link_train_names[] = { "pattern 1", "pattern 2", "idle", "off" }; #endif /* * These are source-specific values; current Intel hardware supports * a maximum voltage of 800mV and a maximum pre-emphasis of 6dB */ static uint8_t intel_dp_voltage_max(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); if (IS_GEN7(dev) && is_cpu_edp(intel_dp)) return DP_TRAIN_VOLTAGE_SWING_800; else if (HAS_PCH_CPT(dev) && !is_cpu_edp(intel_dp)) return DP_TRAIN_VOLTAGE_SWING_1200; else return DP_TRAIN_VOLTAGE_SWING_800; } static uint8_t intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing) { struct drm_device *dev = intel_dp_to_dev(intel_dp); if (IS_HASWELL(dev)) { switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { case DP_TRAIN_VOLTAGE_SWING_400: return DP_TRAIN_PRE_EMPHASIS_9_5; case DP_TRAIN_VOLTAGE_SWING_600: return DP_TRAIN_PRE_EMPHASIS_6; case DP_TRAIN_VOLTAGE_SWING_800: return DP_TRAIN_PRE_EMPHASIS_3_5; case DP_TRAIN_VOLTAGE_SWING_1200: default: return DP_TRAIN_PRE_EMPHASIS_0; } } else if (IS_GEN7(dev) && is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) { switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { case DP_TRAIN_VOLTAGE_SWING_400: return DP_TRAIN_PRE_EMPHASIS_6; case DP_TRAIN_VOLTAGE_SWING_600: case DP_TRAIN_VOLTAGE_SWING_800: return DP_TRAIN_PRE_EMPHASIS_3_5; default: return DP_TRAIN_PRE_EMPHASIS_0; } } else { switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) { case DP_TRAIN_VOLTAGE_SWING_400: return DP_TRAIN_PRE_EMPHASIS_6; case DP_TRAIN_VOLTAGE_SWING_600: return DP_TRAIN_PRE_EMPHASIS_6; case DP_TRAIN_VOLTAGE_SWING_800: return DP_TRAIN_PRE_EMPHASIS_3_5; case DP_TRAIN_VOLTAGE_SWING_1200: default: return DP_TRAIN_PRE_EMPHASIS_0; } } } static void intel_get_adjust_train(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]) { uint8_t v = 0; uint8_t p = 0; int lane; uint8_t voltage_max; uint8_t preemph_max; for (lane = 0; lane < intel_dp->lane_count; lane++) { uint8_t this_v = drm_dp_get_adjust_request_voltage(link_status, lane); uint8_t this_p = drm_dp_get_adjust_request_pre_emphasis(link_status, lane); if (this_v > v) v = this_v; if (this_p > p) p = this_p; } voltage_max = intel_dp_voltage_max(intel_dp); if (v >= voltage_max) v = voltage_max | DP_TRAIN_MAX_SWING_REACHED; preemph_max = intel_dp_pre_emphasis_max(intel_dp, v); if (p >= preemph_max) p = preemph_max | DP_TRAIN_MAX_PRE_EMPHASIS_REACHED; for (lane = 0; lane < 4; lane++) intel_dp->train_set[lane] = v | p; } static uint32_t intel_dp_signal_levels(uint8_t train_set) { uint32_t signal_levels = 0; switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) { case DP_TRAIN_VOLTAGE_SWING_400: default: signal_levels |= DP_VOLTAGE_0_4; break; case DP_TRAIN_VOLTAGE_SWING_600: signal_levels |= DP_VOLTAGE_0_6; break; case DP_TRAIN_VOLTAGE_SWING_800: signal_levels |= DP_VOLTAGE_0_8; break; case DP_TRAIN_VOLTAGE_SWING_1200: signal_levels |= DP_VOLTAGE_1_2; break; } switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) { case DP_TRAIN_PRE_EMPHASIS_0: default: signal_levels |= DP_PRE_EMPHASIS_0; break; case DP_TRAIN_PRE_EMPHASIS_3_5: signal_levels |= DP_PRE_EMPHASIS_3_5; break; case DP_TRAIN_PRE_EMPHASIS_6: signal_levels |= DP_PRE_EMPHASIS_6; break; case DP_TRAIN_PRE_EMPHASIS_9_5: signal_levels |= DP_PRE_EMPHASIS_9_5; break; } return signal_levels; } /* Gen6's DP voltage swing and pre-emphasis control */ static uint32_t intel_gen6_edp_signal_levels(uint8_t train_set) { int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0: case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0: return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5: return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6: case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6: return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5: case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5: return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0: case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0: return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B; default: DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" "0x%x\n", signal_levels); return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; } } /* Gen7's DP voltage swing and pre-emphasis control */ static uint32_t intel_gen7_edp_signal_levels(uint8_t train_set) { int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0: return EDP_LINK_TRAIN_400MV_0DB_IVB; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5: return EDP_LINK_TRAIN_400MV_3_5DB_IVB; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6: return EDP_LINK_TRAIN_400MV_6DB_IVB; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0: return EDP_LINK_TRAIN_600MV_0DB_IVB; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5: return EDP_LINK_TRAIN_600MV_3_5DB_IVB; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0: return EDP_LINK_TRAIN_800MV_0DB_IVB; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5: return EDP_LINK_TRAIN_800MV_3_5DB_IVB; default: DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" "0x%x\n", signal_levels); return EDP_LINK_TRAIN_500MV_0DB_IVB; } } /* Gen7.5's (HSW) DP voltage swing and pre-emphasis control */ static uint32_t intel_dp_signal_levels_hsw(uint8_t train_set) { int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0: return DDI_BUF_EMP_400MV_0DB_HSW; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5: return DDI_BUF_EMP_400MV_3_5DB_HSW; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6: return DDI_BUF_EMP_400MV_6DB_HSW; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_9_5: return DDI_BUF_EMP_400MV_9_5DB_HSW; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0: return DDI_BUF_EMP_600MV_0DB_HSW; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5: return DDI_BUF_EMP_600MV_3_5DB_HSW; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6: return DDI_BUF_EMP_600MV_6DB_HSW; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0: return DDI_BUF_EMP_800MV_0DB_HSW; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5: return DDI_BUF_EMP_800MV_3_5DB_HSW; default: DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" "0x%x\n", signal_levels); return DDI_BUF_EMP_400MV_0DB_HSW; } } static bool intel_dp_set_link_train(struct intel_dp *intel_dp, uint32_t dp_reg_value, uint8_t dp_train_pat) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; enum port port = intel_dig_port->port; int ret; uint32_t temp; if (IS_HASWELL(dev)) { temp = I915_READ(DP_TP_CTL(port)); if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) temp |= DP_TP_CTL_SCRAMBLE_DISABLE; else temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { case DP_TRAINING_PATTERN_DISABLE: if (port != PORT_A) { temp |= DP_TP_CTL_LINK_TRAIN_IDLE; I915_WRITE(DP_TP_CTL(port), temp); if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_IDLE_DONE), 1)) DRM_ERROR("Timed out waiting for DP idle patterns\n"); temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; } temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; break; case DP_TRAINING_PATTERN_1: temp |= DP_TP_CTL_LINK_TRAIN_PAT1; break; case DP_TRAINING_PATTERN_2: temp |= DP_TP_CTL_LINK_TRAIN_PAT2; break; case DP_TRAINING_PATTERN_3: temp |= DP_TP_CTL_LINK_TRAIN_PAT3; break; } I915_WRITE(DP_TP_CTL(port), temp); } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) { dp_reg_value &= ~DP_LINK_TRAIN_MASK_CPT; switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { case DP_TRAINING_PATTERN_DISABLE: dp_reg_value |= DP_LINK_TRAIN_OFF_CPT; break; case DP_TRAINING_PATTERN_1: dp_reg_value |= DP_LINK_TRAIN_PAT_1_CPT; break; case DP_TRAINING_PATTERN_2: dp_reg_value |= DP_LINK_TRAIN_PAT_2_CPT; break; case DP_TRAINING_PATTERN_3: DRM_ERROR("DP training pattern 3 not supported\n"); dp_reg_value |= DP_LINK_TRAIN_PAT_2_CPT; break; } } else { dp_reg_value &= ~DP_LINK_TRAIN_MASK; switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { case DP_TRAINING_PATTERN_DISABLE: dp_reg_value |= DP_LINK_TRAIN_OFF; break; case DP_TRAINING_PATTERN_1: dp_reg_value |= DP_LINK_TRAIN_PAT_1; break; case DP_TRAINING_PATTERN_2: dp_reg_value |= DP_LINK_TRAIN_PAT_2; break; case DP_TRAINING_PATTERN_3: DRM_ERROR("DP training pattern 3 not supported\n"); dp_reg_value |= DP_LINK_TRAIN_PAT_2; break; } } I915_WRITE(intel_dp->output_reg, dp_reg_value); POSTING_READ(intel_dp->output_reg); intel_dp_aux_native_write_1(intel_dp, DP_TRAINING_PATTERN_SET, dp_train_pat); if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) != DP_TRAINING_PATTERN_DISABLE) { ret = intel_dp_aux_native_write(intel_dp, DP_TRAINING_LANE0_SET, intel_dp->train_set, intel_dp->lane_count); if (ret != intel_dp->lane_count) return false; } return true; } /* Enable corresponding port and start training pattern 1 */ void intel_dp_start_link_train(struct intel_dp *intel_dp) { struct drm_encoder *encoder = &dp_to_dig_port(intel_dp)->base.base; struct drm_device *dev = encoder->dev; int i; uint8_t voltage; bool clock_recovery = false; int voltage_tries, loop_tries; uint32_t DP = intel_dp->DP; if (IS_HASWELL(dev)) intel_ddi_prepare_link_retrain(encoder); /* Write the link configuration data */ intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET, intel_dp->link_configuration, DP_LINK_CONFIGURATION_SIZE); DP |= DP_PORT_EN; memset(intel_dp->train_set, 0, 4); voltage = 0xff; voltage_tries = 0; loop_tries = 0; clock_recovery = false; for (;;) { /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */ uint8_t link_status[DP_LINK_STATUS_SIZE]; uint32_t signal_levels; if (IS_HASWELL(dev)) { signal_levels = intel_dp_signal_levels_hsw( intel_dp->train_set[0]); DP = (DP & ~DDI_BUF_EMP_MASK) | signal_levels; } else if (IS_GEN7(dev) && is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) { signal_levels = intel_gen7_edp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_IVB) | signal_levels; } else if (IS_GEN6(dev) && is_cpu_edp(intel_dp)) { signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels; } else { signal_levels = intel_dp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels; } DRM_DEBUG_KMS("training pattern 1 signal levels %08x\n", signal_levels); /* Set training pattern 1 */ if (!intel_dp_set_link_train(intel_dp, DP, DP_TRAINING_PATTERN_1 | DP_LINK_SCRAMBLING_DISABLE)) break; drm_dp_link_train_clock_recovery_delay(intel_dp->dpcd); if (!intel_dp_get_link_status(intel_dp, link_status)) { DRM_ERROR("failed to get link status\n"); break; } if (drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { DRM_DEBUG_KMS("clock recovery OK\n"); clock_recovery = true; break; } /* Check to see if we've tried the max voltage */ for (i = 0; i < intel_dp->lane_count; i++) if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0) break; if (i == intel_dp->lane_count) { ++loop_tries; if (loop_tries == 5) { DRM_DEBUG_KMS("too many full retries, give up\n"); break; } memset(intel_dp->train_set, 0, 4); voltage_tries = 0; continue; } /* Check to see if we've tried the same voltage 5 times */ if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) { ++voltage_tries; if (voltage_tries == 5) { DRM_DEBUG_KMS("too many voltage retries, give up\n"); break; } } else voltage_tries = 0; voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; /* Compute new intel_dp->train_set as requested by target */ intel_get_adjust_train(intel_dp, link_status); } intel_dp->DP = DP; } void intel_dp_complete_link_train(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); bool channel_eq = false; int tries, cr_tries; uint32_t DP = intel_dp->DP; /* channel equalization */ tries = 0; cr_tries = 0; channel_eq = false; for (;;) { /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */ uint32_t signal_levels; uint8_t link_status[DP_LINK_STATUS_SIZE]; if (cr_tries > 5) { DRM_ERROR("failed to train DP, aborting\n"); intel_dp_link_down(intel_dp); break; } if (IS_HASWELL(dev)) { signal_levels = intel_dp_signal_levels_hsw(intel_dp->train_set[0]); DP = (DP & ~DDI_BUF_EMP_MASK) | signal_levels; } else if (IS_GEN7(dev) && is_cpu_edp(intel_dp) && !IS_VALLEYVIEW(dev)) { signal_levels = intel_gen7_edp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_IVB) | signal_levels; } else if (IS_GEN6(dev) && is_cpu_edp(intel_dp)) { signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels; } else { signal_levels = intel_dp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels; } /* channel eq pattern */ if (!intel_dp_set_link_train(intel_dp, DP, DP_TRAINING_PATTERN_2 | DP_LINK_SCRAMBLING_DISABLE)) break; drm_dp_link_train_channel_eq_delay(intel_dp->dpcd); if (!intel_dp_get_link_status(intel_dp, link_status)) break; /* Make sure clock is still ok */ if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) { intel_dp_start_link_train(intel_dp); cr_tries++; continue; } if (drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { channel_eq = true; break; } /* Try 5 times, then try clock recovery if that fails */ if (tries > 5) { intel_dp_link_down(intel_dp); intel_dp_start_link_train(intel_dp); tries = 0; cr_tries++; continue; } /* Compute new intel_dp->train_set as requested by target */ intel_get_adjust_train(intel_dp, link_status); ++tries; } if (channel_eq) - DRM_DEBUG_KMS("Channel EQ done. DP Training successfull\n"); + DRM_DEBUG_KMS("Channel EQ done. DP Training successful\n"); intel_dp_set_link_train(intel_dp, DP, DP_TRAINING_PATTERN_DISABLE); } static void intel_dp_link_down(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct drm_device *dev = intel_dig_port->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; uint32_t DP = intel_dp->DP; /* * DDI code has a strict mode set sequence and we should try to respect * it, otherwise we might hang the machine in many different ways. So we * really should be disabling the port only on a complete crtc_disable * sequence. This function is just called under two conditions on DDI * code: * - Link train failed while doing crtc_enable, and on this case we * really should respect the mode set sequence and wait for a * crtc_disable. * - Someone turned the monitor off and intel_dp_check_link_status * called us. We don't need to disable the whole port on this case, so * when someone turns the monitor on again, * intel_ddi_prepare_link_retrain will take care of redoing the link * train. */ if (IS_HASWELL(dev)) return; if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)) return; DRM_DEBUG_KMS("\n"); if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) { DP &= ~DP_LINK_TRAIN_MASK_CPT; I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT); } else { DP &= ~DP_LINK_TRAIN_MASK; I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE); } POSTING_READ(intel_dp->output_reg); DRM_MSLEEP(17); if (HAS_PCH_IBX(dev) && I915_READ(intel_dp->output_reg) & DP_PIPEB_SELECT) { struct drm_crtc *crtc = intel_dig_port->base.base.crtc; /* Hardware workaround: leaving our transcoder select * set to transcoder B while it's off will prevent the * corresponding HDMI output on transcoder A. * * Combine this with another hardware workaround: * transcoder select bit can only be cleared while the * port is enabled. */ DP &= ~DP_PIPEB_SELECT; I915_WRITE(intel_dp->output_reg, DP); /* Changes to enable or select take place the vblank * after being written. */ if (crtc == NULL) { /* We can arrive here never having been attached * to a CRTC, for instance, due to inheriting * random state from the BIOS. * * If the pipe is not running, play safe and * wait for the clocks to stabilise before * continuing. */ POSTING_READ(intel_dp->output_reg); DRM_MSLEEP(50); } else intel_wait_for_vblank(dev, to_intel_crtc(crtc)->pipe); } DP &= ~DP_AUDIO_OUTPUT_ENABLE; I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); POSTING_READ(intel_dp->output_reg); DRM_MSLEEP(intel_dp->panel_power_down_delay); } static bool intel_dp_get_dpcd(struct intel_dp *intel_dp) { if (intel_dp_aux_native_read_retry(intel_dp, 0x000, intel_dp->dpcd, sizeof(intel_dp->dpcd)) == 0) return false; /* aux transfer failed */ if (intel_dp->dpcd[DP_DPCD_REV] == 0) return false; /* DPCD not present */ if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT)) return true; /* native DP sink */ if (intel_dp->dpcd[DP_DPCD_REV] == 0x10) return true; /* no per-port downstream info */ if (intel_dp_aux_native_read_retry(intel_dp, DP_DOWNSTREAM_PORT_0, intel_dp->downstream_ports, DP_MAX_DOWNSTREAM_PORTS) == 0) return false; /* downstream port status fetch failed */ return true; } static void intel_dp_probe_oui(struct intel_dp *intel_dp) { u8 buf[3]; if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) return; ironlake_edp_panel_vdd_on(intel_dp); if (intel_dp_aux_native_read_retry(intel_dp, DP_SINK_OUI, buf, 3)) DRM_DEBUG_KMS("Sink OUI: %02x%02x%02x\n", buf[0], buf[1], buf[2]); if (intel_dp_aux_native_read_retry(intel_dp, DP_BRANCH_OUI, buf, 3)) DRM_DEBUG_KMS("Branch OUI: %02x%02x%02x\n", buf[0], buf[1], buf[2]); ironlake_edp_panel_vdd_off(intel_dp, false); } static bool intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector) { int ret; ret = intel_dp_aux_native_read_retry(intel_dp, DP_DEVICE_SERVICE_IRQ_VECTOR, sink_irq_vector, 1); if (!ret) return false; return true; } static void intel_dp_handle_test_request(struct intel_dp *intel_dp) { /* NAK by default */ intel_dp_aux_native_write_1(intel_dp, DP_TEST_RESPONSE, DP_TEST_NAK); } /* * According to DP spec * 5.1.2: * 1. Read DPCD * 2. Configure link according to Receiver Capabilities * 3. Use Link Training from 2.5.3.3 and 3.5.1.3 * 4. Check link status on receipt of hot-plug interrupt */ void intel_dp_check_link_status(struct intel_dp *intel_dp) { struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base; u8 sink_irq_vector; u8 link_status[DP_LINK_STATUS_SIZE]; if (!intel_encoder->connectors_active) return; if (WARN_ON(!intel_encoder->base.crtc)) return; /* Try to read receiver status if the link appears to be up */ if (!intel_dp_get_link_status(intel_dp, link_status)) { intel_dp_link_down(intel_dp); return; } /* Now read the DPCD to see if it's actually running */ if (!intel_dp_get_dpcd(intel_dp)) { intel_dp_link_down(intel_dp); return; } /* Try to read the source of the interrupt */ if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 && intel_dp_get_sink_irq(intel_dp, &sink_irq_vector)) { /* Clear interrupt source */ intel_dp_aux_native_write_1(intel_dp, DP_DEVICE_SERVICE_IRQ_VECTOR, sink_irq_vector); if (sink_irq_vector & DP_AUTOMATED_TEST_REQUEST) intel_dp_handle_test_request(intel_dp); if (sink_irq_vector & (DP_CP_IRQ | DP_SINK_SPECIFIC_IRQ)) DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n"); } if (!drm_dp_channel_eq_ok(link_status, intel_dp->lane_count)) { DRM_DEBUG_KMS("%s: channel EQ not ok, retraining\n", drm_get_encoder_name(&intel_encoder->base)); intel_dp_start_link_train(intel_dp); intel_dp_complete_link_train(intel_dp); } } /* XXX this is probably wrong for multiple downstream ports */ static enum drm_connector_status intel_dp_detect_dpcd(struct intel_dp *intel_dp) { uint8_t *dpcd = intel_dp->dpcd; bool hpd; uint8_t type; if (!intel_dp_get_dpcd(intel_dp)) return connector_status_disconnected; /* if there's no downstream port, we're done */ if (!(dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT)) return connector_status_connected; /* If we're HPD-aware, SINK_COUNT changes dynamically */ hpd = !!(intel_dp->downstream_ports[0] & DP_DS_PORT_HPD); if (hpd) { uint8_t reg; if (!intel_dp_aux_native_read_retry(intel_dp, DP_SINK_COUNT, ®, 1)) return connector_status_unknown; return DP_GET_SINK_COUNT(reg) ? connector_status_connected : connector_status_disconnected; } /* If no HPD, poke DDC gently */ if (drm_probe_ddc(intel_dp->adapter)) return connector_status_connected; /* Well we tried, say unknown for unreliable port types */ type = intel_dp->downstream_ports[0] & DP_DS_PORT_TYPE_MASK; if (type == DP_DS_PORT_TYPE_VGA || type == DP_DS_PORT_TYPE_NON_EDID) return connector_status_unknown; /* Anything else is out of spec, warn and ignore */ DRM_DEBUG_KMS("Broken DP branch device, ignoring\n"); return connector_status_disconnected; } static enum drm_connector_status ironlake_dp_detect(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); enum drm_connector_status status; /* Can't disconnect eDP, but you can close the lid... */ if (is_edp(intel_dp)) { status = intel_panel_detect(dev); if (status == connector_status_unknown) status = connector_status_connected; return status; } return intel_dp_detect_dpcd(intel_dp); } static enum drm_connector_status g4x_dp_detect(struct intel_dp *intel_dp) { struct drm_device *dev = intel_dp_to_dev(intel_dp); struct drm_i915_private *dev_priv = dev->dev_private; uint32_t bit; switch (intel_dp->output_reg) { case DP_B: bit = DPB_HOTPLUG_LIVE_STATUS; break; case DP_C: bit = DPC_HOTPLUG_LIVE_STATUS; break; case DP_D: bit = DPD_HOTPLUG_LIVE_STATUS; break; default: return connector_status_unknown; } if ((I915_READ(PORT_HOTPLUG_STAT) & bit) == 0) return connector_status_disconnected; return intel_dp_detect_dpcd(intel_dp); } static struct edid * intel_dp_get_edid(struct drm_connector *connector, device_t adapter) { struct intel_connector *intel_connector = to_intel_connector(connector); /* use cached edid if we have one */ if (intel_connector->edid) { struct edid *edid; int size; /* invalid edid */ if (intel_connector->edid_err) return NULL; size = (intel_connector->edid->extensions + 1) * EDID_LENGTH; edid = malloc(size, DRM_MEM_KMS, M_WAITOK); if (!edid) return NULL; memcpy(edid, intel_connector->edid, size); return edid; } return drm_get_edid(connector, adapter); } static int intel_dp_get_edid_modes(struct drm_connector *connector, device_t adapter) { struct intel_connector *intel_connector = to_intel_connector(connector); /* use cached edid if we have one */ if (intel_connector->edid) { /* invalid edid */ if (intel_connector->edid_err) return 0; return intel_connector_update_modes(connector, intel_connector->edid); } return intel_ddc_get_modes(connector, adapter); } /** * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection. * * \return true if DP port is connected. * \return false if DP port is disconnected. */ static enum drm_connector_status intel_dp_detect(struct drm_connector *connector, bool force) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = connector->dev; enum drm_connector_status status; struct edid *edid = NULL; char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3]; intel_dp->has_audio = false; if (HAS_PCH_SPLIT(dev)) status = ironlake_dp_detect(intel_dp); else status = g4x_dp_detect(intel_dp); hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd), 32, 1, dpcd_hex_dump, sizeof(dpcd_hex_dump), false); DRM_DEBUG_KMS("DPCD: %s\n", dpcd_hex_dump); if (status != connector_status_connected) return status; intel_dp_probe_oui(intel_dp); if (intel_dp->force_audio != HDMI_AUDIO_AUTO) { intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON); } else { edid = intel_dp_get_edid(connector, intel_dp->adapter); if (edid) { intel_dp->has_audio = drm_detect_monitor_audio(edid); free(edid, DRM_MEM_KMS); } } if (intel_encoder->type != INTEL_OUTPUT_EDP) intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; return connector_status_connected; } static int intel_dp_get_modes(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_device *dev = connector->dev; int ret; /* We should parse the EDID data and find out if it has an audio sink */ ret = intel_dp_get_edid_modes(connector, intel_dp->adapter); if (ret) return ret; /* if eDP has no EDID, fall back to fixed mode */ if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) { struct drm_display_mode *mode; mode = drm_mode_duplicate(dev, intel_connector->panel.fixed_mode); if (mode) { drm_mode_probed_add(connector, mode); return 1; } } return 0; } static bool intel_dp_detect_audio(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct edid *edid; bool has_audio = false; edid = intel_dp_get_edid(connector, intel_dp->adapter); if (edid) { has_audio = drm_detect_monitor_audio(edid); free(edid, DRM_MEM_KMS); } return has_audio; } static int intel_dp_set_property(struct drm_connector *connector, struct drm_property *property, uint64_t val) { struct drm_i915_private *dev_priv = connector->dev->dev_private; struct intel_connector *intel_connector = to_intel_connector(connector); struct intel_encoder *intel_encoder = intel_attached_encoder(connector); struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); int ret; ret = drm_object_property_set_value(&connector->base, property, val); if (ret) return ret; if (property == dev_priv->force_audio_property) { int i = val; bool has_audio; if (i == intel_dp->force_audio) return 0; intel_dp->force_audio = i; if (i == HDMI_AUDIO_AUTO) has_audio = intel_dp_detect_audio(connector); else has_audio = (i == HDMI_AUDIO_ON); if (has_audio == intel_dp->has_audio) return 0; intel_dp->has_audio = has_audio; goto done; } if (property == dev_priv->broadcast_rgb_property) { if (val == !!intel_dp->color_range) return 0; intel_dp->color_range = val ? DP_COLOR_RANGE_16_235 : 0; goto done; } if (is_edp(intel_dp) && property == connector->dev->mode_config.scaling_mode_property) { if (val == DRM_MODE_SCALE_NONE) { DRM_DEBUG_KMS("no scaling not supported\n"); return -EINVAL; } if (intel_connector->panel.fitting_mode == val) { /* the eDP scaling property is not changed */ return 0; } intel_connector->panel.fitting_mode = val; goto done; } return -EINVAL; done: if (intel_encoder->base.crtc) { struct drm_crtc *crtc = intel_encoder->base.crtc; intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, crtc->fb); } return 0; } static void intel_dp_destroy(struct drm_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); struct intel_connector *intel_connector = to_intel_connector(connector); free(intel_connector->edid, DRM_MEM_KMS); if (is_edp(intel_dp)) intel_panel_fini(&intel_connector->panel); drm_connector_cleanup(connector); free(connector, DRM_MEM_KMS); } void intel_dp_encoder_destroy(struct drm_encoder *encoder) { struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); struct intel_dp *intel_dp = &intel_dig_port->dp; struct drm_device *dev = intel_dig_port->base.base.dev; if (intel_dp->dp_iic_bus != NULL) { if (intel_dp->adapter != NULL) { device_delete_child(intel_dp->dp_iic_bus, intel_dp->adapter); } device_delete_child(dev->dev, intel_dp->dp_iic_bus); } drm_encoder_cleanup(encoder); if (is_edp(intel_dp)) { struct drm_i915_private *dev_priv = dev->dev_private; taskqueue_cancel_timeout(dev_priv->wq, &intel_dp->panel_vdd_work, NULL); taskqueue_drain_timeout(dev_priv->wq, &intel_dp->panel_vdd_work); ironlake_panel_vdd_off_sync(intel_dp); } free(intel_dig_port, DRM_MEM_KMS); } static const struct drm_encoder_helper_funcs intel_dp_helper_funcs = { .mode_fixup = intel_dp_mode_fixup, .mode_set = intel_dp_mode_set, .disable = intel_encoder_noop, }; static const struct drm_connector_funcs intel_dp_connector_funcs = { .dpms = intel_connector_dpms, .detect = intel_dp_detect, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = intel_dp_set_property, .destroy = intel_dp_destroy, }; static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = { .get_modes = intel_dp_get_modes, .mode_valid = intel_dp_mode_valid, .best_encoder = intel_best_encoder, }; static const struct drm_encoder_funcs intel_dp_enc_funcs = { .destroy = intel_dp_encoder_destroy, }; static void intel_dp_hot_plug(struct intel_encoder *intel_encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base); intel_dp_check_link_status(intel_dp); } /* Return which DP Port should be selected for Transcoder DP control */ int intel_trans_dp_port_sel(struct drm_crtc *crtc) { struct drm_device *dev = crtc->dev; struct intel_encoder *intel_encoder; struct intel_dp *intel_dp; for_each_encoder_on_crtc(dev, crtc, intel_encoder) { intel_dp = enc_to_intel_dp(&intel_encoder->base); if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT || intel_encoder->type == INTEL_OUTPUT_EDP) return intel_dp->output_reg; } return -1; } /* check the VBT to see whether the eDP is on DP-D port */ bool intel_dpd_is_edp(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; struct child_device_config *p_child; int i; if (!dev_priv->child_dev_num) return false; for (i = 0; i < dev_priv->child_dev_num; i++) { p_child = dev_priv->child_dev + i; if (p_child->dvo_port == PORT_IDPD && p_child->device_type == DEVICE_TYPE_eDP) return true; } return false; } static void intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector) { struct intel_connector *intel_connector = to_intel_connector(connector); intel_attach_force_audio_property(connector); intel_attach_broadcast_rgb_property(connector); if (is_edp(intel_dp)) { drm_mode_create_scaling_mode_property(connector->dev); drm_object_attach_property( &connector->base, connector->dev->mode_config.scaling_mode_property, DRM_MODE_SCALE_ASPECT); intel_connector->panel.fitting_mode = DRM_MODE_SCALE_ASPECT; } } static void intel_dp_init_panel_power_sequencer(struct drm_device *dev, struct intel_dp *intel_dp, struct edp_power_seq *out) { struct drm_i915_private *dev_priv = dev->dev_private; struct edp_power_seq cur, vbt, spec, final; u32 pp_on, pp_off, pp_div, pp; /* Workaround: Need to write PP_CONTROL with the unlock key as * the very first thing. */ pp = ironlake_get_pp_control(dev_priv); I915_WRITE(PCH_PP_CONTROL, pp); pp_on = I915_READ(PCH_PP_ON_DELAYS); pp_off = I915_READ(PCH_PP_OFF_DELAYS); pp_div = I915_READ(PCH_PP_DIVISOR); /* Pull timing values out of registers */ cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >> PANEL_POWER_UP_DELAY_SHIFT; cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >> PANEL_LIGHT_ON_DELAY_SHIFT; cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >> PANEL_LIGHT_OFF_DELAY_SHIFT; cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >> PANEL_POWER_DOWN_DELAY_SHIFT; cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >> PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000; DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12); vbt = dev_priv->edp.pps; /* Upper limits from eDP 1.3 spec. Note that we use the clunky units of * our hw here, which are all in 100usec. */ spec.t1_t3 = 210 * 10; spec.t8 = 50 * 10; /* no limit for t8, use t7 instead */ spec.t9 = 50 * 10; /* no limit for t9, make it symmetric with t8 */ spec.t10 = 500 * 10; /* This one is special and actually in units of 100ms, but zero * based in the hw (so we need to add 100 ms). But the sw vbt * table multiplies it with 1000 to make it in units of 100usec, * too. */ spec.t11_t12 = (510 + 100) * 10; DRM_DEBUG_KMS("vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", vbt.t1_t3, vbt.t8, vbt.t9, vbt.t10, vbt.t11_t12); /* Use the max of the register settings and vbt. If both are * unset, fall back to the spec limits. */ #define assign_final(field) final.field = (max(cur.field, vbt.field) == 0 ? \ spec.field : \ max(cur.field, vbt.field)) assign_final(t1_t3); assign_final(t8); assign_final(t9); assign_final(t10); assign_final(t11_t12); #undef assign_final #define get_delay(field) (DIV_ROUND_UP(final.field, 10)) intel_dp->panel_power_up_delay = get_delay(t1_t3); intel_dp->backlight_on_delay = get_delay(t8); intel_dp->backlight_off_delay = get_delay(t9); intel_dp->panel_power_down_delay = get_delay(t10); intel_dp->panel_power_cycle_delay = get_delay(t11_t12); #undef get_delay DRM_DEBUG_KMS("panel power up delay %d, power down delay %d, power cycle delay %d\n", intel_dp->panel_power_up_delay, intel_dp->panel_power_down_delay, intel_dp->panel_power_cycle_delay); DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n", intel_dp->backlight_on_delay, intel_dp->backlight_off_delay); if (out) *out = final; } static void intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev, struct intel_dp *intel_dp, struct edp_power_seq *seq) { struct drm_i915_private *dev_priv = dev->dev_private; u32 pp_on, pp_off, pp_div; /* And finally store the new values in the power sequencer. */ pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) | (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT); pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) | (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT); /* Compute the divisor for the pp clock, simply match the Bspec * formula. */ pp_div = ((100 * intel_pch_rawclk(dev))/2 - 1) << PP_REFERENCE_DIVIDER_SHIFT; pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000) << PANEL_POWER_CYCLE_DELAY_SHIFT); /* Haswell doesn't have any port selection bits for the panel * power sequencer any more. */ if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) { if (is_cpu_edp(intel_dp)) pp_on |= PANEL_POWER_PORT_DP_A; else pp_on |= PANEL_POWER_PORT_DP_D; } I915_WRITE(PCH_PP_ON_DELAYS, pp_on); I915_WRITE(PCH_PP_OFF_DELAYS, pp_off); I915_WRITE(PCH_PP_DIVISOR, pp_div); DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n", I915_READ(PCH_PP_ON_DELAYS), I915_READ(PCH_PP_OFF_DELAYS), I915_READ(PCH_PP_DIVISOR)); } void intel_dp_init_connector(struct intel_digital_port *intel_dig_port, struct intel_connector *intel_connector) { struct drm_connector *connector = &intel_connector->base; struct intel_dp *intel_dp = &intel_dig_port->dp; struct intel_encoder *intel_encoder = &intel_dig_port->base; struct drm_device *dev = intel_encoder->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_display_mode *fixed_mode = NULL; struct edp_power_seq power_seq = { 0 }; enum port port = intel_dig_port->port; const char *name = NULL; int type; /* Preserve the current hw state. */ intel_dp->DP = I915_READ(intel_dp->output_reg); intel_dp->attached_connector = intel_connector; if (HAS_PCH_SPLIT(dev) && port == PORT_D) if (intel_dpd_is_edp(dev)) intel_dp->is_pch_edp = true; /* * FIXME : We need to initialize built-in panels before external panels. * For X0, DP_C is fixed as eDP. Revisit this as part of VLV eDP cleanup */ if (IS_VALLEYVIEW(dev) && port == PORT_C) { type = DRM_MODE_CONNECTOR_eDP; intel_encoder->type = INTEL_OUTPUT_EDP; } else if (port == PORT_A || is_pch_edp(intel_dp)) { type = DRM_MODE_CONNECTOR_eDP; intel_encoder->type = INTEL_OUTPUT_EDP; } else { /* The intel_encoder->type value may be INTEL_OUTPUT_UNKNOWN for * DDI or INTEL_OUTPUT_DISPLAYPORT for the older gens, so don't * rewrite it. */ type = DRM_MODE_CONNECTOR_DisplayPort; } drm_connector_init(dev, connector, &intel_dp_connector_funcs, type); drm_connector_helper_add(connector, &intel_dp_connector_helper_funcs); connector->polled = DRM_CONNECTOR_POLL_HPD; connector->interlace_allowed = true; connector->doublescan_allowed = 0; TIMEOUT_TASK_INIT(dev_priv->wq, &intel_dp->panel_vdd_work, 0, ironlake_panel_vdd_work, intel_dp); intel_connector_attach_encoder(intel_connector, intel_encoder); if (IS_HASWELL(dev)) intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; else intel_connector->get_hw_state = intel_connector_get_hw_state; /* Set up the DDC bus. */ switch (port) { case PORT_A: name = "DPDDC-A"; break; case PORT_B: dev_priv->hotplug_supported_mask |= DPB_HOTPLUG_INT_STATUS; name = "DPDDC-B"; break; case PORT_C: dev_priv->hotplug_supported_mask |= DPC_HOTPLUG_INT_STATUS; name = "DPDDC-C"; break; case PORT_D: dev_priv->hotplug_supported_mask |= DPD_HOTPLUG_INT_STATUS; name = "DPDDC-D"; break; default: WARN(1, "Invalid port %c\n", port_name(port)); break; } if (is_edp(intel_dp)) intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq); intel_dp_i2c_init(intel_dp, intel_connector, name); /* Cache DPCD and EDID for edp. */ if (is_edp(intel_dp)) { bool ret; struct drm_display_mode *scan; struct edid *edid; int edid_err = 0; ironlake_edp_panel_vdd_on(intel_dp); ret = intel_dp_get_dpcd(intel_dp); ironlake_edp_panel_vdd_off(intel_dp, false); if (ret) { if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] & DP_NO_AUX_HANDSHAKE_LINK_TRAINING; } else { /* if this fails, presume the device is a ghost */ DRM_INFO("failed to retrieve link info, disabling eDP\n"); intel_dp_encoder_destroy(&intel_encoder->base); intel_dp_destroy(connector); return; } /* We now know it's not a ghost, init power sequence regs. */ intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, &power_seq); ironlake_edp_panel_vdd_on(intel_dp); edid = drm_get_edid(connector, intel_dp->adapter); if (edid) { if (drm_add_edid_modes(connector, edid)) { drm_mode_connector_update_edid_property(connector, edid); drm_edid_to_eld(connector, edid); } else { free(edid, DRM_MEM_KMS); edid = NULL; edid_err = -EINVAL; } } else { edid = NULL; edid_err = -ENOENT; } intel_connector->edid = edid; intel_connector->edid_err = edid_err; /* prefer fixed mode from EDID if available */ list_for_each_entry(scan, &connector->probed_modes, head) { if ((scan->type & DRM_MODE_TYPE_PREFERRED)) { fixed_mode = drm_mode_duplicate(dev, scan); break; } } /* fallback to VBT if available for eDP */ if (!fixed_mode && dev_priv->lfp_lvds_vbt_mode) { fixed_mode = drm_mode_duplicate(dev, dev_priv->lfp_lvds_vbt_mode); if (fixed_mode) fixed_mode->type |= DRM_MODE_TYPE_PREFERRED; } ironlake_edp_panel_vdd_off(intel_dp, false); } if (is_edp(intel_dp)) { intel_panel_init(&intel_connector->panel, fixed_mode); intel_panel_setup_backlight(connector); } intel_dp_add_properties(intel_dp, connector); /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written * 0xd. Failure to do so will result in spurious interrupts being * generated on the port when a cable is not attached. */ if (IS_G4X(dev) && !IS_GM45(dev)) { u32 temp = I915_READ(PEG_BAND_GAP_DATA); I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd); } } void intel_dp_init(struct drm_device *dev, int output_reg, enum port port) { struct intel_digital_port *intel_dig_port; struct intel_encoder *intel_encoder; struct drm_encoder *encoder; struct intel_connector *intel_connector; intel_dig_port = malloc(sizeof(struct intel_digital_port), DRM_MEM_KMS, M_WAITOK | M_ZERO); if (!intel_dig_port) return; intel_connector = malloc(sizeof(struct intel_connector), DRM_MEM_KMS, M_WAITOK | M_ZERO); if (!intel_connector) { free(intel_dig_port, DRM_MEM_KMS); return; } intel_encoder = &intel_dig_port->base; encoder = &intel_encoder->base; drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs, DRM_MODE_ENCODER_TMDS); drm_encoder_helper_add(&intel_encoder->base, &intel_dp_helper_funcs); intel_encoder->enable = intel_enable_dp; intel_encoder->pre_enable = intel_pre_enable_dp; intel_encoder->disable = intel_disable_dp; intel_encoder->post_disable = intel_post_disable_dp; intel_encoder->get_hw_state = intel_dp_get_hw_state; intel_dig_port->port = port; intel_dig_port->dp.output_reg = output_reg; intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT; intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2); intel_encoder->cloneable = false; intel_encoder->hot_plug = intel_dp_hot_plug; intel_dp_init_connector(intel_dig_port, intel_connector); } Index: head/sys/dev/drm2/radeon/radeon_fb.c =================================================================== --- head/sys/dev/drm2/radeon/radeon_fb.c (revision 298930) +++ head/sys/dev/drm2/radeon/radeon_fb.c (revision 298931) @@ -1,405 +1,405 @@ /* * Copyright © 2007 David Airlie * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: * David Airlie */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "radeon.h" #include /* object hierarchy - this contains a helper + a radeon fb the helper contains a pointer to radeon framebuffer baseclass. */ struct radeon_fbdev { struct drm_fb_helper helper; struct radeon_framebuffer rfb; struct list_head fbdev_list; struct radeon_device *rdev; }; #if defined(__linux__) static struct fb_ops radeonfb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, .fb_fillrect = cfb_fillrect, .fb_copyarea = cfb_copyarea, .fb_imageblit = cfb_imageblit, .fb_pan_display = drm_fb_helper_pan_display, .fb_blank = drm_fb_helper_blank, .fb_setcmap = drm_fb_helper_setcmap, .fb_debug_enter = drm_fb_helper_debug_enter, .fb_debug_leave = drm_fb_helper_debug_leave, }; #endif int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled) { int aligned = width; int align_large = (ASIC_IS_AVIVO(rdev)) || tiled; int pitch_mask = 0; switch (bpp / 8) { case 1: pitch_mask = align_large ? 255 : 127; break; case 2: pitch_mask = align_large ? 127 : 31; break; case 3: case 4: pitch_mask = align_large ? 63 : 15; break; } aligned += pitch_mask; aligned &= ~pitch_mask; return aligned; } static void radeonfb_destroy_pinned_object(struct drm_gem_object *gobj) { struct radeon_bo *rbo = gem_to_radeon_bo(gobj); int ret; ret = radeon_bo_reserve(rbo, false); if (likely(ret == 0)) { radeon_bo_kunmap(rbo); radeon_bo_unpin(rbo); radeon_bo_unreserve(rbo); } drm_gem_object_unreference_unlocked(gobj); } static int radeonfb_create_pinned_object(struct radeon_fbdev *rfbdev, struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **gobj_p) { struct radeon_device *rdev = rfbdev->rdev; struct drm_gem_object *gobj = NULL; struct radeon_bo *rbo = NULL; bool fb_tiled = false; /* useful for testing */ u32 tiling_flags = 0; int ret; int aligned_size, size; int height = mode_cmd->height; u32 bpp, depth; drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp); /* need to align pitch with crtc limits */ mode_cmd->pitches[0] = radeon_align_pitch(rdev, mode_cmd->width, bpp, fb_tiled) * ((bpp + 1) / 8); if (rdev->family >= CHIP_R600) height = roundup2(mode_cmd->height, 8); size = mode_cmd->pitches[0] * height; aligned_size = roundup2(size, PAGE_SIZE); ret = radeon_gem_object_create(rdev, aligned_size, 0, RADEON_GEM_DOMAIN_VRAM, false, true, &gobj); if (ret) { DRM_ERROR("failed to allocate framebuffer (%d)\n", aligned_size); return -ENOMEM; } rbo = gem_to_radeon_bo(gobj); if (fb_tiled) tiling_flags = RADEON_TILING_MACRO; #ifdef __BIG_ENDIAN switch (bpp) { case 32: tiling_flags |= RADEON_TILING_SWAP_32BIT; break; case 16: tiling_flags |= RADEON_TILING_SWAP_16BIT; default: break; } #endif if (tiling_flags) { ret = radeon_bo_set_tiling_flags(rbo, tiling_flags | RADEON_TILING_SURFACE, mode_cmd->pitches[0]); if (ret) dev_err(rdev->dev, "FB failed to set tiling flags\n"); } ret = radeon_bo_reserve(rbo, false); if (unlikely(ret != 0)) goto out_unref; /* Only 27 bit offset for legacy CRTC */ ret = radeon_bo_pin_restricted(rbo, RADEON_GEM_DOMAIN_VRAM, ASIC_IS_AVIVO(rdev) ? 0 : 1 << 27, NULL); if (ret) { radeon_bo_unreserve(rbo); goto out_unref; } if (fb_tiled) radeon_bo_check_tiling(rbo, 0, 0); ret = radeon_bo_kmap(rbo, NULL); radeon_bo_unreserve(rbo); if (ret) { goto out_unref; } *gobj_p = gobj; return 0; out_unref: radeonfb_destroy_pinned_object(gobj); *gobj_p = NULL; return ret; } static int radeonfb_create(struct radeon_fbdev *rfbdev, struct drm_fb_helper_surface_size *sizes) { struct radeon_device *rdev = rfbdev->rdev; struct fb_info *info; struct drm_framebuffer *fb = NULL; struct drm_mode_fb_cmd2 mode_cmd; struct drm_gem_object *gobj = NULL; struct radeon_bo *rbo = NULL; int ret; unsigned long tmp; mode_cmd.width = sizes->surface_width; mode_cmd.height = sizes->surface_height; /* avivo can't scanout real 24bpp */ if ((sizes->surface_bpp == 24) && ASIC_IS_AVIVO(rdev)) sizes->surface_bpp = 32; mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth); ret = radeonfb_create_pinned_object(rfbdev, &mode_cmd, &gobj); if (ret) { DRM_ERROR("failed to create fbcon object %d\n", ret); return ret; } rbo = gem_to_radeon_bo(gobj); /* okay we have an object now allocate the framebuffer */ info = framebuffer_alloc(); if (info == NULL) { ret = -ENOMEM; goto out_unref; } ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->rfb, &mode_cmd, gobj); if (ret) { - DRM_ERROR("failed to initalise framebuffer %d\n", ret); + DRM_ERROR("failed to initialise framebuffer %d\n", ret); goto out_unref; } fb = &rfbdev->rfb.base; /* setup helper */ rfbdev->helper.fb = fb; rfbdev->helper.fbdev = info; memset(rbo->kptr, 0x0, radeon_bo_size(rbo)); drm_fb_helper_fill_fix(info, fb->pitches[0], fb->depth); tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start; info->fb_size = radeon_bo_size(rbo); info->fb_bpp = sizes->surface_bpp; info->fb_pbase = rdev->mc.aper_base + tmp; info->fb_vbase = (vm_offset_t)rbo->kptr; drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height); DRM_INFO("fb mappable at 0x%" PRIXPTR "\n", info->fb_pbase); DRM_INFO("vram apper at 0x%lX\n", (unsigned long)rdev->mc.aper_base); DRM_INFO("size %lu\n", (unsigned long)radeon_bo_size(rbo)); DRM_INFO("fb depth is %d\n", fb->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); return 0; out_unref: if (rbo) { /* TODO? dumbbell@ */ } if (fb && ret) { drm_gem_object_unreference(gobj); drm_framebuffer_cleanup(fb); free(fb, DRM_MEM_DRIVER); /* XXX malloc'd in radeon_user_framebuffer_create? */ } return ret; } static int radeon_fb_find_or_create_single(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { struct radeon_fbdev *rfbdev = (struct radeon_fbdev *)helper; int new_fb = 0; int ret; if (!helper->fb) { ret = radeonfb_create(rfbdev, sizes); if (ret) return ret; new_fb = 1; } return new_fb; } void radeon_fb_output_poll_changed(struct radeon_device *rdev) { drm_fb_helper_hotplug_event(&rdev->mode_info.rfbdev->helper); } static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfbdev) { struct fb_info *info; struct radeon_framebuffer *rfb = &rfbdev->rfb; if (rfbdev->helper.fbdev) { info = rfbdev->helper.fbdev; if (info->fb_fbd_dev != NULL) device_delete_child(dev->dev, info->fb_fbd_dev); framebuffer_release(info); } if (rfb->obj) { radeonfb_destroy_pinned_object(rfb->obj); rfb->obj = NULL; } drm_fb_helper_fini(&rfbdev->helper); drm_framebuffer_cleanup(&rfb->base); return 0; } static struct drm_fb_helper_funcs radeon_fb_helper_funcs = { .gamma_set = radeon_crtc_fb_gamma_set, .gamma_get = radeon_crtc_fb_gamma_get, .fb_probe = radeon_fb_find_or_create_single, }; int radeon_fbdev_init(struct radeon_device *rdev) { struct radeon_fbdev *rfbdev; int bpp_sel = 32; int ret; /* select 8 bpp console on RN50 or 16MB cards */ if (ASIC_IS_RN50(rdev) || rdev->mc.real_vram_size <= (32*1024*1024)) bpp_sel = 8; rfbdev = malloc(sizeof(struct radeon_fbdev), DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); if (!rfbdev) return -ENOMEM; rfbdev->rdev = rdev; rdev->mode_info.rfbdev = rfbdev; rfbdev->helper.funcs = &radeon_fb_helper_funcs; ret = drm_fb_helper_init(rdev->ddev, &rfbdev->helper, rdev->num_crtc, RADEONFB_CONN_LIMIT); if (ret) { free(rfbdev, DRM_MEM_DRIVER); return ret; } drm_fb_helper_single_add_all_connectors(&rfbdev->helper); drm_fb_helper_initial_config(&rfbdev->helper, bpp_sel); return 0; } void radeon_fbdev_fini(struct radeon_device *rdev) { if (!rdev->mode_info.rfbdev) return; radeon_fbdev_destroy(rdev->ddev, rdev->mode_info.rfbdev); free(rdev->mode_info.rfbdev, DRM_MEM_DRIVER); rdev->mode_info.rfbdev = NULL; } void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state) { #ifdef FREEBSD_WIP fb_set_suspend(rdev->mode_info.rfbdev->helper.fbdev, state); #endif /* FREEBSD_WIP */ } int radeon_fbdev_total_size(struct radeon_device *rdev) { struct radeon_bo *robj; int size = 0; robj = gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj); size += radeon_bo_size(robj); return size; } bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj) { if (robj == gem_to_radeon_bo(rdev->mode_info.rfbdev->rfb.obj)) return true; return false; } struct fb_info * radeon_fb_helper_getinfo(device_t kdev) { struct drm_device *dev; struct radeon_device *rdev; struct radeon_fbdev *rfbdev; struct fb_info *info; dev = device_get_softc(kdev); rdev = dev->dev_private; rfbdev = rdev->mode_info.rfbdev; if (rfbdev == NULL) return (NULL); info = rfbdev->helper.fbdev; return (info); } Index: head/sys/dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- head/sys/dev/drm2/ttm/ttm_bo_vm.c (revision 298930) +++ head/sys/dev/drm2/ttm/ttm_bo_vm.c (revision 298931) @@ -1,563 +1,563 @@ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /* * Authors: Thomas Hellstrom */ /* * Copyright (c) 2013 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include #include #include #include #include #include #include #define TTM_BO_VM_NUM_PREFAULT 16 RB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, ttm_bo_cmp_rb_tree_items); int ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, struct ttm_buffer_object *b) { if (a->vm_node->start < b->vm_node->start) { return (-1); } else if (a->vm_node->start > b->vm_node->start) { return (1); } else { return (0); } } static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, unsigned long page_start, unsigned long num_pages) { unsigned long cur_offset; struct ttm_buffer_object *bo; struct ttm_buffer_object *best_bo = NULL; bo = RB_ROOT(&bdev->addr_space_rb); while (bo != NULL) { cur_offset = bo->vm_node->start; if (page_start >= cur_offset) { best_bo = bo; if (page_start == cur_offset) break; bo = RB_RIGHT(bo, vm_rb); } else bo = RB_LEFT(bo, vm_rb); } if (unlikely(best_bo == NULL)) return NULL; if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < (page_start + num_pages))) return NULL; return best_bo; } static int ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, vm_page_t *mres) { struct ttm_buffer_object *bo = vm_obj->handle; struct ttm_bo_device *bdev = bo->bdev; struct ttm_tt *ttm = NULL; vm_page_t m, m1; int ret; int retval = VM_PAGER_OK; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; vm_object_pip_add(vm_obj, 1); if (*mres != NULL) { vm_page_lock(*mres); vm_page_remove(*mres); vm_page_unlock(*mres); } retry: VM_OBJECT_WUNLOCK(vm_obj); m = NULL; reserve: ret = ttm_bo_reserve(bo, false, false, false, 0); if (unlikely(ret != 0)) { if (ret == -EBUSY) { kern_yield(0); goto reserve; } } if (bdev->driver->fault_reserve_notify) { ret = bdev->driver->fault_reserve_notify(bo); switch (ret) { case 0: break; case -EBUSY: case -ERESTARTSYS: case -EINTR: kern_yield(0); goto reserve; default: retval = VM_PAGER_ERROR; goto out_unlock; } } /* * Wait for buffer data in transit, due to a pipelined * move. */ mtx_lock(&bdev->fence_lock); if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { /* * Here, the behavior differs between Linux and FreeBSD. * * On Linux, the wait is interruptible (3rd argument to * ttm_bo_wait). There must be some mechanism to resume * page fault handling, once the signal is processed. * * On FreeBSD, the wait is uninteruptible. This is not a * problem as we can't end up with an unkillable process * here, because the wait will eventually time out. * * An example of this situation is the Xorg process * which uses SIGALRM internally. The signal could * interrupt the wait, causing the page fault to fail * and the process to receive SIGSEGV. */ ret = ttm_bo_wait(bo, false, false, false); mtx_unlock(&bdev->fence_lock); if (unlikely(ret != 0)) { retval = VM_PAGER_ERROR; goto out_unlock; } } else mtx_unlock(&bdev->fence_lock); ret = ttm_mem_io_lock(man, true); if (unlikely(ret != 0)) { retval = VM_PAGER_ERROR; goto out_unlock; } ret = ttm_mem_io_reserve_vm(bo); if (unlikely(ret != 0)) { retval = VM_PAGER_ERROR; goto out_io_unlock; } /* * Strictly, we're not allowed to modify vma->vm_page_prot here, * since the mmap_sem is only held in read mode. However, we * modify only the caching bits of vma->vm_page_prot and * consider those bits protected by * the bo->mutex, as we should be the only writers. * There shouldn't really be any readers of these bits except * within vm_insert_mixed()? fork? * * TODO: Add a list of vmas to the bo, and change the * vma->vm_page_prot when the object changes caching policy, with * the correct locks held. */ if (!bo->mem.bus.is_iomem) { /* Allocate all page at once, most common usage */ ttm = bo->ttm; if (ttm->bdev->driver->ttm_tt_populate(ttm)) { retval = VM_PAGER_ERROR; goto out_io_unlock; } } if (bo->mem.bus.is_iomem) { m = PHYS_TO_VM_PAGE(bo->mem.bus.base + bo->mem.bus.offset + offset); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("physical address %#jx not fictitious", (uintmax_t)(bo->mem.bus.base + bo->mem.bus.offset + offset))); pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement)); } else { ttm = bo->ttm; m = ttm->pages[OFF_TO_IDX(offset)]; if (unlikely(!m)) { retval = VM_PAGER_ERROR; goto out_io_unlock; } pmap_page_set_memattr(m, (bo->mem.placement & TTM_PL_FLAG_CACHED) ? VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement)); } VM_OBJECT_WLOCK(vm_obj); if (vm_page_busied(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); vm_page_busy_sleep(m, "ttmpbs"); VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); goto retry; } m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); if (m1 == NULL) { if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) { VM_OBJECT_WUNLOCK(vm_obj); VM_WAIT; VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); goto retry; } } else { KASSERT(m == m1, ("inconsistent insert bo %p m %p m1 %p offset %jx", bo, m, m1, (uintmax_t)offset)); } m->valid = VM_PAGE_BITS_ALL; vm_page_xbusy(m); if (*mres != NULL) { - KASSERT(*mres != m, ("loosing %p %p", *mres, m)); + KASSERT(*mres != m, ("losing %p %p", *mres, m)); vm_page_lock(*mres); vm_page_free(*mres); vm_page_unlock(*mres); } *mres = m; out_io_unlock1: ttm_mem_io_unlock(man); out_unlock1: ttm_bo_unreserve(bo); vm_object_pip_wakeup(vm_obj); return (retval); out_io_unlock: VM_OBJECT_WLOCK(vm_obj); goto out_io_unlock1; out_unlock: VM_OBJECT_WLOCK(vm_obj); goto out_unlock1; } static int ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { /* * On Linux, a reference to the buffer object is acquired here. * The reason is that this function is not called when the * mmap() is initialized, but only when a process forks for * instance. Therefore on Linux, the reference on the bo is * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's * then released in ttm_bo_vm_close(). * - * Here, this function is called during mmap() intialization. + * Here, this function is called during mmap() initialization. * Thus, the reference acquired in ttm_bo_mmap_single() is * sufficient. */ *color = 0; return (0); } static void ttm_bo_vm_dtor(void *handle) { struct ttm_buffer_object *bo = handle; ttm_bo_unref(&bo); } static struct cdev_pager_ops ttm_pager_ops = { .cdev_pg_fault = ttm_bo_vm_fault, .cdev_pg_ctor = ttm_bo_vm_ctor, .cdev_pg_dtor = ttm_bo_vm_dtor }; int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot) { struct ttm_bo_driver *driver; struct ttm_buffer_object *bo; struct vm_object *vm_obj; int ret; rw_wlock(&bdev->vm_lock); bo = ttm_bo_vm_lookup_rb(bdev, OFF_TO_IDX(*offset), OFF_TO_IDX(size)); if (likely(bo != NULL)) refcount_acquire(&bo->kref); rw_wunlock(&bdev->vm_lock); if (unlikely(bo == NULL)) { printf("[TTM] Could not find buffer object to map\n"); return (-EINVAL); } driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { ret = -EPERM; goto out_unref; } ret = driver->verify_access(bo); if (unlikely(ret != 0)) goto out_unref; vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE, &ttm_pager_ops, size, nprot, 0, curthread->td_ucred); if (vm_obj == NULL) { ret = -EINVAL; goto out_unref; } /* * Note: We're transferring the bo reference to vm_obj->handle here. */ *offset = 0; *obj_res = vm_obj; return 0; out_unref: ttm_bo_unref(&bo); return ret; } void ttm_bo_release_mmap(struct ttm_buffer_object *bo) { vm_object_t vm_obj; vm_page_t m; int i; vm_obj = cdev_pager_lookup(bo); if (vm_obj == NULL) return; VM_OBJECT_WLOCK(vm_obj); retry: for (i = 0; i < bo->num_pages; i++) { m = vm_page_lookup(vm_obj, i); if (m == NULL) continue; if (vm_page_sleep_if_busy(m, "ttm_unm")) goto retry; cdev_pager_free_page(vm_obj, m); } VM_OBJECT_WUNLOCK(vm_obj); vm_object_deallocate(vm_obj); } #if 0 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { if (vma->vm_pgoff != 0) return -EACCES; vma->vm_ops = &ttm_bo_vm_ops; vma->vm_private_data = ttm_bo_reference(bo); vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; return 0; } ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, const char __user *wbuf, char __user *rbuf, size_t count, loff_t *f_pos, bool write) { struct ttm_buffer_object *bo; struct ttm_bo_driver *driver; struct ttm_bo_kmap_obj map; unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); unsigned long kmap_offset; unsigned long kmap_end; unsigned long kmap_num; size_t io_size; unsigned int page_offset; char *virtual; int ret; bool no_wait = false; bool dummy; read_lock(&bdev->vm_lock); bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); if (likely(bo != NULL)) ttm_bo_reference(bo); read_unlock(&bdev->vm_lock); if (unlikely(bo == NULL)) return -EFAULT; driver = bo->bdev->driver; if (unlikely(!driver->verify_access)) { ret = -EPERM; goto out_unref; } ret = driver->verify_access(bo, filp); if (unlikely(ret != 0)) goto out_unref; kmap_offset = dev_offset - bo->vm_node->start; if (unlikely(kmap_offset >= bo->num_pages)) { ret = -EFBIG; goto out_unref; } page_offset = *f_pos & ~PAGE_MASK; io_size = bo->num_pages - kmap_offset; io_size = (io_size << PAGE_SHIFT) - page_offset; if (count < io_size) io_size = count; kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; kmap_num = kmap_end - kmap_offset + 1; ret = ttm_bo_reserve(bo, true, no_wait, false, 0); switch (ret) { case 0: break; case -EBUSY: ret = -EAGAIN; goto out_unref; default: goto out_unref; } ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); if (unlikely(ret != 0)) { ttm_bo_unreserve(bo); goto out_unref; } virtual = ttm_kmap_obj_virtual(&map, &dummy); virtual += page_offset; if (write) ret = copy_from_user(virtual, wbuf, io_size); else ret = copy_to_user(rbuf, virtual, io_size); ttm_bo_kunmap(&map); ttm_bo_unreserve(bo); ttm_bo_unref(&bo); if (unlikely(ret != 0)) return -EFBIG; *f_pos += io_size; return io_size; out_unref: ttm_bo_unref(&bo); return ret; } ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf, char __user *rbuf, size_t count, loff_t *f_pos, bool write) { struct ttm_bo_kmap_obj map; unsigned long kmap_offset; unsigned long kmap_end; unsigned long kmap_num; size_t io_size; unsigned int page_offset; char *virtual; int ret; bool no_wait = false; bool dummy; kmap_offset = (*f_pos >> PAGE_SHIFT); if (unlikely(kmap_offset >= bo->num_pages)) return -EFBIG; page_offset = *f_pos & ~PAGE_MASK; io_size = bo->num_pages - kmap_offset; io_size = (io_size << PAGE_SHIFT) - page_offset; if (count < io_size) io_size = count; kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; kmap_num = kmap_end - kmap_offset + 1; ret = ttm_bo_reserve(bo, true, no_wait, false, 0); switch (ret) { case 0: break; case -EBUSY: return -EAGAIN; default: return ret; } ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); if (unlikely(ret != 0)) { ttm_bo_unreserve(bo); return ret; } virtual = ttm_kmap_obj_virtual(&map, &dummy); virtual += page_offset; if (write) ret = copy_from_user(virtual, wbuf, io_size); else ret = copy_to_user(rbuf, virtual, io_size); ttm_bo_kunmap(&map); ttm_bo_unreserve(bo); ttm_bo_unref(&bo); if (unlikely(ret != 0)) return ret; *f_pos += io_size; return io_size; } #endif Index: head/sys/dev/hptiop/hptiop.c =================================================================== --- head/sys/dev/hptiop/hptiop.c (revision 298930) +++ head/sys/dev/hptiop/hptiop.c (revision 298931) @@ -1,2854 +1,2854 @@ /* * HighPoint RR3xxx/4xxx RAID Driver for FreeBSD * Copyright (C) 2007-2012 HighPoint Technologies, Inc. All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const char driver_name[] = "hptiop"; static const char driver_version[] = "v1.9"; static devclass_t hptiop_devclass; static int hptiop_send_sync_msg(struct hpt_iop_hba *hba, u_int32_t msg, u_int32_t millisec); static void hptiop_request_callback_itl(struct hpt_iop_hba *hba, u_int32_t req); static void hptiop_request_callback_mv(struct hpt_iop_hba *hba, u_int64_t req); static void hptiop_request_callback_mvfrey(struct hpt_iop_hba *hba, u_int32_t req); static void hptiop_os_message_callback(struct hpt_iop_hba *hba, u_int32_t msg); static int hptiop_do_ioctl_itl(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param *pParams); static int hptiop_do_ioctl_mv(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param *pParams); static int hptiop_do_ioctl_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param *pParams); static int hptiop_rescan_bus(struct hpt_iop_hba *hba); static int hptiop_alloc_pci_res_itl(struct hpt_iop_hba *hba); static int hptiop_alloc_pci_res_mv(struct hpt_iop_hba *hba); static int hptiop_alloc_pci_res_mvfrey(struct hpt_iop_hba *hba); static int hptiop_get_config_itl(struct hpt_iop_hba *hba, struct hpt_iop_request_get_config *config); static int hptiop_get_config_mv(struct hpt_iop_hba *hba, struct hpt_iop_request_get_config *config); static int hptiop_get_config_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_request_get_config *config); static int hptiop_set_config_itl(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config); static int hptiop_set_config_mv(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config); static int hptiop_set_config_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config); static int hptiop_internal_memalloc_mv(struct hpt_iop_hba *hba); static int hptiop_internal_memalloc_mvfrey(struct hpt_iop_hba *hba); static int hptiop_internal_memfree_itl(struct hpt_iop_hba *hba); static int hptiop_internal_memfree_mv(struct hpt_iop_hba *hba); static int hptiop_internal_memfree_mvfrey(struct hpt_iop_hba *hba); static int hptiop_post_ioctl_command_itl(struct hpt_iop_hba *hba, u_int32_t req32, struct hpt_iop_ioctl_param *pParams); static int hptiop_post_ioctl_command_mv(struct hpt_iop_hba *hba, struct hpt_iop_request_ioctl_command *req, struct hpt_iop_ioctl_param *pParams); static int hptiop_post_ioctl_command_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_request_ioctl_command *req, struct hpt_iop_ioctl_param *pParams); static void hptiop_post_req_itl(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs); static void hptiop_post_req_mv(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs); static void hptiop_post_req_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs); static void hptiop_post_msg_itl(struct hpt_iop_hba *hba, u_int32_t msg); static void hptiop_post_msg_mv(struct hpt_iop_hba *hba, u_int32_t msg); static void hptiop_post_msg_mvfrey(struct hpt_iop_hba *hba, u_int32_t msg); static void hptiop_enable_intr_itl(struct hpt_iop_hba *hba); static void hptiop_enable_intr_mv(struct hpt_iop_hba *hba); static void hptiop_enable_intr_mvfrey(struct hpt_iop_hba *hba); static void hptiop_disable_intr_itl(struct hpt_iop_hba *hba); static void hptiop_disable_intr_mv(struct hpt_iop_hba *hba); static void hptiop_disable_intr_mvfrey(struct hpt_iop_hba *hba); static void hptiop_free_srb(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb); static int hptiop_os_query_remove_device(struct hpt_iop_hba *hba, int tid); static int hptiop_probe(device_t dev); static int hptiop_attach(device_t dev); static int hptiop_detach(device_t dev); static int hptiop_shutdown(device_t dev); static void hptiop_action(struct cam_sim *sim, union ccb *ccb); static void hptiop_poll(struct cam_sim *sim); static void hptiop_async(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg); static void hptiop_pci_intr(void *arg); static void hptiop_release_resource(struct hpt_iop_hba *hba); static void hptiop_reset_adapter(void *argv); static d_open_t hptiop_open; static d_close_t hptiop_close; static d_ioctl_t hptiop_ioctl; static struct cdevsw hptiop_cdevsw = { .d_open = hptiop_open, .d_close = hptiop_close, .d_ioctl = hptiop_ioctl, .d_name = driver_name, .d_version = D_VERSION, }; #define hba_from_dev(dev) \ ((struct hpt_iop_hba *)devclass_get_softc(hptiop_devclass, dev2unit(dev))) #define BUS_SPACE_WRT4_ITL(offset, value) bus_space_write_4(hba->bar0t,\ hba->bar0h, offsetof(struct hpt_iopmu_itl, offset), (value)) #define BUS_SPACE_RD4_ITL(offset) bus_space_read_4(hba->bar0t,\ hba->bar0h, offsetof(struct hpt_iopmu_itl, offset)) #define BUS_SPACE_WRT4_MV0(offset, value) bus_space_write_4(hba->bar0t,\ hba->bar0h, offsetof(struct hpt_iopmv_regs, offset), value) #define BUS_SPACE_RD4_MV0(offset) bus_space_read_4(hba->bar0t,\ hba->bar0h, offsetof(struct hpt_iopmv_regs, offset)) #define BUS_SPACE_WRT4_MV2(offset, value) bus_space_write_4(hba->bar2t,\ hba->bar2h, offsetof(struct hpt_iopmu_mv, offset), value) #define BUS_SPACE_RD4_MV2(offset) bus_space_read_4(hba->bar2t,\ hba->bar2h, offsetof(struct hpt_iopmu_mv, offset)) #define BUS_SPACE_WRT4_MVFREY2(offset, value) bus_space_write_4(hba->bar2t,\ hba->bar2h, offsetof(struct hpt_iopmu_mvfrey, offset), value) #define BUS_SPACE_RD4_MVFREY2(offset) bus_space_read_4(hba->bar2t,\ hba->bar2h, offsetof(struct hpt_iopmu_mvfrey, offset)) static int hptiop_open(ioctl_dev_t dev, int flags, int devtype, ioctl_thread_t proc) { struct hpt_iop_hba *hba = hba_from_dev(dev); if (hba==NULL) return ENXIO; if (hba->flag & HPT_IOCTL_FLAG_OPEN) return EBUSY; hba->flag |= HPT_IOCTL_FLAG_OPEN; return 0; } static int hptiop_close(ioctl_dev_t dev, int flags, int devtype, ioctl_thread_t proc) { struct hpt_iop_hba *hba = hba_from_dev(dev); hba->flag &= ~(u_int32_t)HPT_IOCTL_FLAG_OPEN; return 0; } static int hptiop_ioctl(ioctl_dev_t dev, u_long cmd, caddr_t data, int flags, ioctl_thread_t proc) { int ret = EFAULT; struct hpt_iop_hba *hba = hba_from_dev(dev); mtx_lock(&Giant); switch (cmd) { case HPT_DO_IOCONTROL: ret = hba->ops->do_ioctl(hba, (struct hpt_iop_ioctl_param *)data); break; case HPT_SCAN_BUS: ret = hptiop_rescan_bus(hba); break; } mtx_unlock(&Giant); return ret; } static u_int64_t hptiop_mv_outbound_read(struct hpt_iop_hba *hba) { u_int64_t p; u_int32_t outbound_tail = BUS_SPACE_RD4_MV2(outbound_tail); u_int32_t outbound_head = BUS_SPACE_RD4_MV2(outbound_head); if (outbound_tail != outbound_head) { bus_space_read_region_4(hba->bar2t, hba->bar2h, offsetof(struct hpt_iopmu_mv, outbound_q[outbound_tail]), (u_int32_t *)&p, 2); outbound_tail++; if (outbound_tail == MVIOP_QUEUE_LEN) outbound_tail = 0; BUS_SPACE_WRT4_MV2(outbound_tail, outbound_tail); return p; } else return 0; } static void hptiop_mv_inbound_write(u_int64_t p, struct hpt_iop_hba *hba) { u_int32_t inbound_head = BUS_SPACE_RD4_MV2(inbound_head); u_int32_t head = inbound_head + 1; if (head == MVIOP_QUEUE_LEN) head = 0; bus_space_write_region_4(hba->bar2t, hba->bar2h, offsetof(struct hpt_iopmu_mv, inbound_q[inbound_head]), (u_int32_t *)&p, 2); BUS_SPACE_WRT4_MV2(inbound_head, head); BUS_SPACE_WRT4_MV0(inbound_doorbell, MVIOP_MU_INBOUND_INT_POSTQUEUE); } static void hptiop_post_msg_itl(struct hpt_iop_hba *hba, u_int32_t msg) { BUS_SPACE_WRT4_ITL(inbound_msgaddr0, msg); BUS_SPACE_RD4_ITL(outbound_intstatus); } static void hptiop_post_msg_mv(struct hpt_iop_hba *hba, u_int32_t msg) { BUS_SPACE_WRT4_MV2(inbound_msg, msg); BUS_SPACE_WRT4_MV0(inbound_doorbell, MVIOP_MU_INBOUND_INT_MSG); BUS_SPACE_RD4_MV0(outbound_intmask); } static void hptiop_post_msg_mvfrey(struct hpt_iop_hba *hba, u_int32_t msg) { BUS_SPACE_WRT4_MVFREY2(f0_to_cpu_msg_a, msg); BUS_SPACE_RD4_MVFREY2(f0_to_cpu_msg_a); } static int hptiop_wait_ready_itl(struct hpt_iop_hba * hba, u_int32_t millisec) { u_int32_t req=0; int i; for (i = 0; i < millisec; i++) { req = BUS_SPACE_RD4_ITL(inbound_queue); if (req != IOPMU_QUEUE_EMPTY) break; DELAY(1000); } if (req!=IOPMU_QUEUE_EMPTY) { BUS_SPACE_WRT4_ITL(outbound_queue, req); BUS_SPACE_RD4_ITL(outbound_intstatus); return 0; } return -1; } static int hptiop_wait_ready_mv(struct hpt_iop_hba * hba, u_int32_t millisec) { if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_NOP, millisec)) return -1; return 0; } static int hptiop_wait_ready_mvfrey(struct hpt_iop_hba * hba, u_int32_t millisec) { if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_NOP, millisec)) return -1; return 0; } static void hptiop_request_callback_itl(struct hpt_iop_hba * hba, u_int32_t index) { struct hpt_iop_srb *srb; struct hpt_iop_request_scsi_command *req=0; union ccb *ccb; u_int8_t *cdb; u_int32_t result, temp, dxfer; u_int64_t temp64; if (index & IOPMU_QUEUE_MASK_HOST_BITS) { /*host req*/ if (hba->firmware_version > 0x01020000 || hba->interface_version > 0x01020000) { srb = hba->srb[index & ~(u_int32_t) (IOPMU_QUEUE_ADDR_HOST_BIT | IOPMU_QUEUE_REQUEST_RESULT_BIT)]; req = (struct hpt_iop_request_scsi_command *)srb; if (index & IOPMU_QUEUE_REQUEST_RESULT_BIT) result = IOP_RESULT_SUCCESS; else result = req->header.result; } else { srb = hba->srb[index & ~(u_int32_t)IOPMU_QUEUE_ADDR_HOST_BIT]; req = (struct hpt_iop_request_scsi_command *)srb; result = req->header.result; } dxfer = req->dataxfer_length; goto srb_complete; } /*iop req*/ temp = bus_space_read_4(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_header, type)); result = bus_space_read_4(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_header, result)); switch(temp) { case IOP_REQUEST_TYPE_IOCTL_COMMAND: { temp64 = 0; bus_space_write_region_4(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_header, context), (u_int32_t *)&temp64, 2); wakeup((void *)((unsigned long)hba->u.itl.mu + index)); break; } case IOP_REQUEST_TYPE_SCSI_COMMAND: bus_space_read_region_4(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_header, context), (u_int32_t *)&temp64, 2); srb = (struct hpt_iop_srb *)(unsigned long)temp64; dxfer = bus_space_read_4(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_scsi_command, dataxfer_length)); srb_complete: ccb = (union ccb *)srb->ccb; if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; if (cdb[0] == SYNCHRONIZE_CACHE) { /* ??? */ ccb->ccb_h.status = CAM_REQ_CMP; goto scsi_done; } switch (result) { case IOP_RESULT_SUCCESS: switch (ccb->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; case CAM_DIR_OUT: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; } ccb->ccb_h.status = CAM_REQ_CMP; break; case IOP_RESULT_BAD_TARGET: ccb->ccb_h.status = CAM_DEV_NOT_THERE; break; case IOP_RESULT_BUSY: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_INVALID_REQUEST: ccb->ccb_h.status = CAM_REQ_INVALID; break; case IOP_RESULT_FAIL: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; case IOP_RESULT_RESET: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_CHECK_CONDITION: memset(&ccb->csio.sense_data, 0, sizeof(ccb->csio.sense_data)); if (dxfer < ccb->csio.sense_len) ccb->csio.sense_resid = ccb->csio.sense_len - dxfer; else ccb->csio.sense_resid = 0; if (srb->srb_flag & HPT_SRB_FLAG_HIGH_MEM_ACESS) {/*iop*/ bus_space_read_region_1(hba->bar0t, hba->bar0h, index + offsetof(struct hpt_iop_request_scsi_command, sg_list), (u_int8_t *)&ccb->csio.sense_data, MIN(dxfer, sizeof(ccb->csio.sense_data))); } else { memcpy(&ccb->csio.sense_data, &req->sg_list, MIN(dxfer, sizeof(ccb->csio.sense_data))); } ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; ccb->ccb_h.status |= CAM_AUTOSNS_VALID; ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; break; default: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; } scsi_done: if (srb->srb_flag & HPT_SRB_FLAG_HIGH_MEM_ACESS) BUS_SPACE_WRT4_ITL(outbound_queue, index); ccb->csio.resid = ccb->csio.dxfer_len - dxfer; hptiop_free_srb(hba, srb); xpt_done(ccb); break; } } static void hptiop_drain_outbound_queue_itl(struct hpt_iop_hba *hba) { u_int32_t req, temp; while ((req = BUS_SPACE_RD4_ITL(outbound_queue)) !=IOPMU_QUEUE_EMPTY) { if (req & IOPMU_QUEUE_MASK_HOST_BITS) hptiop_request_callback_itl(hba, req); else { struct hpt_iop_request_header *p; p = (struct hpt_iop_request_header *) ((char *)hba->u.itl.mu + req); temp = bus_space_read_4(hba->bar0t, hba->bar0h,req + offsetof(struct hpt_iop_request_header, flags)); if (temp & IOP_REQUEST_FLAG_SYNC_REQUEST) { u_int64_t temp64; bus_space_read_region_4(hba->bar0t, hba->bar0h,req + offsetof(struct hpt_iop_request_header, context), (u_int32_t *)&temp64, 2); if (temp64) { hptiop_request_callback_itl(hba, req); } else { temp64 = 1; bus_space_write_region_4(hba->bar0t, hba->bar0h,req + offsetof(struct hpt_iop_request_header, context), (u_int32_t *)&temp64, 2); } } else hptiop_request_callback_itl(hba, req); } } } static int hptiop_intr_itl(struct hpt_iop_hba * hba) { u_int32_t status; int ret = 0; status = BUS_SPACE_RD4_ITL(outbound_intstatus); if (status & IOPMU_OUTBOUND_INT_MSG0) { u_int32_t msg = BUS_SPACE_RD4_ITL(outbound_msgaddr0); KdPrint(("hptiop: received outbound msg %x\n", msg)); BUS_SPACE_WRT4_ITL(outbound_intstatus, IOPMU_OUTBOUND_INT_MSG0); hptiop_os_message_callback(hba, msg); ret = 1; } if (status & IOPMU_OUTBOUND_INT_POSTQUEUE) { hptiop_drain_outbound_queue_itl(hba); ret = 1; } return ret; } static void hptiop_request_callback_mv(struct hpt_iop_hba * hba, u_int64_t _tag) { u_int32_t context = (u_int32_t)_tag; if (context & MVIOP_CMD_TYPE_SCSI) { struct hpt_iop_srb *srb; struct hpt_iop_request_scsi_command *req; union ccb *ccb; u_int8_t *cdb; srb = hba->srb[context >> MVIOP_REQUEST_NUMBER_START_BIT]; req = (struct hpt_iop_request_scsi_command *)srb; ccb = (union ccb *)srb->ccb; if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; if (cdb[0] == SYNCHRONIZE_CACHE) { /* ??? */ ccb->ccb_h.status = CAM_REQ_CMP; goto scsi_done; } if (context & MVIOP_MU_QUEUE_REQUEST_RESULT_BIT) req->header.result = IOP_RESULT_SUCCESS; switch (req->header.result) { case IOP_RESULT_SUCCESS: switch (ccb->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; case CAM_DIR_OUT: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; } ccb->ccb_h.status = CAM_REQ_CMP; break; case IOP_RESULT_BAD_TARGET: ccb->ccb_h.status = CAM_DEV_NOT_THERE; break; case IOP_RESULT_BUSY: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_INVALID_REQUEST: ccb->ccb_h.status = CAM_REQ_INVALID; break; case IOP_RESULT_FAIL: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; case IOP_RESULT_RESET: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_CHECK_CONDITION: memset(&ccb->csio.sense_data, 0, sizeof(ccb->csio.sense_data)); if (req->dataxfer_length < ccb->csio.sense_len) ccb->csio.sense_resid = ccb->csio.sense_len - req->dataxfer_length; else ccb->csio.sense_resid = 0; memcpy(&ccb->csio.sense_data, &req->sg_list, MIN(req->dataxfer_length, sizeof(ccb->csio.sense_data))); ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; ccb->ccb_h.status |= CAM_AUTOSNS_VALID; ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; break; default: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; } scsi_done: ccb->csio.resid = ccb->csio.dxfer_len - req->dataxfer_length; hptiop_free_srb(hba, srb); xpt_done(ccb); } else if (context & MVIOP_CMD_TYPE_IOCTL) { struct hpt_iop_request_ioctl_command *req = hba->ctlcfg_ptr; if (context & MVIOP_MU_QUEUE_REQUEST_RESULT_BIT) hba->config_done = 1; else hba->config_done = -1; wakeup(req); } else if (context & (MVIOP_CMD_TYPE_SET_CONFIG | MVIOP_CMD_TYPE_GET_CONFIG)) hba->config_done = 1; else { device_printf(hba->pcidev, "wrong callback type\n"); } } static void hptiop_request_callback_mvfrey(struct hpt_iop_hba * hba, u_int32_t _tag) { u_int32_t req_type = _tag & 0xf; struct hpt_iop_srb *srb; struct hpt_iop_request_scsi_command *req; union ccb *ccb; u_int8_t *cdb; switch (req_type) { case IOP_REQUEST_TYPE_GET_CONFIG: case IOP_REQUEST_TYPE_SET_CONFIG: hba->config_done = 1; break; case IOP_REQUEST_TYPE_SCSI_COMMAND: srb = hba->srb[(_tag >> 4) & 0xff]; req = (struct hpt_iop_request_scsi_command *)srb; ccb = (union ccb *)srb->ccb; callout_stop(&srb->timeout); if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; if (cdb[0] == SYNCHRONIZE_CACHE) { /* ??? */ ccb->ccb_h.status = CAM_REQ_CMP; goto scsi_done; } if (_tag & MVFREYIOPMU_QUEUE_REQUEST_RESULT_BIT) req->header.result = IOP_RESULT_SUCCESS; switch (req->header.result) { case IOP_RESULT_SUCCESS: switch (ccb->ccb_h.flags & CAM_DIR_MASK) { case CAM_DIR_IN: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; case CAM_DIR_OUT: bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(hba->io_dmat, srb->dma_map); break; } ccb->ccb_h.status = CAM_REQ_CMP; break; case IOP_RESULT_BAD_TARGET: ccb->ccb_h.status = CAM_DEV_NOT_THERE; break; case IOP_RESULT_BUSY: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_INVALID_REQUEST: ccb->ccb_h.status = CAM_REQ_INVALID; break; case IOP_RESULT_FAIL: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; case IOP_RESULT_RESET: ccb->ccb_h.status = CAM_BUSY; break; case IOP_RESULT_CHECK_CONDITION: memset(&ccb->csio.sense_data, 0, sizeof(ccb->csio.sense_data)); if (req->dataxfer_length < ccb->csio.sense_len) ccb->csio.sense_resid = ccb->csio.sense_len - req->dataxfer_length; else ccb->csio.sense_resid = 0; memcpy(&ccb->csio.sense_data, &req->sg_list, MIN(req->dataxfer_length, sizeof(ccb->csio.sense_data))); ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; ccb->ccb_h.status |= CAM_AUTOSNS_VALID; ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; break; default: ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; break; } scsi_done: ccb->csio.resid = ccb->csio.dxfer_len - req->dataxfer_length; hptiop_free_srb(hba, srb); xpt_done(ccb); break; case IOP_REQUEST_TYPE_IOCTL_COMMAND: if (_tag & MVFREYIOPMU_QUEUE_REQUEST_RESULT_BIT) hba->config_done = 1; else hba->config_done = -1; wakeup((struct hpt_iop_request_ioctl_command *)hba->ctlcfg_ptr); break; default: device_printf(hba->pcidev, "wrong callback type\n"); break; } } static void hptiop_drain_outbound_queue_mv(struct hpt_iop_hba * hba) { u_int64_t req; while ((req = hptiop_mv_outbound_read(hba))) { if (req & MVIOP_MU_QUEUE_ADDR_HOST_BIT) { if (req & MVIOP_MU_QUEUE_REQUEST_RETURN_CONTEXT) { hptiop_request_callback_mv(hba, req); } } } } static int hptiop_intr_mv(struct hpt_iop_hba * hba) { u_int32_t status; int ret = 0; status = BUS_SPACE_RD4_MV0(outbound_doorbell); if (status) BUS_SPACE_WRT4_MV0(outbound_doorbell, ~status); if (status & MVIOP_MU_OUTBOUND_INT_MSG) { u_int32_t msg = BUS_SPACE_RD4_MV2(outbound_msg); KdPrint(("hptiop: received outbound msg %x\n", msg)); hptiop_os_message_callback(hba, msg); ret = 1; } if (status & MVIOP_MU_OUTBOUND_INT_POSTQUEUE) { hptiop_drain_outbound_queue_mv(hba); ret = 1; } return ret; } static int hptiop_intr_mvfrey(struct hpt_iop_hba * hba) { u_int32_t status, _tag, cptr; int ret = 0; if (hba->initialized) { BUS_SPACE_WRT4_MVFREY2(pcie_f0_int_enable, 0); } status = BUS_SPACE_RD4_MVFREY2(f0_doorbell); if (status) { BUS_SPACE_WRT4_MVFREY2(f0_doorbell, status); if (status & CPU_TO_F0_DRBL_MSG_A_BIT) { u_int32_t msg = BUS_SPACE_RD4_MVFREY2(cpu_to_f0_msg_a); hptiop_os_message_callback(hba, msg); } ret = 1; } status = BUS_SPACE_RD4_MVFREY2(isr_cause); if (status) { BUS_SPACE_WRT4_MVFREY2(isr_cause, status); do { cptr = *hba->u.mvfrey.outlist_cptr & 0xff; while (hba->u.mvfrey.outlist_rptr != cptr) { hba->u.mvfrey.outlist_rptr++; if (hba->u.mvfrey.outlist_rptr == hba->u.mvfrey.list_count) { hba->u.mvfrey.outlist_rptr = 0; } _tag = hba->u.mvfrey.outlist[hba->u.mvfrey.outlist_rptr].val; hptiop_request_callback_mvfrey(hba, _tag); ret = 2; } } while (cptr != (*hba->u.mvfrey.outlist_cptr & 0xff)); } if (hba->initialized) { BUS_SPACE_WRT4_MVFREY2(pcie_f0_int_enable, 0x1010); } return ret; } static int hptiop_send_sync_request_itl(struct hpt_iop_hba * hba, u_int32_t req32, u_int32_t millisec) { u_int32_t i; u_int64_t temp64; BUS_SPACE_WRT4_ITL(inbound_queue, req32); BUS_SPACE_RD4_ITL(outbound_intstatus); for (i = 0; i < millisec; i++) { hptiop_intr_itl(hba); bus_space_read_region_4(hba->bar0t, hba->bar0h, req32 + offsetof(struct hpt_iop_request_header, context), (u_int32_t *)&temp64, 2); if (temp64) return 0; DELAY(1000); } return -1; } static int hptiop_send_sync_request_mv(struct hpt_iop_hba *hba, void *req, u_int32_t millisec) { u_int32_t i; u_int64_t phy_addr; hba->config_done = 0; phy_addr = hba->ctlcfgcmd_phy | (u_int64_t)MVIOP_MU_QUEUE_ADDR_HOST_BIT; ((struct hpt_iop_request_get_config *)req)->header.flags |= IOP_REQUEST_FLAG_SYNC_REQUEST | IOP_REQUEST_FLAG_OUTPUT_CONTEXT; hptiop_mv_inbound_write(phy_addr, hba); BUS_SPACE_RD4_MV0(outbound_intmask); for (i = 0; i < millisec; i++) { hptiop_intr_mv(hba); if (hba->config_done) return 0; DELAY(1000); } return -1; } static int hptiop_send_sync_request_mvfrey(struct hpt_iop_hba *hba, void *req, u_int32_t millisec) { u_int32_t i, index; u_int64_t phy_addr; struct hpt_iop_request_header *reqhdr = (struct hpt_iop_request_header *)req; hba->config_done = 0; phy_addr = hba->ctlcfgcmd_phy; reqhdr->flags = IOP_REQUEST_FLAG_SYNC_REQUEST | IOP_REQUEST_FLAG_OUTPUT_CONTEXT | IOP_REQUEST_FLAG_ADDR_BITS | ((phy_addr >> 16) & 0xffff0000); reqhdr->context = ((phy_addr & 0xffffffff) << 32 ) | IOPMU_QUEUE_ADDR_HOST_BIT | reqhdr->type; hba->u.mvfrey.inlist_wptr++; index = hba->u.mvfrey.inlist_wptr & 0x3fff; if (index == hba->u.mvfrey.list_count) { index = 0; hba->u.mvfrey.inlist_wptr &= ~0x3fff; hba->u.mvfrey.inlist_wptr ^= CL_POINTER_TOGGLE; } hba->u.mvfrey.inlist[index].addr = phy_addr; hba->u.mvfrey.inlist[index].intrfc_len = (reqhdr->size + 3) / 4; BUS_SPACE_WRT4_MVFREY2(inbound_write_ptr, hba->u.mvfrey.inlist_wptr); BUS_SPACE_RD4_MVFREY2(inbound_write_ptr); for (i = 0; i < millisec; i++) { hptiop_intr_mvfrey(hba); if (hba->config_done) return 0; DELAY(1000); } return -1; } static int hptiop_send_sync_msg(struct hpt_iop_hba *hba, u_int32_t msg, u_int32_t millisec) { u_int32_t i; hba->msg_done = 0; hba->ops->post_msg(hba, msg); for (i=0; iops->iop_intr(hba); if (hba->msg_done) break; DELAY(1000); } return hba->msg_done? 0 : -1; } static int hptiop_get_config_itl(struct hpt_iop_hba * hba, struct hpt_iop_request_get_config * config) { u_int32_t req32; config->header.size = sizeof(struct hpt_iop_request_get_config); config->header.type = IOP_REQUEST_TYPE_GET_CONFIG; config->header.flags = IOP_REQUEST_FLAG_SYNC_REQUEST; config->header.result = IOP_RESULT_PENDING; config->header.context = 0; req32 = BUS_SPACE_RD4_ITL(inbound_queue); if (req32 == IOPMU_QUEUE_EMPTY) return -1; bus_space_write_region_4(hba->bar0t, hba->bar0h, req32, (u_int32_t *)config, sizeof(struct hpt_iop_request_header) >> 2); if (hptiop_send_sync_request_itl(hba, req32, 20000)) { KdPrint(("hptiop: get config send cmd failed")); return -1; } bus_space_read_region_4(hba->bar0t, hba->bar0h, req32, (u_int32_t *)config, sizeof(struct hpt_iop_request_get_config) >> 2); BUS_SPACE_WRT4_ITL(outbound_queue, req32); return 0; } static int hptiop_get_config_mv(struct hpt_iop_hba * hba, struct hpt_iop_request_get_config * config) { struct hpt_iop_request_get_config *req; if (!(req = hba->ctlcfg_ptr)) return -1; req->header.flags = 0; req->header.type = IOP_REQUEST_TYPE_GET_CONFIG; req->header.size = sizeof(struct hpt_iop_request_get_config); req->header.result = IOP_RESULT_PENDING; req->header.context = MVIOP_CMD_TYPE_GET_CONFIG; if (hptiop_send_sync_request_mv(hba, req, 20000)) { KdPrint(("hptiop: get config send cmd failed")); return -1; } *config = *req; return 0; } static int hptiop_get_config_mvfrey(struct hpt_iop_hba * hba, struct hpt_iop_request_get_config * config) { struct hpt_iop_request_get_config *info = hba->u.mvfrey.config; if (info->header.size != sizeof(struct hpt_iop_request_get_config) || info->header.type != IOP_REQUEST_TYPE_GET_CONFIG) { KdPrint(("hptiop: header size %x/%x type %x/%x", info->header.size, (int)sizeof(struct hpt_iop_request_get_config), info->header.type, IOP_REQUEST_TYPE_GET_CONFIG)); return -1; } config->interface_version = info->interface_version; config->firmware_version = info->firmware_version; config->max_requests = info->max_requests; config->request_size = info->request_size; config->max_sg_count = info->max_sg_count; config->data_transfer_length = info->data_transfer_length; config->alignment_mask = info->alignment_mask; config->max_devices = info->max_devices; config->sdram_size = info->sdram_size; KdPrint(("hptiop: maxreq %x reqsz %x datalen %x maxdev %x sdram %x", config->max_requests, config->request_size, config->data_transfer_length, config->max_devices, config->sdram_size)); return 0; } static int hptiop_set_config_itl(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config) { u_int32_t req32; req32 = BUS_SPACE_RD4_ITL(inbound_queue); if (req32 == IOPMU_QUEUE_EMPTY) return -1; config->header.size = sizeof(struct hpt_iop_request_set_config); config->header.type = IOP_REQUEST_TYPE_SET_CONFIG; config->header.flags = IOP_REQUEST_FLAG_SYNC_REQUEST; config->header.result = IOP_RESULT_PENDING; config->header.context = 0; bus_space_write_region_4(hba->bar0t, hba->bar0h, req32, (u_int32_t *)config, sizeof(struct hpt_iop_request_set_config) >> 2); if (hptiop_send_sync_request_itl(hba, req32, 20000)) { KdPrint(("hptiop: set config send cmd failed")); return -1; } BUS_SPACE_WRT4_ITL(outbound_queue, req32); return 0; } static int hptiop_set_config_mv(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config) { struct hpt_iop_request_set_config *req; if (!(req = hba->ctlcfg_ptr)) return -1; memcpy((u_int8_t *)req + sizeof(struct hpt_iop_request_header), (u_int8_t *)config + sizeof(struct hpt_iop_request_header), sizeof(struct hpt_iop_request_set_config) - sizeof(struct hpt_iop_request_header)); req->header.flags = 0; req->header.type = IOP_REQUEST_TYPE_SET_CONFIG; req->header.size = sizeof(struct hpt_iop_request_set_config); req->header.result = IOP_RESULT_PENDING; req->header.context = MVIOP_CMD_TYPE_SET_CONFIG; if (hptiop_send_sync_request_mv(hba, req, 20000)) { KdPrint(("hptiop: set config send cmd failed")); return -1; } return 0; } static int hptiop_set_config_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_request_set_config *config) { struct hpt_iop_request_set_config *req; if (!(req = hba->ctlcfg_ptr)) return -1; memcpy((u_int8_t *)req + sizeof(struct hpt_iop_request_header), (u_int8_t *)config + sizeof(struct hpt_iop_request_header), sizeof(struct hpt_iop_request_set_config) - sizeof(struct hpt_iop_request_header)); req->header.type = IOP_REQUEST_TYPE_SET_CONFIG; req->header.size = sizeof(struct hpt_iop_request_set_config); req->header.result = IOP_RESULT_PENDING; if (hptiop_send_sync_request_mvfrey(hba, req, 20000)) { KdPrint(("hptiop: set config send cmd failed")); return -1; } return 0; } static int hptiop_post_ioctl_command_itl(struct hpt_iop_hba *hba, u_int32_t req32, struct hpt_iop_ioctl_param *pParams) { u_int64_t temp64; struct hpt_iop_request_ioctl_command req; if ((((pParams->nInBufferSize + 3) & ~3) + pParams->nOutBufferSize) > (hba->max_request_size - offsetof(struct hpt_iop_request_ioctl_command, buf))) { device_printf(hba->pcidev, "request size beyond max value"); return -1; } req.header.size = offsetof(struct hpt_iop_request_ioctl_command, buf) + pParams->nInBufferSize; req.header.type = IOP_REQUEST_TYPE_IOCTL_COMMAND; req.header.flags = IOP_REQUEST_FLAG_SYNC_REQUEST; req.header.result = IOP_RESULT_PENDING; req.header.context = req32 + (u_int64_t)(unsigned long)hba->u.itl.mu; req.ioctl_code = HPT_CTL_CODE_BSD_TO_IOP(pParams->dwIoControlCode); req.inbuf_size = pParams->nInBufferSize; req.outbuf_size = pParams->nOutBufferSize; req.bytes_returned = 0; bus_space_write_region_4(hba->bar0t, hba->bar0h, req32, (u_int32_t *)&req, offsetof(struct hpt_iop_request_ioctl_command, buf)>>2); hptiop_lock_adapter(hba); BUS_SPACE_WRT4_ITL(inbound_queue, req32); BUS_SPACE_RD4_ITL(outbound_intstatus); bus_space_read_region_4(hba->bar0t, hba->bar0h, req32 + offsetof(struct hpt_iop_request_ioctl_command, header.context), (u_int32_t *)&temp64, 2); while (temp64) { if (hptiop_sleep(hba, (void *)((unsigned long)hba->u.itl.mu + req32), PPAUSE, "hptctl", HPT_OSM_TIMEOUT)==0) break; hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_RESET, 60000); bus_space_read_region_4(hba->bar0t, hba->bar0h,req32 + offsetof(struct hpt_iop_request_ioctl_command, header.context), (u_int32_t *)&temp64, 2); } hptiop_unlock_adapter(hba); return 0; } static int hptiop_bus_space_copyin(struct hpt_iop_hba *hba, u_int32_t bus, void *user, int size) { unsigned char byte; int i; for (i=0; ibar0t, hba->bar0h, bus + i, byte); } return 0; } static int hptiop_bus_space_copyout(struct hpt_iop_hba *hba, u_int32_t bus, void *user, int size) { unsigned char byte; int i; for (i=0; ibar0t, hba->bar0h, bus + i); if (copyout(&byte, (u_int8_t *)user + i, 1)) return -1; } return 0; } static int hptiop_do_ioctl_itl(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param * pParams) { u_int32_t req32; u_int32_t result; if ((pParams->Magic != HPT_IOCTL_MAGIC) && (pParams->Magic != HPT_IOCTL_MAGIC32)) return EFAULT; req32 = BUS_SPACE_RD4_ITL(inbound_queue); if (req32 == IOPMU_QUEUE_EMPTY) return EFAULT; if (pParams->nInBufferSize) if (hptiop_bus_space_copyin(hba, req32 + offsetof(struct hpt_iop_request_ioctl_command, buf), (void *)pParams->lpInBuffer, pParams->nInBufferSize)) goto invalid; if (hptiop_post_ioctl_command_itl(hba, req32, pParams)) goto invalid; result = bus_space_read_4(hba->bar0t, hba->bar0h, req32 + offsetof(struct hpt_iop_request_ioctl_command, header.result)); if (result == IOP_RESULT_SUCCESS) { if (pParams->nOutBufferSize) if (hptiop_bus_space_copyout(hba, req32 + offsetof(struct hpt_iop_request_ioctl_command, buf) + ((pParams->nInBufferSize + 3) & ~3), (void *)pParams->lpOutBuffer, pParams->nOutBufferSize)) goto invalid; if (pParams->lpBytesReturned) { if (hptiop_bus_space_copyout(hba, req32 + offsetof(struct hpt_iop_request_ioctl_command, bytes_returned), (void *)pParams->lpBytesReturned, sizeof(unsigned long))) goto invalid; } BUS_SPACE_WRT4_ITL(outbound_queue, req32); return 0; } else{ invalid: BUS_SPACE_WRT4_ITL(outbound_queue, req32); return EFAULT; } } static int hptiop_post_ioctl_command_mv(struct hpt_iop_hba *hba, struct hpt_iop_request_ioctl_command *req, struct hpt_iop_ioctl_param *pParams) { u_int64_t req_phy; int size = 0; if ((((pParams->nInBufferSize + 3) & ~3) + pParams->nOutBufferSize) > (hba->max_request_size - offsetof(struct hpt_iop_request_ioctl_command, buf))) { device_printf(hba->pcidev, "request size beyond max value"); return -1; } req->ioctl_code = HPT_CTL_CODE_BSD_TO_IOP(pParams->dwIoControlCode); req->inbuf_size = pParams->nInBufferSize; req->outbuf_size = pParams->nOutBufferSize; req->header.size = offsetof(struct hpt_iop_request_ioctl_command, buf) + pParams->nInBufferSize; req->header.context = (u_int64_t)MVIOP_CMD_TYPE_IOCTL; req->header.type = IOP_REQUEST_TYPE_IOCTL_COMMAND; req->header.result = IOP_RESULT_PENDING; req->header.flags = IOP_REQUEST_FLAG_OUTPUT_CONTEXT; size = req->header.size >> 8; size = size > 3 ? 3 : size; req_phy = hba->ctlcfgcmd_phy | MVIOP_MU_QUEUE_ADDR_HOST_BIT | size; hptiop_mv_inbound_write(req_phy, hba); BUS_SPACE_RD4_MV0(outbound_intmask); while (hba->config_done == 0) { if (hptiop_sleep(hba, req, PPAUSE, "hptctl", HPT_OSM_TIMEOUT)==0) continue; hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_RESET, 60000); } return 0; } static int hptiop_do_ioctl_mv(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param *pParams) { struct hpt_iop_request_ioctl_command *req; if ((pParams->Magic != HPT_IOCTL_MAGIC) && (pParams->Magic != HPT_IOCTL_MAGIC32)) return EFAULT; req = (struct hpt_iop_request_ioctl_command *)(hba->ctlcfg_ptr); hba->config_done = 0; hptiop_lock_adapter(hba); if (pParams->nInBufferSize) if (copyin((void *)pParams->lpInBuffer, req->buf, pParams->nInBufferSize)) goto invalid; if (hptiop_post_ioctl_command_mv(hba, req, pParams)) goto invalid; if (hba->config_done == 1) { if (pParams->nOutBufferSize) if (copyout(req->buf + ((pParams->nInBufferSize + 3) & ~3), (void *)pParams->lpOutBuffer, pParams->nOutBufferSize)) goto invalid; if (pParams->lpBytesReturned) if (copyout(&req->bytes_returned, (void*)pParams->lpBytesReturned, sizeof(u_int32_t))) goto invalid; hptiop_unlock_adapter(hba); return 0; } else{ invalid: hptiop_unlock_adapter(hba); return EFAULT; } } static int hptiop_post_ioctl_command_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_request_ioctl_command *req, struct hpt_iop_ioctl_param *pParams) { u_int64_t phy_addr; u_int32_t index; phy_addr = hba->ctlcfgcmd_phy; if ((((pParams->nInBufferSize + 3) & ~3) + pParams->nOutBufferSize) > (hba->max_request_size - offsetof(struct hpt_iop_request_ioctl_command, buf))) { device_printf(hba->pcidev, "request size beyond max value"); return -1; } req->ioctl_code = HPT_CTL_CODE_BSD_TO_IOP(pParams->dwIoControlCode); req->inbuf_size = pParams->nInBufferSize; req->outbuf_size = pParams->nOutBufferSize; req->header.size = offsetof(struct hpt_iop_request_ioctl_command, buf) + pParams->nInBufferSize; req->header.type = IOP_REQUEST_TYPE_IOCTL_COMMAND; req->header.result = IOP_RESULT_PENDING; req->header.flags = IOP_REQUEST_FLAG_SYNC_REQUEST | IOP_REQUEST_FLAG_OUTPUT_CONTEXT | IOP_REQUEST_FLAG_ADDR_BITS | ((phy_addr >> 16) & 0xffff0000); req->header.context = ((phy_addr & 0xffffffff) << 32 ) | IOPMU_QUEUE_ADDR_HOST_BIT | req->header.type; hba->u.mvfrey.inlist_wptr++; index = hba->u.mvfrey.inlist_wptr & 0x3fff; if (index == hba->u.mvfrey.list_count) { index = 0; hba->u.mvfrey.inlist_wptr &= ~0x3fff; hba->u.mvfrey.inlist_wptr ^= CL_POINTER_TOGGLE; } hba->u.mvfrey.inlist[index].addr = phy_addr; hba->u.mvfrey.inlist[index].intrfc_len = (req->header.size + 3) / 4; BUS_SPACE_WRT4_MVFREY2(inbound_write_ptr, hba->u.mvfrey.inlist_wptr); BUS_SPACE_RD4_MVFREY2(inbound_write_ptr); while (hba->config_done == 0) { if (hptiop_sleep(hba, req, PPAUSE, "hptctl", HPT_OSM_TIMEOUT)==0) continue; hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_RESET, 60000); } return 0; } static int hptiop_do_ioctl_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_ioctl_param *pParams) { struct hpt_iop_request_ioctl_command *req; if ((pParams->Magic != HPT_IOCTL_MAGIC) && (pParams->Magic != HPT_IOCTL_MAGIC32)) return EFAULT; req = (struct hpt_iop_request_ioctl_command *)(hba->ctlcfg_ptr); hba->config_done = 0; hptiop_lock_adapter(hba); if (pParams->nInBufferSize) if (copyin((void *)pParams->lpInBuffer, req->buf, pParams->nInBufferSize)) goto invalid; if (hptiop_post_ioctl_command_mvfrey(hba, req, pParams)) goto invalid; if (hba->config_done == 1) { if (pParams->nOutBufferSize) if (copyout(req->buf + ((pParams->nInBufferSize + 3) & ~3), (void *)pParams->lpOutBuffer, pParams->nOutBufferSize)) goto invalid; if (pParams->lpBytesReturned) if (copyout(&req->bytes_returned, (void*)pParams->lpBytesReturned, sizeof(u_int32_t))) goto invalid; hptiop_unlock_adapter(hba); return 0; } else{ invalid: hptiop_unlock_adapter(hba); return EFAULT; } } static int hptiop_rescan_bus(struct hpt_iop_hba * hba) { union ccb *ccb; if ((ccb = xpt_alloc_ccb()) == NULL) return(ENOMEM); if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(hba->sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { xpt_free_ccb(ccb); return(EIO); } xpt_rescan(ccb); return(0); } static bus_dmamap_callback_t hptiop_map_srb; static bus_dmamap_callback_t hptiop_post_scsi_command; static bus_dmamap_callback_t hptiop_mv_map_ctlcfg; static bus_dmamap_callback_t hptiop_mvfrey_map_ctlcfg; static int hptiop_alloc_pci_res_itl(struct hpt_iop_hba *hba) { hba->bar0_rid = 0x10; hba->bar0_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_MEMORY, &hba->bar0_rid, RF_ACTIVE); if (hba->bar0_res == NULL) { device_printf(hba->pcidev, "failed to get iop base adrress.\n"); return -1; } hba->bar0t = rman_get_bustag(hba->bar0_res); hba->bar0h = rman_get_bushandle(hba->bar0_res); hba->u.itl.mu = (struct hpt_iopmu_itl *) rman_get_virtual(hba->bar0_res); if (!hba->u.itl.mu) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); device_printf(hba->pcidev, "alloc mem res failed\n"); return -1; } return 0; } static int hptiop_alloc_pci_res_mv(struct hpt_iop_hba *hba) { hba->bar0_rid = 0x10; hba->bar0_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_MEMORY, &hba->bar0_rid, RF_ACTIVE); if (hba->bar0_res == NULL) { device_printf(hba->pcidev, "failed to get iop bar0.\n"); return -1; } hba->bar0t = rman_get_bustag(hba->bar0_res); hba->bar0h = rman_get_bushandle(hba->bar0_res); hba->u.mv.regs = (struct hpt_iopmv_regs *) rman_get_virtual(hba->bar0_res); if (!hba->u.mv.regs) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); device_printf(hba->pcidev, "alloc bar0 mem res failed\n"); return -1; } hba->bar2_rid = 0x18; hba->bar2_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_MEMORY, &hba->bar2_rid, RF_ACTIVE); if (hba->bar2_res == NULL) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); device_printf(hba->pcidev, "failed to get iop bar2.\n"); return -1; } hba->bar2t = rman_get_bustag(hba->bar2_res); hba->bar2h = rman_get_bushandle(hba->bar2_res); hba->u.mv.mu = (struct hpt_iopmu_mv *)rman_get_virtual(hba->bar2_res); if (!hba->u.mv.mu) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar2_rid, hba->bar2_res); device_printf(hba->pcidev, "alloc mem bar2 res failed\n"); return -1; } return 0; } static int hptiop_alloc_pci_res_mvfrey(struct hpt_iop_hba *hba) { hba->bar0_rid = 0x10; hba->bar0_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_MEMORY, &hba->bar0_rid, RF_ACTIVE); if (hba->bar0_res == NULL) { device_printf(hba->pcidev, "failed to get iop bar0.\n"); return -1; } hba->bar0t = rman_get_bustag(hba->bar0_res); hba->bar0h = rman_get_bushandle(hba->bar0_res); hba->u.mvfrey.config = (struct hpt_iop_request_get_config *) rman_get_virtual(hba->bar0_res); if (!hba->u.mvfrey.config) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); device_printf(hba->pcidev, "alloc bar0 mem res failed\n"); return -1; } hba->bar2_rid = 0x18; hba->bar2_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_MEMORY, &hba->bar2_rid, RF_ACTIVE); if (hba->bar2_res == NULL) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); device_printf(hba->pcidev, "failed to get iop bar2.\n"); return -1; } hba->bar2t = rman_get_bustag(hba->bar2_res); hba->bar2h = rman_get_bushandle(hba->bar2_res); hba->u.mvfrey.mu = (struct hpt_iopmu_mvfrey *)rman_get_virtual(hba->bar2_res); if (!hba->u.mvfrey.mu) { bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar2_rid, hba->bar2_res); device_printf(hba->pcidev, "alloc mem bar2 res failed\n"); return -1; } return 0; } static void hptiop_release_pci_res_itl(struct hpt_iop_hba *hba) { if (hba->bar0_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); } static void hptiop_release_pci_res_mv(struct hpt_iop_hba *hba) { if (hba->bar0_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); if (hba->bar2_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar2_rid, hba->bar2_res); } static void hptiop_release_pci_res_mvfrey(struct hpt_iop_hba *hba) { if (hba->bar0_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); if (hba->bar2_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar2_rid, hba->bar2_res); } static int hptiop_internal_memalloc_mv(struct hpt_iop_hba *hba) { if (bus_dma_tag_create(hba->parent_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, 0x800 - 0x8, 1, BUS_SPACE_MAXSIZE_32BIT, BUS_DMA_ALLOCNOW, NULL, NULL, &hba->ctlcfg_dmat)) { device_printf(hba->pcidev, "alloc ctlcfg_dmat failed\n"); return -1; } if (bus_dmamem_alloc(hba->ctlcfg_dmat, (void **)&hba->ctlcfg_ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT, &hba->ctlcfg_dmamap) != 0) { device_printf(hba->pcidev, "bus_dmamem_alloc failed!\n"); bus_dma_tag_destroy(hba->ctlcfg_dmat); return -1; } if (bus_dmamap_load(hba->ctlcfg_dmat, hba->ctlcfg_dmamap, hba->ctlcfg_ptr, MVIOP_IOCTLCFG_SIZE, hptiop_mv_map_ctlcfg, hba, 0)) { device_printf(hba->pcidev, "bus_dmamap_load failed!\n"); if (hba->ctlcfg_dmat) { bus_dmamem_free(hba->ctlcfg_dmat, hba->ctlcfg_ptr, hba->ctlcfg_dmamap); bus_dma_tag_destroy(hba->ctlcfg_dmat); } return -1; } return 0; } static int hptiop_internal_memalloc_mvfrey(struct hpt_iop_hba *hba) { u_int32_t list_count = BUS_SPACE_RD4_MVFREY2(inbound_conf_ctl); list_count >>= 16; if (list_count == 0) { return -1; } hba->u.mvfrey.list_count = list_count; hba->u.mvfrey.internal_mem_size = 0x800 + list_count * sizeof(struct mvfrey_inlist_entry) + list_count * sizeof(struct mvfrey_outlist_entry) + sizeof(int); if (bus_dma_tag_create(hba->parent_dmat, 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, hba->u.mvfrey.internal_mem_size, 1, BUS_SPACE_MAXSIZE_32BIT, BUS_DMA_ALLOCNOW, NULL, NULL, &hba->ctlcfg_dmat)) { device_printf(hba->pcidev, "alloc ctlcfg_dmat failed\n"); return -1; } if (bus_dmamem_alloc(hba->ctlcfg_dmat, (void **)&hba->ctlcfg_ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT, &hba->ctlcfg_dmamap) != 0) { device_printf(hba->pcidev, "bus_dmamem_alloc failed!\n"); bus_dma_tag_destroy(hba->ctlcfg_dmat); return -1; } if (bus_dmamap_load(hba->ctlcfg_dmat, hba->ctlcfg_dmamap, hba->ctlcfg_ptr, hba->u.mvfrey.internal_mem_size, hptiop_mvfrey_map_ctlcfg, hba, 0)) { device_printf(hba->pcidev, "bus_dmamap_load failed!\n"); if (hba->ctlcfg_dmat) { bus_dmamem_free(hba->ctlcfg_dmat, hba->ctlcfg_ptr, hba->ctlcfg_dmamap); bus_dma_tag_destroy(hba->ctlcfg_dmat); } return -1; } return 0; } static int hptiop_internal_memfree_itl(struct hpt_iop_hba *hba) { return 0; } static int hptiop_internal_memfree_mv(struct hpt_iop_hba *hba) { if (hba->ctlcfg_dmat) { bus_dmamap_unload(hba->ctlcfg_dmat, hba->ctlcfg_dmamap); bus_dmamem_free(hba->ctlcfg_dmat, hba->ctlcfg_ptr, hba->ctlcfg_dmamap); bus_dma_tag_destroy(hba->ctlcfg_dmat); } return 0; } static int hptiop_internal_memfree_mvfrey(struct hpt_iop_hba *hba) { if (hba->ctlcfg_dmat) { bus_dmamap_unload(hba->ctlcfg_dmat, hba->ctlcfg_dmamap); bus_dmamem_free(hba->ctlcfg_dmat, hba->ctlcfg_ptr, hba->ctlcfg_dmamap); bus_dma_tag_destroy(hba->ctlcfg_dmat); } return 0; } static int hptiop_reset_comm_mvfrey(struct hpt_iop_hba *hba) { u_int32_t i = 100; if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_RESET_COMM, 3000)) return -1; /* wait 100ms for MCU ready */ while(i--) { DELAY(1000); } BUS_SPACE_WRT4_MVFREY2(inbound_base, hba->u.mvfrey.inlist_phy & 0xffffffff); BUS_SPACE_WRT4_MVFREY2(inbound_base_high, (hba->u.mvfrey.inlist_phy >> 16) >> 16); BUS_SPACE_WRT4_MVFREY2(outbound_base, hba->u.mvfrey.outlist_phy & 0xffffffff); BUS_SPACE_WRT4_MVFREY2(outbound_base_high, (hba->u.mvfrey.outlist_phy >> 16) >> 16); BUS_SPACE_WRT4_MVFREY2(outbound_shadow_base, hba->u.mvfrey.outlist_cptr_phy & 0xffffffff); BUS_SPACE_WRT4_MVFREY2(outbound_shadow_base_high, (hba->u.mvfrey.outlist_cptr_phy >> 16) >> 16); hba->u.mvfrey.inlist_wptr = (hba->u.mvfrey.list_count - 1) | CL_POINTER_TOGGLE; *hba->u.mvfrey.outlist_cptr = (hba->u.mvfrey.list_count - 1) | CL_POINTER_TOGGLE; hba->u.mvfrey.outlist_rptr = hba->u.mvfrey.list_count - 1; return 0; } /* * CAM driver interface */ static device_method_t driver_methods[] = { /* Device interface */ DEVMETHOD(device_probe, hptiop_probe), DEVMETHOD(device_attach, hptiop_attach), DEVMETHOD(device_detach, hptiop_detach), DEVMETHOD(device_shutdown, hptiop_shutdown), { 0, 0 } }; static struct hptiop_adapter_ops hptiop_itl_ops = { .family = INTEL_BASED_IOP, .iop_wait_ready = hptiop_wait_ready_itl, .internal_memalloc = 0, .internal_memfree = hptiop_internal_memfree_itl, .alloc_pci_res = hptiop_alloc_pci_res_itl, .release_pci_res = hptiop_release_pci_res_itl, .enable_intr = hptiop_enable_intr_itl, .disable_intr = hptiop_disable_intr_itl, .get_config = hptiop_get_config_itl, .set_config = hptiop_set_config_itl, .iop_intr = hptiop_intr_itl, .post_msg = hptiop_post_msg_itl, .post_req = hptiop_post_req_itl, .do_ioctl = hptiop_do_ioctl_itl, .reset_comm = 0, }; static struct hptiop_adapter_ops hptiop_mv_ops = { .family = MV_BASED_IOP, .iop_wait_ready = hptiop_wait_ready_mv, .internal_memalloc = hptiop_internal_memalloc_mv, .internal_memfree = hptiop_internal_memfree_mv, .alloc_pci_res = hptiop_alloc_pci_res_mv, .release_pci_res = hptiop_release_pci_res_mv, .enable_intr = hptiop_enable_intr_mv, .disable_intr = hptiop_disable_intr_mv, .get_config = hptiop_get_config_mv, .set_config = hptiop_set_config_mv, .iop_intr = hptiop_intr_mv, .post_msg = hptiop_post_msg_mv, .post_req = hptiop_post_req_mv, .do_ioctl = hptiop_do_ioctl_mv, .reset_comm = 0, }; static struct hptiop_adapter_ops hptiop_mvfrey_ops = { .family = MVFREY_BASED_IOP, .iop_wait_ready = hptiop_wait_ready_mvfrey, .internal_memalloc = hptiop_internal_memalloc_mvfrey, .internal_memfree = hptiop_internal_memfree_mvfrey, .alloc_pci_res = hptiop_alloc_pci_res_mvfrey, .release_pci_res = hptiop_release_pci_res_mvfrey, .enable_intr = hptiop_enable_intr_mvfrey, .disable_intr = hptiop_disable_intr_mvfrey, .get_config = hptiop_get_config_mvfrey, .set_config = hptiop_set_config_mvfrey, .iop_intr = hptiop_intr_mvfrey, .post_msg = hptiop_post_msg_mvfrey, .post_req = hptiop_post_req_mvfrey, .do_ioctl = hptiop_do_ioctl_mvfrey, .reset_comm = hptiop_reset_comm_mvfrey, }; static driver_t hptiop_pci_driver = { driver_name, driver_methods, sizeof(struct hpt_iop_hba) }; DRIVER_MODULE(hptiop, pci, hptiop_pci_driver, hptiop_devclass, 0, 0); MODULE_DEPEND(hptiop, cam, 1, 1, 1); static int hptiop_probe(device_t dev) { struct hpt_iop_hba *hba; u_int32_t id; static char buf[256]; int sas = 0; struct hptiop_adapter_ops *ops; if (pci_get_vendor(dev) != 0x1103) return (ENXIO); id = pci_get_device(dev); switch (id) { case 0x4520: case 0x4521: case 0x4522: sas = 1; case 0x3620: case 0x3622: case 0x3640: ops = &hptiop_mvfrey_ops; break; case 0x4210: case 0x4211: case 0x4310: case 0x4311: case 0x4320: case 0x4321: case 0x4322: sas = 1; case 0x3220: case 0x3320: case 0x3410: case 0x3520: case 0x3510: case 0x3511: case 0x3521: case 0x3522: case 0x3530: case 0x3540: case 0x3560: ops = &hptiop_itl_ops; break; case 0x3020: case 0x3120: case 0x3122: ops = &hptiop_mv_ops; break; default: return (ENXIO); } device_printf(dev, "adapter at PCI %d:%d:%d, IRQ %d\n", pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), pci_get_irq(dev)); sprintf(buf, "RocketRAID %x %s Controller\n", id, sas ? "SAS" : "SATA"); device_set_desc_copy(dev, buf); hba = (struct hpt_iop_hba *)device_get_softc(dev); bzero(hba, sizeof(struct hpt_iop_hba)); hba->ops = ops; KdPrint(("hba->ops=%p\n", hba->ops)); return 0; } static int hptiop_attach(device_t dev) { struct hpt_iop_hba *hba = (struct hpt_iop_hba *)device_get_softc(dev); struct hpt_iop_request_get_config iop_config; struct hpt_iop_request_set_config set_config; int rid = 0; struct cam_devq *devq; struct ccb_setasync ccb; u_int32_t unit = device_get_unit(dev); device_printf(dev, "%d RocketRAID 3xxx/4xxx controller driver %s\n", unit, driver_version); KdPrint(("hptiop: attach(%d, %d/%d/%d) ops=%p\n", unit, pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), hba->ops)); pci_enable_busmaster(dev); hba->pcidev = dev; hba->pciunit = unit; if (hba->ops->alloc_pci_res(hba)) return ENXIO; if (hba->ops->iop_wait_ready(hba, 2000)) { device_printf(dev, "adapter is not ready\n"); goto release_pci_res; } mtx_init(&hba->lock, "hptioplock", NULL, MTX_DEF); if (bus_dma_tag_create(bus_get_dma_tag(dev),/* PCI parent */ 1, /* alignment */ 0, /* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &hba->parent_dmat /* tag */)) { device_printf(dev, "alloc parent_dmat failed\n"); goto release_pci_res; } if (hba->ops->family == MV_BASED_IOP) { if (hba->ops->internal_memalloc(hba)) { device_printf(dev, "alloc srb_dmat failed\n"); goto destroy_parent_tag; } } if (hba->ops->get_config(hba, &iop_config)) { device_printf(dev, "get iop config failed.\n"); goto get_config_failed; } hba->firmware_version = iop_config.firmware_version; hba->interface_version = iop_config.interface_version; hba->max_requests = iop_config.max_requests; hba->max_devices = iop_config.max_devices; hba->max_request_size = iop_config.request_size; hba->max_sg_count = iop_config.max_sg_count; if (hba->ops->family == MVFREY_BASED_IOP) { if (hba->ops->internal_memalloc(hba)) { device_printf(dev, "alloc srb_dmat failed\n"); goto destroy_parent_tag; } if (hba->ops->reset_comm(hba)) { device_printf(dev, "reset comm failed\n"); goto get_config_failed; } } if (bus_dma_tag_create(hba->parent_dmat,/* parent */ 4, /* alignment */ BUS_SPACE_MAXADDR_32BIT+1, /* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ PAGE_SIZE * (hba->max_sg_count-1), /* maxsize */ hba->max_sg_count, /* nsegments */ 0x20000, /* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ busdma_lock_mutex, /* lockfunc */ &hba->lock, /* lockfuncarg */ &hba->io_dmat /* tag */)) { device_printf(dev, "alloc io_dmat failed\n"); goto get_config_failed; } if (bus_dma_tag_create(hba->parent_dmat,/* parent */ 1, /* alignment */ 0, /* boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ HPT_SRB_MAX_SIZE * HPT_SRB_MAX_QUEUE_SIZE + 0x20, 1, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &hba->srb_dmat /* tag */)) { device_printf(dev, "alloc srb_dmat failed\n"); goto destroy_io_dmat; } if (bus_dmamem_alloc(hba->srb_dmat, (void **)&hba->uncached_ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT, &hba->srb_dmamap) != 0) { device_printf(dev, "srb bus_dmamem_alloc failed!\n"); goto destroy_srb_dmat; } if (bus_dmamap_load(hba->srb_dmat, hba->srb_dmamap, hba->uncached_ptr, (HPT_SRB_MAX_SIZE * HPT_SRB_MAX_QUEUE_SIZE) + 0x20, hptiop_map_srb, hba, 0)) { device_printf(dev, "bus_dmamap_load failed!\n"); goto srb_dmamem_free; } if ((devq = cam_simq_alloc(hba->max_requests - 1 )) == NULL) { device_printf(dev, "cam_simq_alloc failed\n"); goto srb_dmamap_unload; } hba->sim = cam_sim_alloc(hptiop_action, hptiop_poll, driver_name, hba, unit, &hba->lock, hba->max_requests - 1, 1, devq); if (!hba->sim) { device_printf(dev, "cam_sim_alloc failed\n"); cam_simq_free(devq); goto srb_dmamap_unload; } hptiop_lock_adapter(hba); if (xpt_bus_register(hba->sim, dev, 0) != CAM_SUCCESS) { device_printf(dev, "xpt_bus_register failed\n"); goto free_cam_sim; } if (xpt_create_path(&hba->path, /*periph */ NULL, cam_sim_path(hba->sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { device_printf(dev, "xpt_create_path failed\n"); goto deregister_xpt_bus; } hptiop_unlock_adapter(hba); bzero(&set_config, sizeof(set_config)); set_config.iop_id = unit; set_config.vbus_id = cam_sim_path(hba->sim); set_config.max_host_request_size = HPT_SRB_MAX_REQ_SIZE; if (hba->ops->set_config(hba, &set_config)) { device_printf(dev, "set iop config failed.\n"); goto free_hba_path; } xpt_setup_ccb(&ccb.ccb_h, hba->path, /*priority*/5); ccb.ccb_h.func_code = XPT_SASYNC_CB; ccb.event_enable = (AC_FOUND_DEVICE | AC_LOST_DEVICE); ccb.callback = hptiop_async; ccb.callback_arg = hba->sim; xpt_action((union ccb *)&ccb); rid = 0; if ((hba->irq_res = bus_alloc_resource_any(hba->pcidev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) { device_printf(dev, "allocate irq failed!\n"); goto free_hba_path; } if (bus_setup_intr(hba->pcidev, hba->irq_res, INTR_TYPE_CAM | INTR_MPSAFE, NULL, hptiop_pci_intr, hba, &hba->irq_handle)) { device_printf(dev, "allocate intr function failed!\n"); goto free_irq_resource; } if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_START_BACKGROUND_TASK, 5000)) { device_printf(dev, "fail to start background task\n"); goto teartown_irq_resource; } hba->ops->enable_intr(hba); hba->initialized = 1; hba->ioctl_dev = make_dev(&hptiop_cdevsw, unit, UID_ROOT, GID_WHEEL /*GID_OPERATOR*/, S_IRUSR | S_IWUSR, "%s%d", driver_name, unit); return 0; teartown_irq_resource: bus_teardown_intr(dev, hba->irq_res, hba->irq_handle); free_irq_resource: bus_release_resource(dev, SYS_RES_IRQ, 0, hba->irq_res); hptiop_lock_adapter(hba); free_hba_path: xpt_free_path(hba->path); deregister_xpt_bus: xpt_bus_deregister(cam_sim_path(hba->sim)); free_cam_sim: cam_sim_free(hba->sim, /*free devq*/ TRUE); hptiop_unlock_adapter(hba); srb_dmamap_unload: if (hba->uncached_ptr) bus_dmamap_unload(hba->srb_dmat, hba->srb_dmamap); srb_dmamem_free: if (hba->uncached_ptr) bus_dmamem_free(hba->srb_dmat, hba->uncached_ptr, hba->srb_dmamap); destroy_srb_dmat: if (hba->srb_dmat) bus_dma_tag_destroy(hba->srb_dmat); destroy_io_dmat: if (hba->io_dmat) bus_dma_tag_destroy(hba->io_dmat); get_config_failed: hba->ops->internal_memfree(hba); destroy_parent_tag: if (hba->parent_dmat) bus_dma_tag_destroy(hba->parent_dmat); release_pci_res: if (hba->ops->release_pci_res) hba->ops->release_pci_res(hba); return ENXIO; } static int hptiop_detach(device_t dev) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)device_get_softc(dev); int i; int error = EBUSY; hptiop_lock_adapter(hba); for (i = 0; i < hba->max_devices; i++) if (hptiop_os_query_remove_device(hba, i)) { device_printf(dev, "%d file system is busy. id=%d", hba->pciunit, i); goto out; } if ((error = hptiop_shutdown(dev)) != 0) goto out; if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_STOP_BACKGROUND_TASK, 60000)) goto out; hptiop_unlock_adapter(hba); hptiop_release_resource(hba); return (0); out: hptiop_unlock_adapter(hba); return error; } static int hptiop_shutdown(device_t dev) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)device_get_softc(dev); int error = 0; if (hba->flag & HPT_IOCTL_FLAG_OPEN) { device_printf(dev, "%d device is busy", hba->pciunit); return EBUSY; } hba->ops->disable_intr(hba); if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_SHUTDOWN, 60000)) error = EBUSY; return error; } static void hptiop_pci_intr(void *arg) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)arg; hptiop_lock_adapter(hba); hba->ops->iop_intr(hba); hptiop_unlock_adapter(hba); } static void hptiop_poll(struct cam_sim *sim) { struct hpt_iop_hba *hba; hba = cam_sim_softc(sim); hba->ops->iop_intr(hba); } static void hptiop_async(void * callback_arg, u_int32_t code, struct cam_path * path, void * arg) { } static void hptiop_enable_intr_itl(struct hpt_iop_hba *hba) { BUS_SPACE_WRT4_ITL(outbound_intmask, ~(IOPMU_OUTBOUND_INT_POSTQUEUE | IOPMU_OUTBOUND_INT_MSG0)); } static void hptiop_enable_intr_mv(struct hpt_iop_hba *hba) { u_int32_t int_mask; int_mask = BUS_SPACE_RD4_MV0(outbound_intmask); int_mask |= MVIOP_MU_OUTBOUND_INT_POSTQUEUE | MVIOP_MU_OUTBOUND_INT_MSG; BUS_SPACE_WRT4_MV0(outbound_intmask,int_mask); } static void hptiop_enable_intr_mvfrey(struct hpt_iop_hba *hba) { BUS_SPACE_WRT4_MVFREY2(f0_doorbell_enable, CPU_TO_F0_DRBL_MSG_A_BIT); BUS_SPACE_RD4_MVFREY2(f0_doorbell_enable); BUS_SPACE_WRT4_MVFREY2(isr_enable, 0x1); BUS_SPACE_RD4_MVFREY2(isr_enable); BUS_SPACE_WRT4_MVFREY2(pcie_f0_int_enable, 0x1010); BUS_SPACE_RD4_MVFREY2(pcie_f0_int_enable); } static void hptiop_disable_intr_itl(struct hpt_iop_hba *hba) { u_int32_t int_mask; int_mask = BUS_SPACE_RD4_ITL(outbound_intmask); int_mask |= IOPMU_OUTBOUND_INT_POSTQUEUE | IOPMU_OUTBOUND_INT_MSG0; BUS_SPACE_WRT4_ITL(outbound_intmask, int_mask); BUS_SPACE_RD4_ITL(outbound_intstatus); } static void hptiop_disable_intr_mv(struct hpt_iop_hba *hba) { u_int32_t int_mask; int_mask = BUS_SPACE_RD4_MV0(outbound_intmask); int_mask &= ~(MVIOP_MU_OUTBOUND_INT_MSG | MVIOP_MU_OUTBOUND_INT_POSTQUEUE); BUS_SPACE_WRT4_MV0(outbound_intmask,int_mask); BUS_SPACE_RD4_MV0(outbound_intmask); } static void hptiop_disable_intr_mvfrey(struct hpt_iop_hba *hba) { BUS_SPACE_WRT4_MVFREY2(f0_doorbell_enable, 0); BUS_SPACE_RD4_MVFREY2(f0_doorbell_enable); BUS_SPACE_WRT4_MVFREY2(isr_enable, 0); BUS_SPACE_RD4_MVFREY2(isr_enable); BUS_SPACE_WRT4_MVFREY2(pcie_f0_int_enable, 0); BUS_SPACE_RD4_MVFREY2(pcie_f0_int_enable); } static void hptiop_reset_adapter(void *argv) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)argv; if (hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_RESET, 60000)) return; hptiop_send_sync_msg(hba, IOPMU_INBOUND_MSG0_START_BACKGROUND_TASK, 5000); } static void *hptiop_get_srb(struct hpt_iop_hba * hba) { struct hpt_iop_srb * srb; if (hba->srb_list) { srb = hba->srb_list; hba->srb_list = srb->next; return srb; } return NULL; } static void hptiop_free_srb(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb) { srb->next = hba->srb_list; hba->srb_list = srb; } static void hptiop_action(struct cam_sim *sim, union ccb *ccb) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)cam_sim_softc(sim); struct hpt_iop_srb * srb; int error; switch (ccb->ccb_h.func_code) { case XPT_SCSI_IO: if (ccb->ccb_h.target_lun != 0 || ccb->ccb_h.target_id >= hba->max_devices || (ccb->ccb_h.flags & CAM_CDB_PHYS)) { ccb->ccb_h.status = CAM_TID_INVALID; xpt_done(ccb); return; } if ((srb = hptiop_get_srb(hba)) == NULL) { device_printf(hba->pcidev, "srb allocated failed"); ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(ccb); return; } srb->ccb = ccb; error = bus_dmamap_load_ccb(hba->io_dmat, srb->dma_map, ccb, hptiop_post_scsi_command, srb, 0); if (error && error != EINPROGRESS) { device_printf(hba->pcidev, "%d bus_dmamap_load error %d", hba->pciunit, error); xpt_freeze_simq(hba->sim, 1); ccb->ccb_h.status = CAM_REQ_CMP_ERR; hptiop_free_srb(hba, srb); xpt_done(ccb); return; } return; case XPT_RESET_BUS: device_printf(hba->pcidev, "reset adapter"); hba->msg_done = 0; hptiop_reset_adapter(hba); break; case XPT_GET_TRAN_SETTINGS: case XPT_SET_TRAN_SETTINGS: ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; case XPT_CALC_GEOMETRY: cam_calc_geometry(&ccb->ccg, 1); break; case XPT_PATH_INQ: { struct ccb_pathinq *cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = PI_SDTR_ABLE; cpi->target_sprt = 0; cpi->hba_misc = PIM_NOBUSRESET; cpi->hba_eng_cnt = 0; cpi->max_target = hba->max_devices; cpi->max_lun = 0; cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); cpi->initiator_id = hba->max_devices; cpi->base_transfer_speed = 3300; strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strncpy(cpi->hba_vid, "HPT ", HBA_IDLEN); strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->transport = XPORT_SPI; cpi->transport_version = 2; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_2; cpi->ccb_h.status = CAM_REQ_CMP; break; } default: ccb->ccb_h.status = CAM_REQ_INVALID; break; } xpt_done(ccb); return; } static void hptiop_post_req_itl(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs) { int idx; union ccb *ccb = srb->ccb; u_int8_t *cdb; if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; KdPrint(("ccb=%p %x-%x-%x\n", ccb, *(u_int32_t *)cdb, *((u_int32_t *)cdb+1), *((u_int32_t *)cdb+2))); if (srb->srb_flag & HPT_SRB_FLAG_HIGH_MEM_ACESS) { u_int32_t iop_req32; struct hpt_iop_request_scsi_command req; iop_req32 = BUS_SPACE_RD4_ITL(inbound_queue); if (iop_req32 == IOPMU_QUEUE_EMPTY) { - device_printf(hba->pcidev, "invaild req offset\n"); + device_printf(hba->pcidev, "invalid req offset\n"); ccb->ccb_h.status = CAM_BUSY; bus_dmamap_unload(hba->io_dmat, srb->dma_map); hptiop_free_srb(hba, srb); xpt_done(ccb); return; } if (ccb->csio.dxfer_len && nsegs > 0) { struct hpt_iopsg *psg = req.sg_list; for (idx = 0; idx < nsegs; idx++, psg++) { psg->pci_address = (u_int64_t)segs[idx].ds_addr; psg->size = segs[idx].ds_len; psg->eot = 0; } psg[-1].eot = 1; } bcopy(cdb, req.cdb, ccb->csio.cdb_len); req.header.size = offsetof(struct hpt_iop_request_scsi_command, sg_list) + nsegs*sizeof(struct hpt_iopsg); req.header.type = IOP_REQUEST_TYPE_SCSI_COMMAND; req.header.flags = 0; req.header.result = IOP_RESULT_PENDING; req.header.context = (u_int64_t)(unsigned long)srb; req.dataxfer_length = ccb->csio.dxfer_len; req.channel = 0; req.target = ccb->ccb_h.target_id; req.lun = ccb->ccb_h.target_lun; bus_space_write_region_1(hba->bar0t, hba->bar0h, iop_req32, (u_int8_t *)&req, req.header.size); if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREREAD); } else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREWRITE); BUS_SPACE_WRT4_ITL(inbound_queue,iop_req32); } else { struct hpt_iop_request_scsi_command *req; req = (struct hpt_iop_request_scsi_command *)srb; if (ccb->csio.dxfer_len && nsegs > 0) { struct hpt_iopsg *psg = req->sg_list; for (idx = 0; idx < nsegs; idx++, psg++) { psg->pci_address = (u_int64_t)segs[idx].ds_addr; psg->size = segs[idx].ds_len; psg->eot = 0; } psg[-1].eot = 1; } bcopy(cdb, req->cdb, ccb->csio.cdb_len); req->header.type = IOP_REQUEST_TYPE_SCSI_COMMAND; req->header.result = IOP_RESULT_PENDING; req->dataxfer_length = ccb->csio.dxfer_len; req->channel = 0; req->target = ccb->ccb_h.target_id; req->lun = ccb->ccb_h.target_lun; req->header.size = offsetof(struct hpt_iop_request_scsi_command, sg_list) + nsegs*sizeof(struct hpt_iopsg); req->header.context = (u_int64_t)srb->index | IOPMU_QUEUE_ADDR_HOST_BIT; req->header.flags = IOP_REQUEST_FLAG_OUTPUT_CONTEXT; if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREREAD); }else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREWRITE); } if (hba->firmware_version > 0x01020000 || hba->interface_version > 0x01020000) { u_int32_t size_bits; if (req->header.size < 256) size_bits = IOPMU_QUEUE_REQUEST_SIZE_BIT; else if (req->header.size < 512) size_bits = IOPMU_QUEUE_ADDR_HOST_BIT; else size_bits = IOPMU_QUEUE_REQUEST_SIZE_BIT | IOPMU_QUEUE_ADDR_HOST_BIT; BUS_SPACE_WRT4_ITL(inbound_queue, (u_int32_t)srb->phy_addr | size_bits); } else BUS_SPACE_WRT4_ITL(inbound_queue, (u_int32_t)srb->phy_addr |IOPMU_QUEUE_ADDR_HOST_BIT); } } static void hptiop_post_req_mv(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs) { int idx, size; union ccb *ccb = srb->ccb; u_int8_t *cdb; struct hpt_iop_request_scsi_command *req; u_int64_t req_phy; req = (struct hpt_iop_request_scsi_command *)srb; req_phy = srb->phy_addr; if (ccb->csio.dxfer_len && nsegs > 0) { struct hpt_iopsg *psg = req->sg_list; for (idx = 0; idx < nsegs; idx++, psg++) { psg->pci_address = (u_int64_t)segs[idx].ds_addr; psg->size = segs[idx].ds_len; psg->eot = 0; } psg[-1].eot = 1; } if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; bcopy(cdb, req->cdb, ccb->csio.cdb_len); req->header.type = IOP_REQUEST_TYPE_SCSI_COMMAND; req->header.result = IOP_RESULT_PENDING; req->dataxfer_length = ccb->csio.dxfer_len; req->channel = 0; req->target = ccb->ccb_h.target_id; req->lun = ccb->ccb_h.target_lun; req->header.size = sizeof(struct hpt_iop_request_scsi_command) - sizeof(struct hpt_iopsg) + nsegs * sizeof(struct hpt_iopsg); if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREREAD); } else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREWRITE); req->header.context = (u_int64_t)srb->index << MVIOP_REQUEST_NUMBER_START_BIT | MVIOP_CMD_TYPE_SCSI; req->header.flags = IOP_REQUEST_FLAG_OUTPUT_CONTEXT; size = req->header.size >> 8; hptiop_mv_inbound_write(req_phy | MVIOP_MU_QUEUE_ADDR_HOST_BIT | (size > 3 ? 3 : size), hba); } static void hptiop_post_req_mvfrey(struct hpt_iop_hba *hba, struct hpt_iop_srb *srb, bus_dma_segment_t *segs, int nsegs) { int idx, index; union ccb *ccb = srb->ccb; u_int8_t *cdb; struct hpt_iop_request_scsi_command *req; u_int64_t req_phy; req = (struct hpt_iop_request_scsi_command *)srb; req_phy = srb->phy_addr; if (ccb->csio.dxfer_len && nsegs > 0) { struct hpt_iopsg *psg = req->sg_list; for (idx = 0; idx < nsegs; idx++, psg++) { psg->pci_address = (u_int64_t)segs[idx].ds_addr | 1; psg->size = segs[idx].ds_len; psg->eot = 0; } psg[-1].eot = 1; } if (ccb->ccb_h.flags & CAM_CDB_POINTER) cdb = ccb->csio.cdb_io.cdb_ptr; else cdb = ccb->csio.cdb_io.cdb_bytes; bcopy(cdb, req->cdb, ccb->csio.cdb_len); req->header.type = IOP_REQUEST_TYPE_SCSI_COMMAND; req->header.result = IOP_RESULT_PENDING; req->dataxfer_length = ccb->csio.dxfer_len; req->channel = 0; req->target = ccb->ccb_h.target_id; req->lun = ccb->ccb_h.target_lun; req->header.size = sizeof(struct hpt_iop_request_scsi_command) - sizeof(struct hpt_iopsg) + nsegs * sizeof(struct hpt_iopsg); if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREREAD); } else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) bus_dmamap_sync(hba->io_dmat, srb->dma_map, BUS_DMASYNC_PREWRITE); req->header.flags = IOP_REQUEST_FLAG_OUTPUT_CONTEXT | IOP_REQUEST_FLAG_ADDR_BITS | ((req_phy >> 16) & 0xffff0000); req->header.context = ((req_phy & 0xffffffff) << 32 ) | srb->index << 4 | IOPMU_QUEUE_ADDR_HOST_BIT | req->header.type; hba->u.mvfrey.inlist_wptr++; index = hba->u.mvfrey.inlist_wptr & 0x3fff; if (index == hba->u.mvfrey.list_count) { index = 0; hba->u.mvfrey.inlist_wptr &= ~0x3fff; hba->u.mvfrey.inlist_wptr ^= CL_POINTER_TOGGLE; } hba->u.mvfrey.inlist[index].addr = req_phy; hba->u.mvfrey.inlist[index].intrfc_len = (req->header.size + 3) / 4; BUS_SPACE_WRT4_MVFREY2(inbound_write_ptr, hba->u.mvfrey.inlist_wptr); BUS_SPACE_RD4_MVFREY2(inbound_write_ptr); if (req->header.type == IOP_REQUEST_TYPE_SCSI_COMMAND) { callout_reset(&srb->timeout, 20 * hz, hptiop_reset_adapter, hba); } } static void hptiop_post_scsi_command(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct hpt_iop_srb *srb = (struct hpt_iop_srb *)arg; union ccb *ccb = srb->ccb; struct hpt_iop_hba *hba = srb->hba; if (error || nsegs > hba->max_sg_count) { KdPrint(("hptiop: func_code=%x tid=%x lun=%jx nsegs=%d\n", ccb->ccb_h.func_code, ccb->ccb_h.target_id, (uintmax_t)ccb->ccb_h.target_lun, nsegs)); ccb->ccb_h.status = CAM_BUSY; bus_dmamap_unload(hba->io_dmat, srb->dma_map); hptiop_free_srb(hba, srb); xpt_done(ccb); return; } hba->ops->post_req(hba, srb, segs, nsegs); } static void hptiop_mv_map_ctlcfg(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct hpt_iop_hba *hba = (struct hpt_iop_hba *)arg; hba->ctlcfgcmd_phy = ((u_int64_t)segs->ds_addr + 0x1F) & ~(u_int64_t)0x1F; hba->ctlcfg_ptr = (u_int8_t *)(((unsigned long)hba->ctlcfg_ptr + 0x1F) & ~0x1F); } static void hptiop_mvfrey_map_ctlcfg(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct hpt_iop_hba *hba = (struct hpt_iop_hba *)arg; char *p; u_int64_t phy; u_int32_t list_count = hba->u.mvfrey.list_count; phy = ((u_int64_t)segs->ds_addr + 0x1F) & ~(u_int64_t)0x1F; p = (u_int8_t *)(((unsigned long)hba->ctlcfg_ptr + 0x1F) & ~0x1F); hba->ctlcfgcmd_phy = phy; hba->ctlcfg_ptr = p; p += 0x800; phy += 0x800; hba->u.mvfrey.inlist = (struct mvfrey_inlist_entry *)p; hba->u.mvfrey.inlist_phy = phy; p += list_count * sizeof(struct mvfrey_inlist_entry); phy += list_count * sizeof(struct mvfrey_inlist_entry); hba->u.mvfrey.outlist = (struct mvfrey_outlist_entry *)p; hba->u.mvfrey.outlist_phy = phy; p += list_count * sizeof(struct mvfrey_outlist_entry); phy += list_count * sizeof(struct mvfrey_outlist_entry); hba->u.mvfrey.outlist_cptr = (u_int32_t *)p; hba->u.mvfrey.outlist_cptr_phy = phy; } static void hptiop_map_srb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct hpt_iop_hba * hba = (struct hpt_iop_hba *)arg; bus_addr_t phy_addr = (segs->ds_addr + 0x1F) & ~(bus_addr_t)0x1F; struct hpt_iop_srb *srb, *tmp_srb; int i; if (error || nsegs == 0) { device_printf(hba->pcidev, "hptiop_map_srb error"); return; } /* map srb */ srb = (struct hpt_iop_srb *) (((unsigned long)hba->uncached_ptr + 0x1F) & ~(unsigned long)0x1F); for (i = 0; i < HPT_SRB_MAX_QUEUE_SIZE; i++) { tmp_srb = (struct hpt_iop_srb *) ((char *)srb + i * HPT_SRB_MAX_SIZE); if (((unsigned long)tmp_srb & 0x1F) == 0) { if (bus_dmamap_create(hba->io_dmat, 0, &tmp_srb->dma_map)) { device_printf(hba->pcidev, "dmamap create failed"); return; } bzero(tmp_srb, sizeof(struct hpt_iop_srb)); tmp_srb->hba = hba; tmp_srb->index = i; if (hba->ctlcfg_ptr == 0) {/*itl iop*/ tmp_srb->phy_addr = (u_int64_t)(u_int32_t) (phy_addr >> 5); if (phy_addr & IOPMU_MAX_MEM_SUPPORT_MASK_32G) tmp_srb->srb_flag = HPT_SRB_FLAG_HIGH_MEM_ACESS; } else { tmp_srb->phy_addr = phy_addr; } callout_init_mtx(&tmp_srb->timeout, &hba->lock, 0); hptiop_free_srb(hba, tmp_srb); hba->srb[i] = tmp_srb; phy_addr += HPT_SRB_MAX_SIZE; } else { device_printf(hba->pcidev, "invalid alignment"); return; } } } static void hptiop_os_message_callback(struct hpt_iop_hba * hba, u_int32_t msg) { hba->msg_done = 1; } static int hptiop_os_query_remove_device(struct hpt_iop_hba * hba, int target_id) { struct cam_periph *periph = NULL; struct cam_path *path; int status, retval = 0; status = xpt_create_path(&path, NULL, hba->sim->path_id, target_id, 0); if (status == CAM_REQ_CMP) { if ((periph = cam_periph_find(path, "da")) != NULL) { if (periph->refcount >= 1) { device_printf(hba->pcidev, "%d ," "target_id=0x%x," "refcount=%d", hba->pciunit, target_id, periph->refcount); retval = -1; } } xpt_free_path(path); } return retval; } static void hptiop_release_resource(struct hpt_iop_hba *hba) { int i; if (hba->ioctl_dev) destroy_dev(hba->ioctl_dev); if (hba->path) { struct ccb_setasync ccb; xpt_setup_ccb(&ccb.ccb_h, hba->path, /*priority*/5); ccb.ccb_h.func_code = XPT_SASYNC_CB; ccb.event_enable = 0; ccb.callback = hptiop_async; ccb.callback_arg = hba->sim; xpt_action((union ccb *)&ccb); xpt_free_path(hba->path); } if (hba->irq_handle) bus_teardown_intr(hba->pcidev, hba->irq_res, hba->irq_handle); if (hba->sim) { hptiop_lock_adapter(hba); xpt_bus_deregister(cam_sim_path(hba->sim)); cam_sim_free(hba->sim, TRUE); hptiop_unlock_adapter(hba); } if (hba->ctlcfg_dmat) { bus_dmamap_unload(hba->ctlcfg_dmat, hba->ctlcfg_dmamap); bus_dmamem_free(hba->ctlcfg_dmat, hba->ctlcfg_ptr, hba->ctlcfg_dmamap); bus_dma_tag_destroy(hba->ctlcfg_dmat); } for (i = 0; i < HPT_SRB_MAX_QUEUE_SIZE; i++) { struct hpt_iop_srb *srb = hba->srb[i]; if (srb->dma_map) bus_dmamap_destroy(hba->io_dmat, srb->dma_map); callout_drain(&srb->timeout); } if (hba->srb_dmat) { bus_dmamap_unload(hba->srb_dmat, hba->srb_dmamap); bus_dmamap_destroy(hba->srb_dmat, hba->srb_dmamap); bus_dma_tag_destroy(hba->srb_dmat); } if (hba->io_dmat) bus_dma_tag_destroy(hba->io_dmat); if (hba->parent_dmat) bus_dma_tag_destroy(hba->parent_dmat); if (hba->irq_res) bus_release_resource(hba->pcidev, SYS_RES_IRQ, 0, hba->irq_res); if (hba->bar0_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar0_rid, hba->bar0_res); if (hba->bar2_res) bus_release_resource(hba->pcidev, SYS_RES_MEMORY, hba->bar2_rid, hba->bar2_res); mtx_destroy(&hba->lock); } Index: head/sys/dev/hwpmc/hwpmc_mod.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_mod.c (revision 298930) +++ head/sys/dev/hwpmc/hwpmc_mod.c (revision 298931) @@ -1,5211 +1,5211 @@ /*- * Copyright (c) 2003-2008 Joseph Koshy * Copyright (c) 2007 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by A. Joseph Koshy under * sponsorship from the FreeBSD Foundation and Google, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* needs to be after */ #include #include #include #include #include #include #include #include "hwpmc_soft.h" /* * Types */ enum pmc_flags { PMC_FLAG_NONE = 0x00, /* do nothing */ PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */ PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */ }; /* * The offset in sysent where the syscall is allocated. */ static int pmc_syscall_num = NO_SYSCALL; struct pmc_cpu **pmc_pcpu; /* per-cpu state */ pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */ #define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)] struct mtx_pool *pmc_mtxpool; static int *pmc_pmcdisp; /* PMC row dispositions */ #define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0) #define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0) #define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0) #define PMC_MARK_ROW_FREE(R) do { \ pmc_pmcdisp[(R)] = 0; \ } while (0) #define PMC_MARK_ROW_STANDALONE(R) do { \ KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ atomic_add_int(&pmc_pmcdisp[(R)], -1); \ KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()), \ ("[pmc,%d] row disposition error", __LINE__)); \ } while (0) #define PMC_UNMARK_ROW_STANDALONE(R) do { \ atomic_add_int(&pmc_pmcdisp[(R)], 1); \ KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ } while (0) #define PMC_MARK_ROW_THREAD(R) do { \ KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ atomic_add_int(&pmc_pmcdisp[(R)], 1); \ } while (0) #define PMC_UNMARK_ROW_THREAD(R) do { \ atomic_add_int(&pmc_pmcdisp[(R)], -1); \ KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ __LINE__)); \ } while (0) /* various event handlers */ static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag, pmc_kld_unload_tag; /* Module statistics */ struct pmc_op_getdriverstats pmc_stats; /* Machine/processor dependent operations */ static struct pmc_mdep *md; /* * Hash tables mapping owner processes and target threads to PMCs. */ struct mtx pmc_processhash_mtx; /* spin mutex */ static u_long pmc_processhashmask; static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash; /* * Hash table of PMC owner descriptors. This table is protected by * the shared PMC "sx" lock. */ static u_long pmc_ownerhashmask; static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash; /* * List of PMC owners with system-wide sampling PMCs. */ static LIST_HEAD(, pmc_owner) pmc_ss_owners; /* * A map of row indices to classdep structures. */ static struct pmc_classdep **pmc_rowindex_to_classdep; /* * Prototypes */ #ifdef HWPMC_DEBUG static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS); static int pmc_debugflags_parse(char *newstr, char *fence); #endif static int load(struct module *module, int cmd, void *arg); static int pmc_attach_process(struct proc *p, struct pmc *pm); static struct pmc *pmc_allocate_pmc_descriptor(void); static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); static int pmc_attach_one_process(struct proc *p, struct pmc *pm); static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu); static int pmc_can_attach(struct pmc *pm, struct proc *p); static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf); static void pmc_cleanup(void); static int pmc_detach_process(struct proc *p, struct pmc *pm); static int pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags); static void pmc_destroy_owner_descriptor(struct pmc_owner *po); static void pmc_destroy_pmc_descriptor(struct pmc *pm); static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmc); static struct pmc_process *pmc_find_process_descriptor(struct proc *p, uint32_t mode); static void pmc_force_context_switch(void); static void pmc_link_target_process(struct pmc *pm, struct pmc_process *pp); static void pmc_log_all_process_mappings(struct pmc_owner *po); static void pmc_log_kernel_mappings(struct pmc *pm); static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p); static void pmc_maybe_remove_owner(struct pmc_owner *po); static void pmc_process_csw_in(struct thread *td); static void pmc_process_csw_out(struct thread *td); static void pmc_process_exit(void *arg, struct proc *p); static void pmc_process_fork(void *arg, struct proc *p1, struct proc *p2, int n); static void pmc_process_samples(int cpu, int soft); static void pmc_release_pmc_descriptor(struct pmc *pmc); static void pmc_remove_owner(struct pmc_owner *po); static void pmc_remove_process_descriptor(struct pmc_process *pp); static void pmc_restore_cpu_binding(struct pmc_binding *pb); static void pmc_save_cpu_binding(struct pmc_binding *pb); static void pmc_select_cpu(int cpu); static int pmc_start(struct pmc *pm); static int pmc_stop(struct pmc *pm); static int pmc_syscall_handler(struct thread *td, void *syscall_args); static void pmc_unlink_target_process(struct pmc *pmc, struct pmc_process *pp); static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp); static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp); static struct pmc_mdep *pmc_generic_cpu_initialize(void); static void pmc_generic_cpu_finalize(struct pmc_mdep *md); /* * Kernel tunables and sysctl(8) interface. */ SYSCTL_DECL(_kern_hwpmc); static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN, &pmc_callchaindepth, 0, "depth of call chain records"); #ifdef HWPMC_DEBUG struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; char pmc_debugstr[PMC_DEBUG_STRSIZE]; TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, sizeof(pmc_debugstr)); SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH, 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); #endif /* * kern.hwpmc.hashrows -- determines the number of rows in the * of the hash table used to look up threads */ static int pmc_hashsize = PMC_HASH_SIZE; SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN, &pmc_hashsize, 0, "rows in hash tables"); /* * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU */ static int pmc_nsamples = PMC_NSAMPLES; SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN, &pmc_nsamples, 0, "number of PC samples per CPU"); /* * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. */ static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN, &pmc_mtxpool_size, 0, "size of spin mutex pool"); /* * security.bsd.unprivileged_syspmcs -- allow non-root processes to * allocate system-wide PMCs. * * Allowing unprivileged processes to allocate system PMCs is convenient * if system-wide measurements need to be taken concurrently with other * per-process measurements. This feature is turned off by default. */ static int pmc_unprivileged_syspmcs = 0; SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN, &pmc_unprivileged_syspmcs, 0, "allow unprivileged process to allocate system PMCs"); /* * Hash function. Discard the lower 2 bits of the pointer since * these are always zero for our uses. The hash multiplier is * round((2^LONG_BIT) * ((sqrt(5)-1)/2)). */ #if LONG_BIT == 64 #define _PMC_HM 11400714819323198486u #elif LONG_BIT == 32 #define _PMC_HM 2654435769u #else #error Must know the size of 'long' to compile #endif #define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M)) /* * Syscall structures */ /* The `sysent' for the new syscall */ static struct sysent pmc_sysent = { 2, /* sy_narg */ pmc_syscall_handler /* sy_call */ }; static struct syscall_module_data pmc_syscall_mod = { load, NULL, &pmc_syscall_num, &pmc_sysent, #if (__FreeBSD_version >= 1100000) { 0, NULL }, SY_THR_STATIC_KLD, #else { 0, NULL } #endif }; static moduledata_t pmc_mod = { PMC_MODULE_NAME, syscall_module_handler, &pmc_syscall_mod }; DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY); MODULE_VERSION(pmc, PMC_VERSION); #ifdef HWPMC_DEBUG enum pmc_dbgparse_state { PMCDS_WS, /* in whitespace */ PMCDS_MAJOR, /* seen a major keyword */ PMCDS_MINOR }; static int pmc_debugflags_parse(char *newstr, char *fence) { char c, *p, *q; struct pmc_debugflags *tmpflags; int error, found, *newbits, tmp; size_t kwlen; tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO); p = newstr; error = 0; for (; p < fence && (c = *p); p++) { /* skip white space */ if (c == ' ' || c == '\t') continue; /* look for a keyword followed by "=" */ for (q = p; p < fence && (c = *p) && c != '='; p++) ; if (c != '=') { error = EINVAL; goto done; } kwlen = p - q; newbits = NULL; /* lookup flag group name */ #define DBG_SET_FLAG_MAJ(S,F) \ if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ newbits = &tmpflags->pdb_ ## F; DBG_SET_FLAG_MAJ("cpu", CPU); DBG_SET_FLAG_MAJ("csw", CSW); DBG_SET_FLAG_MAJ("logging", LOG); DBG_SET_FLAG_MAJ("module", MOD); DBG_SET_FLAG_MAJ("md", MDP); DBG_SET_FLAG_MAJ("owner", OWN); DBG_SET_FLAG_MAJ("pmc", PMC); DBG_SET_FLAG_MAJ("process", PRC); DBG_SET_FLAG_MAJ("sampling", SAM); if (newbits == NULL) { error = EINVAL; goto done; } p++; /* skip the '=' */ /* Now parse the individual flags */ tmp = 0; newflag: for (q = p; p < fence && (c = *p); p++) if (c == ' ' || c == '\t' || c == ',') break; /* p == fence or c == ws or c == "," or c == 0 */ if ((kwlen = p - q) == 0) { *newbits = tmp; continue; } found = 0; #define DBG_SET_FLAG_MIN(S,F) \ if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ tmp |= found = (1 << PMC_DEBUG_MIN_ ## F) /* a '*' denotes all possible flags in the group */ if (kwlen == 1 && *q == '*') tmp = found = ~0; /* look for individual flag names */ DBG_SET_FLAG_MIN("allocaterow", ALR); DBG_SET_FLAG_MIN("allocate", ALL); DBG_SET_FLAG_MIN("attach", ATT); DBG_SET_FLAG_MIN("bind", BND); DBG_SET_FLAG_MIN("config", CFG); DBG_SET_FLAG_MIN("exec", EXC); DBG_SET_FLAG_MIN("exit", EXT); DBG_SET_FLAG_MIN("find", FND); DBG_SET_FLAG_MIN("flush", FLS); DBG_SET_FLAG_MIN("fork", FRK); DBG_SET_FLAG_MIN("getbuf", GTB); DBG_SET_FLAG_MIN("hook", PMH); DBG_SET_FLAG_MIN("init", INI); DBG_SET_FLAG_MIN("intr", INT); DBG_SET_FLAG_MIN("linktarget", TLK); DBG_SET_FLAG_MIN("mayberemove", OMR); DBG_SET_FLAG_MIN("ops", OPS); DBG_SET_FLAG_MIN("read", REA); DBG_SET_FLAG_MIN("register", REG); DBG_SET_FLAG_MIN("release", REL); DBG_SET_FLAG_MIN("remove", ORM); DBG_SET_FLAG_MIN("sample", SAM); DBG_SET_FLAG_MIN("scheduleio", SIO); DBG_SET_FLAG_MIN("select", SEL); DBG_SET_FLAG_MIN("signal", SIG); DBG_SET_FLAG_MIN("swi", SWI); DBG_SET_FLAG_MIN("swo", SWO); DBG_SET_FLAG_MIN("start", STA); DBG_SET_FLAG_MIN("stop", STO); DBG_SET_FLAG_MIN("syscall", PMS); DBG_SET_FLAG_MIN("unlinktarget", TUL); DBG_SET_FLAG_MIN("write", WRI); if (found == 0) { /* unrecognized flag name */ error = EINVAL; goto done; } if (c == 0 || c == ' ' || c == '\t') { /* end of flag group */ *newbits = tmp; continue; } p++; goto newflag; } /* save the new flag set */ bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags)); done: free(tmpflags, M_PMC); return error; } static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS) { char *fence, *newstr; int error; unsigned int n; (void) arg1; (void) arg2; /* unused parameters */ n = sizeof(pmc_debugstr); newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO); (void) strlcpy(newstr, pmc_debugstr, n); error = sysctl_handle_string(oidp, newstr, n, req); /* if there is a new string, parse and copy it */ if (error == 0 && req->newptr != NULL) { fence = newstr + (n < req->newlen ? n : req->newlen + 1); if ((error = pmc_debugflags_parse(newstr, fence)) == 0) (void) strlcpy(pmc_debugstr, newstr, sizeof(pmc_debugstr)); } free(newstr, M_PMC); return error; } #endif /* * Map a row index to a classdep structure and return the adjusted row * index for the PMC class index. */ static struct pmc_classdep * pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri) { struct pmc_classdep *pcd; (void) md; KASSERT(ri >= 0 && ri < md->pmd_npmc, ("[pmc,%d] illegal row-index %d", __LINE__, ri)); pcd = pmc_rowindex_to_classdep[ri]; KASSERT(pcd != NULL, ("[pmc,%d] ri %d null pcd", __LINE__, ri)); *adjri = ri - pcd->pcd_ri; KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num, ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri)); return (pcd); } /* * Concurrency Control * * The driver manages the following data structures: * * - target process descriptors, one per target process * - owner process descriptors (and attached lists), one per owner process * - lookup hash tables for owner and target processes * - PMC descriptors (and attached lists) * - per-cpu hardware state * - the 'hook' variable through which the kernel calls into * this module * - the machine hardware state (managed by the MD layer) * * These data structures are accessed from: * * - thread context-switch code * - interrupt handlers (possibly on multiple cpus) * - kernel threads on multiple cpus running on behalf of user * processes doing system calls * - this driver's private kernel threads * * = Locks and Locking strategy = * * The driver uses four locking strategies for its operation: * * - The global SX lock "pmc_sx" is used to protect internal * data structures. * * Calls into the module by syscall() start with this lock being * held in exclusive mode. Depending on the requested operation, * the lock may be downgraded to 'shared' mode to allow more * concurrent readers into the module. Calls into the module from * other parts of the kernel acquire the lock in shared mode. * * This SX lock is held in exclusive mode for any operations that * modify the linkages between the driver's internal data structures. * * The 'pmc_hook' function pointer is also protected by this lock. * It is only examined with the sx lock held in exclusive mode. The * kernel module is allowed to be unloaded only with the sx lock held * in exclusive mode. In normal syscall handling, after acquiring the * pmc_sx lock we first check that 'pmc_hook' is non-null before * proceeding. This prevents races between the thread unloading the module * and other threads seeking to use the module. * * - Lookups of target process structures and owner process structures * cannot use the global "pmc_sx" SX lock because these lookups need * to happen during context switches and in other critical sections * where sleeping is not allowed. We protect these lookup tables * with their own private spin-mutexes, "pmc_processhash_mtx" and * "pmc_ownerhash_mtx". * * - Interrupt handlers work in a lock free manner. At interrupt * time, handlers look at the PMC pointer (phw->phw_pmc) configured * when the PMC was started. If this pointer is NULL, the interrupt * is ignored after updating driver statistics. We ensure that this * pointer is set (using an atomic operation if necessary) before the * PMC hardware is started. Conversely, this pointer is unset atomically * only after the PMC hardware is stopped. * * We ensure that everything needed for the operation of an * interrupt handler is available without it needing to acquire any * locks. We also ensure that a PMC's software state is destroyed only * after the PMC is taken off hardware (on all CPUs). * * - Context-switch handling with process-private PMCs needs more * care. * * A given process may be the target of multiple PMCs. For example, * PMCATTACH and PMCDETACH may be requested by a process on one CPU * while the target process is running on another. A PMC could also * be getting released because its owner is exiting. We tackle * these situations in the following manner: * * - each target process structure 'pmc_process' has an array * of 'struct pmc *' pointers, one for each hardware PMC. * * - At context switch IN time, each "target" PMC in RUNNING state * gets started on hardware and a pointer to each PMC is copied into * the per-cpu phw array. The 'runcount' for the PMC is * incremented. * * - At context switch OUT time, all process-virtual PMCs are stopped * on hardware. The saved value is added to the PMCs value field * only if the PMC is in a non-deleted state (the PMCs state could * have changed during the current time slice). * * Note that since in-between a switch IN on a processor and a switch * OUT, the PMC could have been released on another CPU. Therefore * context switch OUT always looks at the hardware state to turn * OFF PMCs and will update a PMC's saved value only if reachable * from the target process record. * * - OP PMCRELEASE could be called on a PMC at any time (the PMC could * be attached to many processes at the time of the call and could * be active on multiple CPUs). * * We prevent further scheduling of the PMC by marking it as in * state 'DELETED'. If the runcount of the PMC is non-zero then * this PMC is currently running on a CPU somewhere. The thread * doing the PMCRELEASE operation waits by repeatedly doing a * pause() till the runcount comes to zero. * * The contents of a PMC descriptor (struct pmc) are protected using * a spin-mutex. In order to save space, we use a mutex pool. * * In terms of lock types used by witness(4), we use: * - Type "pmc-sx", used by the global SX lock. * - Type "pmc-sleep", for sleep mutexes used by logger threads. * - Type "pmc-per-proc", for protecting PMC owner descriptors. * - Type "pmc-leaf", used for all other spin mutexes. */ /* * save the cpu binding of the current kthread */ static void pmc_save_cpu_binding(struct pmc_binding *pb) { PMCDBG0(CPU,BND,2, "save-cpu"); thread_lock(curthread); pb->pb_bound = sched_is_bound(curthread); pb->pb_cpu = curthread->td_oncpu; thread_unlock(curthread); PMCDBG1(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu); } /* * restore the cpu binding of the current thread */ static void pmc_restore_cpu_binding(struct pmc_binding *pb) { PMCDBG2(CPU,BND,2, "restore-cpu curcpu=%d restore=%d", curthread->td_oncpu, pb->pb_cpu); thread_lock(curthread); if (pb->pb_bound) sched_bind(curthread, pb->pb_cpu); else sched_unbind(curthread); thread_unlock(curthread); PMCDBG0(CPU,BND,2, "restore-cpu done"); } /* * move execution over the specified cpu and bind it there. */ static void pmc_select_cpu(int cpu) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[pmc,%d] bad cpu number %d", __LINE__, cpu)); /* Never move to an inactive CPU. */ KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive " "CPU %d", __LINE__, cpu)); PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d", cpu); thread_lock(curthread); sched_bind(curthread, cpu); thread_unlock(curthread); KASSERT(curthread->td_oncpu == cpu, ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, cpu, curthread->td_oncpu)); PMCDBG1(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); } /* * Force a context switch. * * We do this by pause'ing for 1 tick -- invoking mi_switch() is not * guaranteed to force a context switch. */ static void pmc_force_context_switch(void) { pause("pmcctx", 1); } /* * Get the file name for an executable. This is a simple wrapper * around vn_fullpath(9). */ static void pmc_getfilename(struct vnode *v, char **fullpath, char **freepath) { *fullpath = "unknown"; *freepath = NULL; vn_fullpath(curthread, v, fullpath, freepath); } /* * remove an process owning PMCs */ void pmc_remove_owner(struct pmc_owner *po) { struct pmc *pm, *tmp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG1(OWN,ORM,1, "remove-owner po=%p", po); /* Remove descriptor from the owner hash table */ LIST_REMOVE(po, po_next); /* release all owned PMC descriptors */ LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { PMCDBG1(OWN,ORM,2, "pmc=%p", pm); KASSERT(pm->pm_owner == po, ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); pmc_release_pmc_descriptor(pm); /* will unlink from the list */ pmc_destroy_pmc_descriptor(pm); } KASSERT(po->po_sscount == 0, ("[pmc,%d] SS count not zero", __LINE__)); KASSERT(LIST_EMPTY(&po->po_pmcs), ("[pmc,%d] PMC list not empty", __LINE__)); /* de-configure the log file if present */ if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_deconfigure_log(po); } /* * remove an owner process record if all conditions are met. */ static void pmc_maybe_remove_owner(struct pmc_owner *po) { PMCDBG1(OWN,OMR,1, "maybe-remove-owner po=%p", po); /* * Remove owner record if * - this process does not own any PMCs * - this process has not allocated a system-wide sampling buffer */ if (LIST_EMPTY(&po->po_pmcs) && ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } } /* * Add an association between a target process and a PMC. */ static void pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) { int ri; struct pmc_target *pt; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d", __LINE__, pm, pp->pp_proc->p_pid)); KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1), ("[pmc,%d] Illegal reference count %d for process record %p", __LINE__, pp->pp_refcnt, (void *) pp)); ri = PMC_TO_ROWINDEX(pm); PMCDBG3(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", pm, ri, pp); #ifdef HWPMC_DEBUG LIST_FOREACH(pt, &pm->pm_targets, pt_next) if (pt->pt_process == pp) KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", __LINE__, pp, pm)); #endif pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO); pt->pt_process = pp; LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc, (uintptr_t)pm); if (pm->pm_owner->po_owner == pp->pp_proc) pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; /* * Initialize the per-process values at this row index. */ pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ? pm->pm_sc.pm_reloadcount : 0; pp->pp_refcnt++; } /* * Removes the association between a target process and a PMC. */ static void pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) { int ri; struct proc *p; struct pmc_target *ptgt; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL && pp != NULL, ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc, ("[pmc,%d] Illegal ref count %d on process record %p", __LINE__, pp->pp_refcnt, (void *) pp)); ri = PMC_TO_ROWINDEX(pm); PMCDBG3(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", pm, ri, pp); KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, ri, pm, pp->pp_pmcs[ri].pp_pmc)); pp->pp_pmcs[ri].pp_pmc = NULL; pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; /* Remove owner-specific flags */ if (pm->pm_owner->po_owner == pp->pp_proc) { pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS; pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; } pp->pp_refcnt--; /* Remove the target process from the PMC structure */ LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) if (ptgt->pt_process == pp) break; KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); LIST_REMOVE(ptgt, pt_next); free(ptgt, M_PMC); /* if the PMC now lacks targets, send the owner a SIGIO */ if (LIST_EMPTY(&pm->pm_targets)) { p = pm->pm_owner->po_owner; PROC_LOCK(p); kern_psignal(p, SIGIO); PROC_UNLOCK(p); PMCDBG2(PRC,SIG,2, "signalling proc=%p signal=%d", p, SIGIO); } } /* * Check if PMC 'pm' may be attached to target process 't'. */ static int pmc_can_attach(struct pmc *pm, struct proc *t) { struct proc *o; /* pmc owner */ struct ucred *oc, *tc; /* owner, target credentials */ int decline_attach, i; /* * A PMC's owner can always attach that PMC to itself. */ if ((o = pm->pm_owner->po_owner) == t) return 0; PROC_LOCK(o); oc = o->p_ucred; crhold(oc); PROC_UNLOCK(o); PROC_LOCK(t); tc = t->p_ucred; crhold(tc); PROC_UNLOCK(t); /* * The effective uid of the PMC owner should match at least one * of the {effective,real,saved} uids of the target process. */ decline_attach = oc->cr_uid != tc->cr_uid && oc->cr_uid != tc->cr_svuid && oc->cr_uid != tc->cr_ruid; /* * Every one of the target's group ids, must be in the owner's * group list. */ for (i = 0; !decline_attach && i < tc->cr_ngroups; i++) decline_attach = !groupmember(tc->cr_groups[i], oc); /* check the read and saved gids too */ if (decline_attach == 0) decline_attach = !groupmember(tc->cr_rgid, oc) || !groupmember(tc->cr_svgid, oc); crfree(tc); crfree(oc); return !decline_attach; } /* * Attach a process to a PMC. */ static int pmc_attach_one_process(struct proc *p, struct pmc *pm) { int ri; char *fullpath, *freepath; struct pmc_process *pp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG5(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); /* * Locate the process descriptor corresponding to process 'p', * allocating space as needed. * * Verify that rowindex 'pm_rowindex' is free in the process * descriptor. * * If not, allocate space for a descriptor and link the * process descriptor and PMC. */ ri = PMC_TO_ROWINDEX(pm); if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) return ENOMEM; if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */ return EEXIST; if (pp->pp_pmcs[ri].pp_pmc != NULL) return EBUSY; pmc_link_target_process(pm, pp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) && (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0) pm->pm_flags |= PMC_F_NEEDS_LOGFILE; pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ /* issue an attach event to a configured log file */ if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { if (p->p_flag & P_KPROC) { fullpath = kernelname; freepath = NULL; } else { pmc_getfilename(p->p_textvp, &fullpath, &freepath); pmclog_process_pmcattach(pm, p->p_pid, fullpath); } free(freepath, M_TEMP); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pmc_log_process_mappings(pm->pm_owner, p); } /* mark process as using HWPMCs */ PROC_LOCK(p); p->p_flag |= P_HWPMC; PROC_UNLOCK(p); return 0; } /* * Attach a process and optionally its children */ static int pmc_attach_process(struct proc *p, struct pmc *pm) { int error; struct proc *top; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG5(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); /* * If this PMC successfully allowed a GETMSR operation * in the past, disallow further ATTACHes. */ if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) return EPERM; if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) return pmc_attach_one_process(p, pm); /* * Traverse all child processes, attaching them to * this PMC. */ sx_slock(&proctree_lock); top = p; for (;;) { if ((error = pmc_attach_one_process(p, pm)) != 0) break; if (!LIST_EMPTY(&p->p_children)) p = LIST_FIRST(&p->p_children); else for (;;) { if (p == top) goto done; if (LIST_NEXT(p, p_sibling)) { p = LIST_NEXT(p, p_sibling); break; } p = p->p_pptr; } } if (error) (void) pmc_detach_process(top, pm); done: sx_sunlock(&proctree_lock); return error; } /* * Detach a process from a PMC. If there are no other PMCs tracking * this process, remove the process structure from its hash table. If * 'flags' contains PMC_FLAG_REMOVE, then free the process structure. */ static int pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) { int ri; struct pmc_process *pp; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm != NULL, ("[pmc,%d] null pm pointer", __LINE__)); ri = PMC_TO_ROWINDEX(pm); PMCDBG6(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", pm, ri, p, p->p_pid, p->p_comm, flags); if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) return ESRCH; if (pp->pp_pmcs[ri].pp_pmc != pm) return EINVAL; pmc_unlink_target_process(pm, pp); /* Issue a detach entry if a log file is configured */ if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_pmcdetach(pm, p->p_pid); /* - * If there are no PMCs targetting this process, we remove its + * If there are no PMCs targeting this process, we remove its * descriptor from the target hash table and unset the P_HWPMC * flag in the struct proc. */ KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc, ("[pmc,%d] Illegal refcnt %d for process struct %p", __LINE__, pp->pp_refcnt, pp)); if (pp->pp_refcnt != 0) /* still a target of some PMC */ return 0; pmc_remove_process_descriptor(pp); if (flags & PMC_FLAG_REMOVE) free(pp, M_PMC); PROC_LOCK(p); p->p_flag &= ~P_HWPMC; PROC_UNLOCK(p); return 0; } /* * Detach a process and optionally its descendants from a PMC. */ static int pmc_detach_process(struct proc *p, struct pmc *pm) { struct proc *top; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG5(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); /* * Traverse all children, detaching them from this PMC. We * ignore errors since we could be detaching a PMC from a * partially attached proc tree. */ sx_slock(&proctree_lock); top = p; for (;;) { (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); if (!LIST_EMPTY(&p->p_children)) p = LIST_FIRST(&p->p_children); else for (;;) { if (p == top) goto done; if (LIST_NEXT(p, p_sibling)) { p = LIST_NEXT(p, p_sibling); break; } p = p->p_pptr; } } done: sx_sunlock(&proctree_lock); if (LIST_EMPTY(&pm->pm_targets)) pm->pm_flags &= ~PMC_F_ATTACH_DONE; return 0; } /* * Thread context switch IN */ static void pmc_process_csw_in(struct thread *td) { int cpu; unsigned int adjri, ri; struct pmc *pm; struct proc *p; struct pmc_cpu *pc; struct pmc_hw *phw; pmc_value_t newvalue; struct pmc_process *pp; struct pmc_classdep *pcd; p = td->td_proc; if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) return; KASSERT(pp->pp_proc == td->td_proc, ("[pmc,%d] not my thread state", __LINE__)); critical_enter(); /* no preemption from this point */ cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ PMCDBG5(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), - ("[pmc,%d] wierd CPU id %d", __LINE__, cpu)); + ("[pmc,%d] weird CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; for (ri = 0; ri < md->pmd_npmc; ri++) { if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) continue; KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] Target PMC in non-virtual mode (%d)", __LINE__, PMC_TO_MODE(pm))); KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] Row index mismatch pmc %d != ri %d", __LINE__, PMC_TO_ROWINDEX(pm), ri)); /* * Only PMCs that are marked as 'RUNNING' need * be placed on hardware. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* increment PMC runcount */ atomic_add_rel_int(&pm->pm_runcount, 1); /* configure the HWPMC we are going to use. */ pcd = pmc_ri_to_classdep(md, ri, &adjri); pcd->pcd_config_pmc(cpu, adjri, pm); phw = pc->pc_hwpmcs[ri]; KASSERT(phw != NULL, ("[pmc,%d] null hw pointer", __LINE__)); KASSERT(phw->phw_pmc == pm, ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, phw->phw_pmc, pm)); /* * Write out saved value and start the PMC. * * Sampling PMCs use a per-process value, while * counting mode PMCs use a per-pmc value that is * inherited across descendants. */ if (PMC_TO_MODE(pm) == PMC_MODE_TS) { mtx_pool_lock_spin(pmc_mtxpool, pm); /* * Use the saved value calculated after the most recent * thread switch out to start this counter. Reset * the saved count in case another thread from this * process switches in before any threads switch out. */ newvalue = PMC_PCPU_SAVED(cpu,ri) = pp->pp_pmcs[ri].pp_pmcval; pp->pp_pmcs[ri].pp_pmcval = pm->pm_sc.pm_reloadcount; mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, ("[pmc,%d] illegal mode=%d", __LINE__, PMC_TO_MODE(pm))); mtx_pool_lock_spin(pmc_mtxpool, pm); newvalue = PMC_PCPU_SAVED(cpu, ri) = pm->pm_gv.pm_savedvalue; mtx_pool_unlock_spin(pmc_mtxpool, pm); } PMCDBG3(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); pcd->pcd_write_pmc(cpu, adjri, newvalue); /* If a sampling mode PMC, reset stalled state. */ if (PMC_TO_MODE(pm) == PMC_MODE_TS) CPU_CLR_ATOMIC(cpu, &pm->pm_stalled); /* Indicate that we desire this to run. */ CPU_SET_ATOMIC(cpu, &pm->pm_cpustate); /* Start the PMC. */ pcd->pcd_start_pmc(cpu, adjri); } /* * perform any other architecture/cpu dependent thread * switch-in actions. */ (void) (*md->pmd_switch_in)(pc, pp); critical_exit(); } /* * Thread context switch OUT. */ static void pmc_process_csw_out(struct thread *td) { int cpu; int64_t tmp; struct pmc *pm; struct proc *p; enum pmc_mode mode; struct pmc_cpu *pc; pmc_value_t newvalue; unsigned int adjri, ri; struct pmc_process *pp; struct pmc_classdep *pcd; /* * Locate our process descriptor; this may be NULL if * this process is exiting and we have already removed * the process from the target process table. * * Note that due to kernel preemption, multiple * context switches may happen while the process is * exiting. * * Note also that if the target process cannot be * found we still need to deconfigure any PMCs that * are currently running on hardware. */ p = td->td_proc; pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE); /* * save PMCs */ critical_enter(); cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ PMCDBG5(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, p->p_pid, p->p_comm, pp); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), - ("[pmc,%d wierd CPU id %d", __LINE__, cpu)); + ("[pmc,%d weird CPU id %d", __LINE__, cpu)); pc = pmc_pcpu[cpu]; /* * When a PMC gets unlinked from a target PMC, it will * be removed from the target's pp_pmc[] array. * * However, on a MP system, the target could have been * executing on another CPU at the time of the unlink. * So, at context switch OUT time, we need to look at * the hardware to determine if a PMC is scheduled on * it. */ for (ri = 0; ri < md->pmd_npmc; ri++) { pcd = pmc_ri_to_classdep(md, ri, &adjri); pm = NULL; (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); if (pm == NULL) /* nothing at this row index */ continue; mode = PMC_TO_MODE(pm); if (!PMC_IS_VIRTUAL_MODE(mode)) continue; /* not a process virtual PMC */ KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", __LINE__, PMC_TO_ROWINDEX(pm), ri)); /* * Change desired state, and then stop if not stalled. * This two-step dance should avoid race conditions where * an interrupt re-enables the PMC after this code has * already checked the pm_stalled flag. */ CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate); if (!CPU_ISSET(cpu, &pm->pm_stalled)) pcd->pcd_stop_pmc(cpu, adjri); /* reduce this PMC's runcount */ atomic_subtract_rel_int(&pm->pm_runcount, 1); /* * If this PMC is associated with this process, * save the reading. */ if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) { KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); KASSERT(pp->pp_refcnt > 0, ("[pmc,%d] pp refcnt = %d", __LINE__, pp->pp_refcnt)); pcd->pcd_read_pmc(cpu, adjri, &newvalue); if (mode == PMC_MODE_TS) { PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (samp)", cpu, ri, PMC_PCPU_SAVED(cpu,ri) - newvalue); /* * For sampling process-virtual PMCs, * newvalue is the number of events to be seen * until the next sampling interrupt. * We can just add the events left from this * invocation to the counter, then adjust * in case we overflow our range. * * (Recall that we reload the counter every * time we use it.) */ mtx_pool_lock_spin(pmc_mtxpool, pm); pp->pp_pmcs[ri].pp_pmcval += newvalue; if (pp->pp_pmcs[ri].pp_pmcval > pm->pm_sc.pm_reloadcount) pp->pp_pmcs[ri].pp_pmcval -= pm->pm_sc.pm_reloadcount; KASSERT(pp->pp_pmcs[ri].pp_pmcval > 0 && pp->pp_pmcs[ri].pp_pmcval <= pm->pm_sc.pm_reloadcount, ("[pmc,%d] pp_pmcval outside of expected " "range cpu=%d ri=%d pp_pmcval=%jx " "pm_reloadcount=%jx", __LINE__, cpu, ri, pp->pp_pmcs[ri].pp_pmcval, pm->pm_sc.pm_reloadcount)); mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); PMCDBG3(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd (count)", cpu, ri, tmp); /* * For counting process-virtual PMCs, * we expect the count to be * increasing monotonically, modulo a 64 * bit wraparound. */ KASSERT(tmp >= 0, ("[pmc,%d] negative increment cpu=%d " "ri=%d newvalue=%jx saved=%jx " "incr=%jx", __LINE__, cpu, ri, newvalue, PMC_PCPU_SAVED(cpu,ri), tmp)); mtx_pool_lock_spin(pmc_mtxpool, pm); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; mtx_pool_unlock_spin(pmc_mtxpool, pm); if (pm->pm_flags & PMC_F_LOG_PROCCSW) pmclog_process_proccsw(pm, pp, tmp); } } /* mark hardware as free */ pcd->pcd_config_pmc(cpu, adjri, NULL); } /* * perform any other architecture/cpu dependent thread * switch out functions. */ (void) (*md->pmd_switch_out)(pc, pp); critical_exit(); } /* * A mapping change for a process. */ static void pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm) { int ri; pid_t pid; char *fullpath, *freepath; const struct pmc *pm; struct pmc_owner *po; const struct pmc_process *pp; freepath = fullpath = NULL; pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); pid = td->td_proc->p_pid; /* Inform owners of all system-wide sampling PMCs. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_in(po, pid, pkm->pm_address, fullpath); if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) goto done; /* * Inform sampling PMC owners tracking this process. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pmclog_process_map_in(pm->pm_owner, pid, pkm->pm_address, fullpath); done: if (freepath) free(freepath, M_TEMP); } /* * Log an munmap request. */ static void pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm) { int ri; pid_t pid; struct pmc_owner *po; const struct pmc *pm; const struct pmc_process *pp; pid = td->td_proc->p_pid; LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_out(po, pid, pkm->pm_address, pkm->pm_address + pkm->pm_size); if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) return; for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pmclog_process_map_out(pm->pm_owner, pid, pkm->pm_address, pkm->pm_address + pkm->pm_size); } /* * Log mapping information about the kernel. */ static void pmc_log_kernel_mappings(struct pmc *pm) { struct pmc_owner *po; struct pmckern_map_in *km, *kmbase; sx_assert(&pmc_sx, SX_LOCKED); KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] non-sampling PMC (%p) desires mapping information", __LINE__, (void *) pm)); po = pm->pm_owner; if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE) return; /* * Log the current set of kernel modules. */ kmbase = linker_hwpmc_list_objects(); for (km = kmbase; km->pm_file != NULL; km++) { PMCDBG2(LOG,REG,1,"%s %p", (char *) km->pm_file, (void *) km->pm_address); pmclog_process_map_in(po, (pid_t) -1, km->pm_address, km->pm_file); } free(kmbase, M_LINKER); po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE; } /* * Log the mappings for a single process. */ static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p) { vm_map_t map; struct vnode *vp; struct vmspace *vm; vm_map_entry_t entry; vm_offset_t last_end; u_int last_timestamp; struct vnode *last_vp; vm_offset_t start_addr; vm_object_t obj, lobj, tobj; char *fullpath, *freepath; last_vp = NULL; last_end = (vm_offset_t) 0; fullpath = freepath = NULL; if ((vm = vmspace_acquire_ref(p)) == NULL) return; map = &vm->vm_map; vm_map_lock_read(map); for (entry = map->header.next; entry != &map->header; entry = entry->next) { if (entry == NULL) { PMCDBG2(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly " "NULL! pid=%d vm_map=%p\n", p->p_pid, map); break; } /* * We only care about executable map entries. */ if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) || !(entry->protection & VM_PROT_EXECUTE) || (entry->object.vm_object == NULL)) { continue; } obj = entry->object.vm_object; VM_OBJECT_RLOCK(obj); /* * Walk the backing_object list to find the base * (non-shadowed) vm_object. */ for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) { if (tobj != obj) VM_OBJECT_RLOCK(tobj); if (lobj != obj) VM_OBJECT_RUNLOCK(lobj); lobj = tobj; } /* * At this point lobj is the base vm_object and it is locked. */ if (lobj == NULL) { PMCDBG3(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d " "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj); VM_OBJECT_RUNLOCK(obj); continue; } vp = vm_object_vnode(lobj); if (vp == NULL) { if (lobj != obj) VM_OBJECT_RUNLOCK(lobj); VM_OBJECT_RUNLOCK(obj); continue; } /* * Skip contiguous regions that point to the same * vnode, so we don't emit redundant MAP-IN * directives. */ if (entry->start == last_end && vp == last_vp) { last_end = entry->end; if (lobj != obj) VM_OBJECT_RUNLOCK(lobj); VM_OBJECT_RUNLOCK(obj); continue; } /* * We don't want to keep the proc's vm_map or this * vm_object locked while we walk the pathname, since * vn_fullpath() can sleep. However, if we drop the * lock, it's possible for concurrent activity to * modify the vm_map list. To protect against this, * we save the vm_map timestamp before we release the * lock, and check it after we reacquire the lock * below. */ start_addr = entry->start; last_end = entry->end; last_timestamp = map->timestamp; vm_map_unlock_read(map); vref(vp); if (lobj != obj) VM_OBJECT_RUNLOCK(lobj); VM_OBJECT_RUNLOCK(obj); freepath = NULL; pmc_getfilename(vp, &fullpath, &freepath); last_vp = vp; vrele(vp); vp = NULL; pmclog_process_map_in(po, p->p_pid, start_addr, fullpath); if (freepath) free(freepath, M_TEMP); vm_map_lock_read(map); /* * If our saved timestamp doesn't match, this means * that the vm_map was modified out from under us and * we can't trust our current "entry" pointer. Do a * new lookup for this entry. If there is no entry * for this address range, vm_map_lookup_entry() will * return the previous one, so we always want to go to * entry->next on the next loop iteration. * * There is an edge condition here that can occur if * there is no entry at or before this address. In * this situation, vm_map_lookup_entry returns * &map->header, which would cause our loop to abort * without processing the rest of the map. However, * in practice this will never happen for process * vm_map. This is because the executable's text * segment is the first mapping in the proc's address * space, and this mapping is never removed until the * process exits, so there will always be a non-header * entry at or before the requested address for * vm_map_lookup_entry to return. */ if (map->timestamp != last_timestamp) vm_map_lookup_entry(map, last_end - 1, &entry); } vm_map_unlock_read(map); vmspace_free(vm); return; } /* * Log mappings for all processes in the system. */ static void pmc_log_all_process_mappings(struct pmc_owner *po) { struct proc *p, *top; sx_assert(&pmc_sx, SX_XLOCKED); if ((p = pfind(1)) == NULL) panic("[pmc,%d] Cannot find init", __LINE__); PROC_UNLOCK(p); sx_slock(&proctree_lock); top = p; for (;;) { pmc_log_process_mappings(po, p); if (!LIST_EMPTY(&p->p_children)) p = LIST_FIRST(&p->p_children); else for (;;) { if (p == top) goto done; if (LIST_NEXT(p, p_sibling)) { p = LIST_NEXT(p, p_sibling); break; } p = p->p_pptr; } } done: sx_sunlock(&proctree_lock); } /* * The 'hook' invoked from the kernel proper */ #ifdef HWPMC_DEBUG const char *pmc_hooknames[] = { /* these strings correspond to PMC_FN_* in */ "", "EXEC", "CSW-IN", "CSW-OUT", "SAMPLE", "UNUSED1", "UNUSED2", "MMAP", "MUNMAP", "CALLCHAIN-NMI", "CALLCHAIN-SOFT", "SOFTSAMPLING" }; #endif static int pmc_hook_handler(struct thread *td, int function, void *arg) { PMCDBG4(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, pmc_hooknames[function], arg); switch (function) { /* * Process exec() */ case PMC_FN_PROCESS_EXEC: { char *fullpath, *freepath; unsigned int ri; int is_using_hwpmcs; struct pmc *pm; struct proc *p; struct pmc_owner *po; struct pmc_process *pp; struct pmckern_procexec *pk; sx_assert(&pmc_sx, SX_XLOCKED); p = td->td_proc; pmc_getfilename(p->p_textvp, &fullpath, &freepath); pk = (struct pmckern_procexec *) arg; /* Inform owners of SS mode PMCs of the exec event. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procexec(po, PMC_ID_INVALID, p->p_pid, pk->pm_entryaddr, fullpath); PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; PROC_UNLOCK(p); if (!is_using_hwpmcs) { if (freepath) free(freepath, M_TEMP); break; } /* * PMCs are not inherited across an exec(): remove any * PMCs that this process is the owner of. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } /* * If the process being exec'ed is not the target of any * PMC, we are done. */ if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) { if (freepath) free(freepath, M_TEMP); break; } /* * Log the exec event to all monitoring owners. Skip - * owners who have already recieved the event because + * owners who have already received the event because * they had system sampling PMCs active. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { po = pm->pm_owner; if (po->po_sscount == 0 && po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procexec(po, pm->pm_id, p->p_pid, pk->pm_entryaddr, fullpath); } if (freepath) free(freepath, M_TEMP); PMCDBG4(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d", p, p->p_pid, p->p_comm, pk->pm_credentialschanged); if (pk->pm_credentialschanged == 0) /* no change */ break; /* * If the newly exec()'ed process has a different credential * than before, allow it to be the target of a PMC only if - * the PMC's owner has sufficient priviledge. + * the PMC's owner has sufficient privilege. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) if (pmc_can_attach(pm, td->td_proc) != 0) pmc_detach_one_process(td->td_proc, pm, PMC_FLAG_NONE); KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc, ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__, pp->pp_refcnt, pp)); /* * If this process is no longer the target of any * PMCs, we can remove the process entry and free * up space. */ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); free(pp, M_PMC); break; } } break; case PMC_FN_CSW_IN: pmc_process_csw_in(td); break; case PMC_FN_CSW_OUT: pmc_process_csw_out(td); break; /* * Process accumulated PC samples. * * This function is expected to be called by hardclock() for * each CPU that has accumulated PC samples. * * This function is to be executed on the CPU whose samples * are being processed. */ case PMC_FN_DO_SAMPLES: /* * Clear the cpu specific bit in the CPU mask before * do the rest of the processing. If the NMI handler * gets invoked after the "atomic_clear_int()" call * below but before "pmc_process_samples()" gets * around to processing the interrupt, then we will * come back here at the next hardclock() tick (and * may find nothing to do if "pmc_process_samples()" * had already processed the interrupt). We don't * lose the interrupt sample. */ CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask); pmc_process_samples(PCPU_GET(cpuid), PMC_HR); pmc_process_samples(PCPU_GET(cpuid), PMC_SR); break; case PMC_FN_MMAP: sx_assert(&pmc_sx, SX_LOCKED); pmc_process_mmap(td, (struct pmckern_map_in *) arg); break; case PMC_FN_MUNMAP: sx_assert(&pmc_sx, SX_LOCKED); pmc_process_munmap(td, (struct pmckern_map_out *) arg); break; case PMC_FN_USER_CALLCHAIN: /* * Record a call chain. */ KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR, (struct trapframe *) arg); td->td_pflags &= ~TDP_CALLCHAIN; break; case PMC_FN_USER_CALLCHAIN_SOFT: /* * Record a call chain. */ KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR, (struct trapframe *) arg); td->td_pflags &= ~TDP_CALLCHAIN; break; case PMC_FN_SOFT_SAMPLING: /* * Call soft PMC sampling intr. */ pmc_soft_intr((struct pmckern_soft *) arg); break; default: #ifdef HWPMC_DEBUG KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); #endif break; } return 0; } /* * allocate a 'struct pmc_owner' descriptor in the owner hash table. */ static struct pmc_owner * pmc_allocate_owner_descriptor(struct proc *p) { uint32_t hindex; struct pmc_owner *po; struct pmc_ownerhash *poh; hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); poh = &pmc_ownerhash[hindex]; /* allocate space for N pointers and one descriptor struct */ po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO); po->po_owner = p; LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ TAILQ_INIT(&po->po_logbuffers); mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN); PMCDBG4(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p", p, p->p_pid, p->p_comm, po); return po; } static void pmc_destroy_owner_descriptor(struct pmc_owner *po) { PMCDBG4(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)", po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); mtx_destroy(&po->po_mtx); free(po, M_PMC); } /* * find the descriptor corresponding to process 'p', adding or removing it * as specified by 'mode'. */ static struct pmc_process * pmc_find_process_descriptor(struct proc *p, uint32_t mode) { uint32_t hindex; struct pmc_process *pp, *ppnew; struct pmc_processhash *pph; hindex = PMC_HASH_PTR(p, pmc_processhashmask); pph = &pmc_processhash[hindex]; ppnew = NULL; /* * Pre-allocate memory in the FIND_ALLOCATE case since we * cannot call malloc(9) once we hold a spin lock. */ if (mode & PMC_FLAG_ALLOCATE) ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc * sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO); mtx_lock_spin(&pmc_processhash_mtx); LIST_FOREACH(pp, pph, pp_next) if (pp->pp_proc == p) break; if ((mode & PMC_FLAG_REMOVE) && pp != NULL) LIST_REMOVE(pp, pp_next); if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL && ppnew != NULL) { ppnew->pp_proc = p; LIST_INSERT_HEAD(pph, ppnew, pp_next); pp = ppnew; ppnew = NULL; } mtx_unlock_spin(&pmc_processhash_mtx); if (pp != NULL && ppnew != NULL) free(ppnew, M_PMC); return pp; } /* * remove a process descriptor from the process hash table. */ static void pmc_remove_process_descriptor(struct pmc_process *pp) { KASSERT(pp->pp_refcnt == 0, ("[pmc,%d] Removing process descriptor %p with count %d", __LINE__, pp, pp->pp_refcnt)); mtx_lock_spin(&pmc_processhash_mtx); LIST_REMOVE(pp, pp_next); mtx_unlock_spin(&pmc_processhash_mtx); } /* * find an owner descriptor corresponding to proc 'p' */ static struct pmc_owner * pmc_find_owner_descriptor(struct proc *p) { uint32_t hindex; struct pmc_owner *po; struct pmc_ownerhash *poh; hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); poh = &pmc_ownerhash[hindex]; po = NULL; LIST_FOREACH(po, poh, po_next) if (po->po_owner == p) break; PMCDBG5(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> " "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po); return po; } /* * pmc_allocate_pmc_descriptor * * Allocate a pmc descriptor and initialize its * fields. */ static struct pmc * pmc_allocate_pmc_descriptor(void) { struct pmc *pmc; pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO); PMCDBG1(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); return pmc; } /* * Destroy a pmc descriptor. */ static void pmc_destroy_pmc_descriptor(struct pmc *pm) { KASSERT(pm->pm_state == PMC_STATE_DELETED || pm->pm_state == PMC_STATE_FREE, ("[pmc,%d] destroying non-deleted PMC", __LINE__)); KASSERT(LIST_EMPTY(&pm->pm_targets), ("[pmc,%d] destroying pmc with targets", __LINE__)); KASSERT(pm->pm_owner == NULL, ("[pmc,%d] destroying pmc attached to an owner", __LINE__)); KASSERT(pm->pm_runcount == 0, ("[pmc,%d] pmc has non-zero run count %d", __LINE__, pm->pm_runcount)); free(pm, M_PMC); } static void pmc_wait_for_pmc_idle(struct pmc *pm) { #ifdef HWPMC_DEBUG volatile int maxloop; maxloop = 100 * pmc_cpu_max(); #endif /* * Loop (with a forced context switch) till the PMC's runcount * comes down to zero. */ while (atomic_load_acq_32(&pm->pm_runcount) > 0) { #ifdef HWPMC_DEBUG maxloop--; KASSERT(maxloop > 0, ("[pmc,%d] (ri%d, rc%d) waiting too long for " "pmc to be free", __LINE__, PMC_TO_ROWINDEX(pm), pm->pm_runcount)); #endif pmc_force_context_switch(); } } /* * This function does the following things: * * - detaches the PMC from hardware * - unlinks all target threads that were attached to it * - removes the PMC from its owner's list * - destroys the PMC private mutex * * Once this function completes, the given pmc pointer can be freed by * calling pmc_destroy_pmc_descriptor(). */ static void pmc_release_pmc_descriptor(struct pmc *pm) { enum pmc_mode mode; struct pmc_hw *phw; u_int adjri, ri, cpu; struct pmc_owner *po; struct pmc_binding pb; struct pmc_process *pp; struct pmc_classdep *pcd; struct pmc_target *ptgt, *tmp; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); mode = PMC_TO_MODE(pm); PMCDBG3(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, mode); /* * First, we take the PMC off hardware. */ cpu = 0; if (PMC_IS_SYSTEM_MODE(mode)) { /* * A system mode PMC runs on a specific CPU. Switch * to this CPU and turn hardware off. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); pmc_select_cpu(cpu); /* switch off non-stalled CPUs */ CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate); if (pm->pm_state == PMC_STATE_RUNNING && !CPU_ISSET(cpu, &pm->pm_stalled)) { phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; KASSERT(phw->phw_pmc == pm, ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", __LINE__, ri, phw->phw_pmc, pm)); PMCDBG2(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); critical_enter(); pcd->pcd_stop_pmc(cpu, adjri); critical_exit(); } PMCDBG2(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); critical_enter(); pcd->pcd_config_pmc(cpu, adjri, NULL); critical_exit(); /* adjust the global and process count of SS mode PMCs */ if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { po = pm->pm_owner; po->po_sscount--; if (po->po_sscount == 0) { atomic_subtract_rel_int(&pmc_ss_count, 1); LIST_REMOVE(po, po_ssnext); } } pm->pm_state = PMC_STATE_DELETED; pmc_restore_cpu_binding(&pb); /* * We could have references to this PMC structure in * the per-cpu sample queues. Wait for the queue to * drain. */ pmc_wait_for_pmc_idle(pm); } else if (PMC_IS_VIRTUAL_MODE(mode)) { /* * A virtual PMC could be running on multiple CPUs at * a given instant. * * By marking its state as DELETED, we ensure that * this PMC is never further scheduled on hardware. * * Then we wait till all CPUs are done with this PMC. */ pm->pm_state = PMC_STATE_DELETED; /* Wait for the PMCs runcount to come to zero. */ pmc_wait_for_pmc_idle(pm); /* * At this point the PMC is off all CPUs and cannot be * freshly scheduled onto a CPU. It is now safe to * unlink all targets from this PMC. If a * process-record's refcount falls to zero, we remove * it from the hash table. The module-wide SX lock * protects us from races. */ LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { pp = ptgt->pt_process; pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ PMCDBG1(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt); /* * If the target process record shows that no * PMCs are attached to it, reclaim its space. */ if (pp->pp_refcnt == 0) { pmc_remove_process_descriptor(pp); free(pp, M_PMC); } } cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ } /* * Release any MD resources */ (void) pcd->pcd_release_pmc(cpu, adjri, pm); /* * Update row disposition */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) PMC_UNMARK_ROW_STANDALONE(ri); else PMC_UNMARK_ROW_THREAD(ri); /* unlink from the owner's list */ if (pm->pm_owner) { LIST_REMOVE(pm, pm_next); pm->pm_owner = NULL; } } /* * Register an owner and a pmc. */ static int pmc_register_owner(struct proc *p, struct pmc *pmc) { struct pmc_owner *po; sx_assert(&pmc_sx, SX_XLOCKED); if ((po = pmc_find_owner_descriptor(p)) == NULL) if ((po = pmc_allocate_owner_descriptor(p)) == NULL) return ENOMEM; KASSERT(pmc->pm_owner == NULL, ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); pmc->pm_owner = po; LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next); PROC_LOCK(p); p->p_flag |= P_HWPMC; PROC_UNLOCK(p); if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_pmcallocate(pmc); PMCDBG2(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p", po, pmc); return 0; } /* * Return the current row disposition: * == 0 => FREE * > 0 => PROCESS MODE * < 0 => SYSTEM MODE */ int pmc_getrowdisp(int ri) { return pmc_pmcdisp[ri]; } /* * Check if a PMC at row index 'ri' can be allocated to the current * process. * * Allocation can fail if: * - the current process is already being profiled by a PMC at index 'ri', * attached to it via OP_PMCATTACH. * - the current process has already allocated a PMC at index 'ri' * via OP_ALLOCATE. */ static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu) { enum pmc_mode mode; struct pmc *pm; struct pmc_owner *po; struct pmc_process *pp; PMCDBG5(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d " "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu); /* * We shouldn't have already allocated a process-mode PMC at * row index 'ri'. * * We shouldn't have allocated a system-wide PMC on the same * CPU and same RI. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) LIST_FOREACH(pm, &po->po_pmcs, pm_next) { if (PMC_TO_ROWINDEX(pm) == ri) { mode = PMC_TO_MODE(pm); if (PMC_IS_VIRTUAL_MODE(mode)) return EEXIST; if (PMC_IS_SYSTEM_MODE(mode) && (int) PMC_TO_CPU(pm) == cpu) return EEXIST; } } /* * We also shouldn't be the target of any PMC at this index * since otherwise a PMC_ATTACH to ourselves will fail. */ if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) if (pp->pp_pmcs[ri].pp_pmc) return EEXIST; PMCDBG4(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", p, p->p_pid, p->p_comm, ri); return 0; } /* * Check if a given PMC at row index 'ri' can be currently used in * mode 'mode'. */ static int pmc_can_allocate_row(int ri, enum pmc_mode mode) { enum pmc_disp disp; sx_assert(&pmc_sx, SX_XLOCKED); PMCDBG2(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode); if (PMC_IS_SYSTEM_MODE(mode)) disp = PMC_DISP_STANDALONE; else disp = PMC_DISP_THREAD; /* * check disposition for PMC row 'ri': * * Expected disposition Row-disposition Result * * STANDALONE STANDALONE or FREE proceed * STANDALONE THREAD fail * THREAD THREAD or FREE proceed * THREAD STANDALONE fail */ if (!PMC_ROW_DISP_IS_FREE(ri) && !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) && !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri))) return EBUSY; /* * All OK */ PMCDBG2(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); return 0; } /* * Find a PMC descriptor with user handle 'pmcid' for thread 'td'. */ static struct pmc * pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid) { struct pmc *pm; KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc, ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc)); LIST_FOREACH(pm, &po->po_pmcs, pm_next) if (pm->pm_id == pmcid) return pm; return NULL; } static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) { struct pmc *pm, *opm; struct pmc_owner *po; struct pmc_process *pp; KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc, ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc)); PMCDBG1(PMC,FND,1, "find-pmc id=%d", pmcid); if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) { /* * In case of PMC_F_DESCENDANTS child processes we will not find * the current process in the owners hash list. Find the owner * process first and from there lookup the po. */ if ((pp = pmc_find_process_descriptor(curthread->td_proc, PMC_FLAG_NONE)) == NULL) { return ESRCH; } else { opm = pp->pp_pmcs[PMC_ID_TO_ROWINDEX(pmcid)].pp_pmc; if (opm == NULL) return ESRCH; if ((opm->pm_flags & (PMC_F_ATTACHED_TO_OWNER| PMC_F_DESCENDANTS)) != (PMC_F_ATTACHED_TO_OWNER| PMC_F_DESCENDANTS)) return ESRCH; po = opm->pm_owner; } } if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL) return EINVAL; PMCDBG2(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); *pmc = pm; return 0; } /* * Start a PMC. */ static int pmc_start(struct pmc *pm) { enum pmc_mode mode; struct pmc_owner *po; struct pmc_binding pb; struct pmc_classdep *pcd; int adjri, error, cpu, ri; KASSERT(pm != NULL, ("[pmc,%d] null pm", __LINE__)); mode = PMC_TO_MODE(pm); ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); error = 0; PMCDBG3(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); po = pm->pm_owner; /* * Disallow PMCSTART if a logfile is required but has not been * configured yet. */ if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) return (EDOOFUS); /* programming error */ /* * If this is a sampling mode PMC, log mapping information for * the kernel modules that are currently loaded. */ if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pmc_log_kernel_mappings(pm); if (PMC_IS_VIRTUAL_MODE(mode)) { /* * If a PMCATTACH has never been done on this PMC, * attach it to its owner process. */ if (LIST_EMPTY(&pm->pm_targets)) error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH : pmc_attach_process(po->po_owner, pm); /* * If the PMC is attached to its owner, then force a context * switch to ensure that the MD state gets set correctly. */ if (error == 0) { pm->pm_state = PMC_STATE_RUNNING; if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) pmc_force_context_switch(); } return (error); } /* * A system-wide PMC. * * Add the owner to the global list if this is a system-wide * sampling PMC. */ if (mode == PMC_MODE_SS) { if (po->po_sscount == 0) { LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext); atomic_add_rel_int(&pmc_ss_count, 1); PMCDBG1(PMC,OPS,1, "po=%p in global list", po); } po->po_sscount++; /* * Log mapping information for all existing processes in the * system. Subsequent mappings are logged as they happen; * see pmc_process_mmap(). */ if (po->po_logprocmaps == 0) { pmc_log_all_process_mappings(po); po->po_logprocmaps = 1; } } /* * Move to the CPU associated with this * PMC, and start the hardware. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); if (!pmc_cpu_is_active(cpu)) return (ENXIO); pmc_select_cpu(cpu); /* * global PMCs are configured at allocation time * so write out the initial value and start the PMC. */ pm->pm_state = PMC_STATE_RUNNING; critical_enter(); if ((error = pcd->pcd_write_pmc(cpu, adjri, PMC_IS_SAMPLING_MODE(mode) ? pm->pm_sc.pm_reloadcount : pm->pm_sc.pm_initial)) == 0) { /* If a sampling mode PMC, reset stalled state. */ if (PMC_IS_SAMPLING_MODE(mode)) CPU_CLR_ATOMIC(cpu, &pm->pm_stalled); /* Indicate that we desire this to run. Start it. */ CPU_SET_ATOMIC(cpu, &pm->pm_cpustate); error = pcd->pcd_start_pmc(cpu, adjri); } critical_exit(); pmc_restore_cpu_binding(&pb); return (error); } /* * Stop a PMC. */ static int pmc_stop(struct pmc *pm) { struct pmc_owner *po; struct pmc_binding pb; struct pmc_classdep *pcd; int adjri, cpu, error, ri; KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); PMCDBG3(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); pm->pm_state = PMC_STATE_STOPPED; /* * If the PMC is a virtual mode one, changing the state to * non-RUNNING is enough to ensure that the PMC never gets * scheduled. * * If this PMC is current running on a CPU, then it will * handled correctly at the time its target process is context * switched out. */ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) return 0; /* * A system-mode PMC. Move to the CPU associated with * this PMC, and stop the hardware. We update the * 'initial count' so that a subsequent PMCSTART will * resume counting from the current hardware count. */ pmc_save_cpu_binding(&pb); cpu = PMC_TO_CPU(pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); if (!pmc_cpu_is_active(cpu)) return ENXIO; pmc_select_cpu(cpu); ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate); critical_enter(); if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0) error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial); critical_exit(); pmc_restore_cpu_binding(&pb); po = pm->pm_owner; /* remove this owner from the global list of SS PMC owners */ if (PMC_TO_MODE(pm) == PMC_MODE_SS) { po->po_sscount--; if (po->po_sscount == 0) { atomic_subtract_rel_int(&pmc_ss_count, 1); LIST_REMOVE(po, po_ssnext); PMCDBG1(PMC,OPS,2,"po=%p removed from global list", po); } } return (error); } #ifdef HWPMC_DEBUG static const char *pmc_op_to_name[] = { #undef __PMC_OP #define __PMC_OP(N, D) #N , __PMC_OPS() NULL }; #endif /* * The syscall interface */ #define PMC_GET_SX_XLOCK(...) do { \ sx_xlock(&pmc_sx); \ if (pmc_hook == NULL) { \ sx_xunlock(&pmc_sx); \ return __VA_ARGS__; \ } \ } while (0) #define PMC_DOWNGRADE_SX() do { \ sx_downgrade(&pmc_sx); \ is_sx_downgraded = 1; \ } while (0) static int pmc_syscall_handler(struct thread *td, void *syscall_args) { int error, is_sx_downgraded, is_sx_locked, op; struct pmc_syscall_args *c; void *arg; PMC_GET_SX_XLOCK(ENOSYS); DROP_GIANT(); is_sx_downgraded = 0; is_sx_locked = 1; c = (struct pmc_syscall_args *) syscall_args; op = c->pmop_code; arg = c->pmop_data; PMCDBG3(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op, pmc_op_to_name[op], arg); error = 0; atomic_add_int(&pmc_stats.pm_syscalls, 1); switch(op) { /* * Configure a log file. * * XXX This OP will be reworked. */ case PMC_OP_CONFIGURELOG: { struct proc *p; struct pmc *pm; struct pmc_owner *po; struct pmc_op_configurelog cl; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &cl, sizeof(cl))) != 0) break; /* mark this process as owning a log file */ p = td->td_proc; if ((po = pmc_find_owner_descriptor(p)) == NULL) if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { error = ENOMEM; break; } /* * If a valid fd was passed in, try to configure that, * otherwise if 'fd' was less than zero and there was * a log file configured, flush its buffers and * de-configure it. */ if (cl.pm_logfd >= 0) { sx_xunlock(&pmc_sx); is_sx_locked = 0; error = pmclog_configure_log(md, po, cl.pm_logfd); } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) { pmclog_process_closelog(po); error = pmclog_close(po); if (error == 0) { LIST_FOREACH(pm, &po->po_pmcs, pm_next) if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && pm->pm_state == PMC_STATE_RUNNING) pmc_stop(pm); error = pmclog_deconfigure_log(po); } } else error = EINVAL; if (error) break; } break; /* * Flush a log file. */ case PMC_OP_FLUSHLOG: { struct pmc_owner *po; sx_assert(&pmc_sx, SX_XLOCKED); if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; break; } error = pmclog_flush(po); } break; /* * Close a log file. */ case PMC_OP_CLOSELOG: { struct pmc_owner *po; sx_assert(&pmc_sx, SX_XLOCKED); if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; break; } error = pmclog_close(po); } break; /* * Retrieve hardware configuration. */ case PMC_OP_GETCPUINFO: /* CPU information */ { struct pmc_op_getcpuinfo gci; struct pmc_classinfo *pci; struct pmc_classdep *pcd; int cl; gci.pm_cputype = md->pmd_cputype; gci.pm_ncpu = pmc_cpu_max(); gci.pm_npmc = md->pmd_npmc; gci.pm_nclass = md->pmd_nclass; pci = gci.pm_classes; pcd = md->pmd_classdep; for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) { pci->pm_caps = pcd->pcd_caps; pci->pm_class = pcd->pcd_class; pci->pm_width = pcd->pcd_width; pci->pm_num = pcd->pcd_num; } error = copyout(&gci, arg, sizeof(gci)); } break; /* * Retrieve soft events list. */ case PMC_OP_GETDYNEVENTINFO: { enum pmc_class cl; enum pmc_event ev; struct pmc_op_getdyneventinfo *gei; struct pmc_dyn_event_descr dev; struct pmc_soft *ps; uint32_t nevent; sx_assert(&pmc_sx, SX_LOCKED); gei = (struct pmc_op_getdyneventinfo *) arg; if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0) break; /* Only SOFT class is dynamic. */ if (cl != PMC_CLASS_SOFT) { error = EINVAL; break; } nevent = 0; for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) { ps = pmc_soft_ev_acquire(ev); if (ps == NULL) continue; bcopy(&ps->ps_ev, &dev, sizeof(dev)); pmc_soft_ev_release(ps); error = copyout(&dev, &gei->pm_events[nevent], sizeof(struct pmc_dyn_event_descr)); if (error != 0) break; nevent++; } if (error != 0) break; error = copyout(&nevent, &gei->pm_nevent, sizeof(nevent)); } break; /* * Get module statistics */ case PMC_OP_GETDRIVERSTATS: { struct pmc_op_getdriverstats gms; bcopy(&pmc_stats, &gms, sizeof(gms)); error = copyout(&gms, arg, sizeof(gms)); } break; /* * Retrieve module version number */ case PMC_OP_GETMODULEVERSION: { uint32_t cv, modv; /* retrieve the client's idea of the ABI version */ if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0) break; /* don't service clients newer than our driver */ modv = PMC_VERSION; if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) { error = EPROGMISMATCH; break; } error = copyout(&modv, arg, sizeof(int)); } break; /* * Retrieve the state of all the PMCs on a given * CPU. */ case PMC_OP_GETPMCINFO: { int ari; struct pmc *pm; size_t pmcinfo_size; uint32_t cpu, n, npmc; struct pmc_owner *po; struct pmc_binding pb; struct pmc_classdep *pcd; struct pmc_info *p, *pmcinfo; struct pmc_op_getpmcinfo *gpi; PMC_DOWNGRADE_SX(); gpi = (struct pmc_op_getpmcinfo *) arg; if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) break; if (cpu >= pmc_cpu_max()) { error = EINVAL; break; } if (!pmc_cpu_is_active(cpu)) { error = ENXIO; break; } /* switch to CPU 'cpu' */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); npmc = md->pmd_npmc; pmcinfo_size = npmc * sizeof(struct pmc_info); pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK); p = pmcinfo; for (n = 0; n < md->pmd_npmc; n++, p++) { pcd = pmc_ri_to_classdep(md, n, &ari); KASSERT(pcd != NULL, ("[pmc,%d] null pcd ri=%d", __LINE__, n)); if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) break; if (PMC_ROW_DISP_IS_STANDALONE(n)) p->pm_rowdisp = PMC_DISP_STANDALONE; else if (PMC_ROW_DISP_IS_THREAD(n)) p->pm_rowdisp = PMC_DISP_THREAD; else p->pm_rowdisp = PMC_DISP_FREE; p->pm_ownerpid = -1; if (pm == NULL) /* no PMC associated */ continue; po = pm->pm_owner; KASSERT(po->po_owner != NULL, ("[pmc,%d] pmc_owner had a null proc pointer", __LINE__)); p->pm_ownerpid = po->po_owner->p_pid; p->pm_mode = PMC_TO_MODE(pm); p->pm_event = pm->pm_event; p->pm_flags = pm->pm_flags; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) p->pm_reloadcount = pm->pm_sc.pm_reloadcount; } pmc_restore_cpu_binding(&pb); /* now copy out the PMC info collected */ if (error == 0) error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size); free(pmcinfo, M_PMC); } break; /* * Set the administrative state of a PMC. I.e. whether * the PMC is to be used or not. */ case PMC_OP_PMCADMIN: { int cpu, ri; enum pmc_state request; struct pmc_cpu *pc; struct pmc_hw *phw; struct pmc_op_pmcadmin pma; struct pmc_binding pb; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); error = priv_check(td, PRIV_PMC_MANAGE); if (error) break; if ((error = copyin(arg, &pma, sizeof(pma))) != 0) break; cpu = pma.pm_cpu; if (cpu < 0 || cpu >= (int) pmc_cpu_max()) { error = EINVAL; break; } if (!pmc_cpu_is_active(cpu)) { error = ENXIO; break; } request = pma.pm_state; if (request != PMC_STATE_DISABLED && request != PMC_STATE_FREE) { error = EINVAL; break; } ri = pma.pm_pmc; /* pmc id == row index */ if (ri < 0 || ri >= (int) md->pmd_npmc) { error = EINVAL; break; } /* * We can't disable a PMC with a row-index allocated * for process virtual PMCs. */ if (PMC_ROW_DISP_IS_THREAD(ri) && request == PMC_STATE_DISABLED) { error = EBUSY; break; } /* * otherwise, this PMC on this CPU is either free or * in system-wide mode. */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); pc = pmc_pcpu[cpu]; phw = pc->pc_hwpmcs[ri]; /* * XXX do we need some kind of 'forced' disable? */ if (phw->phw_pmc == NULL) { if (request == PMC_STATE_DISABLED && (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) { phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED; PMC_MARK_ROW_STANDALONE(ri); } else if (request == PMC_STATE_FREE && (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) { phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; PMC_UNMARK_ROW_STANDALONE(ri); } /* other cases are a no-op */ } else error = EBUSY; pmc_restore_cpu_binding(&pb); } break; /* * Allocate a PMC. */ case PMC_OP_PMCALLOCATE: { int adjri, n; u_int cpu; uint32_t caps; struct pmc *pmc; enum pmc_mode mode; struct pmc_hw *phw; struct pmc_binding pb; struct pmc_classdep *pcd; struct pmc_op_pmcallocate pa; if ((error = copyin(arg, &pa, sizeof(pa))) != 0) break; caps = pa.pm_caps; mode = pa.pm_mode; cpu = pa.pm_cpu; if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && mode != PMC_MODE_TS && mode != PMC_MODE_TC) || (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) { error = EINVAL; break; } /* * Virtual PMCs should only ask for a default CPU. * System mode PMCs need to specify a non-default CPU. */ if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) || (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) { error = EINVAL; break; } /* * Check that an inactive CPU is not being asked for. */ if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) { error = ENXIO; break; } /* * Refuse an allocation for a system-wide PMC if this * process has been jailed, or if this process lacks * super-user credentials and the sysctl tunable * 'security.bsd.unprivileged_syspmcs' is zero. */ if (PMC_IS_SYSTEM_MODE(mode)) { if (jailed(curthread->td_ucred)) { error = EPERM; break; } if (!pmc_unprivileged_syspmcs) { error = priv_check(curthread, PRIV_PMC_SYSTEM); if (error) break; } } /* * Look for valid values for 'pm_flags' */ if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) { error = EINVAL; break; } /* process logging options are not allowed for system PMCs */ if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags & (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) { error = EINVAL; break; } /* * All sampling mode PMCs need to be able to interrupt the * CPU. */ if (PMC_IS_SAMPLING_MODE(mode)) caps |= PMC_CAP_INTERRUPT; /* A valid class specifier should have been passed in. */ for (n = 0; n < md->pmd_nclass; n++) if (md->pmd_classdep[n].pcd_class == pa.pm_class) break; if (n == md->pmd_nclass) { error = EINVAL; break; } /* The requested PMC capabilities should be feasible. */ if ((md->pmd_classdep[n].pcd_caps & caps) != caps) { error = EOPNOTSUPP; break; } PMCDBG4(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", pa.pm_ev, caps, mode, cpu); pmc = pmc_allocate_pmc_descriptor(); pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class, PMC_ID_INVALID); pmc->pm_event = pa.pm_ev; pmc->pm_state = PMC_STATE_FREE; pmc->pm_caps = caps; pmc->pm_flags = pa.pm_flags; /* switch thread to CPU 'cpu' */ pmc_save_cpu_binding(&pb); #define PMC_IS_SHAREABLE_PMC(cpu, n) \ (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ PMC_PHW_FLAG_IS_SHAREABLE) #define PMC_IS_UNALLOCATED(cpu, n) \ (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) if (PMC_IS_SYSTEM_MODE(mode)) { pmc_select_cpu(cpu); for (n = 0; n < (int) md->pmd_npmc; n++) { pcd = pmc_ri_to_classdep(md, n, &adjri); if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, cpu) == 0 && (PMC_IS_UNALLOCATED(cpu, n) || PMC_IS_SHAREABLE_PMC(cpu, n)) && pcd->pcd_allocate_pmc(cpu, adjri, pmc, &pa) == 0) break; } } else { /* Process virtual mode */ for (n = 0; n < (int) md->pmd_npmc; n++) { pcd = pmc_ri_to_classdep(md, n, &adjri); if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, PMC_CPU_ANY) == 0 && pcd->pcd_allocate_pmc(curthread->td_oncpu, adjri, pmc, &pa) == 0) break; } } #undef PMC_IS_UNALLOCATED #undef PMC_IS_SHAREABLE_PMC pmc_restore_cpu_binding(&pb); if (n == (int) md->pmd_npmc) { pmc_destroy_pmc_descriptor(pmc); pmc = NULL; error = EINVAL; break; } /* Fill in the correct value in the ID field */ pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n); PMCDBG5(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x", pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id); /* Process mode PMCs with logging enabled need log files */ if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW)) pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; /* All system mode sampling PMCs require a log file */ if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode)) pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; /* * Configure global pmc's immediately */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; pcd = pmc_ri_to_classdep(md, n, &adjri); if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) { (void) pcd->pcd_release_pmc(cpu, adjri, pmc); pmc_destroy_pmc_descriptor(pmc); pmc = NULL; pmc_restore_cpu_binding(&pb); error = EPERM; break; } pmc_restore_cpu_binding(&pb); } pmc->pm_state = PMC_STATE_ALLOCATED; /* * mark row disposition */ if (PMC_IS_SYSTEM_MODE(mode)) PMC_MARK_ROW_STANDALONE(n); else PMC_MARK_ROW_THREAD(n); /* * Register this PMC with the current thread as its owner. */ if ((error = pmc_register_owner(curthread->td_proc, pmc)) != 0) { pmc_release_pmc_descriptor(pmc); pmc_destroy_pmc_descriptor(pmc); pmc = NULL; break; } /* * Return the allocated index. */ pa.pm_pmcid = pmc->pm_id; error = copyout(&pa, arg, sizeof(pa)); } break; /* * Attach a PMC to a process. */ case PMC_OP_PMCATTACH: { struct pmc *pm; struct proc *p; struct pmc_op_pmcattach a; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &a, sizeof(a))) != 0) break; if (a.pm_pid < 0) { error = EINVAL; break; } else if (a.pm_pid == 0) a.pm_pid = td->td_proc->p_pid; if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) break; if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { error = EINVAL; break; } /* PMCs may be (re)attached only when allocated or stopped */ if (pm->pm_state == PMC_STATE_RUNNING) { error = EBUSY; break; } else if (pm->pm_state != PMC_STATE_ALLOCATED && pm->pm_state != PMC_STATE_STOPPED) { error = EINVAL; break; } /* lookup pid */ if ((p = pfind(a.pm_pid)) == NULL) { error = ESRCH; break; } /* * Ignore processes that are working on exiting. */ if (p->p_flag & P_WEXIT) { error = ESRCH; PROC_UNLOCK(p); /* pfind() returns a locked process */ break; } /* * we are allowed to attach a PMC to a process if * we can debug it. */ error = p_candebug(curthread, p); PROC_UNLOCK(p); if (error == 0) error = pmc_attach_process(p, pm); } break; /* * Detach an attached PMC from a process. */ case PMC_OP_PMCDETACH: { struct pmc *pm; struct proc *p; struct pmc_op_pmcattach a; if ((error = copyin(arg, &a, sizeof(a))) != 0) break; if (a.pm_pid < 0) { error = EINVAL; break; } else if (a.pm_pid == 0) a.pm_pid = td->td_proc->p_pid; if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) break; if ((p = pfind(a.pm_pid)) == NULL) { error = ESRCH; break; } /* * Treat processes that are in the process of exiting * as if they were not present. */ if (p->p_flag & P_WEXIT) error = ESRCH; PROC_UNLOCK(p); /* pfind() returns a locked process */ if (error == 0) error = pmc_detach_process(p, pm); } break; /* * Retrieve the MSR number associated with the counter * 'pmc_id'. This allows processes to directly use RDPMC * instructions to read their PMCs, without the overhead of a * system call. */ case PMC_OP_PMCGETMSR: { int adjri, ri; struct pmc *pm; struct pmc_target *pt; struct pmc_op_getmsr gm; struct pmc_classdep *pcd; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &gm, sizeof(gm))) != 0) break; if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) break; /* * The allocated PMC has to be a process virtual PMC, * i.e., of type MODE_T[CS]. Global PMCs can only be * read using the PMCREAD operation since they may be * allocated on a different CPU than the one we could * be running on at the time of the RDPMC instruction. * * The GETMSR operation is not allowed for PMCs that * are inherited across processes. */ if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || (pm->pm_flags & PMC_F_DESCENDANTS)) { error = EINVAL; break; } /* * It only makes sense to use a RDPMC (or its * equivalent instruction on non-x86 architectures) on * a process that has allocated and attached a PMC to * itself. Conversely the PMC is only allowed to have * one process attached to it -- its owner. */ if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || LIST_NEXT(pt, pt_next) != NULL || pt->pt_process->pp_proc != pm->pm_owner->po_owner) { error = EINVAL; break; } ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); /* PMC class has no 'GETMSR' support */ if (pcd->pcd_get_msr == NULL) { error = ENOSYS; break; } if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0) break; if ((error = copyout(&gm, arg, sizeof(gm))) < 0) break; /* * Mark our process as using MSRs. Update machine * state using a forced context switch. */ pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS; pmc_force_context_switch(); } break; /* * Release an allocated PMC */ case PMC_OP_PMCRELEASE: { pmc_id_t pmcid; struct pmc *pm; struct pmc_owner *po; struct pmc_op_simple sp; /* * Find PMC pointer for the named PMC. * * Use pmc_release_pmc_descriptor() to switch off the * PMC, remove all its target threads, and remove the * PMC from its owner's list. * * Remove the owner record if this is the last PMC * owned. * * Free up space. */ if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; po = pm->pm_owner; pmc_release_pmc_descriptor(pm); pmc_maybe_remove_owner(po); pmc_destroy_pmc_descriptor(pm); } break; /* * Read and/or write a PMC. */ case PMC_OP_PMCRW: { int adjri; struct pmc *pm; uint32_t cpu, ri; pmc_value_t oldvalue; struct pmc_binding pb; struct pmc_op_pmcrw prw; struct pmc_classdep *pcd; struct pmc_op_pmcrw *pprw; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &prw, sizeof(prw))) != 0) break; ri = 0; PMCDBG2(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid, prw.pm_flags); /* must have at least one flag set */ if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) { error = EINVAL; break; } /* locate pmc descriptor */ if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0) break; /* Can't read a PMC that hasn't been started. */ if (pm->pm_state != PMC_STATE_ALLOCATED && pm->pm_state != PMC_STATE_STOPPED && pm->pm_state != PMC_STATE_RUNNING) { error = EINVAL; break; } /* writing a new value is allowed only for 'STOPPED' pmcs */ if (pm->pm_state == PMC_STATE_RUNNING && (prw.pm_flags & PMC_F_NEWVALUE)) { error = EBUSY; break; } if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { /* * If this PMC is attached to its owner (i.e., * the process requesting this operation) and * is running, then attempt to get an * upto-date reading from hardware for a READ. * Writes are only allowed when the PMC is * stopped, so only update the saved value * field. * * If the PMC is not running, or is not * attached to its owner, read/write to the * savedvalue field. */ ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); mtx_pool_lock_spin(pmc_mtxpool, pm); cpu = curthread->td_oncpu; if (prw.pm_flags & PMC_F_OLDVALUE) { if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && (pm->pm_state == PMC_STATE_RUNNING)) error = (*pcd->pcd_read_pmc)(cpu, adjri, &oldvalue); else oldvalue = pm->pm_gv.pm_savedvalue; } if (prw.pm_flags & PMC_F_NEWVALUE) pm->pm_gv.pm_savedvalue = prw.pm_value; mtx_pool_unlock_spin(pmc_mtxpool, pm); } else { /* System mode PMCs */ cpu = PMC_TO_CPU(pm); ri = PMC_TO_ROWINDEX(pm); pcd = pmc_ri_to_classdep(md, ri, &adjri); if (!pmc_cpu_is_active(cpu)) { error = ENXIO; break; } /* move this thread to CPU 'cpu' */ pmc_save_cpu_binding(&pb); pmc_select_cpu(cpu); critical_enter(); /* save old value */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, &oldvalue))) goto error; /* write out new value */ if (prw.pm_flags & PMC_F_NEWVALUE) error = (*pcd->pcd_write_pmc)(cpu, adjri, prw.pm_value); error: critical_exit(); pmc_restore_cpu_binding(&pb); if (error) break; } pprw = (struct pmc_op_pmcrw *) arg; #ifdef HWPMC_DEBUG if (prw.pm_flags & PMC_F_NEWVALUE) PMCDBG3(PMC,OPS,2, "rw id=%d new %jx -> old %jx", ri, prw.pm_value, oldvalue); else if (prw.pm_flags & PMC_F_OLDVALUE) PMCDBG2(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue); #endif /* return old value if requested */ if (prw.pm_flags & PMC_F_OLDVALUE) if ((error = copyout(&oldvalue, &pprw->pm_value, sizeof(prw.pm_value)))) break; } break; /* * Set the sampling rate for a sampling mode PMC and the * initial count for a counting mode PMC. */ case PMC_OP_PMCSETCOUNT: { struct pmc *pm; struct pmc_op_pmcsetcount sc; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &sc, sizeof(sc))) != 0) break; if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) break; if (pm->pm_state == PMC_STATE_RUNNING) { error = EBUSY; break; } if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) pm->pm_sc.pm_reloadcount = sc.pm_count; else pm->pm_sc.pm_initial = sc.pm_count; } break; /* * Start a PMC. */ case PMC_OP_PMCSTART: { pmc_id_t pmcid; struct pmc *pm; struct pmc_op_simple sp; sx_assert(&pmc_sx, SX_XLOCKED); if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; KASSERT(pmcid == pm->pm_id, ("[pmc,%d] pmcid %x != id %x", __LINE__, pm->pm_id, pmcid)); if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ break; else if (pm->pm_state != PMC_STATE_STOPPED && pm->pm_state != PMC_STATE_ALLOCATED) { error = EINVAL; break; } error = pmc_start(pm); } break; /* * Stop a PMC. */ case PMC_OP_PMCSTOP: { pmc_id_t pmcid; struct pmc *pm; struct pmc_op_simple sp; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &sp, sizeof(sp))) != 0) break; pmcid = sp.pm_pmcid; /* * Mark the PMC as inactive and invoke the MD stop * routines if needed. */ if ((error = pmc_find_pmc(pmcid, &pm)) != 0) break; KASSERT(pmcid == pm->pm_id, ("[pmc,%d] pmc id %x != pmcid %x", __LINE__, pm->pm_id, pmcid)); if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ break; else if (pm->pm_state != PMC_STATE_RUNNING) { error = EINVAL; break; } error = pmc_stop(pm); } break; /* * Write a user supplied value to the log file. */ case PMC_OP_WRITELOG: { struct pmc_op_writelog wl; struct pmc_owner *po; PMC_DOWNGRADE_SX(); if ((error = copyin(arg, &wl, sizeof(wl))) != 0) break; if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { error = EINVAL; break; } if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) { error = EINVAL; break; } error = pmclog_process_userlog(po, &wl); } break; default: error = EINVAL; break; } if (is_sx_locked != 0) { if (is_sx_downgraded) sx_sunlock(&pmc_sx); else sx_xunlock(&pmc_sx); } if (error) atomic_add_int(&pmc_stats.pm_syscall_errors, 1); PICKUP_GIANT(); return error; } /* * Helper functions */ /* * Mark the thread as needing callchain capture and post an AST. The * actual callchain capture will be done in a context where it is safe * to take page faults. */ static void pmc_post_callchain_callback(void) { struct thread *td; td = curthread; /* * If there is multiple PMCs for the same interrupt ignore new post */ if (td->td_pflags & TDP_CALLCHAIN) return; /* * Mark this thread as needing callchain capture. * `td->td_pflags' will be safe to touch because this thread * was in user space when it was interrupted. */ td->td_pflags |= TDP_CALLCHAIN; /* * Don't let this thread migrate between CPUs until callchain * capture completes. */ sched_pin(); return; } /* * Interrupt processing. * * Find a free slot in the per-cpu array of samples and capture the * current callchain there. If a sample was successfully added, a bit * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook * needs to be invoked from the clock handler. * * This function is meant to be called from an NMI handler. It cannot * use any of the locking primitives supplied by the OS. */ int pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf, int inuserspace) { int error, callchaindepth; struct thread *td; struct pmc_sample *ps; struct pmc_samplebuffer *psb; error = 0; /* * Allocate space for a sample buffer. */ psb = pmc_pcpu[cpu]->pc_sb[ring]; ps = psb->ps_write; if (ps->ps_nsamples) { /* in use, reader hasn't caught up */ CPU_SET_ATOMIC(cpu, &pm->pm_stalled); atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1); PMCDBG6(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, (void *) tf, inuserspace, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); callchaindepth = 1; error = ENOMEM; goto done; } /* Fill in entry. */ PMCDBG6(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, (void *) tf, inuserspace, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); KASSERT(pm->pm_runcount >= 0, ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm, pm->pm_runcount)); atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */ ps->ps_pmc = pm; if ((td = curthread) && td->td_proc) ps->ps_pid = td->td_proc->p_pid; else ps->ps_pid = -1; ps->ps_cpu = cpu; ps->ps_td = td; ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0; callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ? pmc_callchaindepth : 1; if (callchaindepth == 1) ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf); else { /* * Kernel stack traversals can be done immediately, * while we defer to an AST for user space traversals. */ if (!inuserspace) { callchaindepth = pmc_save_kernel_callchain(ps->ps_pc, callchaindepth, tf); } else { pmc_post_callchain_callback(); callchaindepth = PMC_SAMPLE_INUSE; } } ps->ps_nsamples = callchaindepth; /* mark entry as in use */ /* increment write pointer, modulo ring buffer size */ ps++; if (ps == psb->ps_fence) psb->ps_write = psb->ps_samples; else psb->ps_write = ps; done: /* mark CPU as needing processing */ if (callchaindepth != PMC_SAMPLE_INUSE) CPU_SET_ATOMIC(cpu, &pmc_cpumask); return (error); } /* * Capture a user call chain. This function will be called from ast() * before control returns to userland and before the process gets * rescheduled. */ static void pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf) { struct pmc *pm; struct thread *td; struct pmc_sample *ps, *ps_end; struct pmc_samplebuffer *psb; #ifdef INVARIANTS int ncallchains; #endif psb = pmc_pcpu[cpu]->pc_sb[ring]; td = curthread; KASSERT(td->td_pflags & TDP_CALLCHAIN, ("[pmc,%d] Retrieving callchain for thread that doesn't want it", __LINE__)); #ifdef INVARIANTS ncallchains = 0; #endif /* * Iterate through all deferred callchain requests. * Walk from the current read pointer to the current * write pointer. */ ps = psb->ps_read; ps_end = psb->ps_write; do { if (ps->ps_nsamples != PMC_SAMPLE_INUSE) goto next; if (ps->ps_td != td) goto next; KASSERT(ps->ps_cpu == cpu, ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__, ps->ps_cpu, PCPU_GET(cpuid))); pm = ps->ps_pmc; KASSERT(pm->pm_flags & PMC_F_CALLCHAIN, ("[pmc,%d] Retrieving callchain for PMC that doesn't " "want it", __LINE__)); KASSERT(pm->pm_runcount > 0, ("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount)); /* * Retrieve the callchain and mark the sample buffer * as 'processable' by the timer tick sweep code. */ ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc, pmc_callchaindepth, tf); #ifdef INVARIANTS ncallchains++; #endif next: /* increment the pointer, modulo sample ring size */ if (++ps == psb->ps_fence) ps = psb->ps_samples; } while (ps != ps_end); KASSERT(ncallchains > 0, ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__, cpu)); KASSERT(td->td_pinned == 1, ("[pmc,%d] invalid td_pinned value", __LINE__)); sched_unpin(); /* Can migrate safely now. */ /* mark CPU as needing processing */ CPU_SET_ATOMIC(cpu, &pmc_cpumask); return; } /* * Process saved PC samples. */ static void pmc_process_samples(int cpu, int ring) { struct pmc *pm; int adjri, n; struct thread *td; struct pmc_owner *po; struct pmc_sample *ps; struct pmc_classdep *pcd; struct pmc_samplebuffer *psb; KASSERT(PCPU_GET(cpuid) == cpu, ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, PCPU_GET(cpuid), cpu)); psb = pmc_pcpu[cpu]->pc_sb[ring]; for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ ps = psb->ps_read; if (ps->ps_nsamples == PMC_SAMPLE_FREE) break; pm = ps->ps_pmc; KASSERT(pm->pm_runcount > 0, ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm, pm->pm_runcount)); po = pm->pm_owner; KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, pm, PMC_TO_MODE(pm))); /* Ignore PMCs that have been switched off */ if (pm->pm_state != PMC_STATE_RUNNING) goto entrydone; /* If there is a pending AST wait for completion */ if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { /* Need a rescan at a later time. */ CPU_SET_ATOMIC(cpu, &pmc_cpumask); break; } PMCDBG6(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu, pm, ps->ps_nsamples, ps->ps_flags, (int) (psb->ps_write - psb->ps_samples), (int) (psb->ps_read - psb->ps_samples)); /* * If this is a process-mode PMC that is attached to * its owner, and if the PC is in user mode, update * profiling statistics like timer-based profiling * would have done. */ if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { if (ps->ps_flags & PMC_CC_F_USERSPACE) { td = FIRST_THREAD_IN_PROC(po->po_owner); addupc_intr(td, ps->ps_pc[0], 1); } goto entrydone; } /* * Otherwise, this is either a sampling mode PMC that * is attached to a different process than its owner, * or a system-wide sampling PMC. Dispatch a log * entry to the PMC's owner process. */ pmclog_process_callchain(pm, ps); entrydone: ps->ps_nsamples = 0; /* mark entry as free */ atomic_subtract_rel_int(&pm->pm_runcount, 1); /* increment read pointer, modulo sample size */ if (++ps == psb->ps_fence) psb->ps_read = psb->ps_samples; else psb->ps_read = ps; } atomic_add_int(&pmc_stats.pm_log_sweeps, 1); /* Do not re-enable stalled PMCs if we failed to process any samples */ if (n == 0) return; /* * Restart any stalled sampling PMCs on this CPU. * * If the NMI handler sets the pm_stalled field of a PMC after * the check below, we'll end up processing the stalled PMC at * the next hardclock tick. */ for (n = 0; n < md->pmd_npmc; n++) { pcd = pmc_ri_to_classdep(md, n, &adjri); KASSERT(pcd != NULL, ("[pmc,%d] null pcd ri=%d", __LINE__, n)); (void) (*pcd->pcd_get_config)(cpu,adjri,&pm); if (pm == NULL || /* !cfg'ed */ pm->pm_state != PMC_STATE_RUNNING || /* !active */ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ !CPU_ISSET(cpu, &pm->pm_cpustate) || /* !desired */ !CPU_ISSET(cpu, &pm->pm_stalled)) /* !stalled */ continue; CPU_CLR_ATOMIC(cpu, &pm->pm_stalled); (*pcd->pcd_start_pmc)(cpu, adjri); } } /* * Event handlers. */ /* * Handle a process exit. * * Remove this process from all hash tables. If this process * owned any PMCs, turn off those PMCs and deallocate them, * removing any associations with target processes. * * This function will be called by the last 'thread' of a * process. * * XXX This eventhandler gets called early in the exit process. * Consider using a 'hook' invocation from thread_exit() or equivalent * spot. Another negative is that kse_exit doesn't seem to call * exit1() [??]. * */ static void pmc_process_exit(void *arg __unused, struct proc *p) { struct pmc *pm; int adjri, cpu; unsigned int ri; int is_using_hwpmcs; struct pmc_owner *po; struct pmc_process *pp; struct pmc_classdep *pcd; pmc_value_t newvalue, tmp; PROC_LOCK(p); is_using_hwpmcs = p->p_flag & P_HWPMC; PROC_UNLOCK(p); /* * Log a sysexit event to all SS PMC owners. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_sysexit(po, p->p_pid); if (!is_using_hwpmcs) return; PMC_GET_SX_XLOCK(); PMCDBG3(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid, p->p_comm); /* * Since this code is invoked by the last thread in an exiting * process, we would have context switched IN at some prior * point. However, with PREEMPTION, kernel mode context * switches may happen any time, so we want to disable a - * context switch OUT till we get any PMCs targetting this + * context switch OUT till we get any PMCs targeting this * process off the hardware. * * We also need to atomically remove this process' * entry from our target process hash table, using * PMC_FLAG_REMOVE. */ PMCDBG3(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid, p->p_comm); critical_enter(); /* no preemption */ cpu = curthread->td_oncpu; if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_REMOVE)) != NULL) { PMCDBG2(PRC,EXT,2, "process-exit proc=%p pmc-process=%p", p, pp); /* * The exiting process could the target of * some PMCs which will be running on * currently executing CPU. * * We need to turn these PMCs off like we * would do at context switch OUT time. */ for (ri = 0; ri < md->pmd_npmc; ri++) { /* * Pick up the pmc pointer from hardware * state similar to the CSW_OUT code. */ pm = NULL; pcd = pmc_ri_to_classdep(md, ri, &adjri); (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); PMCDBG2(PRC,EXT,2, "ri=%d pm=%p", ri, pm); if (pm == NULL || !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) continue; PMCDBG4(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, pm, pm->pm_state); KASSERT(PMC_TO_ROWINDEX(pm) == ri, ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", __LINE__, PMC_TO_ROWINDEX(pm), ri)); KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); KASSERT(pm->pm_runcount > 0, ("[pmc,%d] bad runcount ri %d rc %d", __LINE__, ri, pm->pm_runcount)); /* * Change desired state, and then stop if not * stalled. This two-step dance should avoid * race conditions where an interrupt re-enables * the PMC after this code has already checked * the pm_stalled flag. */ if (CPU_ISSET(cpu, &pm->pm_cpustate)) { CPU_CLR_ATOMIC(cpu, &pm->pm_cpustate); if (!CPU_ISSET(cpu, &pm->pm_stalled)) { (void) pcd->pcd_stop_pmc(cpu, adjri); pcd->pcd_read_pmc(cpu, adjri, &newvalue); tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); mtx_pool_lock_spin(pmc_mtxpool, pm); pm->pm_gv.pm_savedvalue += tmp; pp->pp_pmcs[ri].pp_pmcval += tmp; mtx_pool_unlock_spin(pmc_mtxpool, pm); } } atomic_subtract_rel_int(&pm->pm_runcount,1); KASSERT((int) pm->pm_runcount >= 0, ("[pmc,%d] runcount is %d", __LINE__, ri)); (void) pcd->pcd_config_pmc(cpu, adjri, NULL); } /* * Inform the MD layer of this pseudo "context switch * out" */ (void) md->pmd_switch_out(pmc_pcpu[cpu], pp); critical_exit(); /* ok to be pre-empted now */ /* * Unlink this process from the PMCs that are - * targetting it. This will send a signal to + * targeting it. This will send a signal to * all PMC owner's whose PMCs are orphaned. * * Log PMC value at exit time if requested. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) pmclog_process_procexit(pm, pp); pmc_unlink_target_process(pm, pp); } free(pp, M_PMC); } else critical_exit(); /* pp == NULL */ /* * If the process owned PMCs, free them up and free up * memory. */ if ((po = pmc_find_owner_descriptor(p)) != NULL) { pmc_remove_owner(po); pmc_destroy_owner_descriptor(po); } sx_xunlock(&pmc_sx); } /* * Handle a process fork. * * If the parent process 'p1' is under HWPMC monitoring, then copy * over any attached PMCs that have 'do_descendants' semantics. */ static void pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, int flags) { int is_using_hwpmcs; unsigned int ri; uint32_t do_descendants; struct pmc *pm; struct pmc_owner *po; struct pmc_process *ppnew, *ppold; (void) flags; /* unused parameter */ PROC_LOCK(p1); is_using_hwpmcs = p1->p_flag & P_HWPMC; PROC_UNLOCK(p1); /* * If there are system-wide sampling PMCs active, we need to * log all fork events to their owner's logs. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); if (!is_using_hwpmcs) return; PMC_GET_SX_XLOCK(); PMCDBG4(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1, p1->p_pid, p1->p_comm, newproc); /* * If the parent process (curthread->td_proc) is a * target of any PMCs, look for PMCs that are to be * inherited, and link these into the new process * descriptor. */ if ((ppold = pmc_find_process_descriptor(curthread->td_proc, PMC_FLAG_NONE)) == NULL) goto done; /* nothing to do */ do_descendants = 0; for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL) do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS; if (do_descendants == 0) /* nothing to do */ goto done; /* allocate a descriptor for the new process */ if ((ppnew = pmc_find_process_descriptor(newproc, PMC_FLAG_ALLOCATE)) == NULL) goto done; /* * Run through all PMCs that were targeting the old process * and which specified F_DESCENDANTS and attach them to the * new process. * * Log the fork event to all owners of PMCs attached to this * process, if not already logged. */ for (ri = 0; ri < md->pmd_npmc; ri++) if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && (pm->pm_flags & PMC_F_DESCENDANTS)) { pmc_link_target_process(pm, ppnew); po = pm->pm_owner; if (po->po_sscount == 0 && po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); } /* * Now mark the new process as being tracked by this driver. */ PROC_LOCK(newproc); newproc->p_flag |= P_HWPMC; PROC_UNLOCK(newproc); done: sx_xunlock(&pmc_sx); } static void pmc_kld_load(void *arg __unused, linker_file_t lf) { struct pmc_owner *po; sx_slock(&pmc_sx); /* * Notify owners of system sampling PMCs about KLD operations. */ LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_in(po, (pid_t) -1, (uintfptr_t) lf->address, lf->filename); /* * TODO: Notify owners of (all) process-sampling PMCs too. */ sx_sunlock(&pmc_sx); } static void pmc_kld_unload(void *arg __unused, const char *filename __unused, caddr_t address, size_t size) { struct pmc_owner *po; sx_slock(&pmc_sx); LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) if (po->po_flags & PMC_PO_OWNS_LOGFILE) pmclog_process_map_out(po, (pid_t) -1, (uintfptr_t) address, (uintfptr_t) address + size); /* * TODO: Notify owners of process-sampling PMCs. */ sx_sunlock(&pmc_sx); } /* * initialization */ static const char * pmc_name_of_pmcclass(enum pmc_class class) { switch (class) { #undef __PMC_CLASS #define __PMC_CLASS(S,V,D) \ case PMC_CLASS_##S: \ return #S; __PMC_CLASSES(); default: return (""); } } /* * Base class initializer: allocate structure and set default classes. */ struct pmc_mdep * pmc_mdep_alloc(int nclasses) { struct pmc_mdep *md; int n; /* SOFT + md classes */ n = 1 + nclasses; md = malloc(sizeof(struct pmc_mdep) + n * sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); md->pmd_nclass = n; /* Add base class. */ pmc_soft_initialize(md); return md; } void pmc_mdep_free(struct pmc_mdep *md) { pmc_soft_finalize(md); free(md, M_PMC); } static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; (void) pp; return (0); } static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { (void) pc; (void) pp; return (0); } static struct pmc_mdep * pmc_generic_cpu_initialize(void) { struct pmc_mdep *md; md = pmc_mdep_alloc(0); md->pmd_cputype = PMC_CPU_GENERIC; md->pmd_pcpu_init = NULL; md->pmd_pcpu_fini = NULL; md->pmd_switch_in = generic_switch_in; md->pmd_switch_out = generic_switch_out; return (md); } static void pmc_generic_cpu_finalize(struct pmc_mdep *md) { (void) md; } static int pmc_initialize(void) { int c, cpu, error, n, ri; unsigned int maxcpu; struct pmc_binding pb; struct pmc_sample *ps; struct pmc_classdep *pcd; struct pmc_samplebuffer *sb; md = NULL; error = 0; #ifdef HWPMC_DEBUG /* parse debug flags first */ if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, sizeof(pmc_debugstr))) pmc_debugflags_parse(pmc_debugstr, pmc_debugstr+strlen(pmc_debugstr)); #endif PMCDBG1(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION); /* check kernel version */ if (pmc_kernel_version != PMC_VERSION) { if (pmc_kernel_version == 0) printf("hwpmc: this kernel has not been compiled with " "'options HWPMC_HOOKS'.\n"); else printf("hwpmc: kernel version (0x%x) does not match " "module version (0x%x).\n", pmc_kernel_version, PMC_VERSION); return EPROGMISMATCH; } /* * check sysctl parameters */ if (pmc_hashsize <= 0) { (void) printf("hwpmc: tunable \"hashsize\"=%d must be " "greater than zero.\n", pmc_hashsize); pmc_hashsize = PMC_HASH_SIZE; } if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { (void) printf("hwpmc: tunable \"nsamples\"=%d out of " "range.\n", pmc_nsamples); pmc_nsamples = PMC_NSAMPLES; } if (pmc_callchaindepth <= 0 || pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) { (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of " "range - using %d.\n", pmc_callchaindepth, PMC_CALLCHAIN_DEPTH_MAX); pmc_callchaindepth = PMC_CALLCHAIN_DEPTH_MAX; } md = pmc_md_initialize(); if (md == NULL) { /* Default to generic CPU. */ md = pmc_generic_cpu_initialize(); if (md == NULL) return (ENOSYS); } KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, ("[pmc,%d] no classes or pmcs", __LINE__)); /* Compute the map from row-indices to classdep pointers. */ pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) * md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); for (n = 0; n < md->pmd_npmc; n++) pmc_rowindex_to_classdep[n] = NULL; for (ri = c = 0; c < md->pmd_nclass; c++) { pcd = &md->pmd_classdep[c]; for (n = 0; n < pcd->pcd_num; n++, ri++) pmc_rowindex_to_classdep[ri] = pcd; } KASSERT(ri == md->pmd_npmc, ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__, ri, md->pmd_npmc)); maxcpu = pmc_cpu_max(); /* allocate space for the per-cpu array */ pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC, M_WAITOK|M_ZERO); /* per-cpu 'saved values' for managing process-mode PMCs */ pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc, M_PMC, M_WAITOK); /* Perform CPU-dependent initialization. */ pmc_save_cpu_binding(&pb); error = 0; for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; pmc_select_cpu(cpu); pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) + md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC, M_WAITOK|M_ZERO); if (md->pmd_pcpu_init) error = md->pmd_pcpu_init(md, cpu); for (n = 0; error == 0 && n < md->pmd_nclass; n++) error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu); } pmc_restore_cpu_binding(&pb); if (error) return (error); /* allocate space for the sample array */ for (cpu = 0; cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; sb = malloc(sizeof(struct pmc_samplebuffer) + pmc_nsamples * sizeof(struct pmc_sample), M_PMC, M_WAITOK|M_ZERO); sb->ps_read = sb->ps_write = sb->ps_samples; sb->ps_fence = sb->ps_samples + pmc_nsamples; KASSERT(pmc_pcpu[cpu] != NULL, ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) ps->ps_pc = sb->ps_callchains + (n * pmc_callchaindepth); pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb; sb = malloc(sizeof(struct pmc_samplebuffer) + pmc_nsamples * sizeof(struct pmc_sample), M_PMC, M_WAITOK|M_ZERO); sb->ps_read = sb->ps_write = sb->ps_samples; sb->ps_fence = sb->ps_samples + pmc_nsamples; KASSERT(pmc_pcpu[cpu] != NULL, ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) ps->ps_pc = sb->ps_callchains + (n * pmc_callchaindepth); pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb; } /* allocate space for the row disposition array */ pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); /* mark all PMCs as available */ for (n = 0; n < (int) md->pmd_npmc; n++) PMC_MARK_ROW_FREE(n); /* allocate thread hash tables */ pmc_ownerhash = hashinit(pmc_hashsize, M_PMC, &pmc_ownerhashmask); pmc_processhash = hashinit(pmc_hashsize, M_PMC, &pmc_processhashmask); mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf", MTX_SPIN); LIST_INIT(&pmc_ss_owners); pmc_ss_count = 0; /* allocate a pool of spin mutexes */ pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size, MTX_SPIN); PMCDBG4(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx " "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, pmc_processhash, pmc_processhashmask); /* register process {exit,fork,exec} handlers */ pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit, pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY); pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork, pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY); /* register kld event handlers */ pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load, NULL, EVENTHANDLER_PRI_ANY); pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload, NULL, EVENTHANDLER_PRI_ANY); /* initialize logging */ pmclog_initialize(); /* set hook functions */ pmc_intr = md->pmd_intr; pmc_hook = pmc_hook_handler; if (error == 0) { printf(PMC_MODULE_NAME ":"); for (n = 0; n < (int) md->pmd_nclass; n++) { pcd = &md->pmd_classdep[n]; printf(" %s/%d/%d/0x%b", pmc_name_of_pmcclass(pcd->pcd_class), pcd->pcd_num, pcd->pcd_width, pcd->pcd_caps, "\20" "\1INT\2USR\3SYS\4EDG\5THR" "\6REA\7WRI\10INV\11QUA\12PRC" "\13TAG\14CSC"); } printf("\n"); } return (error); } /* prepare to be unloaded */ static void pmc_cleanup(void) { int c, cpu; unsigned int maxcpu; struct pmc_ownerhash *ph; struct pmc_owner *po, *tmp; struct pmc_binding pb; #ifdef HWPMC_DEBUG struct pmc_processhash *prh; #endif PMCDBG0(MOD,INI,0, "cleanup"); /* switch off sampling */ CPU_ZERO(&pmc_cpumask); pmc_intr = NULL; sx_xlock(&pmc_sx); if (pmc_hook == NULL) { /* being unloaded already */ sx_xunlock(&pmc_sx); return; } pmc_hook = NULL; /* prevent new threads from entering module */ /* deregister event handlers */ EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag); EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag); EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag); EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag); /* send SIGBUS to all owner threads, free up allocations */ if (pmc_ownerhash) for (ph = pmc_ownerhash; ph <= &pmc_ownerhash[pmc_ownerhashmask]; ph++) { LIST_FOREACH_SAFE(po, ph, po_next, tmp) { pmc_remove_owner(po); /* send SIGBUS to owner processes */ PMCDBG3(MOD,INI,2, "cleanup signal proc=%p " "(%d, %s)", po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); PROC_LOCK(po->po_owner); kern_psignal(po->po_owner, SIGBUS); PROC_UNLOCK(po->po_owner); pmc_destroy_owner_descriptor(po); } } /* reclaim allocated data structures */ if (pmc_mtxpool) mtx_pool_destroy(&pmc_mtxpool); mtx_destroy(&pmc_processhash_mtx); if (pmc_processhash) { #ifdef HWPMC_DEBUG struct pmc_process *pp; PMCDBG0(MOD,INI,3, "destroy process hash"); for (prh = pmc_processhash; prh <= &pmc_processhash[pmc_processhashmask]; prh++) LIST_FOREACH(pp, prh, pp_next) PMCDBG1(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid); #endif hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask); pmc_processhash = NULL; } if (pmc_ownerhash) { PMCDBG0(MOD,INI,3, "destroy owner hash"); hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask); pmc_ownerhash = NULL; } KASSERT(LIST_EMPTY(&pmc_ss_owners), ("[pmc,%d] Global SS owner list not empty", __LINE__)); KASSERT(pmc_ss_count == 0, ("[pmc,%d] Global SS count not empty", __LINE__)); /* do processor and pmc-class dependent cleanup */ maxcpu = pmc_cpu_max(); PMCDBG0(MOD,INI,3, "md cleanup"); if (md) { pmc_save_cpu_binding(&pb); for (cpu = 0; cpu < maxcpu; cpu++) { PMCDBG2(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", cpu, pmc_pcpu[cpu]); if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL) continue; pmc_select_cpu(cpu); for (c = 0; c < md->pmd_nclass; c++) md->pmd_classdep[c].pcd_pcpu_fini(md, cpu); if (md->pmd_pcpu_fini) md->pmd_pcpu_fini(md, cpu); } if (md->pmd_cputype == PMC_CPU_GENERIC) pmc_generic_cpu_finalize(md); else pmc_md_finalize(md); pmc_mdep_free(md); md = NULL; pmc_restore_cpu_binding(&pb); } /* Free per-cpu descriptors. */ for (cpu = 0; cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL, ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__, cpu)); KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL, ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__, cpu)); free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC); free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC); free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC); free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC); free(pmc_pcpu[cpu], M_PMC); } free(pmc_pcpu, M_PMC); pmc_pcpu = NULL; free(pmc_pcpu_saved, M_PMC); pmc_pcpu_saved = NULL; if (pmc_pmcdisp) { free(pmc_pmcdisp, M_PMC); pmc_pmcdisp = NULL; } if (pmc_rowindex_to_classdep) { free(pmc_rowindex_to_classdep, M_PMC); pmc_rowindex_to_classdep = NULL; } pmclog_shutdown(); sx_xunlock(&pmc_sx); /* we are done */ } /* * The function called at load/unload. */ static int load (struct module *module __unused, int cmd, void *arg __unused) { int error; error = 0; switch (cmd) { case MOD_LOAD : /* initialize the subsystem */ error = pmc_initialize(); if (error != 0) break; PMCDBG2(MOD,INI,1, "syscall=%d maxcpu=%d", pmc_syscall_num, pmc_cpu_max()); break; case MOD_UNLOAD : case MOD_SHUTDOWN: pmc_cleanup(); PMCDBG0(MOD,INI,1, "unloaded"); break; default : error = EINVAL; /* XXX should panic(9) */ break; } return error; } /* memory pool */ MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); Index: head/sys/dev/isci/scil/scic_phy.h =================================================================== --- head/sys/dev/isci/scil/scic_phy.h (revision 298930) +++ head/sys/dev/isci/scil/scic_phy.h (revision 298931) @@ -1,462 +1,462 @@ /*- * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. * The full GNU General Public License is included in this distribution * in the file called LICENSE.GPL. * * BSD LICENSE * * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SCIC_PHY_H_ #define _SCIC_PHY_H_ /** * @file * * @brief This file contains all of the interface methods that can be called * by an SCIC user on a phy (SAS or SATA) object. */ #ifdef __cplusplus extern "C" { #endif // __cplusplus #include #include #include #include /** * @struct SCIC_PHY_PROPERTIES * @brief This structure defines the properties common to all phys * that can be retrieved. */ typedef struct SCIC_PHY_PROPERTIES { /** * This field specifies the port that currently contains the * supplied phy. This field may be set to SCI_INVALID_HANDLE * if the phy is not currently contained in a port. */ SCI_PORT_HANDLE_T owning_port; /** * This field specifies the maximum link rate for which this phy * will negotiate. */ SCI_SAS_LINK_RATE max_link_rate; /** * This field specifies the link rate at which the phy is * currently operating. */ SCI_SAS_LINK_RATE negotiated_link_rate; /** * This field indicates the identify address frame that will be * transmitted to the connected phy. */ SCI_SAS_IDENTIFY_ADDRESS_FRAME_T transmit_iaf; /** * This field specifies the index of the phy in relation to other * phys within the controller. This index is zero relative. */ U8 index; } SCIC_PHY_PROPERTIES_T; /** * @struct SCIC_SAS_PHY_PROPERTIES * @brief This structure defines the properties, specific to a * SAS phy, that can be retrieved. */ typedef struct SCIC_SAS_PHY_PROPERTIES { /** * This field delineates the Identify Address Frame received * from the remote end point. */ SCI_SAS_IDENTIFY_ADDRESS_FRAME_T received_iaf; /** * This field delineates the Phy capabilities structure received * from the remote end point. */ SAS_CAPABILITIES_T received_capabilities; } SCIC_SAS_PHY_PROPERTIES_T; /** * @struct SCIC_SATA_PHY_PROPERTIES * @brief This structure defines the properties, specific to a * SATA phy, that can be retrieved. */ typedef struct SCIC_SATA_PHY_PROPERTIES { /** * This field delineates the signature FIS received from the * attached target. */ SATA_FIS_REG_D2H_T signature_fis; /** * This field specifies to the user if a port selector is connected * on the specified phy. */ BOOL is_port_selector_present; } SCIC_SATA_PHY_PROPERTIES_T; /** * @enum SCIC_PHY_COUNTER_ID * @brief This enumeration depicts the various pieces of optional * information that can be retrieved for a specific phy. */ typedef enum SCIC_PHY_COUNTER_ID { /** * This PHY information field tracks the number of frames received. */ SCIC_PHY_COUNTER_RECEIVED_FRAME, /** * This PHY information field tracks the number of frames transmitted. */ SCIC_PHY_COUNTER_TRANSMITTED_FRAME, /** * This PHY information field tracks the number of DWORDs received. */ SCIC_PHY_COUNTER_RECEIVED_FRAME_DWORD, /** * This PHY information field tracks the number of DWORDs transmitted. */ SCIC_PHY_COUNTER_TRANSMITTED_FRAME_DWORD, /** * This PHY information field tracks the number of times DWORD * synchronization was lost. */ SCIC_PHY_COUNTER_LOSS_OF_SYNC_ERROR, /** * This PHY information field tracks the number of received DWORDs with * running disparity errors. */ SCIC_PHY_COUNTER_RECEIVED_DISPARITY_ERROR, /** * This PHY information field tracks the number of received frames with a * CRC error (not including short or truncated frames). */ SCIC_PHY_COUNTER_RECEIVED_FRAME_CRC_ERROR, /** * This PHY information field tracks the number of DONE (ACK/NAK TIMEOUT) * primitives received. */ SCIC_PHY_COUNTER_RECEIVED_DONE_ACK_NAK_TIMEOUT, /** * This PHY information field tracks the number of DONE (ACK/NAK TIMEOUT) * primitives transmitted. */ SCIC_PHY_COUNTER_TRANSMITTED_DONE_ACK_NAK_TIMEOUT, /** * This PHY information field tracks the number of times the inactivity * timer for connections on the phy has been utilized. */ SCIC_PHY_COUNTER_INACTIVITY_TIMER_EXPIRED, /** * This PHY information field tracks the number of DONE (CREDIT TIMEOUT) * primitives received. */ SCIC_PHY_COUNTER_RECEIVED_DONE_CREDIT_TIMEOUT, /** * This PHY information field tracks the number of DONE (CREDIT TIMEOUT) * primitives transmitted. */ SCIC_PHY_COUNTER_TRANSMITTED_DONE_CREDIT_TIMEOUT, /** * This PHY information field tracks the number of CREDIT BLOCKED * primitives received. * @note Depending on remote device implementation, credit blocks * may occur regularly. */ SCIC_PHY_COUNTER_RECEIVED_CREDIT_BLOCKED, /** * This PHY information field contains the number of short frames * received. A short frame is simply a frame smaller then what is * allowed by either the SAS or SATA specification. */ SCIC_PHY_COUNTER_RECEIVED_SHORT_FRAME, /** * This PHY information field contains the number of frames received after * credit has been exhausted. */ SCIC_PHY_COUNTER_RECEIVED_FRAME_WITHOUT_CREDIT, /** * This PHY information field contains the number of frames received after * a DONE has been received. */ SCIC_PHY_COUNTER_RECEIVED_FRAME_AFTER_DONE, /** * This PHY information field contains the number of times the phy * failed to achieve DWORD synchronization during speed negotiation. */ SCIC_PHY_COUNTER_SN_DWORD_SYNC_ERROR } SCIC_PHY_COUNTER_ID_T; /** * @brief This method will enable the user to retrieve information * common to all phys, such as: the negotiated link rate, * the phy id, etc. * * @param[in] phy This parameter specifies the phy for which to retrieve * the properties. * @param[out] properties This parameter specifies the properties * structure into which to copy the requested information. * * @return Indicate if the user specified a valid phy. * @retval SCI_SUCCESS This value is returned if the specified phy was valid. * @retval SCI_FAILURE_INVALID_PHY This value is returned if the specified phy * is not valid. When this value is returned, no data is copied to the * properties output parameter. */ SCI_STATUS scic_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_PHY_PROPERTIES_T * properties ); /** * @brief This method will enable the user to retrieve information * specific to a SAS phy, such as: the received identify * address frame, received phy capabilities, etc. * * @param[in] phy this parameter specifies the phy for which to * retrieve properties. * @param[out] properties This parameter specifies the properties * structure into which to copy the requested information. * * @return This method returns an indication as to whether the SAS * phy properties were successfully retrieved. * @retval SCI_SUCCESS This value is returned if the SAS properties * are successfully retrieved. * @retval SCI_FAILURE This value is returned if the SAS properties * are not successfully retrieved (e.g. It's not a SAS Phy). */ SCI_STATUS scic_sas_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_SAS_PHY_PROPERTIES_T * properties ); /** * @brief This method will enable the user to retrieve information - * specific to a SATA phy, such as: the recieved signature + * specific to a SATA phy, such as: the received signature * FIS, if a port selector is present, etc. * * @param[in] phy this parameter specifies the phy for which to * retrieve properties. * @param[out] properties This parameter specifies the properties * structure into which to copy the requested information. * * @return This method returns an indication as to whether the SATA * phy properties were successfully retrieved. * @retval SCI_SUCCESS This value is returned if the SATA properties * are successfully retrieved. * @retval SCI_FAILURE This value is returned if the SATA properties * are not successfully retrieved (e.g. It's not a SATA Phy). */ SCI_STATUS scic_sata_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_SATA_PHY_PROPERTIES_T * properties ); /** * @brief This method allows the SCIC user to instruct the SCIC * implementation to send the SATA port selection signal. * * @param[in] phy this parameter specifies the phy for which to send * the port selection signal. * * @return An indication of whether the port selection signal was * successfully executed. * @retval SCI_SUCCESS This value is returned if the port selection signal * was successfully transmitted. */ SCI_STATUS scic_sata_phy_send_port_selection_signal( SCI_PHY_HANDLE_T phy ); /** * @brief This method requests the SCI implementation to begin tracking * information specified by the supplied parameters. * * @param[in] phy this parameter specifies the phy for which to enable * the information type. * @param[in] counter_id this parameter specifies the information * type to be enabled. * * @return Indicate if enablement of the information type was successful. * @retval SCI_SUCCESS This value is returned if the information type was * successfully enabled. * @retval SCI_FAILURE_UNSUPPORTED_INFORMATION_FIELD This value is returned * if the supplied information type is not supported. */ SCI_STATUS scic_phy_enable_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ); /** * @brief This method requests the SCI implementation to stop tracking * information specified by the supplied parameters. * * @param[in] phy this parameter specifies the phy for which to disable * the information type. * @param[in] counter_id this parameter specifies the information * type to be disabled. * * @return Indicate if disablement of the information type was successful. * @retval SCI_SUCCESS This value is returned if the information type was * successfully disabled. * @retval SCI_FAILURE_UNSUPPORTED_INFORMATION_FIELD This value is returned * if the supplied information type is not supported. */ SCI_STATUS scic_phy_disable_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ); /** * @brief This method requests the SCI implementation to retrieve * tracking information specified by the supplied parameters. * * @param[in] phy this parameter specifies the phy for which to retrieve * the information type. * @param[in] counter_id this parameter specifies the information * type to be retrieved. * @param[out] data this parameter is a 32-bit pointer to a location * where the data being retrieved is to be placed. * * @return Indicate if retrieval of the information type was successful. * @retval SCI_SUCCESS This value is returned if the information type was * successfully retrieved. * @retval SCI_FAILURE_UNSUPPORTED_INFORMATION_FIELD This value is returned * if the supplied information type is not supported. */ SCI_STATUS scic_phy_get_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id, U32 * data ); /** * @brief This method requests the SCI implementation to clear (reset) * tracking information specified by the supplied parameters. * * @param[in] phy this parameter specifies the phy for which to clear * the information type. * @param[in] counter_id this parameter specifies the information * type to be cleared. * * @return Indicate if clearing of the information type was successful. * @retval SCI_SUCCESS This value is returned if the information type was * successfully cleared. * @retval SCI_FAILURE_UNSUPPORTED_INFORMATION_FIELD This value is returned * if the supplied information type is not supported. */ SCI_STATUS scic_phy_clear_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ); /** * @brief This method will attempt to stop the phy object. * * @param[in] this_phy * * @return SCI_STATUS * @retval SCI_SUCCESS if the phy is going to stop * SCI_INVALID_STATE if the phy is not in a valid state * to stop */ SCI_STATUS scic_phy_stop( SCI_PHY_HANDLE_T phy ); /** * @brief This method will attempt to start the phy object. This * request is only valid when the phy is in the stopped * state * * @param[in] this_phy * * @return SCI_STATUS */ SCI_STATUS scic_phy_start( SCI_PHY_HANDLE_T phy ); #ifdef __cplusplus } #endif // __cplusplus #endif // _SCIC_PHY_H_ Index: head/sys/dev/isci/scil/scic_sds_phy.c =================================================================== --- head/sys/dev/isci/scil/scic_sds_phy.c (revision 298930) +++ head/sys/dev/isci/scil/scic_sds_phy.c (revision 298931) @@ -1,4036 +1,4036 @@ /*- * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. * The full GNU General Public License is included in this distribution * in the file called LICENSE.GPL. * * BSD LICENSE * * Copyright(c) 2008 - 2011 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /** * @file * * @brief This file contains the implementation of the SCIC_SDS_PHY public and * protected methods. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define SCIC_SDS_PHY_MIN_TIMER_COUNT (SCI_MAX_PHYS) #define SCIC_SDS_PHY_MAX_TIMER_COUNT (SCI_MAX_PHYS) // Maximum arbitration wait time in micro-seconds #define SCIC_SDS_PHY_MAX_ARBITRATION_WAIT_TIME (700) #define AFE_REGISTER_WRITE_DELAY 10 //***************************************************************************** //* SCIC SDS PHY Internal Methods //***************************************************************************** /** * @brief This method will initialize the phy transport layer registers * * @param[in] this_phy * @param[in] transport_layer_registers * * @return SCI_STATUS */ static SCI_STATUS scic_sds_phy_transport_layer_initialization( SCIC_SDS_PHY_T *this_phy, SCU_TRANSPORT_LAYER_REGISTERS_T *transport_layer_registers ) { U32 tl_control; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_link_layer_initialization(this_phy:0x%x, link_layer_registers:0x%x)\n", this_phy, transport_layer_registers )); this_phy->transport_layer_registers = transport_layer_registers; SCU_STPTLDARNI_WRITE(this_phy, SCIC_SDS_REMOTE_NODE_CONTEXT_INVALID_INDEX); // Hardware team recommends that we enable the STP prefetch for all transports tl_control = SCU_TLCR_READ(this_phy); tl_control |= SCU_TLCR_GEN_BIT(STP_WRITE_DATA_PREFETCH); SCU_TLCR_WRITE(this_phy, tl_control); return SCI_SUCCESS; } /** * @brief This method will initialize the phy link layer registers * * @param[in] this_phy * @param[in] link_layer_registers * * @return SCI_STATUS */ static SCI_STATUS scic_sds_phy_link_layer_initialization( SCIC_SDS_PHY_T *this_phy, SCU_LINK_LAYER_REGISTERS_T *link_layer_registers ) { U32 phy_configuration; SAS_CAPABILITIES_T phy_capabilities; U32 parity_check = 0; U32 parity_count = 0; U32 link_layer_control; U32 phy_timer_timeout_values; U32 clksm_value = 0; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_link_layer_initialization(this_phy:0x%x, link_layer_registers:0x%x)\n", this_phy, link_layer_registers )); this_phy->link_layer_registers = link_layer_registers; // Set our IDENTIFY frame data #define SCI_END_DEVICE 0x01 SCU_SAS_TIID_WRITE( this_phy, ( SCU_SAS_TIID_GEN_BIT(SMP_INITIATOR) | SCU_SAS_TIID_GEN_BIT(SSP_INITIATOR) | SCU_SAS_TIID_GEN_BIT(STP_INITIATOR) | SCU_SAS_TIID_GEN_BIT(DA_SATA_HOST) | SCU_SAS_TIID_GEN_VAL(DEVICE_TYPE, SCI_END_DEVICE) ) ); // Write the device SAS Address SCU_SAS_TIDNH_WRITE(this_phy, 0xFEDCBA98); SCU_SAS_TIDNL_WRITE(this_phy, this_phy->phy_index); // Write the source SAS Address SCU_SAS_TISSAH_WRITE( this_phy, this_phy->owning_port->owning_controller->oem_parameters.sds1.phys[ this_phy->phy_index].sas_address.sci_format.high ); SCU_SAS_TISSAL_WRITE( this_phy, this_phy->owning_port->owning_controller->oem_parameters.sds1.phys[ this_phy->phy_index].sas_address.sci_format.low ); // Clear and Set the PHY Identifier SCU_SAS_TIPID_WRITE(this_phy, 0x00000000); SCU_SAS_TIPID_WRITE(this_phy, SCU_SAS_TIPID_GEN_VALUE(ID, this_phy->phy_index)); // Change the initial state of the phy configuration register phy_configuration = SCU_SAS_PCFG_READ(this_phy); // Hold OOB state machine in reset phy_configuration |= SCU_SAS_PCFG_GEN_BIT(OOB_RESET); SCU_SAS_PCFG_WRITE(this_phy, phy_configuration); // Configure the SNW capabilities phy_capabilities.u.all = 0; phy_capabilities.u.bits.start = 1; phy_capabilities.u.bits.gen3_without_ssc_supported = 1; phy_capabilities.u.bits.gen2_without_ssc_supported = 1; phy_capabilities.u.bits.gen1_without_ssc_supported = 1; /* * Set up SSC settings according to version of OEM Parameters. */ { U8 header_version, enable_sata, enable_sas, sata_spread, sas_type, sas_spread; OEM_SSC_PARAMETERS_T ssc; header_version = this_phy->owning_port->owning_controller-> oem_parameters_version; ssc.bf.ssc_sata_tx_spread_level = this_phy->owning_port->owning_controller->oem_parameters.sds1.controller.ssc_sata_tx_spread_level; ssc.bf.ssc_sas_tx_spread_level = this_phy->owning_port->owning_controller->oem_parameters.sds1.controller.ssc_sas_tx_spread_level; ssc.bf.ssc_sas_tx_type = this_phy->owning_port->owning_controller->oem_parameters.sds1.controller.ssc_sas_tx_type; enable_sata = enable_sas = sata_spread = sas_type = sas_spread = 0; if (header_version == SCI_OEM_PARAM_VER_1_0) { /* * Version 1.0 is merely turning SSC on to default values.; */ if (ssc.do_enable_ssc != 0) { enable_sas = enable_sata = TRUE; sas_type = 0x0; // Downspreading sata_spread = 0x2; // +0 to -1419 PPM sas_spread = 0x2; // +0 to -1419 PPM } } else // header_version >= SCI_OEM_PARAM_VER_1_1 { /* * Version 1.1 can turn on SAS and SATA independently and * specify spread levels. Also can specify spread type for SAS. */ if ((sata_spread = ssc.bf.ssc_sata_tx_spread_level) != 0) enable_sata = TRUE; // Downspreading only if ((sas_spread = ssc.bf.ssc_sas_tx_spread_level) != 0) { enable_sas = TRUE; sas_type = ssc.bf.ssc_sas_tx_type; } } if (enable_sas == TRUE) { U32 reg_val = scu_afe_register_read( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index]. afe_xcvr_control0); reg_val |= (0x00100000 | (((U32)sas_type) << 19)); scu_afe_register_write( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index].afe_xcvr_control0, reg_val); reg_val = scu_afe_register_read( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index]. afe_tx_ssc_control); reg_val |= (((U32)(sas_spread)) << 8); scu_afe_register_write( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index].afe_tx_ssc_control, reg_val); phy_capabilities.u.bits.gen3_with_ssc_supported = 1; phy_capabilities.u.bits.gen2_with_ssc_supported = 1; phy_capabilities.u.bits.gen1_with_ssc_supported = 1; } if (enable_sata == TRUE) { U32 reg_val = scu_afe_register_read( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index]. afe_tx_ssc_control); reg_val |= (U32)sata_spread; scu_afe_register_write( this_phy->owning_port->owning_controller, scu_afe_xcvr[this_phy->phy_index].afe_tx_ssc_control, reg_val); reg_val = scu_link_layer_register_read( this_phy, stp_control); reg_val |= (U32)(1 << 12); scu_link_layer_register_write( this_phy, stp_control, reg_val); } } // The SAS specification indicates that the phy_capabilities that // are transmitted shall have an even parity. Calculate the parity. parity_check = phy_capabilities.u.all; while (parity_check != 0) { if (parity_check & 0x1) parity_count++; parity_check >>= 1; } // If parity indicates there are an odd number of bits set, then // set the parity bit to 1 in the phy capabilities. if ((parity_count % 2) != 0) phy_capabilities.u.bits.parity = 1; SCU_SAS_PHYCAP_WRITE(this_phy, phy_capabilities.u.all); // Set the enable spinup period but disable the ability to send notify enable spinup SCU_SAS_ENSPINUP_WRITE( this_phy, SCU_ENSPINUP_GEN_VAL( COUNT, this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].notify_enable_spin_up_insertion_frequency ) ); // Write the ALIGN Insertion Ferequency for connected phy and inpendent of connected state clksm_value = SCU_ALIGN_INSERTION_FREQUENCY_GEN_VAL ( CONNECTED, this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].in_connection_align_insertion_frequency ); clksm_value |= SCU_ALIGN_INSERTION_FREQUENCY_GEN_VAL ( GENERAL, this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].align_insertion_frequency ); SCU_SAS_CLKSM_WRITE ( this_phy, clksm_value); #if defined(PBG_HBA_A0_BUILD) || defined(PBG_HBA_A2_BUILD) || defined(PBG_HBA_BETA_BUILD) /// @todo Provide a way to write this register correctly scu_link_layer_register_write(this_phy, afe_lookup_table_control, 0x02108421); #elif defined(PBG_BUILD) if ( (this_phy->owning_port->owning_controller->pci_revision == SCIC_SDS_PCI_REVISION_C0) || (this_phy->owning_port->owning_controller->pci_revision == SCIC_SDS_PCI_REVISION_C1) ) { scu_link_layer_register_write(this_phy, afe_lookup_table_control, 0x04210400); scu_link_layer_register_write(this_phy, sas_primitive_timeout, 0x20A7C05); } else { scu_link_layer_register_write(this_phy, afe_lookup_table_control, 0x02108421); } #else /// @todo Provide a way to write this register correctly scu_link_layer_register_write(this_phy, afe_lookup_table_control, 0x0e739ce7); #endif link_layer_control = SCU_SAS_LLCTL_GEN_VAL( NO_OUTBOUND_TASK_TIMEOUT, (U8) this_phy->owning_port->owning_controller-> user_parameters.sds1.no_outbound_task_timeout ); #if PHY_MAX_LINK_SPEED_GENERATION == SCIC_SDS_PARM_GEN1_SPEED #define COMPILED_MAX_LINK_RATE SCU_SAS_LINK_LAYER_CONTROL_MAX_LINK_RATE_GEN1 #elif PHY_MAX_LINK_SPEED_GENERATION == SCIC_SDS_PARM_GEN2_SPEED #define COMPILED_MAX_LINK_RATE SCU_SAS_LINK_LAYER_CONTROL_MAX_LINK_RATE_GEN2 #else #define COMPILED_MAX_LINK_RATE SCU_SAS_LINK_LAYER_CONTROL_MAX_LINK_RATE_GEN3 #endif // PHY_MAX_LINK_SPEED_GENERATION if (this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].max_speed_generation == SCIC_SDS_PARM_GEN3_SPEED) { link_layer_control |= SCU_SAS_LLCTL_GEN_VAL( MAX_LINK_RATE, COMPILED_MAX_LINK_RATE ); } else if (this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].max_speed_generation == SCIC_SDS_PARM_GEN2_SPEED) { link_layer_control |= SCU_SAS_LLCTL_GEN_VAL( MAX_LINK_RATE, MIN( SCU_SAS_LINK_LAYER_CONTROL_MAX_LINK_RATE_GEN2, COMPILED_MAX_LINK_RATE) ); } else { link_layer_control |= SCU_SAS_LLCTL_GEN_VAL( MAX_LINK_RATE, MIN( SCU_SAS_LINK_LAYER_CONTROL_MAX_LINK_RATE_GEN1, COMPILED_MAX_LINK_RATE) ); } scu_link_layer_register_write( this_phy, link_layer_control, link_layer_control ); phy_timer_timeout_values = scu_link_layer_register_read( this_phy, phy_timer_timeout_values ); // Clear the default 0x36 (54us) RATE_CHANGE timeout value. phy_timer_timeout_values &= ~SCU_SAS_PHYTOV_GEN_VAL(RATE_CHANGE, 0xFF); // Set RATE_CHANGE timeout value to 0x3B (59us). This ensures SCU can // lock with 3Gb drive when SCU max rate is set to 1.5Gb. phy_timer_timeout_values |= SCU_SAS_PHYTOV_GEN_VAL(RATE_CHANGE, 0x3B); scu_link_layer_register_write( this_phy, phy_timer_timeout_values, phy_timer_timeout_values ); // Program the max ARB time for the PHY to 700us so we inter-operate with // the PMC expander which shuts down PHYs if the expander PHY generates too // many breaks. This time value will guarantee that the initiator PHY will // generate the break. #if defined(PBG_HBA_A0_BUILD) || defined(PBG_HBA_A2_BUILD) scu_link_layer_register_write( this_phy, maximum_arbitration_wait_timer_timeout, SCIC_SDS_PHY_MAX_ARBITRATION_WAIT_TIME ); #endif // defined(PBG_HBA_A0_BUILD) || defined(PBG_HBA_A2_BUILD) // Disable the link layer hang detection timer scu_link_layer_register_write( this_phy, link_layer_hang_detection_timeout, 0x00000000 ); // We can exit the initial state to the stopped state sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STOPPED ); return SCI_SUCCESS; } /** * This function will handle the sata SIGNATURE FIS timeout condition. It * will restart the starting substate machine since we dont know what has * actually happening. * * @param[in] cookie This object is cast to the SCIC_SDS_PHY_T object. * * @return none */ void scic_sds_phy_sata_timeout( SCI_OBJECT_HANDLE_T cookie) { SCIC_SDS_PHY_T * this_phy = (SCIC_SDS_PHY_T *)cookie; SCIC_LOG_INFO(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC SDS Phy 0x%x did not receive signature fis before timeout.\n", this_phy )); sci_base_state_machine_stop( scic_sds_phy_get_starting_substate_machine(this_phy)); sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STARTING ); } //***************************************************************************** //* SCIC SDS PHY External Methods //***************************************************************************** /** * @brief This method returns the object size for a phy object. * * @return U32 */ U32 scic_sds_phy_get_object_size(void) { return sizeof(SCIC_SDS_PHY_T); } /** * @brief This method returns the minimum number of timers required for a * phy object. * * @return U32 */ U32 scic_sds_phy_get_min_timer_count(void) { return SCIC_SDS_PHY_MIN_TIMER_COUNT; } /** * @brief This method returns the maximum number of timers required for a * phy object. * * @return U32 */ U32 scic_sds_phy_get_max_timer_count(void) { return SCIC_SDS_PHY_MAX_TIMER_COUNT; } #ifdef SCI_LOGGING static void scic_sds_phy_initialize_state_logging( SCIC_SDS_PHY_T *this_phy ) { sci_base_state_machine_logger_initialize( &this_phy->parent.state_machine_logger, &this_phy->parent.state_machine, &this_phy->parent.parent, scic_cb_logger_log_states, "SCIC_SDS_PHY_T", "base state machine", SCIC_LOG_OBJECT_PHY ); sci_base_state_machine_logger_initialize( &this_phy->starting_substate_machine_logger, &this_phy->starting_substate_machine, &this_phy->parent.parent, scic_cb_logger_log_states, "SCIC_SDS_PHY_T", "starting substate machine", SCIC_LOG_OBJECT_PHY ); } #endif // SCI_LOGGING #ifdef SCIC_DEBUG_ENABLED /** * Debug code to record the state transitions in the phy * * @param our_observer * @param the_state_machine */ void scic_sds_phy_observe_state_change( SCI_BASE_OBSERVER_T * our_observer, SCI_BASE_SUBJECT_T * the_subject ) { SCIC_SDS_PHY_T *this_phy; SCI_BASE_STATE_MACHINE_T *the_state_machine; U8 transition_requestor; U32 base_state_id; U32 starting_substate_id; the_state_machine = (SCI_BASE_STATE_MACHINE_T *)the_subject; this_phy = (SCIC_SDS_PHY_T *)the_state_machine->state_machine_owner; if (the_state_machine == &this_phy->parent.state_machine) { transition_requestor = 0x01; } else if (the_state_machine == &this_phy->starting_substate_machine) { transition_requestor = 0x02; } else { transition_requestor = 0xFF; } base_state_id = sci_base_state_machine_get_state(&this_phy->parent.state_machine); starting_substate_id = sci_base_state_machine_get_state(&this_phy->starting_substate_machine); this_phy->state_record.state_transition_table[ this_phy->state_record.index++] = ( (transition_requestor << 24) | ((U8)base_state_id << 8) | ((U8)starting_substate_id)); this_phy->state_record.index = this_phy->state_record.index & (MAX_STATE_TRANSITION_RECORD - 1); } #endif // SCIC_DEBUG_ENABLED #ifdef SCIC_DEBUG_ENABLED /** * This method initializes the state record debug information for the phy * object. * * @pre The state machines for the phy object must be constructed before this * function is called. * * @param this_phy The phy which is being initialized. */ void scic_sds_phy_initialize_state_recording( SCIC_SDS_PHY_T *this_phy ) { this_phy->state_record.index = 0; sci_base_observer_initialize( &this_phy->state_record.base_state_observer, scic_sds_phy_observe_state_change, &this_phy->parent.state_machine.parent ); sci_base_observer_initialize( &this_phy->state_record.starting_state_observer, scic_sds_phy_observe_state_change, &this_phy->starting_substate_machine.parent ); } #endif // SCIC_DEBUG_ENABLED /** * @brief This method will construct the SCIC_SDS_PHY object * * @param[in] this_phy * @param[in] owning_port * @param[in] phy_index * * @return none */ void scic_sds_phy_construct( SCIC_SDS_PHY_T *this_phy, SCIC_SDS_PORT_T *owning_port, U8 phy_index ) { // Call the base constructor first // Copy the logger from the port (this could be the dummy port) sci_base_phy_construct( &this_phy->parent, sci_base_object_get_logger(owning_port), scic_sds_phy_state_table ); // Copy the rest of the input data to our locals this_phy->owning_port = owning_port; this_phy->phy_index = phy_index; this_phy->bcn_received_while_port_unassigned = FALSE; this_phy->protocol = SCIC_SDS_PHY_PROTOCOL_UNKNOWN; this_phy->link_layer_registers = NULL; this_phy->max_negotiated_speed = SCI_SAS_NO_LINK_RATE; this_phy->sata_timeout_timer = NULL; // Clear out the identification buffer data memset(&this_phy->phy_type, 0, sizeof(this_phy->phy_type)); // Clear out the error counter data memset(this_phy->error_counter, 0, sizeof(this_phy->error_counter)); // Initialize the substate machines sci_base_state_machine_construct( &this_phy->starting_substate_machine, &this_phy->parent.parent, scic_sds_phy_starting_substates, SCIC_SDS_PHY_STARTING_SUBSTATE_INITIAL ); #ifdef SCI_LOGGING scic_sds_phy_initialize_state_logging(this_phy); #endif // SCI_LOGGING #ifdef SCIC_DEBUG_ENABLED scic_sds_phy_initialize_state_recording(this_phy); #endif // SCIC_DEBUG_ENABLED } /** * @brief This method returns the port currently containing this phy. * If the phy is currently contained by the dummy port, then * the phy is considered to not be part of a port. * * @param[in] this_phy This parameter specifies the phy for which to * retrieve the containing port. * * @return This method returns a handle to a port that contains * the supplied phy. * @retval SCI_INVALID_HANDLE This value is returned if the phy is not * part of a real port (i.e. it's contained in the dummy port). * @retval !SCI_INVALID_HANDLE All other values indicate a handle/pointer * to the port containing the phy. */ SCI_PORT_HANDLE_T scic_sds_phy_get_port( SCIC_SDS_PHY_T *this_phy ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_get_port(0x%x) enter\n", this_phy )); if (scic_sds_port_get_index(this_phy->owning_port) == SCIC_SDS_DUMMY_PORT) return SCI_INVALID_HANDLE; return this_phy->owning_port; } /** * @brief This method will assign a port to the phy object. * * @param[in, out] this_phy This parameter specifies the phy for which * to assign a port object. * @param[in] the_port This parameter is the port to assing to the phy. */ void scic_sds_phy_set_port( SCIC_SDS_PHY_T * this_phy, SCIC_SDS_PORT_T * the_port ) { this_phy->owning_port = the_port; if (this_phy->bcn_received_while_port_unassigned) { this_phy->bcn_received_while_port_unassigned = FALSE; scic_sds_port_broadcast_change_received(this_phy->owning_port, this_phy); } } /** * @brief This method will initialize the constructed phy * * @param[in] this_phy * @param[in] link_layer_registers * * @return SCI_STATUS */ SCI_STATUS scic_sds_phy_initialize( SCIC_SDS_PHY_T *this_phy, void *transport_layer_registers, SCU_LINK_LAYER_REGISTERS_T *link_layer_registers ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_initialize(this_phy:0x%x, link_layer_registers:0x%x)\n", this_phy, link_layer_registers )); - // Perfrom the initialization of the TL hardware + // Perform the initialization of the TL hardware scic_sds_phy_transport_layer_initialization(this_phy, transport_layer_registers); // Perofrm the initialization of the PE hardware scic_sds_phy_link_layer_initialization(this_phy, link_layer_registers); // There is nothing that needs to be done in this state just // transition to the stopped state. sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STOPPED ); return SCI_SUCCESS; } /** * This method assigns the direct attached device ID for this phy. * * @param[in] this_phy The phy for which the direct attached device id is to * be assigned. * @param[in] device_id The direct attached device ID to assign to the phy. * This will either be the RNi for the device or an invalid RNi if there * is no current device assigned to the phy. */ void scic_sds_phy_setup_transport( SCIC_SDS_PHY_T * this_phy, U32 device_id ) { U32 tl_control; SCU_STPTLDARNI_WRITE(this_phy, device_id); // The read should guarntee that the first write gets posted // before the next write tl_control = SCU_TLCR_READ(this_phy); tl_control |= SCU_TLCR_GEN_BIT(CLEAR_TCI_NCQ_MAPPING_TABLE); SCU_TLCR_WRITE(this_phy, tl_control); } /** * This function will perform the register reads/writes to suspend the SCU * hardware protocol engine. * * @param[in,out] this_phy The phy object to be suspended. * * @return none */ void scic_sds_phy_suspend( SCIC_SDS_PHY_T * this_phy ) { U32 scu_sas_pcfg_value; scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value |= SCU_SAS_PCFG_GEN_BIT(SUSPEND_PROTOCOL_ENGINE); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); scic_sds_phy_setup_transport( this_phy, SCIC_SDS_REMOTE_NODE_CONTEXT_INVALID_INDEX ); } /** * This function will perform the register reads/writes required to resume the * SCU hardware protocol engine. * * @param[in,out] this_phy The phy object to resume. * * @return none */ void scic_sds_phy_resume( SCIC_SDS_PHY_T * this_phy ) { U32 scu_sas_pcfg_value; scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value &= ~SCU_SAS_PCFG_GEN_BIT(SUSPEND_PROTOCOL_ENGINE); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); } /** * @brief This method returns the local sas address assigned to this phy. * * @param[in] this_phy This parameter specifies the phy for which * to retrieve the local SAS address. * @param[out] sas_address This parameter specifies the location into * which to copy the local SAS address. * * @return none */ void scic_sds_phy_get_sas_address( SCIC_SDS_PHY_T *this_phy, SCI_SAS_ADDRESS_T *sas_address ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_get_sas_address(this_phy:0x%x, sas_address:0x%x)\n", this_phy, sas_address )); sas_address->high = SCU_SAS_TISSAH_READ(this_phy); sas_address->low = SCU_SAS_TISSAL_READ(this_phy); } /** * @brief This method returns the remote end-point (i.e. attached) * sas address assigned to this phy. * * @param[in] this_phy This parameter specifies the phy for which * to retrieve the remote end-point SAS address. * @param[out] sas_address This parameter specifies the location into * which to copy the remote end-point SAS address. * * @return none */ void scic_sds_phy_get_attached_sas_address( SCIC_SDS_PHY_T *this_phy, SCI_SAS_ADDRESS_T *sas_address ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_get_attached_sas_address(0x%x, 0x%x) enter\n", this_phy, sas_address )); sas_address->high = this_phy->phy_type.sas.identify_address_frame_buffer.sas_address.high; sas_address->low = this_phy->phy_type.sas.identify_address_frame_buffer.sas_address.low; } /** * @brief This method returns the supported protocols assigned to * this phy * * @param[in] this_phy * @param[out] protocols */ void scic_sds_phy_get_protocols( SCIC_SDS_PHY_T *this_phy, SCI_SAS_IDENTIFY_ADDRESS_FRAME_PROTOCOLS_T * protocols ) { U32 tiid_value = SCU_SAS_TIID_READ(this_phy); //Check each bit of this register. please refer to //SAS Transmit Identification Register (SAS_TIID). if (tiid_value & 0x2) protocols->u.bits.smp_target = 1; if (tiid_value & 0x4) protocols->u.bits.stp_target = 1; if (tiid_value & 0x8) protocols->u.bits.ssp_target = 1; if (tiid_value & 0x200) protocols->u.bits.smp_initiator = 1; if ((tiid_value & 0x400)) protocols->u.bits.stp_initiator = 1; if (tiid_value & 0x800) protocols->u.bits.ssp_initiator = 1; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_get_protocols(this_phy:0x%x, protocols:0x%x)\n", this_phy, protocols->u.all )); } /** * This method returns the supported protocols for the attached phy. If this * is a SAS phy the protocols are returned from the identify address frame. * If this is a SATA phy then protocols are made up and the target phy is an * STP target phy. * * @note The caller will get the entire set of bits for the protocol value. * * @param[in] this_phy The parameter is the phy object for which the attached * phy protcols are to be returned. * @param[out] protocols The parameter is the returned protocols for the * attached phy. */ void scic_sds_phy_get_attached_phy_protocols( SCIC_SDS_PHY_T *this_phy, SCI_SAS_IDENTIFY_ADDRESS_FRAME_PROTOCOLS_T * protocols ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_get_attached_phy_protocols(this_phy:0x%x, protocols:0x%x[0x%x])\n", this_phy, protocols, protocols->u.all )); protocols->u.all = 0; if (this_phy->protocol == SCIC_SDS_PHY_PROTOCOL_SAS) { protocols->u.all = this_phy->phy_type.sas.identify_address_frame_buffer.protocols.u.all; } else if (this_phy->protocol == SCIC_SDS_PHY_PROTOCOL_SATA) { protocols->u.bits.stp_target = 1; } } /** * @brief This method release resources in for a scic phy. * * @param[in] controller This parameter specifies the core controller, one of * its phy's resources are to be released. * @param[in] this_phy This parameter specifies the phy whose resource is to * be released. */ void scic_sds_phy_release_resource( SCIC_SDS_CONTROLLER_T * controller, SCIC_SDS_PHY_T * this_phy ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_release_resource(0x%x, 0x%x)\n", controller, this_phy )); //Currently, the only resource to be released is a timer. if (this_phy->sata_timeout_timer != NULL) { scic_cb_timer_destroy(controller, this_phy->sata_timeout_timer); this_phy->sata_timeout_timer = NULL; } } //***************************************************************************** //* SCIC SDS PHY Handler Redirects //***************************************************************************** /** * @brief This method will attempt to reset the phy. This * request is only valid when the phy is in an ready * state * * @param[in] this_phy * * @return SCI_STATUS */ SCI_STATUS scic_sds_phy_reset( SCIC_SDS_PHY_T * this_phy ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_reset(this_phy:0x%08x)\n", this_phy )); return this_phy->state_handlers->parent.reset_handler( &this_phy->parent ); } /** - * @brief This method will process the event code recieved. + * @brief This method will process the event code received. * * @param[in] this_phy * @param[in] event_code * * @return SCI_STATUS */ SCI_STATUS scic_sds_phy_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_event_handler(this_phy:0x%08x, event_code:%x)\n", this_phy, event_code )); return this_phy->state_handlers->event_handler(this_phy, event_code); } /** - * @brief This method will process the frame index recieved. + * @brief This method will process the frame index received. * * @param[in] this_phy * @param[in] frame_index * * @return SCI_STATUS */ SCI_STATUS scic_sds_phy_frame_handler( SCIC_SDS_PHY_T *this_phy, U32 frame_index ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_frame_handler(this_phy:0x%08x, frame_index:%d)\n", this_phy, frame_index )); return this_phy->state_handlers->frame_handler(this_phy, frame_index); } /** * @brief This method will give the phy permission to consume power * * @param[in] this_phy * * @return SCI_STATUS */ SCI_STATUS scic_sds_phy_consume_power_handler( SCIC_SDS_PHY_T *this_phy ) { SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sds_phy_consume_power_handler(this_phy:0x%08x)\n", this_phy )); return this_phy->state_handlers->consume_power_handler(this_phy); } //***************************************************************************** //* SCIC PHY Public Methods //***************************************************************************** SCI_STATUS scic_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_PHY_PROPERTIES_T * properties ) { SCIC_SDS_PHY_T *this_phy; U8 max_speed_generation; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_get_properties(0x%x, 0x%x) enter\n", this_phy, properties )); if (phy == SCI_INVALID_HANDLE) { return SCI_FAILURE_INVALID_PHY; } memset(properties, 0, sizeof(SCIC_PHY_PROPERTIES_T)); //get max link rate of this phy set by user. max_speed_generation = this_phy->owning_port->owning_controller->user_parameters.sds1. phys[this_phy->phy_index].max_speed_generation; properties->negotiated_link_rate = this_phy->max_negotiated_speed; if (max_speed_generation == SCIC_SDS_PARM_GEN3_SPEED) properties->max_link_rate = SCI_SAS_600_GB; else if (max_speed_generation == SCIC_SDS_PARM_GEN2_SPEED) properties->max_link_rate = SCI_SAS_300_GB; else properties->max_link_rate = SCI_SAS_150_GB; properties->index = this_phy->phy_index; properties->owning_port = scic_sds_phy_get_port(this_phy); scic_sds_phy_get_protocols(this_phy, &properties->transmit_iaf.protocols); properties->transmit_iaf.sas_address.high = this_phy->owning_port->owning_controller->oem_parameters.sds1. phys[this_phy->phy_index].sas_address.sci_format.high; properties->transmit_iaf.sas_address.low = this_phy->owning_port->owning_controller->oem_parameters.sds1. phys[this_phy->phy_index].sas_address.sci_format.low; return SCI_SUCCESS; } // --------------------------------------------------------------------------- SCI_STATUS scic_sas_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_SAS_PHY_PROPERTIES_T * properties ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sas_phy_get_properties(0x%x, 0x%x) enter\n", this_phy, properties )); if (this_phy->protocol == SCIC_SDS_PHY_PROTOCOL_SAS) { memcpy( &properties->received_iaf, &this_phy->phy_type.sas.identify_address_frame_buffer, sizeof(SCI_SAS_IDENTIFY_ADDRESS_FRAME_T) ); properties->received_capabilities.u.all = SCU_SAS_RECPHYCAP_READ(this_phy); return SCI_SUCCESS; } return SCI_FAILURE; } // --------------------------------------------------------------------------- SCI_STATUS scic_sata_phy_get_properties( SCI_PHY_HANDLE_T phy, SCIC_SATA_PHY_PROPERTIES_T * properties ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sata_phy_get_properties(0x%x, 0x%x) enter\n", this_phy, properties )); if (this_phy->protocol == SCIC_SDS_PHY_PROTOCOL_SATA) { memcpy( &properties->signature_fis, &this_phy->phy_type.sata.signature_fis_buffer, sizeof(SATA_FIS_REG_D2H_T) ); /// @todo add support for port selectors. properties->is_port_selector_present = FALSE; return SCI_SUCCESS; } return SCI_FAILURE; } // --------------------------------------------------------------------------- #if !defined(DISABLE_PORT_SELECTORS) SCI_STATUS scic_sata_phy_send_port_selection_signal( SCI_PHY_HANDLE_T phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_sata_phy_send_port_selection_signals(0x%x) enter\n", this_phy )); /// @todo To be implemented ASSERT(FALSE); return SCI_FAILURE; } #endif // !defined(DISABLE_PORT_SELECTORS) // --------------------------------------------------------------------------- #if !defined(DISABLE_PHY_COUNTERS) SCI_STATUS scic_phy_enable_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ) { SCIC_SDS_PHY_T *this_phy; SCI_STATUS status = SCI_SUCCESS; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_enable_counter(0x%x, 0x%x) enter\n", this_phy, counter_id )); switch(counter_id) { case SCIC_PHY_COUNTER_RECEIVED_DONE_ACK_NAK_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_RX_DONE_ACK_NAK_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_ACK_NAK_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_TX_DONE_ACK_NAK_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_INACTIVITY_TIMER_EXPIRED: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_INACTIVITY_TIMER_EXPIRED_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_RECEIVED_DONE_CREDIT_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_RX_DONE_CREDIT_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_CREDIT_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_TX_DONE_CREDIT_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_RECEIVED_CREDIT_BLOCKED: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control |= (1 << SCU_ERR_CNT_RX_CREDIT_BLOCKED_RECEIVED_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; // These error counters are enabled by default, and cannot be // disabled. Return SCI_SUCCESS to denote that they are // enabled, hiding the fact that enabling the counter is // a no-op. case SCIC_PHY_COUNTER_RECEIVED_FRAME: case SCIC_PHY_COUNTER_TRANSMITTED_FRAME: case SCIC_PHY_COUNTER_RECEIVED_FRAME_DWORD: case SCIC_PHY_COUNTER_TRANSMITTED_FRAME_DWORD: case SCIC_PHY_COUNTER_LOSS_OF_SYNC_ERROR: case SCIC_PHY_COUNTER_RECEIVED_DISPARITY_ERROR: case SCIC_PHY_COUNTER_RECEIVED_FRAME_CRC_ERROR: case SCIC_PHY_COUNTER_RECEIVED_SHORT_FRAME: case SCIC_PHY_COUNTER_RECEIVED_FRAME_WITHOUT_CREDIT: case SCIC_PHY_COUNTER_RECEIVED_FRAME_AFTER_DONE: case SCIC_PHY_COUNTER_SN_DWORD_SYNC_ERROR: break; default: status = SCI_FAILURE; break; } return status; } // --------------------------------------------------------------------------- SCI_STATUS scic_phy_disable_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ) { SCIC_SDS_PHY_T *this_phy; SCI_STATUS status = SCI_SUCCESS; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_disable_counter(0x%x, 0x%x) enter\n", this_phy, counter_id )); switch(counter_id) { case SCIC_PHY_COUNTER_RECEIVED_DONE_ACK_NAK_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_RX_DONE_ACK_NAK_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_ACK_NAK_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_TX_DONE_ACK_NAK_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_INACTIVITY_TIMER_EXPIRED: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_INACTIVITY_TIMER_EXPIRED_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_RECEIVED_DONE_CREDIT_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_RX_DONE_CREDIT_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_CREDIT_TIMEOUT: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_TX_DONE_CREDIT_TIMEOUT_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; case SCIC_PHY_COUNTER_RECEIVED_CREDIT_BLOCKED: { U32 control = SCU_SAS_ECENCR_READ(this_phy); control &= ~(1 << SCU_ERR_CNT_RX_CREDIT_BLOCKED_RECEIVED_INDEX); SCU_SAS_ECENCR_WRITE(this_phy, control); } break; // These error counters cannot be disabled, so return SCI_FAILURE. case SCIC_PHY_COUNTER_RECEIVED_FRAME: case SCIC_PHY_COUNTER_TRANSMITTED_FRAME: case SCIC_PHY_COUNTER_RECEIVED_FRAME_DWORD: case SCIC_PHY_COUNTER_TRANSMITTED_FRAME_DWORD: case SCIC_PHY_COUNTER_LOSS_OF_SYNC_ERROR: case SCIC_PHY_COUNTER_RECEIVED_DISPARITY_ERROR: case SCIC_PHY_COUNTER_RECEIVED_FRAME_CRC_ERROR: case SCIC_PHY_COUNTER_RECEIVED_SHORT_FRAME: case SCIC_PHY_COUNTER_RECEIVED_FRAME_WITHOUT_CREDIT: case SCIC_PHY_COUNTER_RECEIVED_FRAME_AFTER_DONE: case SCIC_PHY_COUNTER_SN_DWORD_SYNC_ERROR: default: status = SCI_FAILURE; break; } return status; } // --------------------------------------------------------------------------- SCI_STATUS scic_phy_get_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id, U32 * data ) { SCIC_SDS_PHY_T *this_phy; SCI_STATUS status = SCI_SUCCESS; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_get_counter(0x%x, 0x%x) enter\n", this_phy, counter_id )); switch(counter_id) { case SCIC_PHY_COUNTER_RECEIVED_FRAME: *data = scu_link_layer_register_read(this_phy, received_frame_count); break; case SCIC_PHY_COUNTER_TRANSMITTED_FRAME: *data = scu_link_layer_register_read(this_phy, transmit_frame_count); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_DWORD: *data = scu_link_layer_register_read(this_phy, received_dword_count); break; case SCIC_PHY_COUNTER_TRANSMITTED_FRAME_DWORD: *data = scu_link_layer_register_read(this_phy, transmit_dword_count); break; case SCIC_PHY_COUNTER_LOSS_OF_SYNC_ERROR: *data = scu_link_layer_register_read(this_phy, loss_of_sync_error_count); break; case SCIC_PHY_COUNTER_RECEIVED_DISPARITY_ERROR: *data = scu_link_layer_register_read(this_phy, running_disparity_error_count); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_CRC_ERROR: *data = scu_link_layer_register_read(this_phy, received_frame_crc_error_count); break; case SCIC_PHY_COUNTER_RECEIVED_DONE_ACK_NAK_TIMEOUT: *data = this_phy->error_counter[SCU_ERR_CNT_RX_DONE_ACK_NAK_TIMEOUT_INDEX]; break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_ACK_NAK_TIMEOUT: *data = this_phy->error_counter[SCU_ERR_CNT_TX_DONE_ACK_NAK_TIMEOUT_INDEX]; break; case SCIC_PHY_COUNTER_INACTIVITY_TIMER_EXPIRED: *data = this_phy->error_counter[SCU_ERR_CNT_INACTIVITY_TIMER_EXPIRED_INDEX]; break; case SCIC_PHY_COUNTER_RECEIVED_DONE_CREDIT_TIMEOUT: *data = this_phy->error_counter[SCU_ERR_CNT_RX_DONE_CREDIT_TIMEOUT_INDEX]; break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_CREDIT_TIMEOUT: *data = this_phy->error_counter[SCU_ERR_CNT_TX_DONE_CREDIT_TIMEOUT_INDEX]; break; case SCIC_PHY_COUNTER_RECEIVED_CREDIT_BLOCKED: *data = this_phy->error_counter[SCU_ERR_CNT_RX_CREDIT_BLOCKED_RECEIVED_INDEX]; break; case SCIC_PHY_COUNTER_RECEIVED_SHORT_FRAME: *data = scu_link_layer_register_read(this_phy, received_short_frame_count); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_WITHOUT_CREDIT: *data = scu_link_layer_register_read(this_phy, received_frame_without_credit_count); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_AFTER_DONE: *data = scu_link_layer_register_read(this_phy, received_frame_after_done_count); break; case SCIC_PHY_COUNTER_SN_DWORD_SYNC_ERROR: *data = scu_link_layer_register_read(this_phy, phy_reset_problem_count); break; default: status = SCI_FAILURE; break; } return status; } // --------------------------------------------------------------------------- SCI_STATUS scic_phy_clear_counter( SCI_PHY_HANDLE_T phy, SCIC_PHY_COUNTER_ID_T counter_id ) { SCIC_SDS_PHY_T *this_phy; SCI_STATUS status = SCI_SUCCESS; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_clear_counter(0x%x, 0x%x) enter\n", this_phy, counter_id )); switch(counter_id) { case SCIC_PHY_COUNTER_RECEIVED_FRAME: scu_link_layer_register_write(this_phy, received_frame_count, 0); break; case SCIC_PHY_COUNTER_TRANSMITTED_FRAME: scu_link_layer_register_write(this_phy, transmit_frame_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_DWORD: scu_link_layer_register_write(this_phy, received_dword_count, 0); break; case SCIC_PHY_COUNTER_TRANSMITTED_FRAME_DWORD: scu_link_layer_register_write(this_phy, transmit_dword_count, 0); break; case SCIC_PHY_COUNTER_LOSS_OF_SYNC_ERROR: scu_link_layer_register_write(this_phy, loss_of_sync_error_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_DISPARITY_ERROR: scu_link_layer_register_write(this_phy, running_disparity_error_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_CRC_ERROR: scu_link_layer_register_write(this_phy, received_frame_crc_error_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_DONE_ACK_NAK_TIMEOUT: this_phy->error_counter[SCU_ERR_CNT_RX_DONE_ACK_NAK_TIMEOUT_INDEX] = 0; break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_ACK_NAK_TIMEOUT: this_phy->error_counter[SCU_ERR_CNT_TX_DONE_ACK_NAK_TIMEOUT_INDEX] = 0; break; case SCIC_PHY_COUNTER_INACTIVITY_TIMER_EXPIRED: this_phy->error_counter[SCU_ERR_CNT_INACTIVITY_TIMER_EXPIRED_INDEX] = 0; break; case SCIC_PHY_COUNTER_RECEIVED_DONE_CREDIT_TIMEOUT: this_phy->error_counter[SCU_ERR_CNT_RX_DONE_CREDIT_TIMEOUT_INDEX] = 0; break; case SCIC_PHY_COUNTER_TRANSMITTED_DONE_CREDIT_TIMEOUT: this_phy->error_counter[SCU_ERR_CNT_TX_DONE_CREDIT_TIMEOUT_INDEX] = 0; break; case SCIC_PHY_COUNTER_RECEIVED_CREDIT_BLOCKED: this_phy->error_counter[SCU_ERR_CNT_RX_CREDIT_BLOCKED_RECEIVED_INDEX] = 0; break; case SCIC_PHY_COUNTER_RECEIVED_SHORT_FRAME: scu_link_layer_register_write(this_phy, received_short_frame_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_WITHOUT_CREDIT: scu_link_layer_register_write(this_phy, received_frame_without_credit_count, 0); break; case SCIC_PHY_COUNTER_RECEIVED_FRAME_AFTER_DONE: scu_link_layer_register_write(this_phy, received_frame_after_done_count, 0); break; case SCIC_PHY_COUNTER_SN_DWORD_SYNC_ERROR: scu_link_layer_register_write(this_phy, phy_reset_problem_count, 0); break; default: status = SCI_FAILURE; } return status; } #endif // !defined(DISABLE_PHY_COUNTERS) SCI_STATUS scic_phy_stop( SCI_PHY_HANDLE_T phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_stop(this_phy:0x%x)\n", this_phy )); return this_phy->state_handlers->parent.stop_handler(&this_phy->parent); } SCI_STATUS scic_phy_start( SCI_PHY_HANDLE_T phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_TRACE(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "scic_phy_start(this_phy:0x%x)\n", this_phy )); return this_phy->state_handlers->parent.start_handler(&this_phy->parent); } //****************************************************************************** //* PHY STATE MACHINE //****************************************************************************** //*************************************************************************** //* DEFAULT HANDLERS //*************************************************************************** /** * This is the default method for phy a start request. It will report a * warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_start_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x requested to start from invalid state %d\n", this_phy, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for phy a stop request. It will report a * warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_stop_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x requested to stop from invalid state %d\n", this_phy, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for phy a reset request. It will report a * warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_reset_handler( SCI_BASE_PHY_T * phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x requested to reset from invalid state %d\n", this_phy, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for phy a destruct request. It will report a * warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_destroy_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; /// @todo Implement something for the default SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x requested to destroy from invalid state %d\n", this_phy, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for a phy frame handling request. It will * report a warning, release the frame and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * @param[in] frame_index This is the frame index that was received from the * SCU hardware. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_frame_handler( SCIC_SDS_PHY_T *this_phy, U32 frame_index ) { SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, - "SCIC Phy 0x%08x recieved unexpected frame data %d while in state %d\n", + "SCIC Phy 0x%08x received unexpected frame data %d while in state %d\n", this_phy, frame_index, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); scic_sds_controller_release_frame( scic_sds_phy_get_controller(this_phy), frame_index); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for a phy event handler. It will report a * warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * @param[in] event_code This is the event code that was received from the SCU * hardware. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x received unexpected event status %x while in state %d\n", this_phy, event_code, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } /** * This is the default method for a phy consume power handler. It will report * a warning and exit. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ SCI_STATUS scic_sds_phy_default_consume_power_handler( SCIC_SDS_PHY_T *this_phy ) { SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY, "SCIC Phy 0x%08x given unexpected permission to consume power while in state %d\n", this_phy, sci_base_state_machine_get_state(&this_phy->parent.state_machine) )); return SCI_FAILURE_INVALID_STATE; } //****************************************************************************** //* PHY STOPPED STATE HANDLERS //****************************************************************************** /** * This method takes the SCIC_SDS_PHY from a stopped state and attempts to * start it. * - The phy state machine is transitioned to the * SCI_BASE_PHY_STATE_STARTING. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_stopped_state_start_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; // Create the SIGNATURE FIS Timeout timer for this phy this_phy->sata_timeout_timer = scic_cb_timer_create( scic_sds_phy_get_controller(this_phy), scic_sds_phy_sata_timeout, this_phy ); if (this_phy->sata_timeout_timer != NULL) { sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STARTING ); } return SCI_SUCCESS; } /** * This method takes the SCIC_SDS_PHY from a stopped state and destroys it. * - This function takes no action. * - * @todo Shouldnt this function transition the SCI_BASE_PHY::state_machine to + * @todo Shouldn't this function transition the SCI_BASE_PHY::state_machine to * the SCI_BASE_PHY_STATE_FINAL? * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_stopped_state_destroy_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; /// @todo what do we actually need to do here? return SCI_SUCCESS; } //****************************************************************************** //* PHY STARTING STATE HANDLERS //****************************************************************************** // All of these state handlers are mapped to the starting sub-state machine //****************************************************************************** //* PHY READY STATE HANDLERS //****************************************************************************** /** * This method takes the SCIC_SDS_PHY from a ready state and attempts to stop * it. * - The phy state machine is transitioned to the SCI_BASE_PHY_STATE_STOPPED. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_ready_state_stop_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STOPPED ); scic_sds_controller_link_down( scic_sds_phy_get_controller(this_phy), scic_sds_phy_get_port(this_phy), this_phy ); return SCI_SUCCESS; } /** * This method takes the SCIC_SDS_PHY from a ready state and attempts to reset * it. * - The phy state machine is transitioned to the SCI_BASE_PHY_STATE_STARTING. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_ready_state_reset_handler( SCI_BASE_PHY_T * phy ) { SCIC_SDS_PHY_T * this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_RESETTING ); return SCI_SUCCESS; } /** * This method request the SCIC_SDS_PHY handle the received event. The only * event that we are interested in while in the ready state is the link * failure event. * - decoded event is a link failure * - transition the SCIC_SDS_PHY back to the SCI_BASE_PHY_STATE_STARTING * state. - * - any other event recived will report a warning message + * - any other event received will report a warning message * * @param[in] phy This is the SCIC_SDS_PHY object which has received the * event. * * @return SCI_STATUS * @retval SCI_SUCCESS if the event received is a link failure * @retval SCI_FAILURE_INVALID_STATE for any other event received. */ static SCI_STATUS scic_sds_phy_ready_state_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { SCI_STATUS result = SCI_FAILURE; switch (scu_get_event_code(event_code)) { case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STARTING ); result = SCI_SUCCESS; break; case SCU_EVENT_BROADCAST_CHANGE: // Broadcast change received. Notify the port. if (scic_sds_phy_get_port(this_phy) != SCI_INVALID_HANDLE) scic_sds_port_broadcast_change_received(this_phy->owning_port, this_phy); else this_phy->bcn_received_while_port_unassigned = TRUE; break; case SCU_EVENT_ERR_CNT(RX_CREDIT_BLOCKED_RECEIVED): case SCU_EVENT_ERR_CNT(TX_DONE_CREDIT_TIMEOUT): case SCU_EVENT_ERR_CNT(RX_DONE_CREDIT_TIMEOUT): case SCU_EVENT_ERR_CNT(INACTIVITY_TIMER_EXPIRED): case SCU_EVENT_ERR_CNT(TX_DONE_ACK_NAK_TIMEOUT): case SCU_EVENT_ERR_CNT(RX_DONE_ACK_NAK_TIMEOUT): { U32 error_counter_index = scu_get_event_specifier(event_code) >> SCU_EVENT_SPECIFIC_CODE_SHIFT; this_phy->error_counter[error_counter_index]++; result = SCI_SUCCESS; } break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "SCIC PHY 0x%x ready state machine recieved unexpected event_code %x\n", + "SCIC PHY 0x%x ready state machine received unexpected event_code %x\n", this_phy, event_code )); result = SCI_FAILURE_INVALID_STATE; break; } return result; } // --------------------------------------------------------------------------- /** * This is the resetting state event handler. * * @param[in] this_phy This is the SCIC_SDS_PHY object which is receiving the * event. * @param[in] event_code This is the event code to be processed. * * @return SCI_STATUS * @retval SCI_FAILURE_INVALID_STATE */ static SCI_STATUS scic_sds_phy_resetting_state_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { SCI_STATUS result = SCI_FAILURE; switch (scu_get_event_code(event_code)) { case SCU_EVENT_HARD_RESET_TRANSMITTED: // Link failure change state back to the starting state sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STARTING ); result = SCI_SUCCESS; break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "SCIC PHY 0x%x resetting state machine recieved unexpected event_code %x\n", + "SCIC PHY 0x%x resetting state machine received unexpected event_code %x\n", this_phy, event_code )); result = SCI_FAILURE_INVALID_STATE; break; } return result; } // --------------------------------------------------------------------------- SCIC_SDS_PHY_STATE_HANDLER_T scic_sds_phy_state_handler_table[SCI_BASE_PHY_MAX_STATES] = { // SCI_BASE_PHY_STATE_INITIAL { { scic_sds_phy_default_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler }, // SCI_BASE_PHY_STATE_STOPPED { { scic_sds_phy_stopped_state_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_stopped_state_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler }, // SCI_BASE_PHY_STATE_STARTING { { scic_sds_phy_default_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler }, // SCI_BASE_PHY_STATE_READY { { scic_sds_phy_default_start_handler, scic_sds_phy_ready_state_stop_handler, scic_sds_phy_ready_state_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_ready_state_event_handler, scic_sds_phy_default_consume_power_handler }, // SCI_BASE_PHY_STATE_RESETTING { { scic_sds_phy_default_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_resetting_state_event_handler, scic_sds_phy_default_consume_power_handler }, // SCI_BASE_PHY_STATE_FINAL { { scic_sds_phy_default_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler } }; //**************************************************************************** //* PHY STATE PRIVATE METHODS //**************************************************************************** /** * This method will stop the SCIC_SDS_PHY object. This does not reset the * protocol engine it just suspends it and places it in a state where it will * not cause the end device to power up. * * @param[in] this_phy This is the SCIC_SDS_PHY object to stop. * * @return none */ static void scu_link_layer_stop_protocol_engine( SCIC_SDS_PHY_T *this_phy ) { U32 scu_sas_pcfg_value; U32 enable_spinup_value; // Suspend the protocol engine and place it in a sata spinup hold state scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value |= ( SCU_SAS_PCFG_GEN_BIT(OOB_RESET) | SCU_SAS_PCFG_GEN_BIT(SUSPEND_PROTOCOL_ENGINE) | SCU_SAS_PCFG_GEN_BIT(SATA_SPINUP_HOLD) ); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); // Disable the notify enable spinup primitives enable_spinup_value = SCU_SAS_ENSPINUP_READ(this_phy); enable_spinup_value &= ~SCU_ENSPINUP_GEN_BIT(ENABLE); SCU_SAS_ENSPINUP_WRITE(this_phy, enable_spinup_value); } /** * This method will start the OOB/SN state machine for this SCIC_SDS_PHY * object. * * @param[in] this_phy This is the SCIC_SDS_PHY object on which to start the * OOB/SN state machine. */ static void scu_link_layer_start_oob( SCIC_SDS_PHY_T *this_phy ) { U32 scu_sas_pcfg_value; /* Reset OOB sequence - start */ scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value &= ~(SCU_SAS_PCFG_GEN_BIT(OOB_RESET) | SCU_SAS_PCFG_GEN_BIT(HARD_RESET)); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); SCU_SAS_PCFG_READ(this_phy); /* Reset OOB sequence - end */ /* Start OOB sequence - start */ scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value |= SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); SCU_SAS_PCFG_READ(this_phy); /* Start OOB sequence - end */ } /** * This method will transmit a hard reset request on the specified phy. The * SCU hardware requires that we reset the OOB state machine and set the hard * reset bit in the phy configuration register. * We then must start OOB over with the hard reset bit set. * * @param[in] this_phy */ static void scu_link_layer_tx_hard_reset( SCIC_SDS_PHY_T *this_phy ) { U32 phy_configuration_value; // SAS Phys must wait for the HARD_RESET_TX event notification to transition // to the starting state. phy_configuration_value = SCU_SAS_PCFG_READ(this_phy); phy_configuration_value |= (SCU_SAS_PCFG_GEN_BIT(HARD_RESET) | SCU_SAS_PCFG_GEN_BIT(OOB_RESET)); SCU_SAS_PCFG_WRITE(this_phy, phy_configuration_value); // Now take the OOB state machine out of reset phy_configuration_value |= SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE); phy_configuration_value &= ~SCU_SAS_PCFG_GEN_BIT(OOB_RESET); SCU_SAS_PCFG_WRITE(this_phy, phy_configuration_value); } //**************************************************************************** //* PHY BASE STATE METHODS //**************************************************************************** /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_INITIAL. * - This function sets the state handlers for the phy object base state * machine initial state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_initial_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_INITIAL); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_INITIAL. * - This function sets the state handlers for the phy object base state * machine initial state. * - The SCU hardware is requested to stop the protocol engine. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_stopped_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; /// @todo We need to get to the controller to place this PE in a reset state scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_STOPPED); if (this_phy->sata_timeout_timer != NULL) { scic_cb_timer_destroy( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer ); this_phy->sata_timeout_timer = NULL; } scu_link_layer_stop_protocol_engine(this_phy); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_STARTING. * - This function sets the state handlers for the phy object base state * machine starting state. * - The SCU hardware is requested to start OOB/SN on this protocol engine. * - The phy starting substate machine is started. * - If the previous state was the ready state then the * SCIC_SDS_CONTROLLER is informed that the phy has gone link down. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_STARTING); scu_link_layer_stop_protocol_engine(this_phy); scu_link_layer_start_oob(this_phy); // We don't know what kind of phy we are going to be just yet this_phy->protocol = SCIC_SDS_PHY_PROTOCOL_UNKNOWN; this_phy->bcn_received_while_port_unassigned = FALSE; // Change over to the starting substate machine to continue sci_base_state_machine_start(&this_phy->starting_substate_machine); if (this_phy->parent.state_machine.previous_state_id == SCI_BASE_PHY_STATE_READY) { scic_sds_controller_link_down( scic_sds_phy_get_controller(this_phy), scic_sds_phy_get_port(this_phy), this_phy ); } } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_READY. * - This function sets the state handlers for the phy object base state * machine ready state. * - The SCU hardware protocol engine is resumed. * - The SCIC_SDS_CONTROLLER is informed that the phy object has gone link * up. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_ready_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_READY); scic_sds_controller_link_up( scic_sds_phy_get_controller(this_phy), scic_sds_phy_get_port(this_phy), this_phy ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCI_BASE_PHY_STATE_INITIAL. This function suspends the SCU * hardware protocol engine represented by this SCIC_SDS_PHY object. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_ready_state_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_suspend(this_phy); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_RESETTING. * - This function sets the state handlers for the phy object base state * machine resetting state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_resetting_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T * this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_RESETTING); // The phy is being reset, therefore deactivate it from the port. // In the resetting state we don't notify the user regarding // link up and link down notifications. scic_sds_port_deactivate_phy(this_phy->owning_port, this_phy, FALSE); if (this_phy->protocol == SCIC_SDS_PHY_PROTOCOL_SAS) { scu_link_layer_tx_hard_reset(this_phy); } else { // The SCU does not need to have a descrete reset state so just go back to // the starting state. sci_base_state_machine_change_state( &this_phy->parent.state_machine, SCI_BASE_PHY_STATE_STARTING ); } } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCI_BASE_PHY_STATE_FINAL. * - This function sets the state handlers for the phy object base state * machine final state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_final_state_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_base_state_handlers(this_phy, SCI_BASE_PHY_STATE_FINAL); // Nothing to do here } // --------------------------------------------------------------------------- SCI_BASE_STATE_T scic_sds_phy_state_table[SCI_BASE_PHY_MAX_STATES] = { { SCI_BASE_PHY_STATE_INITIAL, scic_sds_phy_initial_state_enter, NULL, }, { SCI_BASE_PHY_STATE_STOPPED, scic_sds_phy_stopped_state_enter, NULL, }, { SCI_BASE_PHY_STATE_STARTING, scic_sds_phy_starting_state_enter, NULL, }, { SCI_BASE_PHY_STATE_READY, scic_sds_phy_ready_state_enter, scic_sds_phy_ready_state_exit, }, { SCI_BASE_PHY_STATE_RESETTING, scic_sds_phy_resetting_state_enter, NULL, }, { SCI_BASE_PHY_STATE_FINAL, scic_sds_phy_final_state_enter, NULL, } }; //****************************************************************************** //* PHY STARTING SUB-STATE MACHINE //****************************************************************************** //***************************************************************************** //* SCIC SDS PHY HELPER FUNCTIONS //***************************************************************************** /** * This method continues the link training for the phy as if it were a SAS PHY * instead of a SATA PHY. This is done because the completion queue had a SAS * PHY DETECTED event when the state machine was expecting a SATA PHY event. * * @param[in] this_phy The phy object that received SAS PHY DETECTED. * * @return none */ static void scic_sds_phy_start_sas_link_training( SCIC_SDS_PHY_T * this_phy ) { U32 phy_control; phy_control = SCU_SAS_PCFG_READ(this_phy); phy_control |= SCU_SAS_PCFG_GEN_BIT(SATA_SPINUP_HOLD); SCU_SAS_PCFG_WRITE(this_phy, phy_control); sci_base_state_machine_change_state( &this_phy->starting_substate_machine, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_SPEED_EN ); this_phy->protocol = SCIC_SDS_PHY_PROTOCOL_SAS; } /** * This method continues the link training for the phy as if it were a SATA * PHY instead of a SAS PHY. This is done because the completion queue had a * SATA SPINUP HOLD event when the state machine was expecting a SAS PHY * event. * * @param[in] this_phy The phy object that received a SATA SPINUP HOLD event * * @return none */ static void scic_sds_phy_start_sata_link_training( SCIC_SDS_PHY_T * this_phy ) { sci_base_state_machine_change_state( &this_phy->starting_substate_machine, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER ); this_phy->protocol = SCIC_SDS_PHY_PROTOCOL_SATA; } /** * @brief This method performs processing common to all protocols upon * completion of link training. * * @param[in,out] this_phy This parameter specifies the phy object for which * link training has completed. * @param[in] max_link_rate This parameter specifies the maximum link * rate to be associated with this phy. * @param[in] next_state This parameter specifies the next state for the * phy's starting sub-state machine. * * @return none */ static void scic_sds_phy_complete_link_training( SCIC_SDS_PHY_T * this_phy, SCI_SAS_LINK_RATE max_link_rate, U32 next_state ) { this_phy->max_negotiated_speed = max_link_rate; sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), next_state ); } /** * This method restarts the SCIC_SDS_PHY objects base state machine in the * starting state from any starting substate. * * @param[in] this_phy The SCIC_SDS_PHY object to restart. * * @return none */ void scic_sds_phy_restart_starting_state( SCIC_SDS_PHY_T *this_phy ) { // Stop the current substate machine sci_base_state_machine_stop( scic_sds_phy_get_starting_substate_machine(this_phy) ); // Re-enter the base state machine starting state sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_STARTING ); } //***************************************************************************** //* SCIC SDS PHY general handlers //***************************************************************************** static SCI_STATUS scic_sds_phy_starting_substate_general_stop_handler( SCI_BASE_PHY_T *phy ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)phy; sci_base_state_machine_stop( &this_phy->starting_substate_machine ); sci_base_state_machine_change_state( &phy->state_machine, SCI_BASE_PHY_STATE_STOPPED ); return SCI_SUCCESS; } //***************************************************************************** //* SCIC SDS PHY EVENT_HANDLERS //***************************************************************************** /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SPEED_EN. * - decode the event * - sas phy detected causes a state transition to the wait for speed * event notification. * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on any valid event notification * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_ossp_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_SAS_PHY_DETECTED: scic_sds_phy_start_sas_link_training(this_phy); this_phy->is_in_link_training = TRUE; break; case SCU_EVENT_SATA_SPINUP_HOLD: scic_sds_phy_start_sata_link_training(this_phy); this_phy->is_in_link_training = TRUE; break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SPEED_EN. * - decode the event * - sas phy detected returns us back to this state. * - speed event detected causes a state transition to the wait for iaf. * - identify timeout is an un-expected event and the state machine is * restarted. * - link failure events restart the starting state machine * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on any valid event notification * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sas_phy_speed_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_SAS_PHY_DETECTED: // Why is this being reported again by the controller? // We would re-enter this state so just stay here break; case SCU_EVENT_SAS_15: case SCU_EVENT_SAS_15_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_150_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF ); break; case SCU_EVENT_SAS_30: case SCU_EVENT_SAS_30_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_300_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF ); break; case SCU_EVENT_SAS_60: case SCU_EVENT_SAS_60_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_600_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF ); break; case SCU_EVENT_SATA_SPINUP_HOLD: // We were doing SAS PHY link training and received a SATA PHY event // continue OOB/SN as if this were a SATA PHY scic_sds_phy_start_sata_link_training(this_phy); break; case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF. * - decode the event * - sas phy detected event backs up the state machine to the await * speed notification. * - identify timeout is an un-expected event and the state machine is * restarted. * - link failure events restart the starting state machine * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on any valid event notification * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_iaf_uf_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_SAS_PHY_DETECTED: // Backup the state machine scic_sds_phy_start_sas_link_training(this_phy); break; case SCU_EVENT_SATA_SPINUP_HOLD: // We were doing SAS PHY link training and received a SATA PHY event // continue OOB/SN as if this were a SATA PHY scic_sds_phy_start_sata_link_training(this_phy); break; case SCU_EVENT_RECEIVED_IDENTIFY_TIMEOUT: case SCU_EVENT_LINK_FAILURE: case SCU_EVENT_HARD_RESET_RECEIVED: // Start the oob/sn state machine over again scic_sds_phy_restart_starting_state(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_POWER. * - decode the event * - link failure events restart the starting state machine * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on a link failure event * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sas_power_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER. * - decode the event * - link failure events restart the starting state machine * - sata spinup hold events are ignored since they are expected * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on a link failure event * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sata_power_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; case SCU_EVENT_SATA_SPINUP_HOLD: // These events are received every 10ms and are expected while in this state break; case SCU_EVENT_SAS_PHY_DETECTED: // There has been a change in the phy type before OOB/SN for the // SATA finished start down the SAS link traning path. scic_sds_phy_start_sas_link_training(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN. * - decode the event * - link failure events restart the starting state machine * - sata spinup hold events are ignored since they are expected * - sata phy detected event change to the wait speed event * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on a link failure event * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sata_phy_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; case SCU_EVENT_SATA_SPINUP_HOLD: // These events might be received since we dont know how many may be in // the completion queue while waiting for power break; case SCU_EVENT_SATA_PHY_DETECTED: this_phy->protocol = SCIC_SDS_PHY_PROTOCOL_SATA; // We have received the SATA PHY notification change state sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN ); break; case SCU_EVENT_SAS_PHY_DETECTED: // There has been a change in the phy type before OOB/SN for the // SATA finished start down the SAS link traning path. scic_sds_phy_start_sas_link_training(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state * SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN. * - decode the event * - sata phy detected returns us back to this state. * - speed event detected causes a state transition to the wait for * signature. * - link failure events restart the starting state machine * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on any valid event notification * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sata_speed_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_SATA_PHY_DETECTED: // The hardware reports multiple SATA PHY detected events // ignore the extras break; case SCU_EVENT_SATA_15: case SCU_EVENT_SATA_15_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_150_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF ); break; case SCU_EVENT_SATA_30: case SCU_EVENT_SATA_30_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_300_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF ); break; case SCU_EVENT_SATA_60: case SCU_EVENT_SATA_60_SSC: scic_sds_phy_complete_link_training( this_phy, SCI_SAS_600_GB, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF ); break; case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; case SCU_EVENT_SAS_PHY_DETECTED: // There has been a change in the phy type before OOB/SN for the // SATA finished start down the SAS link traning path. scic_sds_phy_start_sas_link_training(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } /** * This method is called when an event notification is received for the phy * object when in the state SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF. * - decode the event * - sas phy detected event backs up the state machine to the await * speed notification. * - identify timeout is an un-expected event and the state machine is * restarted. * - link failure events restart the starting state machine * - any other events log a warning message and set a failure status * * @param[in] phy This SCIC_SDS_PHY object which has received an event. * @param[in] event_code This is the event code which the phy object is to * decode. * * @return SCI_STATUS * @retval SCI_SUCCESS on any valid event notification * @retval SCI_FAILURE on any unexpected event notifation */ static SCI_STATUS scic_sds_phy_starting_substate_await_sig_fis_event_handler( SCIC_SDS_PHY_T *this_phy, U32 event_code ) { U32 result = SCI_SUCCESS; switch (scu_get_event_code(event_code)) { case SCU_EVENT_SATA_PHY_DETECTED: // Backup the state machine sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN ); break; case SCU_EVENT_LINK_FAILURE: // Link failure change state back to the starting state scic_sds_phy_restart_starting_state(this_phy); break; default: SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_RECEIVED_EVENTS, - "PHY starting substate machine recieved unexpected event_code %x\n", + "PHY starting substate machine received unexpected event_code %x\n", event_code )); result = SCI_FAILURE; break; } return result; } //***************************************************************************** //* SCIC SDS PHY FRAME_HANDLERS //***************************************************************************** /** * This method decodes the unsolicited frame when the SCIC_SDS_PHY is in the * SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF. * - Get the UF Header * - If the UF is an IAF * - Copy IAF data to local phy object IAF data buffer. * - Change starting substate to wait power. * - else * - log warning message of unexpected unsolicted frame * - release frame buffer * * @param[in] phy This is SCIC_SDS_PHY object which is being requested to * decode the frame data. * @param[in] frame_index This is the index of the unsolicited frame which was * received for this phy. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_starting_substate_await_iaf_uf_frame_handler( SCIC_SDS_PHY_T *this_phy, U32 frame_index ) { SCI_STATUS result; U32 *frame_words; SCI_SAS_IDENTIFY_ADDRESS_FRAME_T *identify_frame; result = scic_sds_unsolicited_frame_control_get_header( &(scic_sds_phy_get_controller(this_phy)->uf_control), frame_index, (void **)&frame_words); if (result != SCI_SUCCESS) { return result; } frame_words[0] = SCIC_SWAP_DWORD(frame_words[0]); identify_frame = (SCI_SAS_IDENTIFY_ADDRESS_FRAME_T *)frame_words; if (identify_frame->address_frame_type == 0) { // Byte swap the rest of the frame so we can make // a copy of the buffer frame_words[1] = SCIC_SWAP_DWORD(frame_words[1]); frame_words[2] = SCIC_SWAP_DWORD(frame_words[2]); frame_words[3] = SCIC_SWAP_DWORD(frame_words[3]); frame_words[4] = SCIC_SWAP_DWORD(frame_words[4]); frame_words[5] = SCIC_SWAP_DWORD(frame_words[5]); memcpy( &this_phy->phy_type.sas.identify_address_frame_buffer, identify_frame, sizeof(SCI_SAS_IDENTIFY_ADDRESS_FRAME_T) ); if (identify_frame->protocols.u.bits.smp_target) { // We got the IAF for an expander PHY go to the final state since // there are no power requirements for expander phys. sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL ); } else { // We got the IAF we can now go to the await spinup semaphore state sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER ); } result = SCI_SUCCESS; } else { SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_UNSOLICITED_FRAMES, - "PHY starting substate machine recieved unexpected frame id %x\n", + "PHY starting substate machine received unexpected frame id %x\n", frame_index )); } // Regardless of the result release this frame since we are done with it scic_sds_controller_release_frame( scic_sds_phy_get_controller(this_phy), frame_index ); return result; } /** * This method decodes the unsolicited frame when the SCIC_SDS_PHY is in the * SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF. * - Get the UF Header * - If the UF is an SIGNATURE FIS * - Copy IAF data to local phy object SIGNATURE FIS data buffer. * - else * - log warning message of unexpected unsolicted frame * - release frame buffer * * @param[in] phy This is SCIC_SDS_PHY object which is being requested to * decode the frame data. * @param[in] frame_index This is the index of the unsolicited frame which was * received for this phy. * * @return SCI_STATUS * @retval SCI_SUCCESS * * @todo Must decode the SIGNATURE FIS data */ static SCI_STATUS scic_sds_phy_starting_substate_await_sig_fis_frame_handler( SCIC_SDS_PHY_T *this_phy, U32 frame_index ) { SCI_STATUS result; U32 * frame_words; SATA_FIS_HEADER_T * fis_frame_header; U32 * fis_frame_data; result = scic_sds_unsolicited_frame_control_get_header( &(scic_sds_phy_get_controller(this_phy)->uf_control), frame_index, (void **)&frame_words); if (result != SCI_SUCCESS) { return result; } fis_frame_header = (SATA_FIS_HEADER_T *)frame_words; if ( (fis_frame_header->fis_type == SATA_FIS_TYPE_REGD2H) && !(fis_frame_header->status & ATA_STATUS_REG_BSY_BIT) ) { scic_sds_unsolicited_frame_control_get_buffer( &(scic_sds_phy_get_controller(this_phy)->uf_control), frame_index, (void **)&fis_frame_data ); scic_sds_controller_copy_sata_response( &this_phy->phy_type.sata.signature_fis_buffer, frame_words, fis_frame_data ); // We got the IAF we can now go to the await spinup semaphore state sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL ); result = SCI_SUCCESS; } else { SCIC_LOG_WARNING(( sci_base_object_get_logger(this_phy), SCIC_LOG_OBJECT_PHY | SCIC_LOG_OBJECT_UNSOLICITED_FRAMES, - "PHY starting substate machine recieved unexpected frame id %x\n", + "PHY starting substate machine received unexpected frame id %x\n", frame_index )); } // Regardless of the result release this frame since we are done with it scic_sds_controller_release_frame( scic_sds_phy_get_controller(this_phy), frame_index ); return result; } //***************************************************************************** //* SCIC SDS PHY POWER_HANDLERS //***************************************************************************** /** * This method is called by the SCIC_SDS_CONTROLLER when the phy object is * granted power. * - The notify enable spinups are turned on for this phy object * - The phy state machine is transitioned to the * SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_starting_substate_await_sas_power_consume_power_handler( SCIC_SDS_PHY_T *this_phy ) { U32 enable_spinup; enable_spinup = SCU_SAS_ENSPINUP_READ(this_phy); enable_spinup |= SCU_ENSPINUP_GEN_BIT(ENABLE); SCU_SAS_ENSPINUP_WRITE(this_phy, enable_spinup); // Change state to the final state this substate machine has run to completion sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL ); return SCI_SUCCESS; } /** * This method is called by the SCIC_SDS_CONTROLLER when the phy object is * granted power. * - The phy state machine is transitioned to the * SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN. * * @param[in] phy This is the SCI_BASE_PHY object which is cast into a * SCIC_SDS_PHY object. * * @return SCI_STATUS * @retval SCI_SUCCESS */ static SCI_STATUS scic_sds_phy_starting_substate_await_sata_power_consume_power_handler( SCIC_SDS_PHY_T *this_phy ) { U32 scu_sas_pcfg_value; // Release the spinup hold state and reset the OOB state machine scu_sas_pcfg_value = SCU_SAS_PCFG_READ(this_phy); scu_sas_pcfg_value &= ~(SCU_SAS_PCFG_GEN_BIT(SATA_SPINUP_HOLD) | SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE)); scu_sas_pcfg_value |= SCU_SAS_PCFG_GEN_BIT(OOB_RESET); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); // Now restart the OOB operation scu_sas_pcfg_value &= ~SCU_SAS_PCFG_GEN_BIT(OOB_RESET); scu_sas_pcfg_value |= SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE); SCU_SAS_PCFG_WRITE(this_phy, scu_sas_pcfg_value); // Change state to the final state this substate machine has run to completion sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN ); return SCI_SUCCESS; } // --------------------------------------------------------------------------- SCIC_SDS_PHY_STATE_HANDLER_T scic_sds_phy_starting_substate_handler_table[SCIC_SDS_PHY_STARTING_MAX_SUBSTATES] = { // SCIC_SDS_PHY_STARTING_SUBSTATE_INITIAL { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_OSSP_EN { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_ossp_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_SPEED_EN { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_sas_phy_speed_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF { { scic_sds_phy_default_start_handler, scic_sds_phy_default_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_starting_substate_await_iaf_uf_frame_handler, scic_sds_phy_starting_substate_await_iaf_uf_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_sas_power_event_handler, scic_sds_phy_starting_substate_await_sas_power_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER, { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_sata_power_event_handler, scic_sds_phy_starting_substate_await_sata_power_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN, { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_sata_phy_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN, { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_starting_substate_await_sata_speed_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF, { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_starting_substate_await_sig_fis_frame_handler, scic_sds_phy_starting_substate_await_sig_fis_event_handler, scic_sds_phy_default_consume_power_handler }, // SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL { { scic_sds_phy_default_start_handler, scic_sds_phy_starting_substate_general_stop_handler, scic_sds_phy_default_reset_handler, scic_sds_phy_default_destroy_handler }, scic_sds_phy_default_frame_handler, scic_sds_phy_default_event_handler, scic_sds_phy_default_consume_power_handler } }; /** * This macro sets the starting substate handlers by state_id */ #define scic_sds_phy_set_starting_substate_handlers(phy, state_id) \ scic_sds_phy_set_state_handlers( \ (phy), \ &scic_sds_phy_starting_substate_handler_table[(state_id)] \ ) //**************************************************************************** //* PHY STARTING SUBSTATE METHODS //**************************************************************************** /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_INITIAL. * - The initial state handlers are put in place for the SCIC_SDS_PHY * object. * - The state is changed to the wait phy type event notification. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_initial_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_INITIAL); // This is just an temporary state go off to the starting state sci_base_state_machine_change_state( scic_sds_phy_get_starting_substate_machine(this_phy), SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_OSSP_EN ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_PHY_TYPE_EN. * - Set the SCIC_SDS_PHY object state handlers for this state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_ossp_en_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_OSSP_EN ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SPEED_EN. * - Set the SCIC_SDS_PHY object state handlers for this state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sas_speed_en_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_SPEED_EN ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF. * - Set the SCIC_SDS_PHY object state handlers for this state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_iaf_uf_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER. * - Set the SCIC_SDS_PHY object state handlers for this state. * - Add this phy object to the power control queue * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sas_power_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER ); scic_sds_controller_power_control_queue_insert( scic_sds_phy_get_controller(this_phy), this_phy ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER. * - Remove the SCIC_SDS_PHY object from the power control queue. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sas_power_substate_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_controller_power_control_queue_remove( scic_sds_phy_get_controller(this_phy), this_phy ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER. * - Set the SCIC_SDS_PHY object state handlers for this state. * - Add this phy object to the power control queue * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_power_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER ); scic_sds_controller_power_control_queue_insert( scic_sds_phy_get_controller(this_phy), this_phy ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER. * - Remove the SCIC_SDS_PHY object from the power control queue. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_power_substate_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_controller_power_control_queue_remove( scic_sds_phy_get_controller(this_phy), this_phy ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN. * - Set the SCIC_SDS_PHY object state handlers for this state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_phy_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN ); scic_cb_timer_start( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer, SCIC_SDS_SATA_LINK_TRAINING_TIMEOUT ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN. * - stop the timer that was started on entry to await sata phy * event notification * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_phy_substate_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_cb_timer_stop( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN. * - Set the SCIC_SDS_PHY object state handlers for this state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_speed_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN ); scic_cb_timer_start( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer, SCIC_SDS_SATA_LINK_TRAINING_TIMEOUT ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN. * - stop the timer that was started on entry to await sata phy * event notification * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sata_speed_substate_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_cb_timer_stop( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF. * - Set the SCIC_SDS_PHY object state handlers for this state. * - Start the SIGNATURE FIS timeout timer * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sig_fis_uf_substate_enter( SCI_BASE_OBJECT_T *object ) { BOOL continue_to_ready_state; SCIC_SDS_PHY_T * this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF ); continue_to_ready_state = scic_sds_port_link_detected( this_phy->owning_port, this_phy ); if (continue_to_ready_state) { // Clear the PE suspend condition so we can actually receive SIG FIS // The hardware will not respond to the XRDY until the PE suspend // condition is cleared. scic_sds_phy_resume(this_phy); scic_cb_timer_start( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer, SCIC_SDS_SIGNATURE_FIS_TIMEOUT ); } else { this_phy->is_in_link_training = FALSE; } } /** * This method will perform the actions required by the SCIC_SDS_PHY on * exiting the SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF. * - Stop the SIGNATURE FIS timeout timer. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_await_sig_fis_uf_substate_exit( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_cb_timer_stop( scic_sds_phy_get_controller(this_phy), this_phy->sata_timeout_timer ); } /** * This method will perform the actions required by the SCIC_SDS_PHY on * entering the SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL. * - Set the SCIC_SDS_PHY object state handlers for this state. * - Change base state machine to the ready state. * * @param[in] object This is the SCI_BASE_OBJECT which is cast to a * SCIC_SDS_PHY object. * * @return none */ static void scic_sds_phy_starting_final_substate_enter( SCI_BASE_OBJECT_T *object ) { SCIC_SDS_PHY_T *this_phy; this_phy = (SCIC_SDS_PHY_T *)object; scic_sds_phy_set_starting_substate_handlers( this_phy, SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL ); // State machine has run to completion so exit out and change // the base state machine to the ready state sci_base_state_machine_change_state( scic_sds_phy_get_base_state_machine(this_phy), SCI_BASE_PHY_STATE_READY); } // --------------------------------------------------------------------------- SCI_BASE_STATE_T scic_sds_phy_starting_substates[SCIC_SDS_PHY_STARTING_MAX_SUBSTATES] = { { SCIC_SDS_PHY_STARTING_SUBSTATE_INITIAL, scic_sds_phy_starting_initial_substate_enter, NULL, }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_OSSP_EN, scic_sds_phy_starting_await_ossp_en_substate_enter, NULL, }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_SPEED_EN, scic_sds_phy_starting_await_sas_speed_en_substate_enter, NULL, }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_IAF_UF, scic_sds_phy_starting_await_iaf_uf_substate_enter, NULL, }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SAS_POWER, scic_sds_phy_starting_await_sas_power_substate_enter, scic_sds_phy_starting_await_sas_power_substate_exit, }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_POWER, scic_sds_phy_starting_await_sata_power_substate_enter, scic_sds_phy_starting_await_sata_power_substate_exit }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_PHY_EN, scic_sds_phy_starting_await_sata_phy_substate_enter, scic_sds_phy_starting_await_sata_phy_substate_exit }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SATA_SPEED_EN, scic_sds_phy_starting_await_sata_speed_substate_enter, scic_sds_phy_starting_await_sata_speed_substate_exit }, { SCIC_SDS_PHY_STARTING_SUBSTATE_AWAIT_SIG_FIS_UF, scic_sds_phy_starting_await_sig_fis_uf_substate_enter, scic_sds_phy_starting_await_sig_fis_uf_substate_exit }, { SCIC_SDS_PHY_STARTING_SUBSTATE_FINAL, scic_sds_phy_starting_final_substate_enter, NULL, } }; Index: head/sys/dev/pccbb/pccbb_pci.c =================================================================== --- head/sys/dev/pccbb/pccbb_pci.c (revision 298930) +++ head/sys/dev/pccbb/pccbb_pci.c (revision 298931) @@ -1,968 +1,968 @@ /*- * Copyright (c) 2002-2004 M. Warner Losh. * Copyright (c) 2000-2001 Jonathan Chen. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /*- * Copyright (c) 1998, 1999 and 2000 * HAYAKAWA Koichi. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by HAYAKAWA Koichi. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Driver for PCI to CardBus Bridge chips * * References: * TI Datasheets: * http://www-s.ti.com/cgi-bin/sc/generic2.cgi?family=PCI+CARDBUS+CONTROLLERS * * Written by Jonathan Chen * The author would like to acknowledge: * * HAYAKAWA Koichi: Author of the NetBSD code for the same thing * * Warner Losh: Newbus/newcard guru and author of the pccard side of things * * YAMAMOTO Shigeru: Author of another FreeBSD cardbus driver * * David Cross: Author of the initial ugly hack for a specific cardbus card */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "power_if.h" #include "card_if.h" #include "pcib_if.h" #define DPRINTF(x) do { if (cbb_debug) printf x; } while (0) #define DEVPRINTF(x) do { if (cbb_debug) device_printf x; } while (0) #define PCI_MASK_CONFIG(DEV,REG,MASK,SIZE) \ pci_write_config(DEV, REG, pci_read_config(DEV, REG, SIZE) MASK, SIZE) #define PCI_MASK2_CONFIG(DEV,REG,MASK1,MASK2,SIZE) \ pci_write_config(DEV, REG, ( \ pci_read_config(DEV, REG, SIZE) MASK1) MASK2, SIZE) static void cbb_chipinit(struct cbb_softc *sc); static int cbb_pci_filt(void *arg); static struct yenta_chipinfo { uint32_t yc_id; const char *yc_name; int yc_chiptype; } yc_chipsets[] = { /* Texas Instruments chips */ {PCIC_ID_TI1031, "TI1031 PCI-PC Card Bridge", CB_TI113X}, {PCIC_ID_TI1130, "TI1130 PCI-CardBus Bridge", CB_TI113X}, {PCIC_ID_TI1131, "TI1131 PCI-CardBus Bridge", CB_TI113X}, {PCIC_ID_TI1210, "TI1210 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1211, "TI1211 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1220, "TI1220 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1221, "TI1221 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1225, "TI1225 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1250, "TI1250 PCI-CardBus Bridge", CB_TI125X}, {PCIC_ID_TI1251, "TI1251 PCI-CardBus Bridge", CB_TI125X}, {PCIC_ID_TI1251B,"TI1251B PCI-CardBus Bridge",CB_TI125X}, {PCIC_ID_TI1260, "TI1260 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1260B,"TI1260B PCI-CardBus Bridge",CB_TI12XX}, {PCIC_ID_TI1410, "TI1410 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1420, "TI1420 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1421, "TI1421 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1450, "TI1450 PCI-CardBus Bridge", CB_TI125X}, /*SIC!*/ {PCIC_ID_TI1451, "TI1451 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1510, "TI1510 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI1520, "TI1520 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI4410, "TI4410 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI4450, "TI4450 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI4451, "TI4451 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI4510, "TI4510 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI6411, "TI6411 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI6420, "TI6420 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI6420SC, "TI6420 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7410, "TI7410 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7510, "TI7510 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7610, "TI7610 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7610M, "TI7610 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7610SD, "TI7610 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_TI7610MS, "TI7610 PCI-CardBus Bridge", CB_TI12XX}, /* ENE */ {PCIC_ID_ENE_CB710, "ENE CB710 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_ENE_CB720, "ENE CB720 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_ENE_CB1211, "ENE CB1211 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_ENE_CB1225, "ENE CB1225 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_ENE_CB1410, "ENE CB1410 PCI-CardBus Bridge", CB_TI12XX}, {PCIC_ID_ENE_CB1420, "ENE CB1420 PCI-CardBus Bridge", CB_TI12XX}, /* Ricoh chips */ {PCIC_ID_RICOH_RL5C465, "RF5C465 PCI-CardBus Bridge", CB_RF5C46X}, {PCIC_ID_RICOH_RL5C466, "RF5C466 PCI-CardBus Bridge", CB_RF5C46X}, {PCIC_ID_RICOH_RL5C475, "RF5C475 PCI-CardBus Bridge", CB_RF5C47X}, {PCIC_ID_RICOH_RL5C476, "RF5C476 PCI-CardBus Bridge", CB_RF5C47X}, {PCIC_ID_RICOH_RL5C477, "RF5C477 PCI-CardBus Bridge", CB_RF5C47X}, {PCIC_ID_RICOH_RL5C478, "RF5C478 PCI-CardBus Bridge", CB_RF5C47X}, /* Toshiba products */ {PCIC_ID_TOPIC95, "ToPIC95 PCI-CardBus Bridge", CB_TOPIC95}, {PCIC_ID_TOPIC95B, "ToPIC95B PCI-CardBus Bridge", CB_TOPIC95}, {PCIC_ID_TOPIC97, "ToPIC97 PCI-CardBus Bridge", CB_TOPIC97}, {PCIC_ID_TOPIC100, "ToPIC100 PCI-CardBus Bridge", CB_TOPIC97}, /* Cirrus Logic */ {PCIC_ID_CLPD6832, "CLPD6832 PCI-CardBus Bridge", CB_CIRRUS}, {PCIC_ID_CLPD6833, "CLPD6833 PCI-CardBus Bridge", CB_CIRRUS}, {PCIC_ID_CLPD6834, "CLPD6834 PCI-CardBus Bridge", CB_CIRRUS}, /* 02Micro */ {PCIC_ID_OZ6832, "O2Micro OZ6832/6833 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ6860, "O2Micro OZ6836/6860 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ6872, "O2Micro OZ6812/6872 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ6912, "O2Micro OZ6912/6972 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ6922, "O2Micro OZ6922 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ6933, "O2Micro OZ6933 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711E1, "O2Micro OZ711E1 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711EC1, "O2Micro OZ711EC1/M1 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711E2, "O2Micro OZ711E2 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711M1, "O2Micro OZ711M1 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711M2, "O2Micro OZ711M2 PCI-CardBus Bridge", CB_O2MICRO}, {PCIC_ID_OZ711M3, "O2Micro OZ711M3 PCI-CardBus Bridge", CB_O2MICRO}, /* SMC */ {PCIC_ID_SMC_34C90, "SMC 34C90 PCI-CardBus Bridge", CB_CIRRUS}, /* sentinel */ {0 /* null id */, "unknown", CB_UNKNOWN}, }; /************************************************************************/ /* Probe/Attach */ /************************************************************************/ static int cbb_chipset(uint32_t pci_id, const char **namep) { struct yenta_chipinfo *ycp; for (ycp = yc_chipsets; ycp->yc_id != 0 && pci_id != ycp->yc_id; ++ycp) continue; if (namep != NULL) *namep = ycp->yc_name; return (ycp->yc_chiptype); } static int cbb_pci_probe(device_t brdev) { const char *name; uint32_t progif; uint32_t baseclass; uint32_t subclass; /* * Do we know that we support the chipset? If so, then we * accept the device. */ if (cbb_chipset(pci_get_devid(brdev), &name) != CB_UNKNOWN) { device_set_desc(brdev, name); return (BUS_PROBE_DEFAULT); } /* * We do support generic CardBus bridges. All that we've seen * to date have progif 0 (the Yenta spec, and successors mandate * this). */ baseclass = pci_get_class(brdev); subclass = pci_get_subclass(brdev); progif = pci_get_progif(brdev); if (baseclass == PCIC_BRIDGE && subclass == PCIS_BRIDGE_CARDBUS && progif == 0) { device_set_desc(brdev, "PCI-CardBus Bridge"); return (BUS_PROBE_GENERIC); } return (ENXIO); } /* * Print out the config space */ static void cbb_print_config(device_t dev) { int i; device_printf(dev, "PCI Configuration space:"); for (i = 0; i < 256; i += 4) { if (i % 16 == 0) printf("\n 0x%02x: ", i); printf("0x%08x ", pci_read_config(dev, i, 4)); } printf("\n"); } static int cbb_pci_attach(device_t brdev) { #if !(defined(NEW_PCIB) && defined(PCI_RES_BUS)) static int curr_bus_number = 2; /* XXX EVILE BAD (see below) */ uint32_t pribus; #endif struct cbb_softc *sc = (struct cbb_softc *)device_get_softc(brdev); struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; int rid; device_t parent; parent = device_get_parent(brdev); mtx_init(&sc->mtx, device_get_nameunit(brdev), "cbb", MTX_DEF); sc->chipset = cbb_chipset(pci_get_devid(brdev), NULL); sc->dev = brdev; sc->cbdev = NULL; sc->exca[0].pccarddev = NULL; sc->domain = pci_get_domain(brdev); sc->pribus = pcib_get_bus(parent); #if defined(NEW_PCIB) && defined(PCI_RES_BUS) pci_write_config(brdev, PCIR_PRIBUS_2, sc->pribus, 1); pcib_setup_secbus(brdev, &sc->bus, 1); #else sc->bus.sec = pci_read_config(brdev, PCIR_SECBUS_2, 1); sc->bus.sub = pci_read_config(brdev, PCIR_SUBBUS_2, 1); #endif SLIST_INIT(&sc->rl); rid = CBBR_SOCKBASE; sc->base_res = bus_alloc_resource_any(brdev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!sc->base_res) { device_printf(brdev, "Could not map register memory\n"); mtx_destroy(&sc->mtx); return (ENOMEM); } else { DEVPRINTF((brdev, "Found memory at %jx\n", rman_get_start(sc->base_res))); } sc->bst = rman_get_bustag(sc->base_res); sc->bsh = rman_get_bushandle(sc->base_res); exca_init(&sc->exca[0], brdev, sc->bst, sc->bsh, CBB_EXCA_OFFSET); sc->exca[0].flags |= EXCA_HAS_MEMREG_WIN; sc->exca[0].chipset = EXCA_CARDBUS; sc->chipinit = cbb_chipinit; sc->chipinit(sc); /*Sysctls*/ sctx = device_get_sysctl_ctx(brdev); soid = device_get_sysctl_tree(brdev); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "domain", CTLFLAG_RD, &sc->domain, 0, "Domain number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "pribus", CTLFLAG_RD, &sc->pribus, 0, "Primary bus number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "secbus", CTLFLAG_RD, &sc->bus.sec, 0, "Secondary bus number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "subbus", CTLFLAG_RD, &sc->bus.sub, 0, "Subordinate bus number"); #if 0 SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "memory", CTLFLAG_RD, &sc->subbus, 0, "Memory window open"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "premem", - CTLFLAG_RD, &sc->subbus, 0, "Prefetch memroy window open"); + CTLFLAG_RD, &sc->subbus, 0, "Prefetch memory window open"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "io1", CTLFLAG_RD, &sc->subbus, 0, "io range 1 open"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "io2", CTLFLAG_RD, &sc->subbus, 0, "io range 2 open"); #endif #if !(defined(NEW_PCIB) && defined(PCI_RES_BUS)) /* * This is a gross hack. We should be scanning the entire pci * tree, assigning bus numbers in a way such that we (1) can * reserve 1 extra bus just in case and (2) all sub busses * are in an appropriate range. */ DEVPRINTF((brdev, "Secondary bus is %d\n", sc->bus.sec)); pribus = pci_read_config(brdev, PCIR_PRIBUS_2, 1); if (sc->bus.sec == 0 || sc->pribus != pribus) { if (curr_bus_number <= sc->pribus) curr_bus_number = sc->pribus + 1; if (pribus != sc->pribus) { DEVPRINTF((brdev, "Setting primary bus to %d\n", sc->pribus)); pci_write_config(brdev, PCIR_PRIBUS_2, sc->pribus, 1); } sc->bus.sec = curr_bus_number++; sc->bus.sub = curr_bus_number++; DEVPRINTF((brdev, "Secondary bus set to %d subbus %d\n", sc->bus.sec, sc->bus.sub)); pci_write_config(brdev, PCIR_SECBUS_2, sc->bus.sec, 1); pci_write_config(brdev, PCIR_SUBBUS_2, sc->bus.sub, 1); } #endif /* attach children */ sc->cbdev = device_add_child(brdev, "cardbus", -1); if (sc->cbdev == NULL) DEVPRINTF((brdev, "WARNING: cannot add cardbus bus.\n")); else if (device_probe_and_attach(sc->cbdev) != 0) DEVPRINTF((brdev, "WARNING: cannot attach cardbus bus!\n")); sc->exca[0].pccarddev = device_add_child(brdev, "pccard", -1); if (sc->exca[0].pccarddev == NULL) DEVPRINTF((brdev, "WARNING: cannot add pccard bus.\n")); else if (device_probe_and_attach(sc->exca[0].pccarddev) != 0) DEVPRINTF((brdev, "WARNING: cannot attach pccard bus.\n")); /* Map and establish the interrupt. */ rid = 0; sc->irq_res = bus_alloc_resource_any(brdev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (sc->irq_res == NULL) { device_printf(brdev, "Unable to map IRQ...\n"); goto err; } if (bus_setup_intr(brdev, sc->irq_res, INTR_TYPE_AV | INTR_MPSAFE, cbb_pci_filt, NULL, sc, &sc->intrhand)) { device_printf(brdev, "couldn't establish interrupt\n"); goto err; } /* reset 16-bit pcmcia bus */ exca_clrb(&sc->exca[0], EXCA_INTR, EXCA_INTR_RESET); /* turn off power */ cbb_power(brdev, CARD_OFF); /* CSC Interrupt: Card detect interrupt on */ cbb_setb(sc, CBB_SOCKET_MASK, CBB_SOCKET_MASK_CD); /* reset interrupt */ cbb_set(sc, CBB_SOCKET_EVENT, cbb_get(sc, CBB_SOCKET_EVENT)); if (bootverbose) cbb_print_config(brdev); /* Start the thread */ if (kproc_create(cbb_event_thread, sc, &sc->event_thread, 0, 0, "%s event thread", device_get_nameunit(brdev))) { device_printf(brdev, "unable to create event thread.\n"); panic("cbb_create_event_thread"); } sc->sc_root_token = root_mount_hold(device_get_nameunit(sc->dev)); return (0); err: if (sc->irq_res) bus_release_resource(brdev, SYS_RES_IRQ, 0, sc->irq_res); if (sc->base_res) { bus_release_resource(brdev, SYS_RES_MEMORY, CBBR_SOCKBASE, sc->base_res); } mtx_destroy(&sc->mtx); return (ENOMEM); } static void cbb_chipinit(struct cbb_softc *sc) { uint32_t mux, sysctrl, reg; /* Set CardBus latency timer */ if (pci_read_config(sc->dev, PCIR_SECLAT_2, 1) < 0x20) pci_write_config(sc->dev, PCIR_SECLAT_2, 0x20, 1); /* Set PCI latency timer */ if (pci_read_config(sc->dev, PCIR_LATTIMER, 1) < 0x20) pci_write_config(sc->dev, PCIR_LATTIMER, 0x20, 1); - /* Enable DMA, memory access for this card and I/O acces for children */ + /* Enable DMA, memory access for this card and I/O access for children */ pci_enable_busmaster(sc->dev); pci_enable_io(sc->dev, SYS_RES_IOPORT); pci_enable_io(sc->dev, SYS_RES_MEMORY); /* disable Legacy IO */ switch (sc->chipset) { case CB_RF5C46X: PCI_MASK_CONFIG(sc->dev, CBBR_BRIDGECTRL, & ~(CBBM_BRIDGECTRL_RL_3E0_EN | CBBM_BRIDGECTRL_RL_3E2_EN), 2); break; default: pci_write_config(sc->dev, CBBR_LEGACY, 0x0, 4); break; } /* Use PCI interrupt for interrupt routing */ PCI_MASK2_CONFIG(sc->dev, CBBR_BRIDGECTRL, & ~(CBBM_BRIDGECTRL_MASTER_ABORT | CBBM_BRIDGECTRL_INTR_IREQ_ISA_EN), | CBBM_BRIDGECTRL_WRITE_POST_EN, 2); /* * XXX this should be a function table, ala OLDCARD. This means * that we could more easily support ISA interrupts for pccard * cards if we had to. */ switch (sc->chipset) { case CB_TI113X: /* * The TI 1031, TI 1130 and TI 1131 all require another bit * be set to enable PCI routing of interrupts, and then * a bit for each of the CSC and Function interrupts we * want routed. */ PCI_MASK_CONFIG(sc->dev, CBBR_CBCTRL, | CBBM_CBCTRL_113X_PCI_INTR | CBBM_CBCTRL_113X_PCI_CSC | CBBM_CBCTRL_113X_PCI_IRQ_EN, 1); PCI_MASK_CONFIG(sc->dev, CBBR_DEVCTRL, & ~(CBBM_DEVCTRL_INT_SERIAL | CBBM_DEVCTRL_INT_PCI), 1); break; case CB_TI12XX: /* * Some TI 12xx (and [14][45]xx) based pci cards * sometimes have issues with the MFUNC register not * being initialized due to a bad EEPROM on board. * Laptops that this matters on have this register * properly initialized. * * The TI125X parts have a different register. * * Note: Only the lower two nibbles matter. When set * to 0, the MFUNC{0,1} pins are GPIO, which isn't * going to work out too well because we specifically * program these parts to parallel interrupt signalling * elsewhere. We preserve the upper bits of this * register since changing them have subtle side effects * for different variants of the card and are * extremely difficult to exaustively test. * * Also, the TI 1510/1520 changed the default for the MFUNC * register from 0x0 to 0x1000 to enable IRQSER by default. * We want to be careful to avoid overriding that, and the * below test will do that. Should this check prove to be * too permissive, we should just check against 0 and 0x1000 * and not touch it otherwise. */ mux = pci_read_config(sc->dev, CBBR_MFUNC, 4); sysctrl = pci_read_config(sc->dev, CBBR_SYSCTRL, 4); if ((mux & (CBBM_MFUNC_PIN0 | CBBM_MFUNC_PIN1)) == 0) { mux = (mux & ~CBBM_MFUNC_PIN0) | CBBM_MFUNC_PIN0_INTA; if ((sysctrl & CBBM_SYSCTRL_INTRTIE) == 0) mux = (mux & ~CBBM_MFUNC_PIN1) | CBBM_MFUNC_PIN1_INTB; pci_write_config(sc->dev, CBBR_MFUNC, mux, 4); } /*FALLTHROUGH*/ case CB_TI125X: /* * Disable zoom video. Some machines initialize this * improperly and exerpience has shown that this helps * prevent strange behavior. We don't support zoom * video anyway, so no harm can come from this. */ pci_write_config(sc->dev, CBBR_MMCTRL, 0, 4); break; case CB_O2MICRO: /* * Issue #1: INT# generated at the same time as * selected ISA IRQ. When IREQ# or STSCHG# is active, * in addition to the ISA IRQ being generated, INT# * will also be generated at the same time. * * Some of the older controllers have an issue in * which the slot's PCI INT# will be asserted whenever * IREQ# or STSCGH# is asserted even if ExCA registers * 03h or 05h have an ISA IRQ selected. * * The fix for this issue, which will work for any * controller (old or new), is to set ExCA registers * 3Ah (slot 0) & 7Ah (slot 1) bits 7:4 = 1010b. * These bits are undocumented. By setting this * register (of each slot) to '1010xxxxb' a routing of * IREQ# to INTC# and STSCHG# to INTC# is selected. * Since INTC# isn't connected there will be no * unexpected PCI INT when IREQ# or STSCHG# is active. * However, INTA# (slot 0) or INTB# (slot 1) will * still be correctly generated if NO ISA IRQ is * selected (ExCA regs 03h or 05h are cleared). */ reg = exca_getb(&sc->exca[0], EXCA_O2MICRO_CTRL_C); reg = (reg & 0x0f) | EXCA_O2CC_IREQ_INTC | EXCA_O2CC_STSCHG_INTC; exca_putb(&sc->exca[0], EXCA_O2MICRO_CTRL_C, reg); break; case CB_TOPIC97: /* * Disable Zoom Video, ToPIC 97, 100. */ pci_write_config(sc->dev, TOPIC97_ZV_CONTROL, 0, 1); /* * ToPIC 97, 100 * At offset 0xa1: INTERRUPT CONTROL register * 0x1: Turn on INT interrupts. */ PCI_MASK_CONFIG(sc->dev, TOPIC_INTCTRL, | TOPIC97_INTCTRL_INTIRQSEL, 1); /* * ToPIC97, 100 * Need to assert support for low voltage cards */ exca_setb(&sc->exca[0], EXCA_TOPIC97_CTRL, EXCA_TOPIC97_CTRL_LV_MASK); goto topic_common; case CB_TOPIC95: /* * SOCKETCTRL appears to be TOPIC 95/B specific */ PCI_MASK_CONFIG(sc->dev, TOPIC95_SOCKETCTRL, | TOPIC95_SOCKETCTRL_SCR_IRQSEL, 4); topic_common:; /* * At offset 0xa0: SLOT CONTROL * 0x80 Enable CardBus Functionality * 0x40 Enable CardBus and PC Card registers * 0x20 Lock ID in exca regs * 0x10 Write protect ID in config regs * Clear the rest of the bits, which defaults the slot * in legacy mode to 0x3e0 and offset 0. (legacy * mode is determined elsewhere) */ pci_write_config(sc->dev, TOPIC_SLOTCTRL, TOPIC_SLOTCTRL_SLOTON | TOPIC_SLOTCTRL_SLOTEN | TOPIC_SLOTCTRL_ID_LOCK | TOPIC_SLOTCTRL_ID_WP, 1); /* * At offset 0xa3 Card Detect Control Register * 0x80 CARDBUS enbale * 0x01 Cleared for hardware change detect */ PCI_MASK2_CONFIG(sc->dev, TOPIC_CDC, | TOPIC_CDC_CARDBUS, & ~TOPIC_CDC_SWDETECT, 4); break; } /* * Need to tell ExCA registers to CSC interrupts route via PCI * interrupts. There are two ways to do this. One is to set * INTR_ENABLE and the other is to set CSC to 0. Since both * methods are mutually compatible, we do both. */ exca_putb(&sc->exca[0], EXCA_INTR, EXCA_INTR_ENABLE); exca_putb(&sc->exca[0], EXCA_CSC_INTR, 0); cbb_disable_func_intr(sc); /* close all memory and io windows */ pci_write_config(sc->dev, CBBR_MEMBASE0, 0xffffffff, 4); pci_write_config(sc->dev, CBBR_MEMLIMIT0, 0, 4); pci_write_config(sc->dev, CBBR_MEMBASE1, 0xffffffff, 4); pci_write_config(sc->dev, CBBR_MEMLIMIT1, 0, 4); pci_write_config(sc->dev, CBBR_IOBASE0, 0xffffffff, 4); pci_write_config(sc->dev, CBBR_IOLIMIT0, 0, 4); pci_write_config(sc->dev, CBBR_IOBASE1, 0xffffffff, 4); pci_write_config(sc->dev, CBBR_IOLIMIT1, 0, 4); } static int cbb_route_interrupt(device_t pcib, device_t dev, int pin) { struct cbb_softc *sc = (struct cbb_softc *)device_get_softc(pcib); return (rman_get_start(sc->irq_res)); } static int cbb_pci_shutdown(device_t brdev) { struct cbb_softc *sc = (struct cbb_softc *)device_get_softc(brdev); /* * We're about to pull the rug out from the card, so mark it as * gone to prevent harm. */ sc->cardok = 0; /* * Place the cards in reset, turn off the interrupts and power * down the socket. */ PCI_MASK_CONFIG(brdev, CBBR_BRIDGECTRL, |CBBM_BRIDGECTRL_RESET, 2); exca_clrb(&sc->exca[0], EXCA_INTR, EXCA_INTR_RESET); cbb_set(sc, CBB_SOCKET_MASK, 0); cbb_set(sc, CBB_SOCKET_EVENT, 0xffffffff); cbb_power(brdev, CARD_OFF); /* * For paranoia, turn off all address decoding. Really not needed, * it seems, but it can't hurt */ exca_putb(&sc->exca[0], EXCA_ADDRWIN_ENABLE, 0); pci_write_config(brdev, CBBR_MEMBASE0, 0, 4); pci_write_config(brdev, CBBR_MEMLIMIT0, 0, 4); pci_write_config(brdev, CBBR_MEMBASE1, 0, 4); pci_write_config(brdev, CBBR_MEMLIMIT1, 0, 4); pci_write_config(brdev, CBBR_IOBASE0, 0, 4); pci_write_config(brdev, CBBR_IOLIMIT0, 0, 4); pci_write_config(brdev, CBBR_IOBASE1, 0, 4); pci_write_config(brdev, CBBR_IOLIMIT1, 0, 4); return (0); } static int cbb_pci_filt(void *arg) { struct cbb_softc *sc = arg; uint32_t sockevent; uint8_t csc; int retval = FILTER_STRAY; /* * Some chips also require us to read the old ExCA registe for card * status change when we route CSC vis PCI. This isn't supposed to be * required, but it clears the interrupt state on some chipsets. * Maybe there's a setting that would obviate its need. Maybe we * should test the status bits and deal with them, but so far we've * not found any machines that don't also give us the socket status * indication above. * * This call used to be unconditional. However, further research * suggests that we hit this condition when the card READY interrupt * fired. So now we only read it for 16-bit cards, and we only claim * the interrupt if READY is set. If this still causes problems, then * the next step would be to read this if we have a 16-bit card *OR* * we have no card. We treat the READY signal as if it were the power * completion signal. Some bridges may double signal things here, bit * signalling twice should be OK since we only sleep on the powerintr * in one place and a double wakeup would be benign there. */ if (sc->flags & CBB_16BIT_CARD) { csc = exca_getb(&sc->exca[0], EXCA_CSC); if (csc & EXCA_CSC_READY) { atomic_add_int(&sc->powerintr, 1); wakeup((void *)&sc->powerintr); retval = FILTER_HANDLED; } } /* * Read the socket event. Sometimes, the theory goes, the PCI bus is * so loaded that it cannot satisfy the read request, so we get * garbage back from the following read. We have to filter out the * garbage so that we don't spontaneously reset the card under high * load. PCI isn't supposed to act like this. No doubt this is a bug * in the PCI bridge chipset (or cbb brige) that's being used in * certain amd64 laptops today. Work around the issue by assuming * that any bits we don't know about being set means that we got * garbage. */ sockevent = cbb_get(sc, CBB_SOCKET_EVENT); if (sockevent != 0 && (sockevent & ~CBB_SOCKET_EVENT_VALID_MASK) == 0) { /* * If anything has happened to the socket, we assume that the * card is no longer OK, and we shouldn't call its ISR. We * set cardok as soon as we've attached the card. This helps * in a noisy eject, which happens all too often when users * are ejecting their PC Cards. * * We use this method in preference to checking to see if the * card is still there because the check suffers from a race * condition in the bouncing case. */ #define DELTA (CBB_SOCKET_MASK_CD) if (sockevent & DELTA) { cbb_clrb(sc, CBB_SOCKET_MASK, DELTA); cbb_set(sc, CBB_SOCKET_EVENT, DELTA); sc->cardok = 0; cbb_disable_func_intr(sc); wakeup(&sc->intrhand); } #undef DELTA /* * Wakeup anybody waiting for a power interrupt. We have to * use atomic_add_int for wakups on other cores. */ if (sockevent & CBB_SOCKET_EVENT_POWER) { cbb_clrb(sc, CBB_SOCKET_MASK, CBB_SOCKET_EVENT_POWER); cbb_set(sc, CBB_SOCKET_EVENT, CBB_SOCKET_EVENT_POWER); atomic_add_int(&sc->powerintr, 1); wakeup((void *)&sc->powerintr); } /* * Status change interrupts aren't presently used in the * rest of the driver. For now, just ACK them. */ if (sockevent & CBB_SOCKET_EVENT_CSTS) cbb_set(sc, CBB_SOCKET_EVENT, CBB_SOCKET_EVENT_CSTS); retval = FILTER_HANDLED; } return retval; } #if defined(NEW_PCIB) && defined(PCI_RES_BUS) static struct resource * cbb_pci_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct cbb_softc *sc; sc = device_get_softc(bus); if (type == PCI_RES_BUS) return (pcib_alloc_subbus(&sc->bus, child, rid, start, end, count, flags)); return (cbb_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static int cbb_pci_adjust_resource(device_t bus, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end) { struct cbb_softc *sc; sc = device_get_softc(bus); if (type == PCI_RES_BUS) { if (!rman_is_region_manager(r, &sc->bus.rman)) return (EINVAL); return (rman_adjust_resource(r, start, end)); } return (bus_generic_adjust_resource(bus, child, type, r, start, end)); } static int cbb_pci_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { struct cbb_softc *sc; int error; sc = device_get_softc(bus); if (type == PCI_RES_BUS) { if (!rman_is_region_manager(r, &sc->bus.rman)) return (EINVAL); if (rman_get_flags(r) & RF_ACTIVE) { error = bus_deactivate_resource(child, type, rid, r); if (error) return (error); } return (rman_release_resource(r)); } return (cbb_release_resource(bus, child, type, rid, r)); } #endif /************************************************************************/ /* PCI compat methods */ /************************************************************************/ static int cbb_maxslots(device_t brdev) { return (0); } static uint32_t cbb_read_config(device_t brdev, u_int b, u_int s, u_int f, u_int reg, int width) { /* * Pass through to the next ppb up the chain (i.e. our grandparent). */ return (PCIB_READ_CONFIG(device_get_parent(device_get_parent(brdev)), b, s, f, reg, width)); } static void cbb_write_config(device_t brdev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width) { /* * Pass through to the next ppb up the chain (i.e. our grandparent). */ PCIB_WRITE_CONFIG(device_get_parent(device_get_parent(brdev)), b, s, f, reg, val, width); } static int cbb_pci_suspend(device_t brdev) { int error = 0; struct cbb_softc *sc = device_get_softc(brdev); error = bus_generic_suspend(brdev); if (error != 0) return (error); cbb_set(sc, CBB_SOCKET_MASK, 0); /* Quiet hardware */ sc->cardok = 0; /* Card is bogus now */ return (0); } static int cbb_pci_resume(device_t brdev) { int error = 0; struct cbb_softc *sc = (struct cbb_softc *)device_get_softc(brdev); uint32_t tmp; /* * In the APM and early ACPI era, BIOSes saved the PCI config * registers. As chips became more complicated, that functionality moved * into the ACPI code / tables. We must therefore, restore the settings * we made here to make sure the device come back. Transitions to Dx * from D0 and back to D0 cause the bridge to lose its config space, so * all the bus mappings and such are preserved. * * The PCI layer handles standard PCI registers like the * command register and BARs, but cbb-specific registers are * handled here. */ sc->chipinit(sc); /* reset interrupt -- Do we really need to do this? */ tmp = cbb_get(sc, CBB_SOCKET_EVENT); cbb_set(sc, CBB_SOCKET_EVENT, tmp); /* CSC Interrupt: Card detect interrupt on */ cbb_setb(sc, CBB_SOCKET_MASK, CBB_SOCKET_MASK_CD); /* Signal the thread to wakeup. */ wakeup(&sc->intrhand); error = bus_generic_resume(brdev); return (error); } static device_method_t cbb_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cbb_pci_probe), DEVMETHOD(device_attach, cbb_pci_attach), DEVMETHOD(device_detach, cbb_detach), DEVMETHOD(device_shutdown, cbb_pci_shutdown), DEVMETHOD(device_suspend, cbb_pci_suspend), DEVMETHOD(device_resume, cbb_pci_resume), /* bus methods */ DEVMETHOD(bus_read_ivar, cbb_read_ivar), DEVMETHOD(bus_write_ivar, cbb_write_ivar), #if defined(NEW_PCIB) && defined(PCI_RES_BUS) DEVMETHOD(bus_alloc_resource, cbb_pci_alloc_resource), DEVMETHOD(bus_adjust_resource, cbb_pci_adjust_resource), DEVMETHOD(bus_release_resource, cbb_pci_release_resource), #else DEVMETHOD(bus_alloc_resource, cbb_alloc_resource), DEVMETHOD(bus_release_resource, cbb_release_resource), #endif DEVMETHOD(bus_activate_resource, cbb_activate_resource), DEVMETHOD(bus_deactivate_resource, cbb_deactivate_resource), DEVMETHOD(bus_driver_added, cbb_driver_added), DEVMETHOD(bus_child_detached, cbb_child_detached), DEVMETHOD(bus_setup_intr, cbb_setup_intr), DEVMETHOD(bus_teardown_intr, cbb_teardown_intr), DEVMETHOD(bus_child_present, cbb_child_present), /* 16-bit card interface */ DEVMETHOD(card_set_res_flags, cbb_pcic_set_res_flags), DEVMETHOD(card_set_memory_offset, cbb_pcic_set_memory_offset), /* power interface */ DEVMETHOD(power_enable_socket, cbb_power_enable_socket), DEVMETHOD(power_disable_socket, cbb_power_disable_socket), /* pcib compatibility interface */ DEVMETHOD(pcib_maxslots, cbb_maxslots), DEVMETHOD(pcib_read_config, cbb_read_config), DEVMETHOD(pcib_write_config, cbb_write_config), DEVMETHOD(pcib_route_interrupt, cbb_route_interrupt), DEVMETHOD_END }; static driver_t cbb_driver = { "cbb", cbb_methods, sizeof(struct cbb_softc) }; DRIVER_MODULE(cbb, pci, cbb_driver, cbb_devclass, 0, 0); MODULE_DEPEND(cbb, exca, 1, 1, 1); Index: head/sys/dev/wbwd/wbwd.c =================================================================== --- head/sys/dev/wbwd/wbwd.c (revision 298930) +++ head/sys/dev/wbwd/wbwd.c (revision 298931) @@ -1,925 +1,925 @@ /*- * Copyright (c) 2011 Sandvine Incorporated ULC. * Copyright (c) 2012 iXsystems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Support for Winbond watchdog. * * With minor abstractions it might be possible to add support for other * different Winbond Super I/O chips as well. Winbond seems to have four * different types of chips, four different ways to get into extended config * mode. * * Note: there is no serialization between the debugging sysctl handlers and * the watchdog functions and possibly others poking the registers at the same * time. For that at least possibly interfering sysctls are hidden by default. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Global registers. */ #define WB_DEVICE_ID_REG 0x20 /* Device ID */ #define WB_DEVICE_REV_REG 0x21 /* Device revision */ #define WB_CR26 0x26 /* Bit6: HEFRAS (base port selector) */ /* LDN selection. */ #define WB_LDN_REG 0x07 #define WB_LDN_REG_LDN8 0x08 /* GPIO 2, Watchdog */ /* * LDN8 (GPIO 2, Watchdog) specific registers and options. */ /* CR30: LDN8 activation control. */ #define WB_LDN8_CR30 0x30 #define WB_LDN8_CR30_ACTIVE 0x01 /* 1: LD active */ /* CRF5: Watchdog scale, P20. Mapped to reg_1. */ #define WB_LDN8_CRF5 0xF5 #define WB_LDN8_CRF5_SCALE 0x08 /* 0: 1s, 1: 60s */ #define WB_LDN8_CRF5_KEYB_P20 0x04 /* 1: keyb P20 forces timeout */ #define WB_LDN8_CRF5_KBRST 0x02 /* 1: timeout causes pin60 kbd reset */ /* CRF6: Watchdog Timeout (0 == off). Mapped to reg_timeout. */ #define WB_LDN8_CRF6 0xF6 /* CRF7: Watchdog mouse, keyb, force, .. Mapped to reg_2. */ #define WB_LDN8_CRF7 0xF7 #define WB_LDN8_CRF7_MOUSE 0x80 /* 1: mouse irq resets wd timer */ #define WB_LDN8_CRF7_KEYB 0x40 /* 1: keyb irq resets wd timer */ #define WB_LDN8_CRF7_FORCE 0x20 /* 1: force timeout (self-clear) */ #define WB_LDN8_CRF7_TS 0x10 /* 0: counting, 1: fired */ #define WB_LDN8_CRF7_IRQS 0x0f /* irq source for watchdog, 2 == SMI */ enum chips { w83627hf, w83627s, w83697hf, w83697ug, w83637hf, w83627thf, w83687thf, w83627ehf, w83627dhg, w83627uhg, w83667hg, w83627dhg_p, w83667hg_b, nct6775, nct6776, nct6779, nct6791, nct6792, nct6102 }; struct wb_softc { device_t dev; struct resource *portres; bus_space_tag_t bst; bus_space_handle_t bsh; int rid; eventhandler_tag ev_tag; int (*ext_cfg_enter_f)(struct wb_softc *, u_short); void (*ext_cfg_exit_f)(struct wb_softc *, u_short); enum chips chip; uint8_t ctl_reg; uint8_t time_reg; uint8_t csr_reg; int debug_verbose; /* * Special feature to let the watchdog fire at a different * timeout as set by watchdog(4) but still use that API to * re-load it periodically. */ unsigned int timeout_override; /* * Space to save current state temporary and for sysctls. * We want to know the timeout value and usually need two * additional registers for options. Do not name them by * register as these might be different by chip. */ uint8_t reg_timeout; uint8_t reg_1; uint8_t reg_2; }; static int ext_cfg_enter_0x87_0x87(struct wb_softc *, u_short); static void ext_cfg_exit_0xaa(struct wb_softc *, u_short); struct winbond_superio_cfg { uint8_t efer; /* and efir */ int (*ext_cfg_enter_f)(struct wb_softc *, u_short); void (*ext_cfg_exit_f)(struct wb_softc *, u_short); } probe_addrs[] = { { .efer = 0x2e, .ext_cfg_enter_f = ext_cfg_enter_0x87_0x87, .ext_cfg_exit_f = ext_cfg_exit_0xaa, }, { .efer = 0x4e, .ext_cfg_enter_f = ext_cfg_enter_0x87_0x87, .ext_cfg_exit_f = ext_cfg_exit_0xaa, }, }; struct winbond_vendor_device_id { uint8_t device_id; enum chips chip; const char * descr; } wb_devs[] = { { .device_id = 0x52, .chip = w83627hf, .descr = "Winbond 83627HF/F/HG/G", }, { .device_id = 0x59, .chip = w83627s, .descr = "Winbond 83627S", }, { .device_id = 0x60, .chip = w83697hf, .descr = "Winbond 83697HF", }, { .device_id = 0x68, .chip = w83697ug, .descr = "Winbond 83697UG", }, { .device_id = 0x70, .chip = w83637hf, .descr = "Winbond 83637HF", }, { .device_id = 0x82, .chip = w83627thf, .descr = "Winbond 83627THF", }, { .device_id = 0x85, .chip = w83687thf, .descr = "Winbond 83687THF", }, { .device_id = 0x88, .chip = w83627ehf, .descr = "Winbond 83627EHF", }, { .device_id = 0xa0, .chip = w83627dhg, .descr = "Winbond 83627DHG", }, { .device_id = 0xa2, .chip = w83627uhg, .descr = "Winbond 83627UHG", }, { .device_id = 0xa5, .chip = w83667hg, .descr = "Winbond 83667HG", }, { .device_id = 0xb0, .chip = w83627dhg_p, .descr = "Winbond 83627DHG-P", }, { .device_id = 0xb3, .chip = w83667hg_b, .descr = "Winbond 83667HG-B", }, { .device_id = 0xb4, .chip = nct6775, .descr = "Nuvoton NCT6775", }, { .device_id = 0xc3, .chip = nct6776, .descr = "Nuvoton NCT6776", }, { .device_id = 0xc4, .chip = nct6102, .descr = "Nuvoton NCT6102", }, { .device_id = 0xc5, .chip = nct6779, .descr = "Nuvoton NCT6779", }, { .device_id = 0xc8, .chip = nct6791, .descr = "Nuvoton NCT6791", }, { .device_id = 0xc9, .chip = nct6792, .descr = "Nuvoton NCT6792", }, }; static void write_efir_1(struct wb_softc *sc, u_short baseport, uint8_t value) { MPASS(sc != NULL || baseport != 0); if (sc != NULL) bus_space_write_1((sc)->bst, (sc)->bsh, 0, (value)); else outb(baseport, value); } static uint8_t __unused read_efir_1(struct wb_softc *sc, u_short baseport) { MPASS(sc != NULL || baseport != 0); if (sc != NULL) return (bus_space_read_1((sc)->bst, (sc)->bsh, 0)); else return (inb(baseport)); } static void write_efdr_1(struct wb_softc *sc, u_short baseport, uint8_t value) { MPASS(sc != NULL || baseport != 0); if (sc != NULL) bus_space_write_1((sc)->bst, (sc)->bsh, 1, (value)); else outb(baseport + 1, value); } static uint8_t read_efdr_1(struct wb_softc *sc, u_short baseport) { MPASS(sc != NULL || baseport != 0); if (sc != NULL) return (bus_space_read_1((sc)->bst, (sc)->bsh, 1)); else return (inb(baseport + 1)); } static void write_reg(struct wb_softc *sc, uint8_t reg, uint8_t value) { write_efir_1(sc, 0, reg); write_efdr_1(sc, 0, value); } static uint8_t read_reg(struct wb_softc *sc, uint8_t reg) { write_efir_1(sc, 0, reg); return (read_efdr_1(sc, 0)); } /* * Return the watchdog related registers as we last read them. This will * usually not give the current timeout or state on whether the watchdog * fired. */ static int sysctl_wb_debug(SYSCTL_HANDLER_ARGS) { struct wb_softc *sc; struct sbuf sb; int error; sc = arg1; sbuf_new_for_sysctl(&sb, NULL, 64, req); sbuf_printf(&sb, "LDN8 (GPIO2, Watchdog): "); sbuf_printf(&sb, "CR%02X 0x%02x ", sc->ctl_reg, sc->reg_1); sbuf_printf(&sb, "CR%02X 0x%02x ", sc->time_reg, sc->reg_timeout); sbuf_printf(&sb, "CR%02X 0x%02x", sc->csr_reg, sc->reg_2); error = sbuf_finish(&sb); sbuf_delete(&sb); return (error); } /* * Read the current values before returning them. Given this might poke * the registers the same time as the watchdog, this sysctl handler should * be marked CTLFLAG_SKIP to not show up by default. */ static int sysctl_wb_debug_current(SYSCTL_HANDLER_ARGS) { struct wb_softc *sc; sc = arg1; if ((*sc->ext_cfg_enter_f)(sc, 0) != 0) return (ENXIO); /* Watchdog is configured as part of LDN 8 (GPIO Port2, Watchdog). */ write_reg(sc, WB_LDN_REG, WB_LDN_REG_LDN8); sc->reg_1 = read_reg(sc, sc->ctl_reg); sc->reg_timeout = read_reg(sc, sc->time_reg); sc->reg_2 = read_reg(sc, sc->csr_reg); (*sc->ext_cfg_exit_f)(sc, 0); return (sysctl_wb_debug(oidp, arg1, arg2, req)); } /* * Sysctl handlers to force a watchdog timeout or to test the NMI functionality * works as expetced. * For testing we could set a test_nmi flag in the softc that, in case of NMI, a * callback function from trap.c could check whether we fired and not report the * timeout but clear the flag for the sysctl again. This is interesting given a * lot of boards have jumpers to change the action on watchdog timeout or * disable the watchdog completely. * XXX-BZ notyet: currently no general infrastructure exists to do this. */ static int sysctl_wb_force_test_nmi(SYSCTL_HANDLER_ARGS) { struct wb_softc *sc; int error, test, val; sc = arg1; test = arg2; #ifdef notyet val = sc->test_nmi; #else val = 0; #endif error = sysctl_handle_int(oidp, &val, 0, req); if (error || !req->newptr) return (error); #ifdef notyet /* Manually clear the test for a value of 0 and do nothing else. */ if (test && val == 0) { sc->test_nmi = 0; return (0); } #endif if ((*sc->ext_cfg_enter_f)(sc, 0) != 0) return (ENXIO); #ifdef notyet /* * If we are testing the NMI functionality, set the flag before * forcing the timeout. */ if (test) sc->test_nmi = 1; #endif /* Watchdog is configured as part of LDN 8 (GPIO Port2, Watchdog). */ write_reg(sc, WB_LDN_REG, WB_LDN_REG_LDN8); /* Force watchdog to fire. */ sc->reg_2 = read_reg(sc, sc->csr_reg); sc->reg_2 |= WB_LDN8_CRF7_FORCE; write_reg(sc, sc->csr_reg, sc->reg_2); (*sc->ext_cfg_exit_f)(sc, 0); return (0); } /* * Print current watchdog state. * * Note: it is the responsibility of the caller to update the registers * upfront. */ static void wb_print_state(struct wb_softc *sc, const char *msg) { device_printf(sc->dev, "%s%sWatchdog %sabled. %s" "Scaling by %ds, timer at %d (%s=%ds%s). " "CRF5 0x%02x CRF7 0x%02x\n", (msg != NULL) ? msg : "", (msg != NULL) ? ": " : "", (sc->reg_timeout > 0x00) ? "en" : "dis", (sc->reg_2 & WB_LDN8_CRF7_TS) ? "Watchdog fired. " : "", (sc->reg_1 & WB_LDN8_CRF5_SCALE) ? 60 : 1, sc->reg_timeout, (sc->reg_timeout > 0x00) ? "<" : "", sc->reg_timeout * ((sc->reg_1 & WB_LDN8_CRF5_SCALE) ? 60 : 1), (sc->reg_timeout > 0x00) ? " left" : "", sc->reg_1, sc->reg_2); } /* * Functions to enter and exit extended function mode. Possibly shared * between different chips. */ static int ext_cfg_enter_0x87_0x87(struct wb_softc *sc, u_short baseport) { /* * Enable extended function mode. * Winbond does not allow us to validate so always return success. */ write_efir_1(sc, baseport, 0x87); write_efir_1(sc, baseport, 0x87); return (0); } static void ext_cfg_exit_0xaa(struct wb_softc *sc, u_short baseport) { write_efir_1(sc, baseport, 0xaa); } /* * (Re)load the watchdog counter depending on timeout. A timeout of 0 will * disable the watchdog. */ static int wb_set_watchdog(struct wb_softc *sc, unsigned int timeout) { if (timeout != 0) { /* * In case an override is set, let it override. It may lead * to strange results as we do not check the input of the sysctl. */ if (sc->timeout_override > 0) timeout = sc->timeout_override; /* Make sure we support the requested timeout. */ if (timeout > 255 * 60) return (EINVAL); } if (sc->debug_verbose) wb_print_state(sc, "Before watchdog counter (re)load"); if ((*sc->ext_cfg_enter_f)(sc, 0) != 0) return (ENXIO); /* Watchdog is configured as part of LDN 8 (GPIO Port2, Watchdog) */ write_reg(sc, WB_LDN_REG, WB_LDN_REG_LDN8); if (timeout == 0) { /* Disable watchdog. */ sc->reg_timeout = 0; write_reg(sc, sc->time_reg, sc->reg_timeout); } else { /* Read current scaling factor. */ sc->reg_1 = read_reg(sc, sc->ctl_reg); if (timeout > 255) { /* Set scaling factor to 60s. */ sc->reg_1 |= WB_LDN8_CRF5_SCALE; sc->reg_timeout = (timeout / 60); if (timeout % 60) sc->reg_timeout++; } else { /* Set scaling factor to 1s. */ sc->reg_1 &= ~WB_LDN8_CRF5_SCALE; sc->reg_timeout = timeout; } /* In case we fired before we need to clear to fire again. */ sc->reg_2 = read_reg(sc, sc->csr_reg); if (sc->reg_2 & WB_LDN8_CRF7_TS) { sc->reg_2 &= ~WB_LDN8_CRF7_TS; write_reg(sc, sc->csr_reg, sc->reg_2); } /* Write back scaling factor. */ write_reg(sc, sc->ctl_reg, sc->reg_1); /* Set timer and arm/reset the watchdog. */ write_reg(sc, sc->time_reg, sc->reg_timeout); } (*sc->ext_cfg_exit_f)(sc, 0); if (sc->debug_verbose) wb_print_state(sc, "After watchdog counter (re)load"); return (0); } /* * watchdog(9) EVENTHANDLER function implementation to (re)load the counter * with the given timeout or disable the watchdog. */ static void wb_watchdog_fn(void *private, u_int cmd, int *error) { struct wb_softc *sc; unsigned int timeout; int e; sc = private; KASSERT(sc != NULL, ("%s: watchdog handler function called without " "softc.", __func__)); cmd &= WD_INTERVAL; if (cmd > 0 && cmd <= 63) { /* Reset (and arm) watchdog. */ timeout = ((uint64_t)1 << cmd) / 1000000000; if (timeout == 0) timeout = 1; e = wb_set_watchdog(sc, timeout); if (e == 0) { if (error != NULL) *error = 0; } else { /* On error, try to make sure the WD is disabled. */ wb_set_watchdog(sc, 0); } } else { /* Disable watchdog. */ e = wb_set_watchdog(sc, 0); if (e != 0 && cmd == 0 && error != NULL) { /* Failed to disable watchdog. */ *error = EOPNOTSUPP; } } } /* * Probe/attach the Winbond Super I/O chip. * * Initial abstraction to possibly support more chips: * - Iterate over the well known base ports, try to enable extended function * mode and read and match the device ID and device revision. Unfortunately * the Vendor ID is in the hardware monitoring section accessible by different * base ports only. * - Also HEFRAS, which would tell use the base port, is only accessible after * entering extended function mode, for which the base port is needed. * At least check HEFRAS to match the current base port we are probing. * - On match set the description, remember functions to enter/exit extended * function mode as well as the base port. */ static int wb_probe_enable(device_t dev, int probe) { struct wb_softc *sc; int error, found, i, j; uint8_t dev_id, dev_rev, cr26; char buf[128]; if (dev == NULL) sc = NULL; else { sc = device_get_softc(dev); bzero(sc, sizeof(*sc)); sc->dev = dev; } error = ENXIO; found = 0; for (i = 0; i < nitems(probe_addrs); i++) { if (sc != NULL) { /* Allocate bus resources for IO index/data register access. */ sc->portres = bus_alloc_resource(dev, SYS_RES_IOPORT, &sc->rid, probe_addrs[i].efer, probe_addrs[i].efer + 1, 2, RF_ACTIVE); if (sc->portres == NULL) continue; sc->bst = rman_get_bustag(sc->portres); sc->bsh = rman_get_bushandle(sc->portres); } error = (*probe_addrs[i].ext_cfg_enter_f)(sc, probe_addrs[i].efer); if (error != 0) goto cleanup; /* Identify the SuperIO chip. */ write_efir_1(sc, probe_addrs[i].efer, WB_DEVICE_ID_REG); dev_id = read_efdr_1(sc, probe_addrs[i].efer); write_efir_1(sc, probe_addrs[i].efer, WB_DEVICE_REV_REG); dev_rev = read_efdr_1(sc, probe_addrs[i].efer); write_efir_1(sc, probe_addrs[i].efer, WB_CR26); cr26 = read_efdr_1(sc, probe_addrs[i].efer); if (dev_id == 0xff && dev_rev == 0xff) goto cleanup; /* HEFRAS of 0 means EFER at 0x2e, 1 means EFER at 0x4e. */ if (((cr26 & 0x40) == 0x00 && probe_addrs[i].efer != 0x2e) || ((cr26 & 0x40) == 0x40 && probe_addrs[i].efer != 0x4e)) { if (dev != NULL) device_printf(dev, "HEFRAS and EFER do not " "align: EFER 0x%02x DevID 0x%02x DevRev " "0x%02x CR26 0x%02x\n", probe_addrs[i].efer, dev_id, dev_rev, cr26); goto cleanup; } for (j = 0; j < nitems(wb_devs); j++) { if (wb_devs[j].device_id == dev_id) { found = 1; break; } } if (probe && dev != NULL) { snprintf(buf, sizeof(buf), "%s (0x%02x/0x%02x) Watchdog Timer", found ? wb_devs[j].descr : "Unknown Winbond/Nuvoton", dev_id, dev_rev); device_set_desc_copy(dev, buf); } /* If this is hinted attach, try to guess the model. */ if (dev != NULL && !found) { found = 1; j = 0; } cleanup: if (probe || !found) { (*probe_addrs[i].ext_cfg_exit_f)(sc, probe_addrs[i].efer); if (sc != NULL) (void) bus_release_resource(dev, SYS_RES_IOPORT, sc->rid, sc->portres); } /* * Stop probing if have successfully identified the SuperIO. * Remember the extended function mode enter/exit functions * for operations. */ if (found) { if (sc != NULL) { sc->ext_cfg_enter_f = probe_addrs[i].ext_cfg_enter_f; sc->ext_cfg_exit_f = probe_addrs[i].ext_cfg_exit_f; sc->chip = wb_devs[j].chip; sc->ctl_reg = 0xf5; sc->time_reg = 0xf6; sc->csr_reg = 0xf7; if (sc->chip == w83697hf || sc->chip == w83697ug) { sc->ctl_reg = 0xf3; sc->time_reg = 0xf4; } else if (sc->chip == nct6102) { sc->ctl_reg = 0xf0; sc->time_reg = 0xf1; sc->csr_reg = 0xf2; } } return (BUS_PROBE_SPECIFIC); } else error = ENXIO; } return (error); } static void wb_identify(driver_t *driver, device_t parent) { if (device_find_child(parent, driver->name, 0) == NULL) { if (wb_probe_enable(NULL, 1) <= 0) BUS_ADD_CHILD(parent, 0, driver->name, 0); } } static int wb_probe(device_t dev) { /* Make sure we do not claim some ISA PNP device. */ if (isa_get_logicalid(dev) != 0) return (ENXIO); return (wb_probe_enable(dev, 1)); } static int wb_attach(device_t dev) { struct wb_softc *sc; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; unsigned long timeout; int error; uint8_t t; error = wb_probe_enable(dev, 0); if (error > 0) return (ENXIO); sc = device_get_softc(dev); KASSERT(sc->ext_cfg_enter_f != NULL && sc->ext_cfg_exit_f != NULL, - ("%s: successfull probe result but not setup correctly", __func__)); + ("%s: successful probe result but not setup correctly", __func__)); /* Watchdog is configured as part of LDN 8 (GPIO Port2, Watchdog). */ write_reg(sc, WB_LDN_REG, WB_LDN_REG_LDN8); /* Make sure WDT is enabled. */ write_reg(sc, WB_LDN8_CR30, read_reg(sc, WB_LDN8_CR30) | WB_LDN8_CR30_ACTIVE); switch (sc->chip) { case w83627hf: case w83627s: t = read_reg(sc, 0x2B) & ~0x10; write_reg(sc, 0x2B, t); /* set GPIO24 to WDT0 */ break; case w83697hf: /* Set pin 119 to WDTO# mode (= CR29, WDT0) */ t = read_reg(sc, 0x29) & ~0x60; t |= 0x20; write_reg(sc, 0x29, t); break; case w83697ug: /* Set pin 118 to WDTO# mode */ t = read_reg(sc, 0x2b) & ~0x04; write_reg(sc, 0x2b, t); break; case w83627thf: t = (read_reg(sc, 0x2B) & ~0x08) | 0x04; write_reg(sc, 0x2B, t); /* set GPIO3 to WDT0 */ break; case w83627dhg: case w83627dhg_p: t = read_reg(sc, 0x2D) & ~0x01; /* PIN77 -> WDT0# */ write_reg(sc, 0x2D, t); /* set GPIO5 to WDT0 */ t = read_reg(sc, sc->ctl_reg); t |= 0x02; /* enable the WDTO# output low pulse * to the KBRST# pin */ write_reg(sc, sc->ctl_reg, t); break; case w83637hf: break; case w83687thf: t = read_reg(sc, 0x2C) & ~0x80; /* PIN47 -> WDT0# */ write_reg(sc, 0x2C, t); break; case w83627ehf: case w83627uhg: case w83667hg: case w83667hg_b: case nct6775: case nct6776: case nct6779: case nct6791: case nct6792: case nct6102: /* * These chips have a fixed WDTO# output pin (W83627UHG), * or support more than one WDTO# output pin. * Don't touch its configuration, and hope the BIOS * does the right thing. */ t = read_reg(sc, sc->ctl_reg); t |= 0x02; /* enable the WDTO# output low pulse * to the KBRST# pin */ write_reg(sc, sc->ctl_reg, t); break; default: break; } /* Read the current watchdog configuration. */ sc->reg_1 = read_reg(sc, sc->ctl_reg); sc->reg_timeout = read_reg(sc, sc->time_reg); sc->reg_2 = read_reg(sc, sc->csr_reg); /* Print current state if bootverbose or watchdog already enabled. */ if (bootverbose || (sc->reg_timeout > 0x00)) wb_print_state(sc, "Before watchdog attach"); sc->reg_1 &= ~WB_LDN8_CRF5_KEYB_P20; sc->reg_1 |= WB_LDN8_CRF5_KBRST; write_reg(sc, sc->ctl_reg, sc->reg_1); /* * Clear a previous watchdog timeout event (if still set). * Disable timer reset on mouse interrupts. Leave reset on keyboard, * since one of my boards is getting stuck in reboot without it. */ sc->reg_2 &= ~(WB_LDN8_CRF7_MOUSE|WB_LDN8_CRF7_TS); write_reg(sc, sc->csr_reg, sc->reg_2); (*sc->ext_cfg_exit_f)(sc, 0); /* Read global timeout override tunable, Add per device sysctls. */ if (TUNABLE_ULONG_FETCH("hw.wbwd.timeout_override", &timeout)) { if (timeout > 0) sc->timeout_override = timeout; } sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "timeout_override", CTLFLAG_RW, &sc->timeout_override, 0, "Timeout in seconds overriding default watchdog timeout"); SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "debug_verbose", CTLFLAG_RW, &sc->debug_verbose, 0, "Enables extra debugging information"); SYSCTL_ADD_PROC(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "debug", CTLTYPE_STRING|CTLFLAG_RD, sc, 0, sysctl_wb_debug, "A", "Selected register information from last change by driver"); SYSCTL_ADD_PROC(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "debug_current", CTLTYPE_STRING|CTLFLAG_RD|CTLFLAG_SKIP, sc, 0, sysctl_wb_debug_current, "A", "Selected register information (may interfere)"); SYSCTL_ADD_PROC(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "force_timeout", CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_SKIP, sc, 0, sysctl_wb_force_test_nmi, "I", "Enable to force watchdog to fire."); /* Register watchdog. */ sc->ev_tag = EVENTHANDLER_REGISTER(watchdog_list, wb_watchdog_fn, sc, 0); if (bootverbose) wb_print_state(sc, "After watchdog attach"); return (0); } static int wb_detach(device_t dev) { struct wb_softc *sc; sc = device_get_softc(dev); /* Unregister and stop the watchdog if running. */ if (sc->ev_tag) EVENTHANDLER_DEREGISTER(watchdog_list, sc->ev_tag); wb_set_watchdog(sc, 0); /* Disable extended function mode. */ (*sc->ext_cfg_exit_f)(sc, 0); /* Cleanup resources. */ (void) bus_release_resource(dev, SYS_RES_IOPORT, sc->rid, sc->portres); /* Bus subroutines take care of sysctls already. */ return (0); } static device_method_t wb_methods[] = { /* Device interface */ DEVMETHOD(device_identify, wb_identify), DEVMETHOD(device_probe, wb_probe), DEVMETHOD(device_attach, wb_attach), DEVMETHOD(device_detach, wb_detach), DEVMETHOD_END }; static driver_t wb_isa_driver = { "wbwd", wb_methods, sizeof(struct wb_softc) }; static devclass_t wb_devclass; DRIVER_MODULE(wb, isa, wb_isa_driver, wb_devclass, NULL, NULL); Index: head/sys/dev/wtap/if_wtap.c =================================================================== --- head/sys/dev/wtap/if_wtap.c (revision 298930) +++ head/sys/dev/wtap/if_wtap.c (revision 298931) @@ -1,740 +1,740 @@ /*- * Copyright (c) 2010-2011 Monthadar Al Jaberi, TerraNet AB * All rights reserved. * * Copyright (c) 2002-2009 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ #include "if_wtapvar.h" #include /* uio struct */ #include #include #include #include #include "if_medium.h" /* * This _requires_ vimage to be useful. */ #ifndef VIMAGE #error if_wtap requires VIMAGE. #endif /* VIMAGE */ /* device for IOCTL and read/write for debuggin purposes */ /* Function prototypes */ static d_open_t wtap_node_open; static d_close_t wtap_node_close; static d_write_t wtap_node_write; static d_ioctl_t wtap_node_ioctl; static struct cdevsw wtap_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_open = wtap_node_open, .d_close = wtap_node_close, .d_write = wtap_node_write, .d_ioctl = wtap_node_ioctl, .d_name = "wtapnode", }; static int wtap_node_open(struct cdev *dev, int oflags, int devtype, struct thread *p) { int err = 0; uprintf("Opened device \"echo\" successfully.\n"); return(err); } static int wtap_node_close(struct cdev *dev, int fflag, int devtype, struct thread *p) { uprintf("Closing device \"echo.\"\n"); return(0); } static int wtap_node_write(struct cdev *dev, struct uio *uio, int ioflag) { int err = 0; struct mbuf *m; struct ifnet *ifp; struct wtap_softc *sc; uint8_t buf[1024]; int buf_len; uprintf("write device %s \"echo.\"\n", devtoname(dev)); buf_len = MIN(uio->uio_iov->iov_len, 1024); err = copyin(uio->uio_iov->iov_base, buf, buf_len); if (err != 0) { uprintf("Write failed: bad address!\n"); return (err); } MGETHDR(m, M_NOWAIT, MT_DATA); m_copyback(m, 0, buf_len, buf); CURVNET_SET(TD_TO_VNET(curthread)); IFNET_RLOCK_NOSLEEP(); TAILQ_FOREACH(ifp, &V_ifnet, if_link) { printf("ifp->if_xname = %s\n", ifp->if_xname); if(strcmp(devtoname(dev), ifp->if_xname) == 0){ printf("found match, correspoding wtap = %s\n", ifp->if_xname); sc = (struct wtap_softc *)ifp->if_softc; printf("wtap id = %d\n", sc->id); wtap_inject(sc, m); } } IFNET_RUNLOCK_NOSLEEP(); CURVNET_RESTORE(); return(err); } int wtap_node_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { int error = 0; switch(cmd) { default: - DWTAP_PRINTF("Unkown WTAP IOCTL\n"); + DWTAP_PRINTF("Unknown WTAP IOCTL\n"); error = EINVAL; } return error; } static int wtap_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_bpf_params *params); static int wtap_medium_enqueue(struct wtap_vap *avp, struct mbuf *m) { return medium_transmit(avp->av_md, avp->id, m); } static int wtap_media_change(struct ifnet *ifp) { DWTAP_PRINTF("%s\n", __func__); int error = ieee80211_media_change(ifp); /* NB: only the fixed rate can change and that doesn't need a reset */ return (error == ENETRESET ? 0 : error); } /* * Intercept management frames to collect beacon rssi data * and to do ibss merges. */ static void wtap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m, int subtype, const struct ieee80211_rx_stats *stats, int rssi, int nf) { struct ieee80211vap *vap = ni->ni_vap; #if 0 DWTAP_PRINTF("[%d] %s\n", myath_id(ni), __func__); #endif WTAP_VAP(vap)->av_recv_mgmt(ni, m, subtype, stats, rssi, nf); } static int wtap_reset_vap(struct ieee80211vap *vap, u_long cmd) { DWTAP_PRINTF("%s\n", __func__); return 0; } static void wtap_beacon_update(struct ieee80211vap *vap, int item) { struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off; DWTAP_PRINTF("%s\n", __func__); setbit(bo->bo_flags, item); } /* * Allocate and setup an initial beacon frame. */ static int wtap_beacon_alloc(struct wtap_softc *sc, struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct wtap_vap *avp = WTAP_VAP(vap); DWTAP_PRINTF("[%s] %s\n", ether_sprintf(ni->ni_macaddr), __func__); /* * NB: the beacon data buffer must be 32-bit aligned; * we assume the mbuf routines will return us something * with this alignment (perhaps should assert). */ avp->beacon = ieee80211_beacon_alloc(ni); if (avp->beacon == NULL) { printf("%s: cannot get mbuf\n", __func__); return ENOMEM; } callout_init(&avp->av_swba, 0); avp->bf_node = ieee80211_ref_node(ni); return 0; } static void wtap_beacon_config(struct wtap_softc *sc, struct ieee80211vap *vap) { DWTAP_PRINTF("%s\n", __func__); } static void wtap_beacon_intrp(void *arg) { struct wtap_vap *avp = arg; struct ieee80211vap *vap = arg; struct mbuf *m; if (vap->iv_state < IEEE80211_S_RUN) { DWTAP_PRINTF("Skip beacon, not running, state %d", vap->iv_state); return ; } DWTAP_PRINTF("[%d] beacon intrp\n", avp->id); //burst mode /* * Update dynamic beacon contents. If this returns * non-zero then we need to remap the memory because * the beacon frame changed size (probably because * of the TIM bitmap). */ m = m_dup(avp->beacon, M_NOWAIT); if (ieee80211_beacon_update(avp->bf_node, m, 0)) { printf("%s, need to remap the memory because the beacon frame" " changed size.\n",__func__); } if (ieee80211_radiotap_active_vap(vap)) ieee80211_radiotap_tx(vap, m); #if 0 medium_transmit(avp->av_md, avp->id, m); #endif wtap_medium_enqueue(avp, m); callout_schedule(&avp->av_swba, avp->av_bcinterval); } static int wtap_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct ieee80211com *ic = vap->iv_ic; struct wtap_softc *sc = ic->ic_softc; struct wtap_vap *avp = WTAP_VAP(vap); struct ieee80211_node *ni = NULL; int error; DWTAP_PRINTF("%s\n", __func__); ni = ieee80211_ref_node(vap->iv_bss); /* * Invoke the parent method to do net80211 work. */ error = avp->av_newstate(vap, nstate, arg); if (error != 0) goto bad; if (nstate == IEEE80211_S_RUN) { /* NB: collect bss node again, it may have changed */ ieee80211_free_node(ni); ni = ieee80211_ref_node(vap->iv_bss); switch (vap->iv_opmode) { case IEEE80211_M_MBSS: error = wtap_beacon_alloc(sc, ni); if (error != 0) goto bad; wtap_beacon_config(sc, vap); callout_reset(&avp->av_swba, avp->av_bcinterval, wtap_beacon_intrp, vap); break; default: goto bad; } } else if (nstate == IEEE80211_S_INIT) { callout_stop(&avp->av_swba); } ieee80211_free_node(ni); return 0; bad: printf("%s: bad\n", __func__); ieee80211_free_node(ni); return error; } static void wtap_bmiss(struct ieee80211vap *vap) { struct wtap_vap *avp = (struct wtap_vap *)vap; DWTAP_PRINTF("%s\n", __func__); avp->av_bmiss(vap); } static struct ieee80211vap * wtap_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN], const uint8_t mac[IEEE80211_ADDR_LEN]) { struct wtap_softc *sc = ic->ic_softc; struct ieee80211vap *vap; struct wtap_vap *avp; int error; struct ieee80211_node *ni; DWTAP_PRINTF("%s\n", __func__); avp = malloc(sizeof(struct wtap_vap), M_80211_VAP, M_WAITOK | M_ZERO); avp->id = sc->id; avp->av_md = sc->sc_md; avp->av_bcinterval = msecs_to_ticks(BEACON_INTRERVAL + 100*sc->id); vap = (struct ieee80211vap *) avp; error = ieee80211_vap_setup(ic, vap, name, unit, IEEE80211_M_MBSS, flags | IEEE80211_CLONE_NOBEACONS, bssid); if (error) { free(avp, M_80211_VAP); return (NULL); } /* override various methods */ avp->av_recv_mgmt = vap->iv_recv_mgmt; vap->iv_recv_mgmt = wtap_recv_mgmt; vap->iv_reset = wtap_reset_vap; vap->iv_update_beacon = wtap_beacon_update; avp->av_newstate = vap->iv_newstate; vap->iv_newstate = wtap_newstate; avp->av_bmiss = vap->iv_bmiss; vap->iv_bmiss = wtap_bmiss; /* complete setup */ ieee80211_vap_attach(vap, wtap_media_change, ieee80211_media_status, mac); avp->av_dev = make_dev(&wtap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "%s", (const char *)sc->name); /* TODO this is a hack to force it to choose the rate we want */ ni = ieee80211_ref_node(vap->iv_bss); ni->ni_txrate = 130; ieee80211_free_node(ni); return vap; } static void wtap_vap_delete(struct ieee80211vap *vap) { struct wtap_vap *avp = WTAP_VAP(vap); DWTAP_PRINTF("%s\n", __func__); destroy_dev(avp->av_dev); callout_stop(&avp->av_swba); ieee80211_vap_detach(vap); free((struct wtap_vap*) vap, M_80211_VAP); } static void wtap_parent(struct ieee80211com *ic) { struct wtap_softc *sc = ic->ic_softc; if (ic->ic_nrunning > 0) { sc->up = 1; ieee80211_start_all(ic); } else sc->up = 0; } static void wtap_scan_start(struct ieee80211com *ic) { #if 0 DWTAP_PRINTF("%s\n", __func__); #endif } static void wtap_scan_end(struct ieee80211com *ic) { #if 0 DWTAP_PRINTF("%s\n", __func__); #endif } static void wtap_set_channel(struct ieee80211com *ic) { #if 0 DWTAP_PRINTF("%s\n", __func__); #endif } static int wtap_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_bpf_params *params) { #if 0 DWTAP_PRINTF("%s, %p\n", __func__, m); #endif struct ieee80211vap *vap = ni->ni_vap; struct wtap_vap *avp = WTAP_VAP(vap); if (ieee80211_radiotap_active_vap(vap)) { ieee80211_radiotap_tx(vap, m); } if (m->m_flags & M_TXCB) ieee80211_process_callback(ni, m, 0); ieee80211_free_node(ni); return wtap_medium_enqueue(avp, m); } void wtap_inject(struct wtap_softc *sc, struct mbuf *m) { struct wtap_buf *bf = (struct wtap_buf *)malloc(sizeof(struct wtap_buf), M_WTAP_RXBUF, M_NOWAIT | M_ZERO); KASSERT(bf != NULL, ("could not allocated a new wtap_buf\n")); bf->m = m; mtx_lock(&sc->sc_mtx); STAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list); taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask); mtx_unlock(&sc->sc_mtx); } void wtap_rx_deliver(struct wtap_softc *sc, struct mbuf *m) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni; int type; #if 0 DWTAP_PRINTF("%s\n", __func__); #endif DWTAP_PRINTF("[%d] receiving m=%p\n", sc->id, m); if (m == NULL) { /* NB: shouldn't happen */ ic_printf(ic, "%s: no mbuf!\n", __func__); } ieee80211_dump_pkt(ic, mtod(m, caddr_t), 0,0,0); /* * Locate the node for sender, track state, and then * pass the (referenced) node up to the 802.11 layer * for its use. */ ni = ieee80211_find_rxnode_withkey(ic, mtod(m, const struct ieee80211_frame_min *),IEEE80211_KEYIX_NONE); if (ni != NULL) { /* * Sending station is known, dispatch directly. */ type = ieee80211_input(ni, m, 1<<7, 10); ieee80211_free_node(ni); } else { type = ieee80211_input_all(ic, m, 1<<7, 10); } } static void wtap_rx_proc(void *arg, int npending) { struct wtap_softc *sc = (struct wtap_softc *)arg; struct ieee80211com *ic = &sc->sc_ic; struct mbuf *m; struct ieee80211_node *ni; int type; struct wtap_buf *bf; #if 0 DWTAP_PRINTF("%s\n", __func__); #endif for(;;) { mtx_lock(&sc->sc_mtx); bf = STAILQ_FIRST(&sc->sc_rxbuf); if (bf == NULL) { mtx_unlock(&sc->sc_mtx); return; } STAILQ_REMOVE_HEAD(&sc->sc_rxbuf, bf_list); mtx_unlock(&sc->sc_mtx); KASSERT(bf != NULL, ("wtap_buf is NULL\n")); m = bf->m; DWTAP_PRINTF("[%d] receiving m=%p\n", sc->id, bf->m); if (m == NULL) { /* NB: shouldn't happen */ ic_printf(ic, "%s: no mbuf!\n", __func__); free(bf, M_WTAP_RXBUF); return; } #if 0 ieee80211_dump_pkt(ic, mtod(m, caddr_t), 0,0,0); #endif /* * Locate the node for sender, track state, and then * pass the (referenced) node up to the 802.11 layer * for its use. */ ni = ieee80211_find_rxnode_withkey(ic, mtod(m, const struct ieee80211_frame_min *), IEEE80211_KEYIX_NONE); if (ni != NULL) { /* * Sending station is known, dispatch directly. */ type = ieee80211_input(ni, m, 1<<7, 10); ieee80211_free_node(ni); } else { type = ieee80211_input_all(ic, m, 1<<7, 10); } /* The mbufs are freed by the Net80211 stack */ free(bf, M_WTAP_RXBUF); } } static void wtap_newassoc(struct ieee80211_node *ni, int isnew) { DWTAP_PRINTF("%s\n", __func__); } /* * Callback from the 802.11 layer to update WME parameters. */ static int wtap_wme_update(struct ieee80211com *ic) { DWTAP_PRINTF("%s\n", __func__); return 0; } static void wtap_update_mcast(struct ieee80211com *ic) { DWTAP_PRINTF("%s\n", __func__); } static void wtap_update_promisc(struct ieee80211com *ic) { DWTAP_PRINTF("%s\n", __func__); } static int wtap_transmit(struct ieee80211com *ic, struct mbuf *m) { struct ieee80211_node *ni = (struct ieee80211_node *) m->m_pkthdr.rcvif; struct ieee80211vap *vap = ni->ni_vap; struct wtap_vap *avp = WTAP_VAP(vap); if(ni == NULL){ printf("m->m_pkthdr.rcvif is NULL we cant radiotap_tx\n"); }else{ if (ieee80211_radiotap_active_vap(vap)) ieee80211_radiotap_tx(vap, m); } if (m->m_flags & M_TXCB) ieee80211_process_callback(ni, m, 0); ieee80211_free_node(ni); return wtap_medium_enqueue(avp, m); } static struct ieee80211_node * wtap_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN]) { struct ieee80211_node *ni; DWTAP_PRINTF("%s\n", __func__); ni = malloc(sizeof(struct ieee80211_node), M_80211_NODE, M_NOWAIT|M_ZERO); ni->ni_txrate = 130; return ni; } static void wtap_node_free(struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; struct wtap_softc *sc = ic->ic_softc; DWTAP_PRINTF("%s\n", __func__); sc->sc_node_free(ni); } int32_t wtap_attach(struct wtap_softc *sc, const uint8_t *macaddr) { struct ieee80211com *ic = &sc->sc_ic; DWTAP_PRINTF("%s\n", __func__); sc->up = 0; STAILQ_INIT(&sc->sc_rxbuf); sc->sc_tq = taskqueue_create("wtap_taskq", M_NOWAIT | M_ZERO, taskqueue_thread_enqueue, &sc->sc_tq); taskqueue_start_threads(&sc->sc_tq, 1, PI_SOFT, "%s taskQ", sc->name); TASK_INIT(&sc->sc_rxtask, 0, wtap_rx_proc, sc); ic->ic_softc = sc; ic->ic_name = sc->name; ic->ic_phytype = IEEE80211_T_DS; ic->ic_opmode = IEEE80211_M_MBSS; ic->ic_caps = IEEE80211_C_MBSS; ic->ic_max_keyix = 128; /* A value read from Atheros ATH_KEYMAX */ ic->ic_regdomain.regdomain = SKU_ETSI; ic->ic_regdomain.country = CTRY_SWEDEN; ic->ic_regdomain.location = 1; /* Indoors */ ic->ic_regdomain.isocc[0] = 'S'; ic->ic_regdomain.isocc[1] = 'E'; ic->ic_nchans = 1; ic->ic_channels[0].ic_flags = IEEE80211_CHAN_B; ic->ic_channels[0].ic_freq = 2412; IEEE80211_ADDR_COPY(ic->ic_macaddr, macaddr); ieee80211_ifattach(ic); /* override default methods */ ic->ic_newassoc = wtap_newassoc; ic->ic_wme.wme_update = wtap_wme_update; ic->ic_vap_create = wtap_vap_create; ic->ic_vap_delete = wtap_vap_delete; ic->ic_raw_xmit = wtap_raw_xmit; ic->ic_update_mcast = wtap_update_mcast; ic->ic_update_promisc = wtap_update_promisc; ic->ic_transmit = wtap_transmit; ic->ic_parent = wtap_parent; sc->sc_node_alloc = ic->ic_node_alloc; ic->ic_node_alloc = wtap_node_alloc; sc->sc_node_free = ic->ic_node_free; ic->ic_node_free = wtap_node_free; ic->ic_scan_start = wtap_scan_start; ic->ic_scan_end = wtap_scan_end; ic->ic_set_channel = wtap_set_channel; ieee80211_radiotap_attach(ic, &sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th), WTAP_TX_RADIOTAP_PRESENT, &sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th), WTAP_RX_RADIOTAP_PRESENT); /* Work here, we must find a way to populate the rate table */ #if 0 if(ic->ic_rt == NULL){ printf("no table for ic_curchan\n"); ic->ic_rt = ieee80211_get_ratetable(&ic->ic_channels[0]); } printf("ic->ic_rt =%p\n", ic->ic_rt); printf("rate count %d\n", ic->ic_rt->rateCount); uint8_t code = ic->ic_rt->info[0].dot11Rate; uint8_t cix = ic->ic_rt->info[0].ctlRateIndex; uint8_t ctl_rate = ic->ic_rt->info[cix].dot11Rate; printf("code=%d, cix=%d, ctl_rate=%d\n", code, cix, ctl_rate); uint8_t rix0 = ic->ic_rt->rateCodeToIndex[130]; uint8_t rix1 = ic->ic_rt->rateCodeToIndex[132]; uint8_t rix2 = ic->ic_rt->rateCodeToIndex[139]; uint8_t rix3 = ic->ic_rt->rateCodeToIndex[150]; printf("rix0 %u,rix1 %u,rix2 %u,rix3 %u\n", rix0,rix1,rix2,rix3); printf("lpAckDuration=%u\n", ic->ic_rt->info[0].lpAckDuration); printf("rate=%d\n", ic->ic_rt->info[0].rateKbps); #endif return 0; } int32_t wtap_detach(struct wtap_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; DWTAP_PRINTF("%s\n", __func__); ieee80211_ageq_drain(&ic->ic_stageq); ieee80211_ifdetach(ic); return 0; } void wtap_resume(struct wtap_softc *sc) { DWTAP_PRINTF("%s\n", __func__); } void wtap_suspend(struct wtap_softc *sc) { DWTAP_PRINTF("%s\n", __func__); } void wtap_shutdown(struct wtap_softc *sc) { DWTAP_PRINTF("%s\n", __func__); } void wtap_intr(struct wtap_softc *sc) { DWTAP_PRINTF("%s\n", __func__); } Index: head/sys/dev/wtap/if_wtap_module.c =================================================================== --- head/sys/dev/wtap/if_wtap_module.c (revision 298930) +++ head/sys/dev/wtap/if_wtap_module.c (revision 298931) @@ -1,186 +1,186 @@ /*- * Copyright (c) 2010-2011 Monthadar Al Jaberi, TerraNet AB * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* cdevsw struct */ #include /* uio struct */ #include #include #include "if_wtapvar.h" #include "if_wtapioctl.h" #include "if_medium.h" #include "wtap_hal/hal.h" /* WTAP PLUGINS */ #include "plugins/visibility.h" MALLOC_DEFINE(M_WTAP, "wtap", "wtap wireless simulator"); MALLOC_DEFINE(M_WTAP_PACKET, "wtap packet", "wtap wireless simulator packet"); MALLOC_DEFINE(M_WTAP_RXBUF, "wtap rxbuf", - "wtap wireless simulator recieve buffer"); + "wtap wireless simulator receive buffer"); MALLOC_DEFINE(M_WTAP_PLUGIN, "wtap plugin", "wtap wireless simulator plugin"); static struct wtap_hal *hal; /* Function prototypes */ static d_ioctl_t wtap_ioctl; static struct cdev *sdev; static struct cdevsw wtap_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_ioctl = wtap_ioctl, .d_name = "wtapctl", }; int wtap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { int error = 0; CURVNET_SET(CRED_TO_VNET(curthread->td_ucred)); switch(cmd) { case WTAPIOCTLCRT: if(new_wtap(hal, *(int *)data)) error = EINVAL; break; case WTAPIOCTLDEL: if(free_wtap(hal, *(int *)data)) error = EINVAL; break; default: - DWTAP_PRINTF("Unkown WTAP IOCTL\n"); + DWTAP_PRINTF("Unknown WTAP IOCTL\n"); error = EINVAL; } CURVNET_RESTORE(); return error; } /* The function called at load/unload. */ static int event_handler(module_t module, int event, void *arg) { struct visibility_plugin *plugin; int e = 0; /* Error, 0 for normal return status */ switch (event) { case MOD_LOAD: sdev = make_dev(&wtap_cdevsw,0,UID_ROOT, GID_WHEEL,0600,(const char *)"wtapctl"); hal = (struct wtap_hal *)malloc(sizeof(struct wtap_hal), M_WTAP, M_NOWAIT | M_ZERO); bzero(hal, sizeof(struct wtap_hal)); init_hal(hal); /* Setting up a simple plugin */ plugin = (struct visibility_plugin *)malloc (sizeof(struct visibility_plugin), M_WTAP_PLUGIN, M_NOWAIT | M_ZERO); plugin->base.wp_hal = hal; plugin->base.init = visibility_init; plugin->base.deinit = visibility_deinit; plugin->base.work = visibility_work; register_plugin(hal, (struct wtap_plugin *)plugin); printf("Loaded wtap wireless simulator\n"); break; case MOD_UNLOAD: destroy_dev(sdev); deregister_plugin(hal); deinit_hal(hal); free(hal, M_WTAP); printf("Unloading wtap wireless simulator\n"); break; default: e = EOPNOTSUPP; /* Error, Operation Not Supported */ break; } return(e); } /* The second argument of DECLARE_MODULE. */ static moduledata_t wtap_conf = { "wtap", /* module name */ event_handler, /* event handler */ NULL /* extra data */ }; DECLARE_MODULE(wtap, wtap_conf, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); MODULE_DEPEND(wtap, wlan, 1, 1, 1); /* 802.11 media layer */ Index: head/sys/dev/wtap/plugins/visibility.c =================================================================== --- head/sys/dev/wtap/plugins/visibility.c (revision 298930) +++ head/sys/dev/wtap/plugins/visibility.c (revision 298931) @@ -1,240 +1,240 @@ /*- * Copyright (c) 2010-2011 Monthadar Al Jaberi, TerraNet AB * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* cdevsw struct */ #include /* uio struct */ #include #include #include "visibility.h" /* Function prototypes */ static d_ioctl_t vis_ioctl; static struct cdevsw vis_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_ioctl = vis_ioctl, .d_name = "visctl", }; void visibility_init(struct wtap_plugin *plugin) { struct visibility_plugin *vis_plugin; vis_plugin = (struct visibility_plugin *) plugin; plugin->wp_sdev = make_dev(&vis_cdevsw,0,UID_ROOT,GID_WHEEL,0600, (const char *)"visctl"); plugin->wp_sdev->si_drv1 = vis_plugin; mtx_init(&vis_plugin->pl_mtx, "visibility_plugin mtx", NULL, MTX_DEF | MTX_RECURSE); printf("Using visibility wtap plugin...\n"); } void visibility_deinit(struct wtap_plugin *plugin) { struct visibility_plugin *vis_plugin; vis_plugin = (struct visibility_plugin *) plugin; destroy_dev(plugin->wp_sdev); mtx_destroy(&vis_plugin->pl_mtx); free(vis_plugin, M_WTAP_PLUGIN); printf("Removing visibility wtap plugin...\n"); } /* We need to use a mutex lock when we read out a visibility map * and when we change visibility map from user space through IOCTL */ void visibility_work(struct wtap_plugin *plugin, struct packet *p) { struct visibility_plugin *vis_plugin = (struct visibility_plugin *) plugin; struct wtap_hal *hal = (struct wtap_hal *)vis_plugin->base.wp_hal; struct vis_map *map; KASSERT(mtod(p->m, const char *) != (const char *) 0xdeadc0de || mtod(p->m, const char *) != NULL, ("[%s] got a corrupt packet from master queue, p->m=%p, p->id=%d\n", __func__, p->m, p->id)); DWTAP_PRINTF("[%d] BROADCASTING m=%p\n", p->id, p->m); mtx_lock(&vis_plugin->pl_mtx); map = &vis_plugin->pl_node[p->id]; mtx_unlock(&vis_plugin->pl_mtx); /* This is O(n*n) which is not optimal for large * number of nodes. Another way of doing it is * creating groups of nodes that hear each other. * Atleast for this simple static node plugin. */ for(int i=0; imap[i]; for(int j=0; j<32; ++j){ int vis = index & 0x01; if(vis){ int k = i*ARRAY_SIZE + j; if(hal->hal_devs[k] != NULL && hal->hal_devs[k]->up == 1){ struct wtap_softc *sc = hal->hal_devs[k]; struct mbuf *m = m_dup(p->m, M_NOWAIT); DWTAP_PRINTF("[%d] duplicated old_m=%p" "to new_m=%p\n", p->id, p->m, m); #if 0 printf("[%d] sending to %d\n", p->id, k); #endif wtap_inject(sc, m); } } index = index >> 1; } } } static void add_link(struct visibility_plugin *vis_plugin, struct link *l) { mtx_lock(&vis_plugin->pl_mtx); struct vis_map *map = &vis_plugin->pl_node[l->id1]; int index = l->id2/ARRAY_SIZE; int bit = l->id2 % ARRAY_SIZE; uint32_t value = 1 << bit; map->map[index] = map->map[index] | value; mtx_unlock(&vis_plugin->pl_mtx); #if 0 printf("l->id1=%d, l->id2=%d, map->map[%d] = %u, bit=%d\n", l->id1, l->id2, index, map->map[index], bit); #endif } static void del_link(struct visibility_plugin *vis_plugin, struct link *l) { mtx_lock(&vis_plugin->pl_mtx); struct vis_map *map = &vis_plugin->pl_node[l->id1]; int index = l->id2/ARRAY_SIZE; int bit = l->id2 % ARRAY_SIZE; uint32_t value = 1 << bit; map->map[index] = map->map[index] & ~value; mtx_unlock(&vis_plugin->pl_mtx); #if 0 printf("map->map[index] = %u\n", map->map[index]); #endif } int vis_ioctl(struct cdev *sdev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct visibility_plugin *vis_plugin = (struct visibility_plugin *) sdev->si_drv1; struct wtap_hal *hal = vis_plugin->base.wp_hal; struct link l; int op; int error = 0; CURVNET_SET(CRED_TO_VNET(curthread->td_ucred)); switch(cmd) { case VISIOCTLOPEN: op = *(int *)data; if(op == 0) medium_close(hal->hal_md); else medium_open(hal->hal_md); break; case VISIOCTLLINK: l = *(struct link *)data; if(l.op == 0) del_link(vis_plugin, &l); else add_link(vis_plugin, &l); #if 0 printf("op=%d, id1=%d, id2=%d\n", l.op, l.id1, l.id2); #endif break; default: - DWTAP_PRINTF("Unkown WTAP IOCTL\n"); + DWTAP_PRINTF("Unknown WTAP IOCTL\n"); error = EINVAL; } CURVNET_RESTORE(); return error; }