Index: head/sys/cam/ata/ata_xpt.c =================================================================== --- head/sys/cam/ata/ata_xpt.c (revision 298410) +++ head/sys/cam/ata/ata_xpt.c (revision 298411) @@ -1,2093 +1,2090 @@ /*- * Copyright (c) 2009 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for xpt_print below */ #include "opt_cam.h" struct ata_quirk_entry { struct scsi_inquiry_pattern inq_pat; u_int8_t quirks; #define CAM_QUIRK_MAXTAGS 0x01 u_int mintags; u_int maxtags; }; static periph_init_t probe_periph_init; static struct periph_driver probe_driver = { probe_periph_init, "aprobe", TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0, CAM_PERIPH_DRV_EARLY }; PERIPHDRIVER_DECLARE(aprobe, probe_driver); typedef enum { PROBE_RESET, PROBE_IDENTIFY, PROBE_SPINUP, PROBE_SETMODE, PROBE_SETPM, PROBE_SETAPST, PROBE_SETDMAAA, PROBE_SETAN, PROBE_SET_MULTI, PROBE_INQUIRY, PROBE_FULL_INQUIRY, PROBE_PM_PID, PROBE_PM_PRV, PROBE_IDENTIFY_SES, PROBE_IDENTIFY_SAFTE, PROBE_DONE, PROBE_INVALID } probe_action; static char *probe_action_text[] = { "PROBE_RESET", "PROBE_IDENTIFY", "PROBE_SPINUP", "PROBE_SETMODE", "PROBE_SETPM", "PROBE_SETAPST", "PROBE_SETDMAAA", "PROBE_SETAN", "PROBE_SET_MULTI", "PROBE_INQUIRY", "PROBE_FULL_INQUIRY", "PROBE_PM_PID", "PROBE_PM_PRV", "PROBE_IDENTIFY_SES", "PROBE_IDENTIFY_SAFTE", "PROBE_DONE", "PROBE_INVALID" }; #define PROBE_SET_ACTION(softc, newaction) \ do { \ char **text; \ text = probe_action_text; \ CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE, \ ("Probe %s to %s\n", text[(softc)->action], \ text[(newaction)])); \ (softc)->action = (newaction); \ } while(0) typedef enum { PROBE_NO_ANNOUNCE = 0x04 } probe_flags; typedef struct { TAILQ_HEAD(, ccb_hdr) request_ccbs; struct ata_params ident_data; probe_action action; probe_flags flags; uint32_t pm_pid; uint32_t pm_prv; int restart; int spinup; int faults; u_int caps; struct cam_periph *periph; } probe_softc; static struct ata_quirk_entry ata_quirk_table[] = { { /* Default tagged queuing parameters for all devices */ { T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED, /*vendor*/"*", /*product*/"*", /*revision*/"*" }, /*quirks*/0, /*mintags*/0, /*maxtags*/0 }, }; -static const int ata_quirk_table_size = - sizeof(ata_quirk_table) / sizeof(*ata_quirk_table); - static cam_status proberegister(struct cam_periph *periph, void *arg); static void probeschedule(struct cam_periph *probe_periph); static void probestart(struct cam_periph *periph, union ccb *start_ccb); static void proberequestdefaultnegotiation(struct cam_periph *periph); static void probedone(struct cam_periph *periph, union ccb *done_ccb); static void probecleanup(struct cam_periph *periph); static void ata_find_quirk(struct cam_ed *device); static void ata_scan_bus(struct cam_periph *periph, union ccb *ccb); static void ata_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_flags flags, union ccb *ccb); static void xptscandone(struct cam_periph *periph, union ccb *done_ccb); static struct cam_ed * ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id); static void ata_device_transport(struct cam_path *path); static void ata_get_transfer_settings(struct ccb_trans_settings *cts); static void ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path, int async_update); static void ata_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, struct cam_ed *device, void *async_arg); static void ata_action(union ccb *start_ccb); static void ata_announce_periph(struct cam_periph *periph); static int ata_dma = 1; static int atapi_dma = 1; TUNABLE_INT("hw.ata.ata_dma", &ata_dma); TUNABLE_INT("hw.ata.atapi_dma", &atapi_dma); static struct xpt_xport ata_xport = { .alloc_device = ata_alloc_device, .action = ata_action, .async = ata_dev_async, .announce = ata_announce_periph, }; struct xpt_xport * ata_get_xport(void) { return (&ata_xport); } static void probe_periph_init() { } static cam_status proberegister(struct cam_periph *periph, void *arg) { union ccb *request_ccb; /* CCB representing the probe request */ cam_status status; probe_softc *softc; request_ccb = (union ccb *)arg; if (request_ccb == NULL) { printf("proberegister: no probe CCB, " "can't register device\n"); return(CAM_REQ_CMP_ERR); } softc = (probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT); if (softc == NULL) { printf("proberegister: Unable to probe new device. " "Unable to allocate softc\n"); return(CAM_REQ_CMP_ERR); } TAILQ_INIT(&softc->request_ccbs); TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h, periph_links.tqe); softc->flags = 0; periph->softc = softc; softc->periph = periph; softc->action = PROBE_INVALID; status = cam_periph_acquire(periph); if (status != CAM_REQ_CMP) { return (status); } CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n")); ata_device_transport(periph->path); probeschedule(periph); return(CAM_REQ_CMP); } static void probeschedule(struct cam_periph *periph) { union ccb *ccb; probe_softc *softc; softc = (probe_softc *)periph->softc; ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs); if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) || periph->path->device->protocol == PROTO_SATAPM || periph->path->device->protocol == PROTO_SEMB) PROBE_SET_ACTION(softc, PROBE_RESET); else PROBE_SET_ACTION(softc, PROBE_IDENTIFY); if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE) softc->flags |= PROBE_NO_ANNOUNCE; else softc->flags &= ~PROBE_NO_ANNOUNCE; xpt_schedule(periph, CAM_PRIORITY_XPT); } static void probestart(struct cam_periph *periph, union ccb *start_ccb) { struct ccb_trans_settings cts; struct ccb_ataio *ataio; struct ccb_scsiio *csio; probe_softc *softc; struct cam_path *path; struct ata_params *ident_buf; CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n")); softc = (probe_softc *)periph->softc; path = start_ccb->ccb_h.path; ataio = &start_ccb->ataio; csio = &start_ccb->csio; ident_buf = &periph->path->device->ident_data; if (softc->restart) { softc->restart = 0; if ((path->device->flags & CAM_DEV_UNCONFIGURED) || path->device->protocol == PROTO_SATAPM || path->device->protocol == PROTO_SEMB) softc->action = PROBE_RESET; else softc->action = PROBE_IDENTIFY; } switch (softc->action) { case PROBE_RESET: cam_fill_ataio(ataio, 0, probedone, /*flags*/CAM_DIR_NONE, 0, /*data_ptr*/NULL, /*dxfer_len*/0, 15 * 1000); ata_reset_cmd(ataio); break; case PROBE_IDENTIFY: cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_IN, 0, /*data_ptr*/(u_int8_t *)&softc->ident_data, /*dxfer_len*/sizeof(softc->ident_data), 30 * 1000); if (periph->path->device->protocol == PROTO_ATA) ata_28bit_cmd(ataio, ATA_ATA_IDENTIFY, 0, 0, 0); else ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0); break; case PROBE_SPINUP: if (bootverbose) xpt_print(path, "Spinning up device\n"); cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_NONE | CAM_HIGH_POWER, 0, /*data_ptr*/NULL, /*dxfer_len*/0, 30 * 1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_PUIS_SPINUP, 0, 0); break; case PROBE_SETMODE: { int mode, wantmode; mode = 0; /* Fetch user modes from SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_ATA) { if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE) mode = cts.xport_specific.ata.mode; } else { if (cts.xport_specific.sata.valid & CTS_SATA_VALID_MODE) mode = cts.xport_specific.sata.mode; } if (periph->path->device->protocol == PROTO_ATA) { if (ata_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX)) mode = ATA_PIO_MAX; } else { if (atapi_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX)) mode = ATA_PIO_MAX; } negotiate: /* Honor device capabilities. */ wantmode = mode = ata_max_mode(ident_buf, mode); /* Report modes to SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; if (path->device->transport == XPORT_ATA) { cts.xport_specific.ata.mode = mode; cts.xport_specific.ata.valid = CTS_ATA_VALID_MODE; } else { cts.xport_specific.sata.mode = mode; cts.xport_specific.sata.valid = CTS_SATA_VALID_MODE; } xpt_action((union ccb *)&cts); /* Fetch current modes from SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_ATA) { if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE) mode = cts.xport_specific.ata.mode; } else { if (cts.xport_specific.ata.valid & CTS_SATA_VALID_MODE) mode = cts.xport_specific.sata.mode; } /* If SIM disagree - renegotiate. */ if (mode != wantmode) goto negotiate; /* Remember what transport thinks about DMA. */ if (mode < ATA_DMA) path->device->inq_flags &= ~SID_DMA; else path->device->inq_flags |= SID_DMA; xpt_async(AC_GETDEV_CHANGED, path, NULL); cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_NONE, 0, /*data_ptr*/NULL, /*dxfer_len*/0, 30 * 1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0, mode); break; } case PROBE_SETPM: cam_fill_ataio(ataio, 1, probedone, CAM_DIR_NONE, 0, NULL, 0, 30*1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, (softc->caps & CTS_SATA_CAPS_H_PMREQ) ? 0x10 : 0x90, 0, 0x03); break; case PROBE_SETAPST: cam_fill_ataio(ataio, 1, probedone, CAM_DIR_NONE, 0, NULL, 0, 30*1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, (softc->caps & CTS_SATA_CAPS_H_APST) ? 0x10 : 0x90, 0, 0x07); break; case PROBE_SETDMAAA: cam_fill_ataio(ataio, 1, probedone, CAM_DIR_NONE, 0, NULL, 0, 30*1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, (softc->caps & CTS_SATA_CAPS_H_DMAAA) ? 0x10 : 0x90, 0, 0x02); break; case PROBE_SETAN: /* Remember what transport thinks about AEN. */ if (softc->caps & CTS_SATA_CAPS_H_AN) path->device->inq_flags |= SID_AEN; else path->device->inq_flags &= ~SID_AEN; xpt_async(AC_GETDEV_CHANGED, path, NULL); cam_fill_ataio(ataio, 1, probedone, CAM_DIR_NONE, 0, NULL, 0, 30*1000); ata_28bit_cmd(ataio, ATA_SETFEATURES, (softc->caps & CTS_SATA_CAPS_H_AN) ? 0x10 : 0x90, 0, 0x05); break; case PROBE_SET_MULTI: { u_int sectors, bytecount; bytecount = 8192; /* SATA maximum */ /* Fetch user bytecount from SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_ATA) { if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.ata.bytecount; } else { if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.sata.bytecount; } /* Honor device capabilities. */ sectors = max(1, min(ident_buf->sectors_intr & 0xff, bytecount / ata_logical_sector_size(ident_buf))); /* Report bytecount to SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; if (path->device->transport == XPORT_ATA) { cts.xport_specific.ata.bytecount = sectors * ata_logical_sector_size(ident_buf); cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT; } else { cts.xport_specific.sata.bytecount = sectors * ata_logical_sector_size(ident_buf); cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT; } xpt_action((union ccb *)&cts); /* Fetch current bytecount from SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_ATA) { if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.ata.bytecount; } else { if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.sata.bytecount; } sectors = bytecount / ata_logical_sector_size(ident_buf); cam_fill_ataio(ataio, 1, probedone, CAM_DIR_NONE, 0, NULL, 0, 30*1000); ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors); break; } case PROBE_INQUIRY: { u_int bytecount; bytecount = 8192; /* SATA maximum */ /* Fetch user bytecount from SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_ATA) { if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.ata.bytecount; } else { if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) bytecount = cts.xport_specific.sata.bytecount; } /* Honor device capabilities. */ bytecount &= ~1; bytecount = max(2, min(65534, bytecount)); if (ident_buf->satacapabilities != 0x0000 && ident_buf->satacapabilities != 0xffff) { bytecount = min(8192, bytecount); } /* Report bytecount to SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; if (path->device->transport == XPORT_ATA) { cts.xport_specific.ata.bytecount = bytecount; cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT; } else { cts.xport_specific.sata.bytecount = bytecount; cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT; } xpt_action((union ccb *)&cts); /* FALLTHROUGH */ } case PROBE_FULL_INQUIRY: { u_int inquiry_len; struct scsi_inquiry_data *inq_buf = &periph->path->device->inq_data; if (softc->action == PROBE_INQUIRY) inquiry_len = SHORT_INQUIRY_LENGTH; else inquiry_len = SID_ADDITIONAL_LENGTH(inq_buf); /* * Some parallel SCSI devices fail to send an * ignore wide residue message when dealing with * odd length inquiry requests. Round up to be * safe. */ inquiry_len = roundup2(inquiry_len, 2); scsi_inquiry(csio, /*retries*/1, probedone, MSG_SIMPLE_Q_TAG, (u_int8_t *)inq_buf, inquiry_len, /*evpd*/FALSE, /*page_code*/0, SSD_MIN_SIZE, /*timeout*/60 * 1000); break; } case PROBE_PM_PID: cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_NONE, 0, /*data_ptr*/NULL, /*dxfer_len*/0, 10 * 1000); ata_pm_read_cmd(ataio, 0, 15); break; case PROBE_PM_PRV: cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_NONE, 0, /*data_ptr*/NULL, /*dxfer_len*/0, 10 * 1000); ata_pm_read_cmd(ataio, 1, 15); break; case PROBE_IDENTIFY_SES: cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_IN, 0, /*data_ptr*/(u_int8_t *)&softc->ident_data, /*dxfer_len*/sizeof(softc->ident_data), 30 * 1000); ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x02, sizeof(softc->ident_data) / 4); break; case PROBE_IDENTIFY_SAFTE: cam_fill_ataio(ataio, 1, probedone, /*flags*/CAM_DIR_IN, 0, /*data_ptr*/(u_int8_t *)&softc->ident_data, /*dxfer_len*/sizeof(softc->ident_data), 30 * 1000); ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x00, sizeof(softc->ident_data) / 4); break; default: panic("probestart: invalid action state 0x%x\n", softc->action); } start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE; xpt_action(start_ccb); } static void proberequestdefaultnegotiation(struct cam_periph *periph) { struct ccb_trans_settings cts; xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) return; cts.xport_specific.valid = 0; cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb *)&cts); } static void probedone(struct cam_periph *periph, union ccb *done_ccb) { struct ccb_trans_settings cts; struct ata_params *ident_buf; struct scsi_inquiry_data *inq_buf; probe_softc *softc; struct cam_path *path; cam_status status; u_int32_t priority; u_int caps; int changed = 1, found = 1; static const uint8_t fake_device_id_hdr[8] = {0, SVPD_DEVICE_ID, 0, 12, SVPD_ID_CODESET_BINARY, SVPD_ID_TYPE_NAA, 0, 8}; CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probedone\n")); softc = (probe_softc *)periph->softc; path = done_ccb->ccb_h.path; priority = done_ccb->ccb_h.pinfo.priority; ident_buf = &path->device->ident_data; inq_buf = &path->device->inq_data; if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { if (cam_periph_error(done_ccb, 0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0, NULL) == ERESTART) { out: /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ cam_release_devq(path, 0, 0, 0, FALSE); return; } if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { /* Don't wedge the queue */ xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE); } status = done_ccb->ccb_h.status & CAM_STATUS_MASK; if (softc->restart) { softc->faults++; if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_CMD_TIMEOUT) softc->faults += 4; if (softc->faults < 10) goto done; else softc->restart = 0; /* Old PIO2 devices may not support mode setting. */ } else if (softc->action == PROBE_SETMODE && status == CAM_ATA_STATUS_ERROR && ata_max_pmode(ident_buf) <= ATA_PIO2 && (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0) { goto noerror; /* * Some old WD SATA disks report supported and enabled * device-initiated interface power management, but return * ABORT on attempt to disable it. */ } else if (softc->action == PROBE_SETPM && status == CAM_ATA_STATUS_ERROR) { goto noerror; /* * Some HP SATA disks report supported DMA Auto-Activation, * but return ABORT on attempt to enable it. */ } else if (softc->action == PROBE_SETDMAAA && status == CAM_ATA_STATUS_ERROR) { goto noerror; /* * SES and SAF-TE SEPs have different IDENTIFY commands, * but SATA specification doesn't tell how to identify them. * Until better way found, just try another if first fail. */ } else if (softc->action == PROBE_IDENTIFY_SES && status == CAM_ATA_STATUS_ERROR) { PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SAFTE); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } /* * If we get to this point, we got an error status back * from the inquiry and the error status doesn't require * automatically retrying the command. Therefore, the * inquiry failed. If we had inquiry information before * for this device, but this latest inquiry command failed, * the device has probably gone away. If this device isn't * already marked unconfigured, notify the peripheral * drivers that this device is no more. */ device_fail: if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0) xpt_async(AC_LOST_DEVICE, path, NULL); PROBE_SET_ACTION(softc, PROBE_INVALID); found = 0; goto done; } noerror: if (softc->restart) goto done; switch (softc->action) { case PROBE_RESET: { int sign = (done_ccb->ataio.res.lba_high << 8) + done_ccb->ataio.res.lba_mid; CAM_DEBUG(path, CAM_DEBUG_PROBE, ("SIGNATURE: %04x\n", sign)); if (sign == 0x0000 && done_ccb->ccb_h.target_id != 15) { path->device->protocol = PROTO_ATA; PROBE_SET_ACTION(softc, PROBE_IDENTIFY); } else if (sign == 0x9669 && done_ccb->ccb_h.target_id == 15) { /* Report SIM that PM is present. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.xport_specific.sata.pm_present = 1; cts.xport_specific.sata.valid = CTS_SATA_VALID_PM; xpt_action((union ccb *)&cts); path->device->protocol = PROTO_SATAPM; PROBE_SET_ACTION(softc, PROBE_PM_PID); } else if (sign == 0xc33c && done_ccb->ccb_h.target_id != 15) { path->device->protocol = PROTO_SEMB; PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SES); } else if (sign == 0xeb14 && done_ccb->ccb_h.target_id != 15) { path->device->protocol = PROTO_SCSI; PROBE_SET_ACTION(softc, PROBE_IDENTIFY); } else { if (done_ccb->ccb_h.target_id != 15) { xpt_print(path, "Unexpected signature 0x%04x\n", sign); } goto device_fail; } xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } case PROBE_IDENTIFY: { struct ccb_pathinq cpi; int16_t *ptr; ident_buf = &softc->ident_data; for (ptr = (int16_t *)ident_buf; ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) { *ptr = le16toh(*ptr); } if (strncmp(ident_buf->model, "FX", 2) && strncmp(ident_buf->model, "NEC", 3) && strncmp(ident_buf->model, "Pioneer", 7) && strncmp(ident_buf->model, "SHARP", 5)) { ata_bswap(ident_buf->model, sizeof(ident_buf->model)); ata_bswap(ident_buf->revision, sizeof(ident_buf->revision)); ata_bswap(ident_buf->serial, sizeof(ident_buf->serial)); } ata_btrim(ident_buf->model, sizeof(ident_buf->model)); ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model)); ata_btrim(ident_buf->revision, sizeof(ident_buf->revision)); ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision)); ata_btrim(ident_buf->serial, sizeof(ident_buf->serial)); ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial)); /* Device may need spin-up before IDENTIFY become valid. */ if ((ident_buf->specconf == 0x37c8 || ident_buf->specconf == 0x738c) && ((ident_buf->config & ATA_RESP_INCOMPLETE) || softc->spinup == 0)) { PROBE_SET_ACTION(softc, PROBE_SPINUP); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } ident_buf = &path->device->ident_data; if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) { /* Check that it is the same device. */ if (bcmp(softc->ident_data.model, ident_buf->model, sizeof(ident_buf->model)) || bcmp(softc->ident_data.revision, ident_buf->revision, sizeof(ident_buf->revision)) || bcmp(softc->ident_data.serial, ident_buf->serial, sizeof(ident_buf->serial))) { /* Device changed. */ xpt_async(AC_LOST_DEVICE, path, NULL); } else { bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params)); changed = 0; } } if (changed) { bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params)); /* Clean up from previous instance of this device */ if (path->device->serial_num != NULL) { free(path->device->serial_num, M_CAMXPT); path->device->serial_num = NULL; path->device->serial_num_len = 0; } if (path->device->device_id != NULL) { free(path->device->device_id, M_CAMXPT); path->device->device_id = NULL; path->device->device_id_len = 0; } path->device->serial_num = (u_int8_t *)malloc((sizeof(ident_buf->serial) + 1), M_CAMXPT, M_NOWAIT); if (path->device->serial_num != NULL) { bcopy(ident_buf->serial, path->device->serial_num, sizeof(ident_buf->serial)); path->device->serial_num[sizeof(ident_buf->serial)] = '\0'; path->device->serial_num_len = strlen(path->device->serial_num); } if (ident_buf->enabled.extension & ATA_SUPPORT_64BITWWN) { path->device->device_id = malloc(16, M_CAMXPT, M_NOWAIT); if (path->device->device_id != NULL) { path->device->device_id_len = 16; bcopy(&fake_device_id_hdr, path->device->device_id, 8); bcopy(ident_buf->wwn, path->device->device_id + 8, 8); ata_bswap(path->device->device_id + 8, 8); } } path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID; xpt_async(AC_GETDEV_CHANGED, path, NULL); } if (ident_buf->satacapabilities & ATA_SUPPORT_NCQ) { path->device->mintags = 2; path->device->maxtags = ATA_QUEUE_LEN(ident_buf->queue) + 1; } ata_find_quirk(path->device); if (path->device->mintags != 0 && path->bus->sim->max_tagged_dev_openings != 0) { /* Check if the SIM does not want queued commands. */ bzero(&cpi, sizeof(cpi)); xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_inquiry & PI_TAG_ABLE)) { /* Report SIM which tags are allowed. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.xport_specific.sata.tags = path->device->maxtags; cts.xport_specific.sata.valid = CTS_SATA_VALID_TAGS; xpt_action((union ccb *)&cts); } } ata_device_transport(path); if (changed) proberequestdefaultnegotiation(periph); PROBE_SET_ACTION(softc, PROBE_SETMODE); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } case PROBE_SPINUP: if (bootverbose) xpt_print(path, "Spin-up done\n"); softc->spinup = 1; PROBE_SET_ACTION(softc, PROBE_IDENTIFY); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; case PROBE_SETMODE: /* Set supported bits. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_SATA && cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS) caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H; else if (path->device->transport == XPORT_ATA && cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) caps = cts.xport_specific.ata.caps & CTS_ATA_CAPS_H; else caps = 0; if (path->device->transport == XPORT_SATA && ident_buf->satacapabilities != 0xffff) { if (ident_buf->satacapabilities & ATA_SUPPORT_IFPWRMNGTRCV) caps |= CTS_SATA_CAPS_D_PMREQ; if (ident_buf->satacapabilities & ATA_SUPPORT_HAPST) caps |= CTS_SATA_CAPS_D_APST; } /* Mask unwanted bits. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if (path->device->transport == XPORT_SATA && cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS) caps &= cts.xport_specific.sata.caps; else if (path->device->transport == XPORT_ATA && cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) caps &= cts.xport_specific.ata.caps; else caps = 0; /* * Remember what transport thinks about 48-bit DMA. If * capability information is not provided or transport is * SATA, we take support for granted. */ if (!(path->device->inq_flags & SID_DMA) || (path->device->transport == XPORT_ATA && (cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) && !(caps & CTS_ATA_CAPS_H_DMA48))) path->device->inq_flags &= ~SID_DMA48; else path->device->inq_flags |= SID_DMA48; /* Store result to SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; if (path->device->transport == XPORT_SATA) { cts.xport_specific.sata.caps = caps; cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS; } else { cts.xport_specific.ata.caps = caps; cts.xport_specific.ata.valid = CTS_ATA_VALID_CAPS; } xpt_action((union ccb *)&cts); softc->caps = caps; if (path->device->transport != XPORT_SATA) goto notsata; if ((ident_buf->satasupport & ATA_SUPPORT_IFPWRMNGT) && (!(softc->caps & CTS_SATA_CAPS_H_PMREQ)) != (!(ident_buf->sataenabled & ATA_SUPPORT_IFPWRMNGT))) { PROBE_SET_ACTION(softc, PROBE_SETPM); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } /* FALLTHROUGH */ case PROBE_SETPM: if (ident_buf->satacapabilities != 0xffff && (ident_buf->satacapabilities & ATA_SUPPORT_DAPST) && (!(softc->caps & CTS_SATA_CAPS_H_APST)) != (!(ident_buf->sataenabled & ATA_ENABLED_DAPST))) { PROBE_SET_ACTION(softc, PROBE_SETAPST); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } /* FALLTHROUGH */ case PROBE_SETAPST: if ((ident_buf->satasupport & ATA_SUPPORT_AUTOACTIVATE) && (!(softc->caps & CTS_SATA_CAPS_H_DMAAA)) != (!(ident_buf->sataenabled & ATA_SUPPORT_AUTOACTIVATE))) { PROBE_SET_ACTION(softc, PROBE_SETDMAAA); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } /* FALLTHROUGH */ case PROBE_SETDMAAA: if (path->device->protocol != PROTO_ATA && (ident_buf->satasupport & ATA_SUPPORT_ASYNCNOTIF) && (!(softc->caps & CTS_SATA_CAPS_H_AN)) != (!(ident_buf->sataenabled & ATA_SUPPORT_ASYNCNOTIF))) { PROBE_SET_ACTION(softc, PROBE_SETAN); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } /* FALLTHROUGH */ case PROBE_SETAN: notsata: if (path->device->protocol == PROTO_ATA) { PROBE_SET_ACTION(softc, PROBE_SET_MULTI); } else { PROBE_SET_ACTION(softc, PROBE_INQUIRY); } xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; case PROBE_SET_MULTI: if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) { path->device->flags &= ~CAM_DEV_UNCONFIGURED; xpt_acquire_device(path->device); done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_FOUND_DEVICE, path, done_ccb); } PROBE_SET_ACTION(softc, PROBE_DONE); break; case PROBE_INQUIRY: case PROBE_FULL_INQUIRY: { u_int8_t periph_qual, len; path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID; periph_qual = SID_QUAL(inq_buf); if (periph_qual != SID_QUAL_LU_CONNECTED && periph_qual != SID_QUAL_LU_OFFLINE) break; /* * We conservatively request only * SHORT_INQUIRY_LEN bytes of inquiry * information during our first try * at sending an INQUIRY. If the device * has more information to give, * perform a second request specifying * the amount of information the device * is willing to give. */ len = inq_buf->additional_length + offsetof(struct scsi_inquiry_data, additional_length) + 1; if (softc->action == PROBE_INQUIRY && len > SHORT_INQUIRY_LENGTH) { PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; } ata_device_transport(path); if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) { path->device->flags &= ~CAM_DEV_UNCONFIGURED; xpt_acquire_device(path->device); done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_FOUND_DEVICE, path, done_ccb); } PROBE_SET_ACTION(softc, PROBE_DONE); break; } case PROBE_PM_PID: if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0) bzero(ident_buf, sizeof(*ident_buf)); softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) + (done_ccb->ataio.res.lba_mid << 16) + (done_ccb->ataio.res.lba_low << 8) + done_ccb->ataio.res.sector_count; ((uint32_t *)ident_buf)[0] = softc->pm_pid; snprintf(ident_buf->model, sizeof(ident_buf->model), "Port Multiplier %08x", softc->pm_pid); PROBE_SET_ACTION(softc, PROBE_PM_PRV); xpt_release_ccb(done_ccb); xpt_schedule(periph, priority); goto out; case PROBE_PM_PRV: softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) + (done_ccb->ataio.res.lba_mid << 16) + (done_ccb->ataio.res.lba_low << 8) + done_ccb->ataio.res.sector_count; ((uint32_t *)ident_buf)[1] = softc->pm_prv; snprintf(ident_buf->revision, sizeof(ident_buf->revision), "%04x", softc->pm_prv); path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID; ata_device_transport(path); if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) proberequestdefaultnegotiation(periph); /* Set supported bits. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb *)&cts); if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS) caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H; else caps = 0; /* All PMPs must support PM requests. */ caps |= CTS_SATA_CAPS_D_PMREQ; /* Mask unwanted bits. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; xpt_action((union ccb *)&cts); if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS) caps &= cts.xport_specific.sata.caps; else caps = 0; /* Remember what transport thinks about AEN. */ if ((caps & CTS_SATA_CAPS_H_AN) && path->device->protocol != PROTO_ATA) path->device->inq_flags |= SID_AEN; else path->device->inq_flags &= ~SID_AEN; /* Store result to SIM. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.xport_specific.sata.caps = caps; cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS; xpt_action((union ccb *)&cts); softc->caps = caps; xpt_async(AC_GETDEV_CHANGED, path, NULL); if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) { path->device->flags &= ~CAM_DEV_UNCONFIGURED; xpt_acquire_device(path->device); done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_FOUND_DEVICE, path, done_ccb); } else { done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_SCSI_AEN, path, done_ccb); } PROBE_SET_ACTION(softc, PROBE_DONE); break; case PROBE_IDENTIFY_SES: case PROBE_IDENTIFY_SAFTE: if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) { /* Check that it is the same device. */ if (bcmp(&softc->ident_data, ident_buf, 53)) { /* Device changed. */ xpt_async(AC_LOST_DEVICE, path, NULL); } else { bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params)); changed = 0; } } if (changed) { bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params)); /* Clean up from previous instance of this device */ if (path->device->device_id != NULL) { free(path->device->device_id, M_CAMXPT); path->device->device_id = NULL; path->device->device_id_len = 0; } path->device->device_id = malloc(16, M_CAMXPT, M_NOWAIT); if (path->device->device_id != NULL) { path->device->device_id_len = 16; bcopy(&fake_device_id_hdr, path->device->device_id, 8); bcopy(((uint8_t*)ident_buf) + 2, path->device->device_id + 8, 8); } path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID; } ata_device_transport(path); if (changed) proberequestdefaultnegotiation(periph); if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) { path->device->flags &= ~CAM_DEV_UNCONFIGURED; xpt_acquire_device(path->device); done_ccb->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action(done_ccb); xpt_async(AC_FOUND_DEVICE, path, done_ccb); } PROBE_SET_ACTION(softc, PROBE_DONE); break; default: panic("probedone: invalid action state 0x%x\n", softc->action); } done: if (softc->restart) { softc->restart = 0; xpt_release_ccb(done_ccb); probeschedule(periph); goto out; } xpt_release_ccb(done_ccb); CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n")); while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) { TAILQ_REMOVE(&softc->request_ccbs, &done_ccb->ccb_h, periph_links.tqe); done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR; xpt_done(done_ccb); } /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ cam_release_devq(path, 0, 0, 0, FALSE); cam_periph_invalidate(periph); cam_periph_release_locked(periph); } static void probecleanup(struct cam_periph *periph) { free(periph->softc, M_CAMXPT); } static void ata_find_quirk(struct cam_ed *device) { struct ata_quirk_entry *quirk; caddr_t match; match = cam_quirkmatch((caddr_t)&device->ident_data, (caddr_t)ata_quirk_table, - ata_quirk_table_size, + nitems(ata_quirk_table), sizeof(*ata_quirk_table), ata_identify_match); if (match == NULL) panic("xpt_find_quirk: device didn't match wildcard entry!!"); quirk = (struct ata_quirk_entry *)match; device->quirk = quirk; if (quirk->quirks & CAM_QUIRK_MAXTAGS) { device->mintags = quirk->mintags; device->maxtags = quirk->maxtags; } } typedef struct { union ccb *request_ccb; struct ccb_pathinq *cpi; int counter; } ata_scan_bus_info; /* * To start a scan, request_ccb is an XPT_SCAN_BUS ccb. * As the scan progresses, xpt_scan_bus is used as the * callback on completion function. */ static void ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb) { struct cam_path *path; ata_scan_bus_info *scan_info; union ccb *work_ccb, *reset_ccb; struct mtx *mtx; cam_status status; CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_scan_bus\n")); switch (request_ccb->ccb_h.func_code) { case XPT_SCAN_BUS: case XPT_SCAN_TGT: /* Find out the characteristics of the bus */ work_ccb = xpt_alloc_ccb_nowait(); if (work_ccb == NULL) { request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_done(request_ccb); return; } xpt_setup_ccb(&work_ccb->ccb_h, request_ccb->ccb_h.path, request_ccb->ccb_h.pinfo.priority); work_ccb->ccb_h.func_code = XPT_PATH_INQ; xpt_action(work_ccb); if (work_ccb->ccb_h.status != CAM_REQ_CMP) { request_ccb->ccb_h.status = work_ccb->ccb_h.status; xpt_free_ccb(work_ccb); xpt_done(request_ccb); return; } /* We may need to reset bus first, if we haven't done it yet. */ if ((work_ccb->cpi.hba_inquiry & (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE)) && !(work_ccb->cpi.hba_misc & PIM_NOBUSRESET) && !timevalisset(&request_ccb->ccb_h.path->bus->last_reset)) { reset_ccb = xpt_alloc_ccb_nowait(); if (reset_ccb == NULL) { request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_free_ccb(work_ccb); xpt_done(request_ccb); return; } xpt_setup_ccb(&reset_ccb->ccb_h, request_ccb->ccb_h.path, CAM_PRIORITY_NONE); reset_ccb->ccb_h.func_code = XPT_RESET_BUS; xpt_action(reset_ccb); if (reset_ccb->ccb_h.status != CAM_REQ_CMP) { request_ccb->ccb_h.status = reset_ccb->ccb_h.status; xpt_free_ccb(reset_ccb); xpt_free_ccb(work_ccb); xpt_done(request_ccb); return; } xpt_free_ccb(reset_ccb); } /* Save some state for use while we probe for devices */ scan_info = (ata_scan_bus_info *) malloc(sizeof(ata_scan_bus_info), M_CAMXPT, M_NOWAIT); if (scan_info == NULL) { request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_free_ccb(work_ccb); xpt_done(request_ccb); return; } scan_info->request_ccb = request_ccb; scan_info->cpi = &work_ccb->cpi; /* If PM supported, probe it first. */ if (scan_info->cpi->hba_inquiry & PI_SATAPM) scan_info->counter = scan_info->cpi->max_target; else scan_info->counter = 0; work_ccb = xpt_alloc_ccb_nowait(); if (work_ccb == NULL) { free(scan_info, M_CAMXPT); request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL; xpt_done(request_ccb); break; } mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); goto scan_next; case XPT_SCAN_LUN: work_ccb = request_ccb; /* Reuse the same CCB to query if a device was really found */ scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0; mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); mtx_lock(mtx); /* If there is PMP... */ if ((scan_info->cpi->hba_inquiry & PI_SATAPM) && (scan_info->counter == scan_info->cpi->max_target)) { if (work_ccb->ccb_h.status == CAM_REQ_CMP) { /* everything else will be probed by it */ /* Free the current request path- we're done with it. */ xpt_free_path(work_ccb->ccb_h.path); goto done; } else { struct ccb_trans_settings cts; /* Report SIM that PM is absent. */ bzero(&cts, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, work_ccb->ccb_h.path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.xport_specific.sata.pm_present = 0; cts.xport_specific.sata.valid = CTS_SATA_VALID_PM; xpt_action((union ccb *)&cts); } } /* Free the current request path- we're done with it. */ xpt_free_path(work_ccb->ccb_h.path); if (scan_info->counter == ((scan_info->cpi->hba_inquiry & PI_SATAPM) ? 0 : scan_info->cpi->max_target)) { done: mtx_unlock(mtx); xpt_free_ccb(work_ccb); xpt_free_ccb((union ccb *)scan_info->cpi); request_ccb = scan_info->request_ccb; free(scan_info, M_CAMXPT); request_ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(request_ccb); break; } /* Take next device. Wrap from max (PMP) to 0. */ scan_info->counter = (scan_info->counter + 1 ) % (scan_info->cpi->max_target + 1); scan_next: status = xpt_create_path(&path, NULL, scan_info->request_ccb->ccb_h.path_id, scan_info->counter, 0); if (status != CAM_REQ_CMP) { if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) mtx_unlock(mtx); printf("xpt_scan_bus: xpt_create_path failed" " with status %#x, bus scan halted\n", status); xpt_free_ccb(work_ccb); xpt_free_ccb((union ccb *)scan_info->cpi); request_ccb = scan_info->request_ccb; free(scan_info, M_CAMXPT); request_ccb->ccb_h.status = status; xpt_done(request_ccb); break; } xpt_setup_ccb(&work_ccb->ccb_h, path, scan_info->request_ccb->ccb_h.pinfo.priority); work_ccb->ccb_h.func_code = XPT_SCAN_LUN; work_ccb->ccb_h.cbfcnp = ata_scan_bus; work_ccb->ccb_h.flags |= CAM_UNLOCKED; work_ccb->ccb_h.ppriv_ptr0 = scan_info; work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags; mtx_unlock(mtx); if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) mtx = NULL; xpt_action(work_ccb); if (mtx != NULL) mtx_lock(mtx); break; default: break; } } static void ata_scan_lun(struct cam_periph *periph, struct cam_path *path, cam_flags flags, union ccb *request_ccb) { struct ccb_pathinq cpi; cam_status status; struct cam_path *new_path; struct cam_periph *old_periph; int lock; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n")); xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); if (cpi.ccb_h.status != CAM_REQ_CMP) { if (request_ccb != NULL) { request_ccb->ccb_h.status = cpi.ccb_h.status; xpt_done(request_ccb); } return; } if (request_ccb == NULL) { request_ccb = xpt_alloc_ccb_nowait(); if (request_ccb == NULL) { xpt_print(path, "xpt_scan_lun: can't allocate CCB, " "can't continue\n"); return; } status = xpt_create_path(&new_path, NULL, path->bus->path_id, path->target->target_id, path->device->lun_id); if (status != CAM_REQ_CMP) { xpt_print(path, "xpt_scan_lun: can't create path, " "can't continue\n"); xpt_free_ccb(request_ccb); return; } xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT); request_ccb->ccb_h.cbfcnp = xptscandone; request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->ccb_h.func_code = XPT_SCAN_LUN; request_ccb->crcn.flags = flags; } lock = (xpt_path_owned(path) == 0); if (lock) xpt_path_lock(path); if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) { if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) { probe_softc *softc; softc = (probe_softc *)old_periph->softc; TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h, periph_links.tqe); softc->restart = 1; } else { request_ccb->ccb_h.status = CAM_REQ_CMP_ERR; xpt_done(request_ccb); } } else { status = cam_periph_alloc(proberegister, NULL, probecleanup, probestart, "aprobe", CAM_PERIPH_BIO, request_ccb->ccb_h.path, NULL, 0, request_ccb); if (status != CAM_REQ_CMP) { xpt_print(path, "xpt_scan_lun: cam_alloc_periph " "returned an error, can't continue probe\n"); request_ccb->ccb_h.status = status; xpt_done(request_ccb); } } if (lock) xpt_path_unlock(path); } static void xptscandone(struct cam_periph *periph, union ccb *done_ccb) { xpt_free_path(done_ccb->ccb_h.path); xpt_free_ccb(done_ccb); } static struct cam_ed * ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { struct ata_quirk_entry *quirk; struct cam_ed *device; device = xpt_alloc_device(bus, target, lun_id); if (device == NULL) return (NULL); /* * Take the default quirk entry until we have inquiry * data and can determine a better quirk to use. */ - quirk = &ata_quirk_table[ata_quirk_table_size - 1]; + quirk = &ata_quirk_table[nitems(ata_quirk_table) - 1]; device->quirk = (void *)quirk; device->mintags = 0; device->maxtags = 0; bzero(&device->inq_data, sizeof(device->inq_data)); device->inq_flags = 0; device->queue_flags = 0; device->serial_num = NULL; device->serial_num_len = 0; return (device); } static void ata_device_transport(struct cam_path *path) { struct ccb_pathinq cpi; struct ccb_trans_settings cts; struct scsi_inquiry_data *inq_buf = NULL; struct ata_params *ident_buf = NULL; /* Get transport information from the SIM */ xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); path->device->transport = cpi.transport; if ((path->device->flags & CAM_DEV_INQUIRY_DATA_VALID) != 0) inq_buf = &path->device->inq_data; if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) != 0) ident_buf = &path->device->ident_data; if (path->device->protocol == PROTO_ATA) { path->device->protocol_version = ident_buf ? ata_version(ident_buf->version_major) : cpi.protocol_version; } else if (path->device->protocol == PROTO_SCSI) { path->device->protocol_version = inq_buf ? SID_ANSI_REV(inq_buf) : cpi.protocol_version; } path->device->transport_version = ident_buf ? ata_version(ident_buf->version_major) : cpi.transport_version; /* Tell the controller what we think */ xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; cts.transport = path->device->transport; cts.transport_version = path->device->transport_version; cts.protocol = path->device->protocol; cts.protocol_version = path->device->protocol_version; cts.proto_specific.valid = 0; if (ident_buf) { if (path->device->transport == XPORT_ATA) { cts.xport_specific.ata.atapi = (ident_buf->config == ATA_PROTO_CFA) ? 0 : ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 : ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0; cts.xport_specific.ata.valid = CTS_ATA_VALID_ATAPI; } else { cts.xport_specific.sata.atapi = (ident_buf->config == ATA_PROTO_CFA) ? 0 : ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 : ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0; cts.xport_specific.sata.valid = CTS_SATA_VALID_ATAPI; } } else cts.xport_specific.valid = 0; xpt_action((union ccb *)&cts); } static void ata_dev_advinfo(union ccb *start_ccb) { struct cam_ed *device; struct ccb_dev_advinfo *cdai; off_t amt; start_ccb->ccb_h.status = CAM_REQ_INVALID; device = start_ccb->ccb_h.path->device; cdai = &start_ccb->cdai; switch(cdai->buftype) { case CDAI_TYPE_SCSI_DEVID: if (cdai->flags & CDAI_FLAG_STORE) return; cdai->provsiz = device->device_id_len; if (device->device_id_len == 0) break; amt = device->device_id_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->device_id, amt); break; case CDAI_TYPE_SERIAL_NUM: if (cdai->flags & CDAI_FLAG_STORE) return; cdai->provsiz = device->serial_num_len; if (device->serial_num_len == 0) break; amt = device->serial_num_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->serial_num, amt); break; case CDAI_TYPE_PHYS_PATH: if (cdai->flags & CDAI_FLAG_STORE) { if (device->physpath != NULL) free(device->physpath, M_CAMXPT); device->physpath_len = cdai->bufsiz; /* Clear existing buffer if zero length */ if (cdai->bufsiz == 0) break; device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT); if (device->physpath == NULL) { start_ccb->ccb_h.status = CAM_REQ_ABORTED; return; } memcpy(device->physpath, cdai->buf, cdai->bufsiz); } else { cdai->provsiz = device->physpath_len; if (device->physpath_len == 0) break; amt = device->physpath_len; if (cdai->provsiz > cdai->bufsiz) amt = cdai->bufsiz; memcpy(cdai->buf, device->physpath, amt); } break; default: return; } start_ccb->ccb_h.status = CAM_REQ_CMP; if (cdai->flags & CDAI_FLAG_STORE) { xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path, (void *)(uintptr_t)cdai->buftype); } } static void ata_action(union ccb *start_ccb) { switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: { ata_set_transfer_settings(&start_ccb->cts, start_ccb->ccb_h.path, /*async_update*/FALSE); break; } case XPT_SCAN_BUS: case XPT_SCAN_TGT: ata_scan_bus(start_ccb->ccb_h.path->periph, start_ccb); break; case XPT_SCAN_LUN: ata_scan_lun(start_ccb->ccb_h.path->periph, start_ccb->ccb_h.path, start_ccb->crcn.flags, start_ccb); break; case XPT_GET_TRAN_SETTINGS: { ata_get_transfer_settings(&start_ccb->cts); break; } case XPT_SCSI_IO: { struct cam_ed *device; u_int maxlen = 0; device = start_ccb->ccb_h.path->device; if (device->protocol == PROTO_SCSI && (device->flags & CAM_DEV_IDENTIFY_DATA_VALID)) { uint16_t p = device->ident_data.config & ATA_PROTO_MASK; maxlen = (device->ident_data.config == ATA_PROTO_CFA) ? 0 : (p == ATA_PROTO_ATAPI_16) ? 16 : (p == ATA_PROTO_ATAPI_12) ? 12 : 0; } if (start_ccb->csio.cdb_len > maxlen) { start_ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(start_ccb); break; } xpt_action_default(start_ccb); break; } case XPT_DEV_ADVINFO: { ata_dev_advinfo(start_ccb); break; } default: xpt_action_default(start_ccb); break; } } static void ata_get_transfer_settings(struct ccb_trans_settings *cts) { struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; struct cam_ed *device; device = cts->ccb_h.path->device; xpt_action_default((union ccb *)cts); if (cts->protocol == PROTO_UNKNOWN || cts->protocol == PROTO_UNSPECIFIED) { cts->protocol = device->protocol; cts->protocol_version = device->protocol_version; } if (cts->protocol == PROTO_ATA) { ata = &cts->proto_specific.ata; if ((ata->valid & CTS_ATA_VALID_TQ) == 0) { ata->valid |= CTS_ATA_VALID_TQ; if (cts->type == CTS_TYPE_USER_SETTINGS || (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 || (device->inq_flags & SID_CmdQue) != 0) ata->flags |= CTS_ATA_FLAGS_TAG_ENB; } } if (cts->protocol == PROTO_SCSI) { scsi = &cts->proto_specific.scsi; if ((scsi->valid & CTS_SCSI_VALID_TQ) == 0) { scsi->valid |= CTS_SCSI_VALID_TQ; if (cts->type == CTS_TYPE_USER_SETTINGS || (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 || (device->inq_flags & SID_CmdQue) != 0) scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB; } } if (cts->transport == XPORT_UNKNOWN || cts->transport == XPORT_UNSPECIFIED) { cts->transport = device->transport; cts->transport_version = device->transport_version; } } static void ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path, int async_update) { struct ccb_pathinq cpi; struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; struct ata_params *ident_data; struct scsi_inquiry_data *inq_data; struct cam_ed *device; if (path == NULL || (device = path->device) == NULL) { cts->ccb_h.status = CAM_PATH_INVALID; xpt_done((union ccb *)cts); return; } if (cts->protocol == PROTO_UNKNOWN || cts->protocol == PROTO_UNSPECIFIED) { cts->protocol = device->protocol; cts->protocol_version = device->protocol_version; } if (cts->protocol_version == PROTO_VERSION_UNKNOWN || cts->protocol_version == PROTO_VERSION_UNSPECIFIED) cts->protocol_version = device->protocol_version; if (cts->protocol != device->protocol) { xpt_print(path, "Uninitialized Protocol %x:%x?\n", cts->protocol, device->protocol); cts->protocol = device->protocol; } if (cts->protocol_version > device->protocol_version) { if (bootverbose) { xpt_print(path, "Down reving Protocol " "Version from %d to %d?\n", cts->protocol_version, device->protocol_version); } cts->protocol_version = device->protocol_version; } if (cts->transport == XPORT_UNKNOWN || cts->transport == XPORT_UNSPECIFIED) { cts->transport = device->transport; cts->transport_version = device->transport_version; } if (cts->transport_version == XPORT_VERSION_UNKNOWN || cts->transport_version == XPORT_VERSION_UNSPECIFIED) cts->transport_version = device->transport_version; if (cts->transport != device->transport) { xpt_print(path, "Uninitialized Transport %x:%x?\n", cts->transport, device->transport); cts->transport = device->transport; } if (cts->transport_version > device->transport_version) { if (bootverbose) { xpt_print(path, "Down reving Transport " "Version from %d to %d?\n", cts->transport_version, device->transport_version); } cts->transport_version = device->transport_version; } ident_data = &device->ident_data; inq_data = &device->inq_data; if (cts->protocol == PROTO_ATA) ata = &cts->proto_specific.ata; else ata = NULL; if (cts->protocol == PROTO_SCSI) scsi = &cts->proto_specific.scsi; else scsi = NULL; xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); /* Sanity checking */ if ((cpi.hba_inquiry & PI_TAG_ABLE) == 0 || (ata && (ident_data->satacapabilities & ATA_SUPPORT_NCQ) == 0) || (scsi && (INQ_DATA_TQ_ENABLED(inq_data)) == 0) || (device->queue_flags & SCP_QUEUE_DQUE) != 0 || (device->mintags == 0)) { /* * Can't tag on hardware that doesn't support tags, * doesn't have it enabled, or has broken tag support. */ if (ata) ata->flags &= ~CTS_ATA_FLAGS_TAG_ENB; if (scsi) scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB; } /* Start/stop tags use. */ if (cts->type == CTS_TYPE_CURRENT_SETTINGS && ((ata && (ata->valid & CTS_ATA_VALID_TQ) != 0) || (scsi && (scsi->valid & CTS_SCSI_VALID_TQ) != 0))) { int nowt, newt = 0; nowt = ((device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 || (device->inq_flags & SID_CmdQue) != 0); if (ata) newt = (ata->flags & CTS_ATA_FLAGS_TAG_ENB) != 0; if (scsi) newt = (scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0; if (newt && !nowt) { /* * Delay change to use tags until after a * few commands have gone to this device so * the controller has time to perform transfer * negotiations without tagged messages getting * in the way. */ device->tag_delay_count = CAM_TAG_DELAY_COUNT; device->flags |= CAM_DEV_TAG_AFTER_COUNT; } else if (nowt && !newt) xpt_stop_tags(path); } if (async_update == FALSE) xpt_action_default((union ccb *)cts); } /* * Handle any per-device event notifications that require action by the XPT. */ static void ata_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target, struct cam_ed *device, void *async_arg) { cam_status status; struct cam_path newpath; /* * We only need to handle events for real devices. */ if (target->target_id == CAM_TARGET_WILDCARD || device->lun_id == CAM_LUN_WILDCARD) return; /* * We need our own path with wildcards expanded to * handle certain types of events. */ if ((async_code == AC_SENT_BDR) || (async_code == AC_BUS_RESET) || (async_code == AC_INQ_CHANGED)) status = xpt_compile_path(&newpath, NULL, bus->path_id, target->target_id, device->lun_id); else status = CAM_REQ_CMP_ERR; if (status == CAM_REQ_CMP) { if (async_code == AC_INQ_CHANGED) { /* * We've sent a start unit command, or * something similar to a device that * may have caused its inquiry data to * change. So we re-scan the device to * refresh the inquiry data for it. */ ata_scan_lun(newpath.periph, &newpath, CAM_EXPECT_INQ_CHANGE, NULL); } else { /* We need to reinitialize device after reset. */ ata_scan_lun(newpath.periph, &newpath, 0, NULL); } xpt_release_path(&newpath); } else if (async_code == AC_LOST_DEVICE && (device->flags & CAM_DEV_UNCONFIGURED) == 0) { device->flags |= CAM_DEV_UNCONFIGURED; xpt_release_device(device); } else if (async_code == AC_TRANSFER_NEG) { struct ccb_trans_settings *settings; struct cam_path path; settings = (struct ccb_trans_settings *)async_arg; xpt_compile_path(&path, NULL, bus->path_id, target->target_id, device->lun_id); ata_set_transfer_settings(settings, &path, /*async_update*/TRUE); xpt_release_path(&path); } } static void ata_announce_periph(struct cam_periph *periph) { struct ccb_pathinq cpi; struct ccb_trans_settings cts; struct cam_path *path = periph->path; u_int speed; u_int mb; cam_periph_assert(periph, MA_OWNED); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; xpt_action((union ccb*)&cts); if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) return; /* Ask the SIM for its base transfer speed */ xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); /* Report connection speed */ speed = cpi.base_transfer_speed; if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) { struct ccb_trans_settings_pata *pata = &cts.xport_specific.ata; if (pata->valid & CTS_ATA_VALID_MODE) speed = ata_mode2speed(pata->mode); } if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) { struct ccb_trans_settings_sata *sata = &cts.xport_specific.sata; if (sata->valid & CTS_SATA_VALID_REVISION) speed = ata_revision2speed(sata->revision); } mb = speed / 1000; if (mb > 0) printf("%s%d: %d.%03dMB/s transfers", periph->periph_name, periph->unit_number, mb, speed % 1000); else printf("%s%d: %dKB/s transfers", periph->periph_name, periph->unit_number, speed); /* Report additional information about connection */ if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) { struct ccb_trans_settings_pata *pata = &cts.xport_specific.ata; printf(" ("); if (pata->valid & CTS_ATA_VALID_MODE) printf("%s, ", ata_mode2string(pata->mode)); if ((pata->valid & CTS_ATA_VALID_ATAPI) && pata->atapi != 0) printf("ATAPI %dbytes, ", pata->atapi); if (pata->valid & CTS_ATA_VALID_BYTECOUNT) printf("PIO %dbytes", pata->bytecount); printf(")"); } if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) { struct ccb_trans_settings_sata *sata = &cts.xport_specific.sata; printf(" ("); if (sata->valid & CTS_SATA_VALID_REVISION) printf("SATA %d.x, ", sata->revision); else printf("SATA, "); if (sata->valid & CTS_SATA_VALID_MODE) printf("%s, ", ata_mode2string(sata->mode)); if ((sata->valid & CTS_ATA_VALID_ATAPI) && sata->atapi != 0) printf("ATAPI %dbytes, ", sata->atapi); if (sata->valid & CTS_SATA_VALID_BYTECOUNT) printf("PIO %dbytes", sata->bytecount); printf(")"); } printf("\n"); } Index: head/sys/cam/cam.c =================================================================== --- head/sys/cam/cam.c (revision 298410) +++ head/sys/cam/cam.c (revision 298411) @@ -1,534 +1,531 @@ /*- * Generic utility routines for the Common Access Method layer. * * Copyright (c) 1997 Justin T. Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #ifdef _KERNEL #include #include #include #else /* _KERNEL */ #include #include #include #include #endif /* _KERNEL */ #include #include #include #include #include #ifdef _KERNEL #include #include #include FEATURE(scbus, "SCSI devices support"); #endif static int camstatusentrycomp(const void *key, const void *member); const struct cam_status_entry cam_status_table[] = { { CAM_REQ_INPROG, "CCB request is in progress" }, { CAM_REQ_CMP, "CCB request completed without error" }, { CAM_REQ_ABORTED, "CCB request aborted by the host" }, { CAM_UA_ABORT, "Unable to abort CCB request" }, { CAM_REQ_CMP_ERR, "CCB request completed with an error" }, { CAM_BUSY, "CAM subsystem is busy" }, { CAM_REQ_INVALID, "CCB request was invalid" }, { CAM_PATH_INVALID, "Supplied Path ID is invalid" }, { CAM_DEV_NOT_THERE, "Device Not Present" }, { CAM_UA_TERMIO, "Unable to terminate I/O CCB request" }, { CAM_SEL_TIMEOUT, "Selection Timeout" }, { CAM_CMD_TIMEOUT, "Command timeout" }, { CAM_SCSI_STATUS_ERROR, "SCSI Status Error" }, { CAM_MSG_REJECT_REC, "Message Reject Reveived" }, { CAM_SCSI_BUS_RESET, "SCSI Bus Reset Sent/Received" }, { CAM_UNCOR_PARITY, "Uncorrectable parity/CRC error" }, { CAM_AUTOSENSE_FAIL, "Auto-Sense Retrieval Failed" }, { CAM_NO_HBA, "No HBA Detected" }, { CAM_DATA_RUN_ERR, "Data Overrun error" }, { CAM_UNEXP_BUSFREE, "Unexpected Bus Free" }, { CAM_SEQUENCE_FAIL, "Target Bus Phase Sequence Failure" }, { CAM_CCB_LEN_ERR, "CCB length supplied is inadequate" }, { CAM_PROVIDE_FAIL, "Unable to provide requested capability" }, { CAM_BDR_SENT, "SCSI BDR Message Sent" }, { CAM_REQ_TERMIO, "CCB request terminated by the host" }, { CAM_UNREC_HBA_ERROR, "Unrecoverable Host Bus Adapter Error" }, { CAM_REQ_TOO_BIG, "The request was too large for this host" }, { CAM_REQUEUE_REQ, "Unconditionally Re-queue Request", }, { CAM_ATA_STATUS_ERROR, "ATA Status Error" }, { CAM_SCSI_IT_NEXUS_LOST,"Initiator/Target Nexus Lost" }, { CAM_SMP_STATUS_ERROR, "SMP Status Error" }, { CAM_IDE, "Initiator Detected Error Message Received" }, { CAM_RESRC_UNAVAIL, "Resource Unavailable" }, { CAM_UNACKED_EVENT, "Unacknowledged Event by Host" }, { CAM_MESSAGE_RECV, "Message Received in Host Target Mode" }, { CAM_INVALID_CDB, "Invalid CDB received in Host Target Mode" }, { CAM_LUN_INVALID, "Invalid Lun" }, { CAM_TID_INVALID, "Invalid Target ID" }, { CAM_FUNC_NOTAVAIL, "Function Not Available" }, { CAM_NO_NEXUS, "Nexus Not Established" }, { CAM_IID_INVALID, "Invalid Initiator ID" }, { CAM_CDB_RECVD, "CDB Received" }, { CAM_LUN_ALRDY_ENA, "LUN Already Enabled for Target Mode" }, { CAM_SCSI_BUSY, "SCSI Bus Busy" }, }; -const int num_cam_status_entries = - sizeof(cam_status_table)/sizeof(*cam_status_table); - #ifdef _KERNEL SYSCTL_NODE(_kern, OID_AUTO, cam, CTLFLAG_RD, 0, "CAM Subsystem"); #ifndef CAM_DEFAULT_SORT_IO_QUEUES #define CAM_DEFAULT_SORT_IO_QUEUES 1 #endif int cam_sort_io_queues = CAM_DEFAULT_SORT_IO_QUEUES; SYSCTL_INT(_kern_cam, OID_AUTO, sort_io_queues, CTLFLAG_RWTUN, &cam_sort_io_queues, 0, "Sort IO queues to try and optimise disk access patterns"); #endif void cam_strvis(u_int8_t *dst, const u_int8_t *src, int srclen, int dstlen) { /* Trim leading/trailing spaces, nulls. */ while (srclen > 0 && src[0] == ' ') src++, srclen--; while (srclen > 0 && (src[srclen-1] == ' ' || src[srclen-1] == '\0')) srclen--; while (srclen > 0 && dstlen > 1) { u_int8_t *cur_pos = dst; if (*src < 0x20 || *src >= 0x80) { /* SCSI-II Specifies that these should never occur. */ /* non-printable character */ if (dstlen > 4) { *cur_pos++ = '\\'; *cur_pos++ = ((*src & 0300) >> 6) + '0'; *cur_pos++ = ((*src & 0070) >> 3) + '0'; *cur_pos++ = ((*src & 0007) >> 0) + '0'; } else { *cur_pos++ = '?'; } } else { /* normal character */ *cur_pos++ = *src; } src++; srclen--; dstlen -= cur_pos - dst; dst = cur_pos; } *dst = '\0'; } void cam_strvis_sbuf(struct sbuf *sb, const u_int8_t *src, int srclen, uint32_t flags) { /* Trim leading/trailing spaces, nulls. */ while (srclen > 0 && src[0] == ' ') src++, srclen--; while (srclen > 0 && (src[srclen-1] == ' ' || src[srclen-1] == '\0')) srclen--; while (srclen > 0) { if (*src < 0x20 || *src >= 0x80) { /* SCSI-II Specifies that these should never occur. */ /* non-printable character */ switch (flags & CAM_STRVIS_FLAG_NONASCII_MASK) { case CAM_STRVIS_FLAG_NONASCII_ESC: sbuf_printf(sb, "\\%c%c%c", ((*src & 0300) >> 6) + '0', ((*src & 0070) >> 3) + '0', ((*src & 0007) >> 0) + '0'); break; case CAM_STRVIS_FLAG_NONASCII_RAW: /* * If we run into a NUL, just transform it * into a space. */ if (*src != 0x00) sbuf_putc(sb, *src); else sbuf_putc(sb, ' '); break; case CAM_STRVIS_FLAG_NONASCII_SPC: sbuf_putc(sb, ' '); break; case CAM_STRVIS_FLAG_NONASCII_TRIM: default: break; } } else { /* normal character */ sbuf_putc(sb, *src); } src++; srclen--; } } /* * Compare string with pattern, returning 0 on match. * Short pattern matches trailing blanks in name, * wildcard '*' in pattern matches rest of name, * wildcard '?' matches a single non-space character. */ int cam_strmatch(const u_int8_t *str, const u_int8_t *pattern, int str_len) { while (*pattern != '\0'&& str_len > 0) { if (*pattern == '*') { return (0); } if ((*pattern != *str) && (*pattern != '?' || *str == ' ')) { return (1); } pattern++; str++; str_len--; } while (str_len > 0 && *str == ' ') { str++; str_len--; } if (str_len > 0 && *str == 0) str_len = 0; return (str_len); } caddr_t cam_quirkmatch(caddr_t target, caddr_t quirk_table, int num_entries, int entry_size, cam_quirkmatch_t *comp_func) { for (; num_entries > 0; num_entries--, quirk_table += entry_size) { if ((*comp_func)(target, quirk_table) == 0) return (quirk_table); } return (NULL); } const struct cam_status_entry* cam_fetch_status_entry(cam_status status) { status &= CAM_STATUS_MASK; return (bsearch(&status, &cam_status_table, - num_cam_status_entries, + nitems(cam_status_table), sizeof(*cam_status_table), camstatusentrycomp)); } static int camstatusentrycomp(const void *key, const void *member) { cam_status status; const struct cam_status_entry *table_entry; status = *(const cam_status *)key; table_entry = (const struct cam_status_entry *)member; return (status - table_entry->status_code); } #ifdef _KERNEL char * cam_error_string(union ccb *ccb, char *str, int str_len, cam_error_string_flags flags, cam_error_proto_flags proto_flags) #else /* !_KERNEL */ char * cam_error_string(struct cam_device *device, union ccb *ccb, char *str, int str_len, cam_error_string_flags flags, cam_error_proto_flags proto_flags) #endif /* _KERNEL/!_KERNEL */ { char path_str[64]; struct sbuf sb; if ((ccb == NULL) || (str == NULL) || (str_len <= 0)) return(NULL); if (flags == CAM_ESF_NONE) return(NULL); switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: case CAM_EPF_NORMAL: proto_flags |= CAM_EAF_PRINT_RESULT; /* FALLTHROUGH */ case CAM_EPF_MINIMAL: proto_flags |= CAM_EAF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; case XPT_SCSI_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: case CAM_EPF_NORMAL: proto_flags |= CAM_ESF_PRINT_SENSE; /* FALLTHROUGH */ case CAM_EPF_MINIMAL: proto_flags |= CAM_ESF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; case XPT_SMP_IO: switch (proto_flags & CAM_EPF_LEVEL_MASK) { case CAM_EPF_NONE: break; case CAM_EPF_ALL: proto_flags |= CAM_ESMF_PRINT_FULL_CMD; /* FALLTHROUGH */ case CAM_EPF_NORMAL: case CAM_EPF_MINIMAL: proto_flags |= CAM_ESMF_PRINT_STATUS; /* FALLTHROUGH */ default: break; } break; default: break; } #ifdef _KERNEL xpt_path_string(ccb->csio.ccb_h.path, path_str, sizeof(path_str)); #else /* !_KERNEL */ cam_path_string(device, path_str, sizeof(path_str)); #endif /* _KERNEL/!_KERNEL */ sbuf_new(&sb, str, str_len, 0); if (flags & CAM_ESF_COMMAND) { sbuf_cat(&sb, path_str); switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: ata_command_sbuf(&ccb->ataio, &sb); sbuf_printf(&sb, "\n"); break; case XPT_SCSI_IO: #ifdef _KERNEL scsi_command_string(&ccb->csio, &sb); #else /* !_KERNEL */ scsi_command_string(device, &ccb->csio, &sb); #endif /* _KERNEL/!_KERNEL */ sbuf_printf(&sb, "\n"); break; case XPT_SMP_IO: smp_command_sbuf(&ccb->smpio, &sb, path_str, 79 - strlen(path_str), (proto_flags & CAM_ESMF_PRINT_FULL_CMD) ? 79 : 0); sbuf_printf(&sb, "\n"); break; default: break; } } if (flags & CAM_ESF_CAM_STATUS) { cam_status status; const struct cam_status_entry *entry; sbuf_cat(&sb, path_str); status = ccb->ccb_h.status & CAM_STATUS_MASK; entry = cam_fetch_status_entry(status); if (entry == NULL) sbuf_printf(&sb, "CAM status: Unknown (%#x)\n", ccb->ccb_h.status); else sbuf_printf(&sb, "CAM status: %s\n", entry->status_text); } if (flags & CAM_ESF_PROTO_STATUS) { switch (ccb->ccb_h.func_code) { case XPT_ATA_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_ATA_STATUS_ERROR) break; if (proto_flags & CAM_EAF_PRINT_STATUS) { sbuf_cat(&sb, path_str); ata_status_sbuf(&ccb->ataio, &sb); sbuf_printf(&sb, "\n"); } if (proto_flags & CAM_EAF_PRINT_RESULT) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "RES: "); ata_res_sbuf(&ccb->ataio.res, &sb); sbuf_printf(&sb, "\n"); } break; case XPT_SCSI_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_SCSI_STATUS_ERROR) break; if (proto_flags & CAM_ESF_PRINT_STATUS) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "SCSI status: %s\n", scsi_status_string(&ccb->csio)); } if ((proto_flags & CAM_ESF_PRINT_SENSE) && (ccb->csio.scsi_status == SCSI_STATUS_CHECK_COND) && (ccb->ccb_h.status & CAM_AUTOSNS_VALID)) { #ifdef _KERNEL scsi_sense_sbuf(&ccb->csio, &sb, SSS_FLAG_NONE); #else /* !_KERNEL */ scsi_sense_sbuf(device, &ccb->csio, &sb, SSS_FLAG_NONE); #endif /* _KERNEL/!_KERNEL */ } break; case XPT_SMP_IO: if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_SMP_STATUS_ERROR) break; if (proto_flags & CAM_ESF_PRINT_STATUS) { sbuf_cat(&sb, path_str); sbuf_printf(&sb, "SMP status: %s (%#x)\n", smp_error_desc(ccb->smpio.smp_response[2]), ccb->smpio.smp_response[2]); } /* There is no SMP equivalent to SCSI sense. */ break; default: break; } } sbuf_finish(&sb); return(sbuf_data(&sb)); } #ifdef _KERNEL void cam_error_print(union ccb *ccb, cam_error_string_flags flags, cam_error_proto_flags proto_flags) { char str[512]; printf("%s", cam_error_string(ccb, str, sizeof(str), flags, proto_flags)); } #else /* !_KERNEL */ void cam_error_print(struct cam_device *device, union ccb *ccb, cam_error_string_flags flags, cam_error_proto_flags proto_flags, FILE *ofile) { char str[512]; if ((device == NULL) || (ccb == NULL) || (ofile == NULL)) return; fprintf(ofile, "%s", cam_error_string(device, ccb, str, sizeof(str), flags, proto_flags)); } #endif /* _KERNEL/!_KERNEL */ /* * Common calculate geometry fuction * * Caller should set ccg->volume_size and block_size. * The extended parameter should be zero if extended translation * should not be used. */ void cam_calc_geometry(struct ccb_calc_geometry *ccg, int extended) { uint32_t size_mb, secs_per_cylinder; if (ccg->block_size == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } size_mb = (1024L * 1024L) / ccg->block_size; if (size_mb == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } size_mb = ccg->volume_size / size_mb; if (size_mb > 1024 && extended) { ccg->heads = 255; ccg->secs_per_track = 63; } else { ccg->heads = 64; ccg->secs_per_track = 32; } secs_per_cylinder = ccg->heads * ccg->secs_per_track; if (secs_per_cylinder == 0) { ccg->ccb_h.status = CAM_REQ_CMP_ERR; return; } ccg->cylinders = ccg->volume_size / secs_per_cylinder; ccg->ccb_h.status = CAM_REQ_CMP; } Index: head/sys/cam/scsi/scsi_all.c =================================================================== --- head/sys/cam/scsi/scsi_all.c (revision 298410) +++ head/sys/cam/scsi/scsi_all.c (revision 298411) @@ -1,8841 +1,8838 @@ /*- * Implementation of Utility functions for all SCSI device types. * * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs. * Copyright (c) 1997, 1998, 2003 Kenneth D. Merry. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #ifdef _KERNEL #include #include #include #include #include #include #include #include #include #else #include #include #include #include #include #endif #include #include #include #include #include #include #include #ifdef _KERNEL #include #include #include #include #else #include #include #ifndef FALSE #define FALSE 0 #endif /* FALSE */ #ifndef TRUE #define TRUE 1 #endif /* TRUE */ #define ERESTART -1 /* restart syscall */ #define EJUSTRETURN -2 /* don't modify regs, just return */ #endif /* !_KERNEL */ /* * This is the default number of milliseconds we wait for devices to settle * after a SCSI bus reset. */ #ifndef SCSI_DELAY #define SCSI_DELAY 2000 #endif /* * All devices need _some_ sort of bus settle delay, so we'll set it to * a minimum value of 100ms. Note that this is pertinent only for SPI- * not transport like Fibre Channel or iSCSI where 'delay' is completely * meaningless. */ #ifndef SCSI_MIN_DELAY #define SCSI_MIN_DELAY 100 #endif /* * Make sure the user isn't using seconds instead of milliseconds. */ #if (SCSI_DELAY < SCSI_MIN_DELAY && SCSI_DELAY != 0) #error "SCSI_DELAY is in milliseconds, not seconds! Please use a larger value" #endif int scsi_delay; static int ascentrycomp(const void *key, const void *member); static int senseentrycomp(const void *key, const void *member); static void fetchtableentries(int sense_key, int asc, int ascq, struct scsi_inquiry_data *, const struct sense_key_table_entry **, const struct asc_table_entry **); #ifdef _KERNEL static void init_scsi_delay(void); static int sysctl_scsi_delay(SYSCTL_HANDLER_ARGS); static int set_scsi_delay(int delay); #endif #if !defined(SCSI_NO_OP_STRINGS) #define D (1 << T_DIRECT) #define T (1 << T_SEQUENTIAL) #define L (1 << T_PRINTER) #define P (1 << T_PROCESSOR) #define W (1 << T_WORM) #define R (1 << T_CDROM) #define O (1 << T_OPTICAL) #define M (1 << T_CHANGER) #define A (1 << T_STORARRAY) #define E (1 << T_ENCLOSURE) #define B (1 << T_RBC) #define K (1 << T_OCRW) #define V (1 << T_ADC) #define F (1 << T_OSD) #define S (1 << T_SCANNER) #define C (1 << T_COMM) #define ALL (D | T | L | P | W | R | O | M | A | E | B | K | V | F | S | C) static struct op_table_entry plextor_cd_ops[] = { { 0xD8, R, "CD-DA READ" } }; static struct scsi_op_quirk_entry scsi_op_quirk_table[] = { { /* * I believe that 0xD8 is the Plextor proprietary command * to read CD-DA data. I'm not sure which Plextor CDROM * models support the command, though. I know for sure * that the 4X, 8X, and 12X models do, and presumably the * 12-20X does. I don't know about any earlier models, * though. If anyone has any more complete information, * feel free to change this quirk entry. */ {T_CDROM, SIP_MEDIA_REMOVABLE, "PLEXTOR", "CD-ROM PX*", "*"}, sizeof(plextor_cd_ops)/sizeof(struct op_table_entry), plextor_cd_ops } }; static struct op_table_entry scsi_op_codes[] = { /* * From: http://www.t10.org/lists/op-num.txt * Modifications by Kenneth Merry (ken@FreeBSD.ORG) * and Jung-uk Kim (jkim@FreeBSD.org) * * Note: order is important in this table, scsi_op_desc() currently * depends on the opcodes in the table being in order to save * search time. * Note: scanner and comm. devices are carried over from the previous * version because they were removed in the latest spec. */ /* File: OP-NUM.TXT * * SCSI Operation Codes * Numeric Sorted Listing * as of 5/26/15 * * D - DIRECT ACCESS DEVICE (SBC-2) device column key * .T - SEQUENTIAL ACCESS DEVICE (SSC-2) ----------------- * . L - PRINTER DEVICE (SSC) M = Mandatory * . P - PROCESSOR DEVICE (SPC) O = Optional * . .W - WRITE ONCE READ MULTIPLE DEVICE (SBC-2) V = Vendor spec. * . . R - CD/DVE DEVICE (MMC-3) Z = Obsolete * . . O - OPTICAL MEMORY DEVICE (SBC-2) * . . .M - MEDIA CHANGER DEVICE (SMC-2) * . . . A - STORAGE ARRAY DEVICE (SCC-2) * . . . .E - ENCLOSURE SERVICES DEVICE (SES) * . . . .B - SIMPLIFIED DIRECT-ACCESS DEVICE (RBC) * . . . . K - OPTICAL CARD READER/WRITER DEVICE (OCRW) * . . . . V - AUTOMATION/DRIVE INTERFACE (ADC) * . . . . .F - OBJECT-BASED STORAGE (OSD) * OP DTLPWROMAEBKVF Description * -- -------------- ---------------------------------------------- */ /* 00 MMMMMMMMMMMMMM TEST UNIT READY */ { 0x00, ALL, "TEST UNIT READY" }, /* 01 M REWIND */ { 0x01, T, "REWIND" }, /* 01 Z V ZZZZ REZERO UNIT */ { 0x01, D | W | R | O | M, "REZERO UNIT" }, /* 02 VVVVVV V */ /* 03 MMMMMMMMMMOMMM REQUEST SENSE */ { 0x03, ALL, "REQUEST SENSE" }, /* 04 M OO FORMAT UNIT */ { 0x04, D | R | O, "FORMAT UNIT" }, /* 04 O FORMAT MEDIUM */ { 0x04, T, "FORMAT MEDIUM" }, /* 04 O FORMAT */ { 0x04, L, "FORMAT" }, /* 05 VMVVVV V READ BLOCK LIMITS */ { 0x05, T, "READ BLOCK LIMITS" }, /* 06 VVVVVV V */ /* 07 OVV O OV REASSIGN BLOCKS */ { 0x07, D | W | O, "REASSIGN BLOCKS" }, /* 07 O INITIALIZE ELEMENT STATUS */ { 0x07, M, "INITIALIZE ELEMENT STATUS" }, /* 08 MOV O OV READ(6) */ { 0x08, D | T | W | O, "READ(6)" }, /* 08 O RECEIVE */ { 0x08, P, "RECEIVE" }, /* 08 GET MESSAGE(6) */ { 0x08, C, "GET MESSAGE(6)" }, /* 09 VVVVVV V */ /* 0A OO O OV WRITE(6) */ { 0x0A, D | T | W | O, "WRITE(6)" }, /* 0A M SEND(6) */ { 0x0A, P, "SEND(6)" }, /* 0A SEND MESSAGE(6) */ { 0x0A, C, "SEND MESSAGE(6)" }, /* 0A M PRINT */ { 0x0A, L, "PRINT" }, /* 0B Z ZOZV SEEK(6) */ { 0x0B, D | W | R | O, "SEEK(6)" }, /* 0B O SET CAPACITY */ { 0x0B, T, "SET CAPACITY" }, /* 0B O SLEW AND PRINT */ { 0x0B, L, "SLEW AND PRINT" }, /* 0C VVVVVV V */ /* 0D VVVVVV V */ /* 0E VVVVVV V */ /* 0F VOVVVV V READ REVERSE(6) */ { 0x0F, T, "READ REVERSE(6)" }, /* 10 VM VVV WRITE FILEMARKS(6) */ { 0x10, T, "WRITE FILEMARKS(6)" }, /* 10 O SYNCHRONIZE BUFFER */ { 0x10, L, "SYNCHRONIZE BUFFER" }, /* 11 VMVVVV SPACE(6) */ { 0x11, T, "SPACE(6)" }, /* 12 MMMMMMMMMMMMMM INQUIRY */ { 0x12, ALL, "INQUIRY" }, /* 13 V VVVV */ /* 13 O VERIFY(6) */ { 0x13, T, "VERIFY(6)" }, /* 14 VOOVVV RECOVER BUFFERED DATA */ { 0x14, T | L, "RECOVER BUFFERED DATA" }, /* 15 OMO O OOOO OO MODE SELECT(6) */ { 0x15, ALL & ~(P | R | B | F), "MODE SELECT(6)" }, /* 16 ZZMZO OOOZ O RESERVE(6) */ { 0x16, ALL & ~(R | B | V | F | C), "RESERVE(6)" }, /* 16 Z RESERVE ELEMENT(6) */ { 0x16, M, "RESERVE ELEMENT(6)" }, /* 17 ZZMZO OOOZ O RELEASE(6) */ { 0x17, ALL & ~(R | B | V | F | C), "RELEASE(6)" }, /* 17 Z RELEASE ELEMENT(6) */ { 0x17, M, "RELEASE ELEMENT(6)" }, /* 18 ZZZZOZO Z COPY */ { 0x18, D | T | L | P | W | R | O | K | S, "COPY" }, /* 19 VMVVVV ERASE(6) */ { 0x19, T, "ERASE(6)" }, /* 1A OMO O OOOO OO MODE SENSE(6) */ { 0x1A, ALL & ~(P | R | B | F), "MODE SENSE(6)" }, /* 1B O OOO O MO O START STOP UNIT */ { 0x1B, D | W | R | O | A | B | K | F, "START STOP UNIT" }, /* 1B O M LOAD UNLOAD */ { 0x1B, T | V, "LOAD UNLOAD" }, /* 1B SCAN */ { 0x1B, S, "SCAN" }, /* 1B O STOP PRINT */ { 0x1B, L, "STOP PRINT" }, /* 1B O OPEN/CLOSE IMPORT/EXPORT ELEMENT */ { 0x1B, M, "OPEN/CLOSE IMPORT/EXPORT ELEMENT" }, /* 1C OOOOO OOOM OOO RECEIVE DIAGNOSTIC RESULTS */ { 0x1C, ALL & ~(R | B), "RECEIVE DIAGNOSTIC RESULTS" }, /* 1D MMMMM MMOM MMM SEND DIAGNOSTIC */ { 0x1D, ALL & ~(R | B), "SEND DIAGNOSTIC" }, /* 1E OO OOOO O O PREVENT ALLOW MEDIUM REMOVAL */ { 0x1E, D | T | W | R | O | M | K | F, "PREVENT ALLOW MEDIUM REMOVAL" }, /* 1F */ /* 20 V VVV V */ /* 21 V VVV V */ /* 22 V VVV V */ /* 23 V V V V */ /* 23 O READ FORMAT CAPACITIES */ { 0x23, R, "READ FORMAT CAPACITIES" }, /* 24 V VV SET WINDOW */ { 0x24, S, "SET WINDOW" }, /* 25 M M M M READ CAPACITY(10) */ { 0x25, D | W | O | B, "READ CAPACITY(10)" }, /* 25 O READ CAPACITY */ { 0x25, R, "READ CAPACITY" }, /* 25 M READ CARD CAPACITY */ { 0x25, K, "READ CARD CAPACITY" }, /* 25 GET WINDOW */ { 0x25, S, "GET WINDOW" }, /* 26 V VV */ /* 27 V VV */ /* 28 M MOM MM READ(10) */ { 0x28, D | W | R | O | B | K | S, "READ(10)" }, /* 28 GET MESSAGE(10) */ { 0x28, C, "GET MESSAGE(10)" }, /* 29 V VVO READ GENERATION */ { 0x29, O, "READ GENERATION" }, /* 2A O MOM MO WRITE(10) */ { 0x2A, D | W | R | O | B | K, "WRITE(10)" }, /* 2A SEND(10) */ { 0x2A, S, "SEND(10)" }, /* 2A SEND MESSAGE(10) */ { 0x2A, C, "SEND MESSAGE(10)" }, /* 2B Z OOO O SEEK(10) */ { 0x2B, D | W | R | O | K, "SEEK(10)" }, /* 2B O LOCATE(10) */ { 0x2B, T, "LOCATE(10)" }, /* 2B O POSITION TO ELEMENT */ { 0x2B, M, "POSITION TO ELEMENT" }, /* 2C V OO ERASE(10) */ { 0x2C, R | O, "ERASE(10)" }, /* 2D O READ UPDATED BLOCK */ { 0x2D, O, "READ UPDATED BLOCK" }, /* 2D V */ /* 2E O OOO MO WRITE AND VERIFY(10) */ { 0x2E, D | W | R | O | B | K, "WRITE AND VERIFY(10)" }, /* 2F O OOO VERIFY(10) */ { 0x2F, D | W | R | O, "VERIFY(10)" }, /* 30 Z ZZZ SEARCH DATA HIGH(10) */ { 0x30, D | W | R | O, "SEARCH DATA HIGH(10)" }, /* 31 Z ZZZ SEARCH DATA EQUAL(10) */ { 0x31, D | W | R | O, "SEARCH DATA EQUAL(10)" }, /* 31 OBJECT POSITION */ { 0x31, S, "OBJECT POSITION" }, /* 32 Z ZZZ SEARCH DATA LOW(10) */ { 0x32, D | W | R | O, "SEARCH DATA LOW(10)" }, /* 33 Z OZO SET LIMITS(10) */ { 0x33, D | W | R | O, "SET LIMITS(10)" }, /* 34 O O O O PRE-FETCH(10) */ { 0x34, D | W | O | K, "PRE-FETCH(10)" }, /* 34 M READ POSITION */ { 0x34, T, "READ POSITION" }, /* 34 GET DATA BUFFER STATUS */ { 0x34, S, "GET DATA BUFFER STATUS" }, /* 35 O OOO MO SYNCHRONIZE CACHE(10) */ { 0x35, D | W | R | O | B | K, "SYNCHRONIZE CACHE(10)" }, /* 36 Z O O O LOCK UNLOCK CACHE(10) */ { 0x36, D | W | O | K, "LOCK UNLOCK CACHE(10)" }, /* 37 O O READ DEFECT DATA(10) */ { 0x37, D | O, "READ DEFECT DATA(10)" }, /* 37 O INITIALIZE ELEMENT STATUS WITH RANGE */ { 0x37, M, "INITIALIZE ELEMENT STATUS WITH RANGE" }, /* 38 O O O MEDIUM SCAN */ { 0x38, W | O | K, "MEDIUM SCAN" }, /* 39 ZZZZOZO Z COMPARE */ { 0x39, D | T | L | P | W | R | O | K | S, "COMPARE" }, /* 3A ZZZZOZO Z COPY AND VERIFY */ { 0x3A, D | T | L | P | W | R | O | K | S, "COPY AND VERIFY" }, /* 3B OOOOOOOOOOMOOO WRITE BUFFER */ { 0x3B, ALL, "WRITE BUFFER" }, /* 3C OOOOOOOOOO OOO READ BUFFER */ { 0x3C, ALL & ~(B), "READ BUFFER" }, /* 3D O UPDATE BLOCK */ { 0x3D, O, "UPDATE BLOCK" }, /* 3E O O O READ LONG(10) */ { 0x3E, D | W | O, "READ LONG(10)" }, /* 3F O O O WRITE LONG(10) */ { 0x3F, D | W | O, "WRITE LONG(10)" }, /* 40 ZZZZOZOZ CHANGE DEFINITION */ { 0x40, D | T | L | P | W | R | O | M | S | C, "CHANGE DEFINITION" }, /* 41 O WRITE SAME(10) */ { 0x41, D, "WRITE SAME(10)" }, /* 42 O UNMAP */ { 0x42, D, "UNMAP" }, /* 42 O READ SUB-CHANNEL */ { 0x42, R, "READ SUB-CHANNEL" }, /* 43 O READ TOC/PMA/ATIP */ { 0x43, R, "READ TOC/PMA/ATIP" }, /* 44 M M REPORT DENSITY SUPPORT */ { 0x44, T | V, "REPORT DENSITY SUPPORT" }, /* 44 READ HEADER */ /* 45 O PLAY AUDIO(10) */ { 0x45, R, "PLAY AUDIO(10)" }, /* 46 M GET CONFIGURATION */ { 0x46, R, "GET CONFIGURATION" }, /* 47 O PLAY AUDIO MSF */ { 0x47, R, "PLAY AUDIO MSF" }, /* 48 */ /* 49 */ /* 4A M GET EVENT STATUS NOTIFICATION */ { 0x4A, R, "GET EVENT STATUS NOTIFICATION" }, /* 4B O PAUSE/RESUME */ { 0x4B, R, "PAUSE/RESUME" }, /* 4C OOOOO OOOO OOO LOG SELECT */ { 0x4C, ALL & ~(R | B), "LOG SELECT" }, /* 4D OOOOO OOOO OMO LOG SENSE */ { 0x4D, ALL & ~(R | B), "LOG SENSE" }, /* 4E O STOP PLAY/SCAN */ { 0x4E, R, "STOP PLAY/SCAN" }, /* 4F */ /* 50 O XDWRITE(10) */ { 0x50, D, "XDWRITE(10)" }, /* 51 O XPWRITE(10) */ { 0x51, D, "XPWRITE(10)" }, /* 51 O READ DISC INFORMATION */ { 0x51, R, "READ DISC INFORMATION" }, /* 52 O XDREAD(10) */ { 0x52, D, "XDREAD(10)" }, /* 52 O READ TRACK INFORMATION */ { 0x52, R, "READ TRACK INFORMATION" }, /* 53 O RESERVE TRACK */ { 0x53, R, "RESERVE TRACK" }, /* 54 O SEND OPC INFORMATION */ { 0x54, R, "SEND OPC INFORMATION" }, /* 55 OOO OMOOOOMOMO MODE SELECT(10) */ { 0x55, ALL & ~(P), "MODE SELECT(10)" }, /* 56 ZZMZO OOOZ RESERVE(10) */ { 0x56, ALL & ~(R | B | K | V | F | C), "RESERVE(10)" }, /* 56 Z RESERVE ELEMENT(10) */ { 0x56, M, "RESERVE ELEMENT(10)" }, /* 57 ZZMZO OOOZ RELEASE(10) */ { 0x57, ALL & ~(R | B | K | V | F | C), "RELEASE(10)" }, /* 57 Z RELEASE ELEMENT(10) */ { 0x57, M, "RELEASE ELEMENT(10)" }, /* 58 O REPAIR TRACK */ { 0x58, R, "REPAIR TRACK" }, /* 59 */ /* 5A OOO OMOOOOMOMO MODE SENSE(10) */ { 0x5A, ALL & ~(P), "MODE SENSE(10)" }, /* 5B O CLOSE TRACK/SESSION */ { 0x5B, R, "CLOSE TRACK/SESSION" }, /* 5C O READ BUFFER CAPACITY */ { 0x5C, R, "READ BUFFER CAPACITY" }, /* 5D O SEND CUE SHEET */ { 0x5D, R, "SEND CUE SHEET" }, /* 5E OOOOO OOOO M PERSISTENT RESERVE IN */ { 0x5E, ALL & ~(R | B | K | V | C), "PERSISTENT RESERVE IN" }, /* 5F OOOOO OOOO M PERSISTENT RESERVE OUT */ { 0x5F, ALL & ~(R | B | K | V | C), "PERSISTENT RESERVE OUT" }, /* 7E OO O OOOO O extended CDB */ { 0x7E, D | T | R | M | A | E | B | V, "extended CDB" }, /* 7F O M variable length CDB (more than 16 bytes) */ { 0x7F, D | F, "variable length CDB (more than 16 bytes)" }, /* 80 Z XDWRITE EXTENDED(16) */ { 0x80, D, "XDWRITE EXTENDED(16)" }, /* 80 M WRITE FILEMARKS(16) */ { 0x80, T, "WRITE FILEMARKS(16)" }, /* 81 Z REBUILD(16) */ { 0x81, D, "REBUILD(16)" }, /* 81 O READ REVERSE(16) */ { 0x81, T, "READ REVERSE(16)" }, /* 82 Z REGENERATE(16) */ { 0x82, D, "REGENERATE(16)" }, /* 83 OOOOO O OO EXTENDED COPY */ { 0x83, D | T | L | P | W | O | K | V, "EXTENDED COPY" }, /* 84 OOOOO O OO RECEIVE COPY RESULTS */ { 0x84, D | T | L | P | W | O | K | V, "RECEIVE COPY RESULTS" }, /* 85 O O O ATA COMMAND PASS THROUGH(16) */ { 0x85, D | R | B, "ATA COMMAND PASS THROUGH(16)" }, /* 86 OO OO OOOOOOO ACCESS CONTROL IN */ { 0x86, ALL & ~(L | R | F), "ACCESS CONTROL IN" }, /* 87 OO OO OOOOOOO ACCESS CONTROL OUT */ { 0x87, ALL & ~(L | R | F), "ACCESS CONTROL OUT" }, /* * XXX READ(16)/WRITE(16) were not listed for CD/DVE in op-num.txt * but we had it since r1.40. Do we really want them? */ /* 88 MM O O O READ(16) */ { 0x88, D | T | W | O | B, "READ(16)" }, /* 89 O COMPARE AND WRITE*/ { 0x89, D, "COMPARE AND WRITE" }, /* 8A OM O O O WRITE(16) */ { 0x8A, D | T | W | O | B, "WRITE(16)" }, /* 8B O ORWRITE */ { 0x8B, D, "ORWRITE" }, /* 8C OO O OO O M READ ATTRIBUTE */ { 0x8C, D | T | W | O | M | B | V, "READ ATTRIBUTE" }, /* 8D OO O OO O O WRITE ATTRIBUTE */ { 0x8D, D | T | W | O | M | B | V, "WRITE ATTRIBUTE" }, /* 8E O O O O WRITE AND VERIFY(16) */ { 0x8E, D | W | O | B, "WRITE AND VERIFY(16)" }, /* 8F OO O O O VERIFY(16) */ { 0x8F, D | T | W | O | B, "VERIFY(16)" }, /* 90 O O O O PRE-FETCH(16) */ { 0x90, D | W | O | B, "PRE-FETCH(16)" }, /* 91 O O O O SYNCHRONIZE CACHE(16) */ { 0x91, D | W | O | B, "SYNCHRONIZE CACHE(16)" }, /* 91 O SPACE(16) */ { 0x91, T, "SPACE(16)" }, /* 92 Z O O LOCK UNLOCK CACHE(16) */ { 0x92, D | W | O, "LOCK UNLOCK CACHE(16)" }, /* 92 O LOCATE(16) */ { 0x92, T, "LOCATE(16)" }, /* 93 O WRITE SAME(16) */ { 0x93, D, "WRITE SAME(16)" }, /* 93 M ERASE(16) */ { 0x93, T, "ERASE(16)" }, /* 94 O ZBC OUT */ { 0x94, D, "ZBC OUT" }, /* 95 O ZBC OUT */ { 0x95, D, "ZBC OUT" }, /* 96 */ /* 97 */ /* 98 */ /* 99 */ /* 9A O WRITE STREAM(16) */ { 0x9A, D, "WRITE STREAM(16)" }, /* 9B OOOOOOOOOO OOO READ BUFFER(16) */ { 0x9B, ALL & ~(B) , "READ BUFFER(16)" }, /* 9C O WRITE ATOMIC(16) */ { 0x9C, D, "WRITE ATOMIC(16)" }, /* 9D SERVICE ACTION BIDIRECTIONAL */ { 0x9D, ALL, "SERVICE ACTION BIDIRECTIONAL" }, /* XXX KDM ALL for this? op-num.txt defines it for none.. */ /* 9E SERVICE ACTION IN(16) */ { 0x9E, ALL, "SERVICE ACTION IN(16)" }, /* XXX KDM ALL for this? op-num.txt defines it for ADC.. */ /* 9F M SERVICE ACTION OUT(16) */ { 0x9F, ALL, "SERVICE ACTION OUT(16)" }, /* A0 MMOOO OMMM OMO REPORT LUNS */ { 0xA0, ALL & ~(R | B), "REPORT LUNS" }, /* A1 O BLANK */ { 0xA1, R, "BLANK" }, /* A1 O O ATA COMMAND PASS THROUGH(12) */ { 0xA1, D | B, "ATA COMMAND PASS THROUGH(12)" }, /* A2 OO O O SECURITY PROTOCOL IN */ { 0xA2, D | T | R | V, "SECURITY PROTOCOL IN" }, /* A3 OOO O OOMOOOM MAINTENANCE (IN) */ { 0xA3, ALL & ~(P | R | F), "MAINTENANCE (IN)" }, /* A3 O SEND KEY */ { 0xA3, R, "SEND KEY" }, /* A4 OOO O OOOOOOO MAINTENANCE (OUT) */ { 0xA4, ALL & ~(P | R | F), "MAINTENANCE (OUT)" }, /* A4 O REPORT KEY */ { 0xA4, R, "REPORT KEY" }, /* A5 O O OM MOVE MEDIUM */ { 0xA5, T | W | O | M, "MOVE MEDIUM" }, /* A5 O PLAY AUDIO(12) */ { 0xA5, R, "PLAY AUDIO(12)" }, /* A6 O EXCHANGE MEDIUM */ { 0xA6, M, "EXCHANGE MEDIUM" }, /* A6 O LOAD/UNLOAD C/DVD */ { 0xA6, R, "LOAD/UNLOAD C/DVD" }, /* A7 ZZ O O MOVE MEDIUM ATTACHED */ { 0xA7, D | T | W | O, "MOVE MEDIUM ATTACHED" }, /* A7 O SET READ AHEAD */ { 0xA7, R, "SET READ AHEAD" }, /* A8 O OOO READ(12) */ { 0xA8, D | W | R | O, "READ(12)" }, /* A8 GET MESSAGE(12) */ { 0xA8, C, "GET MESSAGE(12)" }, /* A9 O SERVICE ACTION OUT(12) */ { 0xA9, V, "SERVICE ACTION OUT(12)" }, /* AA O OOO WRITE(12) */ { 0xAA, D | W | R | O, "WRITE(12)" }, /* AA SEND MESSAGE(12) */ { 0xAA, C, "SEND MESSAGE(12)" }, /* AB O O SERVICE ACTION IN(12) */ { 0xAB, R | V, "SERVICE ACTION IN(12)" }, /* AC O ERASE(12) */ { 0xAC, O, "ERASE(12)" }, /* AC O GET PERFORMANCE */ { 0xAC, R, "GET PERFORMANCE" }, /* AD O READ DVD STRUCTURE */ { 0xAD, R, "READ DVD STRUCTURE" }, /* AE O O O WRITE AND VERIFY(12) */ { 0xAE, D | W | O, "WRITE AND VERIFY(12)" }, /* AF O OZO VERIFY(12) */ { 0xAF, D | W | R | O, "VERIFY(12)" }, /* B0 ZZZ SEARCH DATA HIGH(12) */ { 0xB0, W | R | O, "SEARCH DATA HIGH(12)" }, /* B1 ZZZ SEARCH DATA EQUAL(12) */ { 0xB1, W | R | O, "SEARCH DATA EQUAL(12)" }, /* B2 ZZZ SEARCH DATA LOW(12) */ { 0xB2, W | R | O, "SEARCH DATA LOW(12)" }, /* B3 Z OZO SET LIMITS(12) */ { 0xB3, D | W | R | O, "SET LIMITS(12)" }, /* B4 ZZ OZO READ ELEMENT STATUS ATTACHED */ { 0xB4, D | T | W | R | O, "READ ELEMENT STATUS ATTACHED" }, /* B5 OO O O SECURITY PROTOCOL OUT */ { 0xB5, D | T | R | V, "SECURITY PROTOCOL OUT" }, /* B5 O REQUEST VOLUME ELEMENT ADDRESS */ { 0xB5, M, "REQUEST VOLUME ELEMENT ADDRESS" }, /* B6 O SEND VOLUME TAG */ { 0xB6, M, "SEND VOLUME TAG" }, /* B6 O SET STREAMING */ { 0xB6, R, "SET STREAMING" }, /* B7 O O READ DEFECT DATA(12) */ { 0xB7, D | O, "READ DEFECT DATA(12)" }, /* B8 O OZOM READ ELEMENT STATUS */ { 0xB8, T | W | R | O | M, "READ ELEMENT STATUS" }, /* B9 O READ CD MSF */ { 0xB9, R, "READ CD MSF" }, /* BA O O OOMO REDUNDANCY GROUP (IN) */ { 0xBA, D | W | O | M | A | E, "REDUNDANCY GROUP (IN)" }, /* BA O SCAN */ { 0xBA, R, "SCAN" }, /* BB O O OOOO REDUNDANCY GROUP (OUT) */ { 0xBB, D | W | O | M | A | E, "REDUNDANCY GROUP (OUT)" }, /* BB O SET CD SPEED */ { 0xBB, R, "SET CD SPEED" }, /* BC O O OOMO SPARE (IN) */ { 0xBC, D | W | O | M | A | E, "SPARE (IN)" }, /* BD O O OOOO SPARE (OUT) */ { 0xBD, D | W | O | M | A | E, "SPARE (OUT)" }, /* BD O MECHANISM STATUS */ { 0xBD, R, "MECHANISM STATUS" }, /* BE O O OOMO VOLUME SET (IN) */ { 0xBE, D | W | O | M | A | E, "VOLUME SET (IN)" }, /* BE O READ CD */ { 0xBE, R, "READ CD" }, /* BF O O OOOO VOLUME SET (OUT) */ { 0xBF, D | W | O | M | A | E, "VOLUME SET (OUT)" }, /* BF O SEND DVD STRUCTURE */ { 0xBF, R, "SEND DVD STRUCTURE" } }; const char * scsi_op_desc(u_int16_t opcode, struct scsi_inquiry_data *inq_data) { caddr_t match; int i, j; u_int32_t opmask; u_int16_t pd_type; int num_ops[2]; struct op_table_entry *table[2]; int num_tables; /* * If we've got inquiry data, use it to determine what type of * device we're dealing with here. Otherwise, assume direct * access. */ if (inq_data == NULL) { pd_type = T_DIRECT; match = NULL; } else { pd_type = SID_TYPE(inq_data); match = cam_quirkmatch((caddr_t)inq_data, (caddr_t)scsi_op_quirk_table, sizeof(scsi_op_quirk_table)/ sizeof(*scsi_op_quirk_table), sizeof(*scsi_op_quirk_table), scsi_inquiry_match); } if (match != NULL) { table[0] = ((struct scsi_op_quirk_entry *)match)->op_table; num_ops[0] = ((struct scsi_op_quirk_entry *)match)->num_ops; table[1] = scsi_op_codes; num_ops[1] = sizeof(scsi_op_codes)/sizeof(scsi_op_codes[0]); num_tables = 2; } else { /* * If this is true, we have a vendor specific opcode that * wasn't covered in the quirk table. */ if ((opcode > 0xBF) || ((opcode > 0x5F) && (opcode < 0x80))) return("Vendor Specific Command"); table[0] = scsi_op_codes; num_ops[0] = sizeof(scsi_op_codes)/sizeof(scsi_op_codes[0]); num_tables = 1; } /* RBC is 'Simplified' Direct Access Device */ if (pd_type == T_RBC) pd_type = T_DIRECT; /* Map NODEVICE to Direct Access Device to handle REPORT LUNS, etc. */ if (pd_type == T_NODEVICE) pd_type = T_DIRECT; opmask = 1 << pd_type; for (j = 0; j < num_tables; j++) { for (i = 0;i < num_ops[j] && table[j][i].opcode <= opcode; i++){ if ((table[j][i].opcode == opcode) && ((table[j][i].opmask & opmask) != 0)) return(table[j][i].desc); } } /* * If we can't find a match for the command in the table, we just * assume it's a vendor specifc command. */ return("Vendor Specific Command"); } #else /* SCSI_NO_OP_STRINGS */ const char * scsi_op_desc(u_int16_t opcode, struct scsi_inquiry_data *inq_data) { return(""); } #endif #if !defined(SCSI_NO_SENSE_STRINGS) #define SST(asc, ascq, action, desc) \ asc, ascq, action, desc #else const char empty_string[] = ""; #define SST(asc, ascq, action, desc) \ asc, ascq, action, empty_string #endif const struct sense_key_table_entry sense_key_table[] = { { SSD_KEY_NO_SENSE, SS_NOP, "NO SENSE" }, { SSD_KEY_RECOVERED_ERROR, SS_NOP|SSQ_PRINT_SENSE, "RECOVERED ERROR" }, { SSD_KEY_NOT_READY, SS_RDEF, "NOT READY" }, { SSD_KEY_MEDIUM_ERROR, SS_RDEF, "MEDIUM ERROR" }, { SSD_KEY_HARDWARE_ERROR, SS_RDEF, "HARDWARE FAILURE" }, { SSD_KEY_ILLEGAL_REQUEST, SS_FATAL|EINVAL, "ILLEGAL REQUEST" }, { SSD_KEY_UNIT_ATTENTION, SS_FATAL|ENXIO, "UNIT ATTENTION" }, { SSD_KEY_DATA_PROTECT, SS_FATAL|EACCES, "DATA PROTECT" }, { SSD_KEY_BLANK_CHECK, SS_FATAL|ENOSPC, "BLANK CHECK" }, { SSD_KEY_Vendor_Specific, SS_FATAL|EIO, "Vendor Specific" }, { SSD_KEY_COPY_ABORTED, SS_FATAL|EIO, "COPY ABORTED" }, { SSD_KEY_ABORTED_COMMAND, SS_RDEF, "ABORTED COMMAND" }, { SSD_KEY_EQUAL, SS_NOP, "EQUAL" }, { SSD_KEY_VOLUME_OVERFLOW, SS_FATAL|EIO, "VOLUME OVERFLOW" }, { SSD_KEY_MISCOMPARE, SS_NOP, "MISCOMPARE" }, { SSD_KEY_COMPLETED, SS_NOP, "COMPLETED" } }; -const int sense_key_table_size = - sizeof(sense_key_table)/sizeof(sense_key_table[0]); - static struct asc_table_entry quantum_fireball_entries[] = { { SST(0x04, 0x0b, SS_START | SSQ_DECREMENT_COUNT | ENXIO, "Logical unit not ready, initializing cmd. required") } }; static struct asc_table_entry sony_mo_entries[] = { { SST(0x04, 0x00, SS_START | SSQ_DECREMENT_COUNT | ENXIO, "Logical unit not ready, cause not reportable") } }; static struct asc_table_entry hgst_entries[] = { { SST(0x04, 0xF0, SS_RDEF, "Vendor Unique - Logical Unit Not Ready") }, { SST(0x0A, 0x01, SS_RDEF, "Unrecovered Super Certification Log Write Error") }, { SST(0x0A, 0x02, SS_RDEF, "Unrecovered Super Certification Log Read Error") }, { SST(0x15, 0x03, SS_RDEF, "Unrecovered Sector Error") }, { SST(0x3E, 0x04, SS_RDEF, "Unrecovered Self-Test Hard-Cache Test Fail") }, { SST(0x3E, 0x05, SS_RDEF, "Unrecovered Self-Test OTF-Cache Fail") }, { SST(0x40, 0x00, SS_RDEF, "Unrecovered SAT No Buffer Overflow Error") }, { SST(0x40, 0x01, SS_RDEF, "Unrecovered SAT Buffer Overflow Error") }, { SST(0x40, 0x02, SS_RDEF, "Unrecovered SAT No Buffer Overflow With ECS Fault") }, { SST(0x40, 0x03, SS_RDEF, "Unrecovered SAT Buffer Overflow With ECS Fault") }, { SST(0x40, 0x81, SS_RDEF, "DRAM Failure") }, { SST(0x44, 0x0B, SS_RDEF, "Vendor Unique - Internal Target Failure") }, { SST(0x44, 0xF2, SS_RDEF, "Vendor Unique - Internal Target Failure") }, { SST(0x44, 0xF6, SS_RDEF, "Vendor Unique - Internal Target Failure") }, { SST(0x44, 0xF9, SS_RDEF, "Vendor Unique - Internal Target Failure") }, { SST(0x44, 0xFA, SS_RDEF, "Vendor Unique - Internal Target Failure") }, { SST(0x5D, 0x22, SS_RDEF, "Extreme Over-Temperature Warning") }, { SST(0x5D, 0x50, SS_RDEF, "Load/Unload cycle Count Warning") }, { SST(0x81, 0x00, SS_RDEF, "Vendor Unique - Internal Logic Error") }, { SST(0x85, 0x00, SS_RDEF, "Vendor Unique - Internal Key Seed Error") }, }; static struct asc_table_entry seagate_entries[] = { { SST(0x04, 0xF0, SS_RDEF, "Logical Unit Not Ready, super certify in Progress") }, { SST(0x08, 0x86, SS_RDEF, "Write Fault Data Corruption") }, { SST(0x09, 0x0D, SS_RDEF, "Tracking Failure") }, { SST(0x09, 0x0E, SS_RDEF, "ETF Failure") }, { SST(0x0B, 0x5D, SS_RDEF, "Pre-SMART Warning") }, { SST(0x0B, 0x85, SS_RDEF, "5V Voltage Warning") }, { SST(0x0B, 0x8C, SS_RDEF, "12V Voltage Warning") }, { SST(0x0C, 0xFF, SS_RDEF, "Write Error - Too many error recovery revs") }, { SST(0x11, 0xFF, SS_RDEF, "Unrecovered Read Error - Too many error recovery revs") }, { SST(0x19, 0x0E, SS_RDEF, "Fewer than 1/2 defect list copies") }, { SST(0x20, 0xF3, SS_RDEF, "Illegal CDB linked to skip mask cmd") }, { SST(0x24, 0xF0, SS_RDEF, "Illegal byte in CDB, LBA not matching") }, { SST(0x24, 0xF1, SS_RDEF, "Illegal byte in CDB, LEN not matching") }, { SST(0x24, 0xF2, SS_RDEF, "Mask not matching transfer length") }, { SST(0x24, 0xF3, SS_RDEF, "Drive formatted without plist") }, { SST(0x26, 0x95, SS_RDEF, "Invalid Field Parameter - CAP File") }, { SST(0x26, 0x96, SS_RDEF, "Invalid Field Parameter - RAP File") }, { SST(0x26, 0x97, SS_RDEF, "Invalid Field Parameter - TMS Firmware Tag") }, { SST(0x26, 0x98, SS_RDEF, "Invalid Field Parameter - Check Sum") }, { SST(0x26, 0x99, SS_RDEF, "Invalid Field Parameter - Firmware Tag") }, { SST(0x29, 0x08, SS_RDEF, "Write Log Dump data") }, { SST(0x29, 0x09, SS_RDEF, "Write Log Dump data") }, { SST(0x29, 0x0A, SS_RDEF, "Reserved disk space") }, { SST(0x29, 0x0B, SS_RDEF, "SDBP") }, { SST(0x29, 0x0C, SS_RDEF, "SDBP") }, { SST(0x31, 0x91, SS_RDEF, "Format Corrupted World Wide Name (WWN) is Invalid") }, { SST(0x32, 0x03, SS_RDEF, "Defect List - Length exceeds Command Allocated Length") }, { SST(0x33, 0x00, SS_RDEF, "Flash not ready for access") }, { SST(0x3F, 0x70, SS_RDEF, "Invalid RAP block") }, { SST(0x3F, 0x71, SS_RDEF, "RAP/ETF mismatch") }, { SST(0x3F, 0x90, SS_RDEF, "Invalid CAP block") }, { SST(0x3F, 0x91, SS_RDEF, "World Wide Name (WWN) Mismatch") }, { SST(0x40, 0x01, SS_RDEF, "DRAM Parity Error") }, { SST(0x40, 0x02, SS_RDEF, "DRAM Parity Error") }, { SST(0x42, 0x0A, SS_RDEF, "Loopback Test") }, { SST(0x42, 0x0B, SS_RDEF, "Loopback Test") }, { SST(0x44, 0xF2, SS_RDEF, "Compare error during data integrity check") }, { SST(0x44, 0xF6, SS_RDEF, "Unrecoverable error during data integrity check") }, { SST(0x47, 0x80, SS_RDEF, "Fibre Channel Sequence Error") }, { SST(0x4E, 0x01, SS_RDEF, "Information Unit Too Short") }, { SST(0x80, 0x00, SS_RDEF, "General Firmware Error / Command Timeout") }, { SST(0x80, 0x01, SS_RDEF, "Command Timeout") }, { SST(0x80, 0x02, SS_RDEF, "Command Timeout") }, { SST(0x80, 0x80, SS_RDEF, "FC FIFO Error During Read Transfer") }, { SST(0x80, 0x81, SS_RDEF, "FC FIFO Error During Write Transfer") }, { SST(0x80, 0x82, SS_RDEF, "DISC FIFO Error During Read Transfer") }, { SST(0x80, 0x83, SS_RDEF, "DISC FIFO Error During Write Transfer") }, { SST(0x80, 0x84, SS_RDEF, "LBA Seeded LRC Error on Read") }, { SST(0x80, 0x85, SS_RDEF, "LBA Seeded LRC Error on Write") }, { SST(0x80, 0x86, SS_RDEF, "IOEDC Error on Read") }, { SST(0x80, 0x87, SS_RDEF, "IOEDC Error on Write") }, { SST(0x80, 0x88, SS_RDEF, "Host Parity Check Failed") }, { SST(0x80, 0x89, SS_RDEF, "IOEDC error on read detected by formatter") }, { SST(0x80, 0x8A, SS_RDEF, "Host Parity Errors / Host FIFO Initialization Failed") }, { SST(0x80, 0x8B, SS_RDEF, "Host Parity Errors") }, { SST(0x80, 0x8C, SS_RDEF, "Host Parity Errors") }, { SST(0x80, 0x8D, SS_RDEF, "Host Parity Errors") }, { SST(0x81, 0x00, SS_RDEF, "LA Check Failed") }, { SST(0x82, 0x00, SS_RDEF, "Internal client detected insufficient buffer") }, { SST(0x84, 0x00, SS_RDEF, "Scheduled Diagnostic And Repair") }, }; static struct scsi_sense_quirk_entry sense_quirk_table[] = { { /* * XXX The Quantum Fireball ST and SE like to return 0x04 0x0b * when they really should return 0x04 0x02. */ {T_DIRECT, SIP_MEDIA_FIXED, "QUANTUM", "FIREBALL S*", "*"}, /*num_sense_keys*/0, sizeof(quantum_fireball_entries)/sizeof(struct asc_table_entry), /*sense key entries*/NULL, quantum_fireball_entries }, { /* * This Sony MO drive likes to return 0x04, 0x00 when it * isn't spun up. */ {T_DIRECT, SIP_MEDIA_REMOVABLE, "SONY", "SMO-*", "*"}, /*num_sense_keys*/0, sizeof(sony_mo_entries)/sizeof(struct asc_table_entry), /*sense key entries*/NULL, sony_mo_entries }, { /* * HGST vendor-specific error codes */ {T_DIRECT, SIP_MEDIA_FIXED, "HGST", "*", "*"}, /*num_sense_keys*/0, sizeof(hgst_entries)/sizeof(struct asc_table_entry), /*sense key entries*/NULL, hgst_entries }, { /* * SEAGATE vendor-specific error codes */ {T_DIRECT, SIP_MEDIA_FIXED, "SEAGATE", "*", "*"}, /*num_sense_keys*/0, sizeof(seagate_entries)/sizeof(struct asc_table_entry), /*sense key entries*/NULL, seagate_entries } }; const int sense_quirk_table_size = sizeof(sense_quirk_table)/sizeof(sense_quirk_table[0]); static struct asc_table_entry asc_table[] = { /* * From: http://www.t10.org/lists/asc-num.txt * Modifications by Jung-uk Kim (jkim@FreeBSD.org) */ /* * File: ASC-NUM.TXT * * SCSI ASC/ASCQ Assignments * Numeric Sorted Listing * as of 8/12/15 * * D - DIRECT ACCESS DEVICE (SBC-2) device column key * .T - SEQUENTIAL ACCESS DEVICE (SSC) ------------------- * . L - PRINTER DEVICE (SSC) blank = reserved * . P - PROCESSOR DEVICE (SPC) not blank = allowed * . .W - WRITE ONCE READ MULTIPLE DEVICE (SBC-2) * . . R - CD DEVICE (MMC) * . . O - OPTICAL MEMORY DEVICE (SBC-2) * . . .M - MEDIA CHANGER DEVICE (SMC) * . . . A - STORAGE ARRAY DEVICE (SCC) * . . . E - ENCLOSURE SERVICES DEVICE (SES) * . . . .B - SIMPLIFIED DIRECT-ACCESS DEVICE (RBC) * . . . . K - OPTICAL CARD READER/WRITER DEVICE (OCRW) * . . . . V - AUTOMATION/DRIVE INTERFACE (ADC) * . . . . .F - OBJECT-BASED STORAGE (OSD) * DTLPWROMAEBKVF * ASC ASCQ Action * Description */ /* DTLPWROMAEBKVF */ { SST(0x00, 0x00, SS_NOP, "No additional sense information") }, /* T */ { SST(0x00, 0x01, SS_RDEF, "Filemark detected") }, /* T */ { SST(0x00, 0x02, SS_RDEF, "End-of-partition/medium detected") }, /* T */ { SST(0x00, 0x03, SS_RDEF, "Setmark detected") }, /* T */ { SST(0x00, 0x04, SS_RDEF, "Beginning-of-partition/medium detected") }, /* TL */ { SST(0x00, 0x05, SS_RDEF, "End-of-data detected") }, /* DTLPWROMAEBKVF */ { SST(0x00, 0x06, SS_RDEF, "I/O process terminated") }, /* T */ { SST(0x00, 0x07, SS_RDEF, /* XXX TBD */ "Programmable early warning detected") }, /* R */ { SST(0x00, 0x11, SS_FATAL | EBUSY, "Audio play operation in progress") }, /* R */ { SST(0x00, 0x12, SS_NOP, "Audio play operation paused") }, /* R */ { SST(0x00, 0x13, SS_NOP, "Audio play operation successfully completed") }, /* R */ { SST(0x00, 0x14, SS_RDEF, "Audio play operation stopped due to error") }, /* R */ { SST(0x00, 0x15, SS_NOP, "No current audio status to return") }, /* DTLPWROMAEBKVF */ { SST(0x00, 0x16, SS_FATAL | EBUSY, "Operation in progress") }, /* DTL WROMAEBKVF */ { SST(0x00, 0x17, SS_RDEF, "Cleaning requested") }, /* T */ { SST(0x00, 0x18, SS_RDEF, /* XXX TBD */ "Erase operation in progress") }, /* T */ { SST(0x00, 0x19, SS_RDEF, /* XXX TBD */ "Locate operation in progress") }, /* T */ { SST(0x00, 0x1A, SS_RDEF, /* XXX TBD */ "Rewind operation in progress") }, /* T */ { SST(0x00, 0x1B, SS_RDEF, /* XXX TBD */ "Set capacity operation in progress") }, /* T */ { SST(0x00, 0x1C, SS_RDEF, /* XXX TBD */ "Verify operation in progress") }, /* DT B */ { SST(0x00, 0x1D, SS_RDEF, /* XXX TBD */ "ATA pass through information available") }, /* DT R MAEBKV */ { SST(0x00, 0x1E, SS_RDEF, /* XXX TBD */ "Conflicting SA creation request") }, /* DT B */ { SST(0x00, 0x1F, SS_RDEF, /* XXX TBD */ "Logical unit transitioning to another power condition") }, /* DT P B */ { SST(0x00, 0x20, SS_RDEF, /* XXX TBD */ "Extended copy information available") }, /* D */ { SST(0x00, 0x21, SS_RDEF, /* XXX TBD */ "Atomic command aborted due to ACA") }, /* D W O BK */ { SST(0x01, 0x00, SS_RDEF, "No index/sector signal") }, /* D WRO BK */ { SST(0x02, 0x00, SS_RDEF, "No seek complete") }, /* DTL W O BK */ { SST(0x03, 0x00, SS_RDEF, "Peripheral device write fault") }, /* T */ { SST(0x03, 0x01, SS_RDEF, "No write current") }, /* T */ { SST(0x03, 0x02, SS_RDEF, "Excessive write errors") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x00, SS_RDEF, "Logical unit not ready, cause not reportable") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x01, SS_WAIT | EBUSY, "Logical unit is in process of becoming ready") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x02, SS_START | SSQ_DECREMENT_COUNT | ENXIO, "Logical unit not ready, initializing command required") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x03, SS_FATAL | ENXIO, "Logical unit not ready, manual intervention required") }, /* DTL RO B */ { SST(0x04, 0x04, SS_FATAL | EBUSY, "Logical unit not ready, format in progress") }, /* DT W O A BK F */ { SST(0x04, 0x05, SS_FATAL | EBUSY, "Logical unit not ready, rebuild in progress") }, /* DT W O A BK */ { SST(0x04, 0x06, SS_FATAL | EBUSY, "Logical unit not ready, recalculation in progress") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x07, SS_FATAL | EBUSY, "Logical unit not ready, operation in progress") }, /* R */ { SST(0x04, 0x08, SS_FATAL | EBUSY, "Logical unit not ready, long write in progress") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x09, SS_RDEF, /* XXX TBD */ "Logical unit not ready, self-test in progress") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x0A, SS_WAIT | ENXIO, "Logical unit not accessible, asymmetric access state transition")}, /* DTLPWROMAEBKVF */ { SST(0x04, 0x0B, SS_FATAL | ENXIO, "Logical unit not accessible, target port in standby state") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x0C, SS_FATAL | ENXIO, "Logical unit not accessible, target port in unavailable state") }, /* F */ { SST(0x04, 0x0D, SS_RDEF, /* XXX TBD */ "Logical unit not ready, structure check required") }, /* DTL WR MAEBKVF */ { SST(0x04, 0x0E, SS_RDEF, /* XXX TBD */ "Logical unit not ready, security session in progress") }, /* DT WROM B */ { SST(0x04, 0x10, SS_RDEF, /* XXX TBD */ "Logical unit not ready, auxiliary memory not accessible") }, /* DT WRO AEB VF */ { SST(0x04, 0x11, SS_WAIT | EBUSY, "Logical unit not ready, notify (enable spinup) required") }, /* M V */ { SST(0x04, 0x12, SS_RDEF, /* XXX TBD */ "Logical unit not ready, offline") }, /* DT R MAEBKV */ { SST(0x04, 0x13, SS_RDEF, /* XXX TBD */ "Logical unit not ready, SA creation in progress") }, /* D B */ { SST(0x04, 0x14, SS_RDEF, /* XXX TBD */ "Logical unit not ready, space allocation in progress") }, /* M */ { SST(0x04, 0x15, SS_RDEF, /* XXX TBD */ "Logical unit not ready, robotics disabled") }, /* M */ { SST(0x04, 0x16, SS_RDEF, /* XXX TBD */ "Logical unit not ready, configuration required") }, /* M */ { SST(0x04, 0x17, SS_RDEF, /* XXX TBD */ "Logical unit not ready, calibration required") }, /* M */ { SST(0x04, 0x18, SS_RDEF, /* XXX TBD */ "Logical unit not ready, a door is open") }, /* M */ { SST(0x04, 0x19, SS_RDEF, /* XXX TBD */ "Logical unit not ready, operating in sequential mode") }, /* DT B */ { SST(0x04, 0x1A, SS_RDEF, /* XXX TBD */ "Logical unit not ready, START/STOP UNIT command in progress") }, /* D B */ { SST(0x04, 0x1B, SS_RDEF, /* XXX TBD */ "Logical unit not ready, sanitize in progress") }, /* DT MAEB */ { SST(0x04, 0x1C, SS_RDEF, /* XXX TBD */ "Logical unit not ready, additional power use not yet granted") }, /* D */ { SST(0x04, 0x1D, SS_RDEF, /* XXX TBD */ "Logical unit not ready, configuration in progress") }, /* D */ { SST(0x04, 0x1E, SS_FATAL | ENXIO, "Logical unit not ready, microcode activation required") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x1F, SS_FATAL | ENXIO, "Logical unit not ready, microcode download required") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x20, SS_RDEF, /* XXX TBD */ "Logical unit not ready, logical unit reset required") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x21, SS_RDEF, /* XXX TBD */ "Logical unit not ready, hard reset required") }, /* DTLPWROMAEBKVF */ { SST(0x04, 0x22, SS_RDEF, /* XXX TBD */ "Logical unit not ready, power cycle required") }, /* DTL WROMAEBKVF */ { SST(0x05, 0x00, SS_RDEF, "Logical unit does not respond to selection") }, /* D WROM BK */ { SST(0x06, 0x00, SS_RDEF, "No reference position found") }, /* DTL WROM BK */ { SST(0x07, 0x00, SS_RDEF, "Multiple peripheral devices selected") }, /* DTL WROMAEBKVF */ { SST(0x08, 0x00, SS_RDEF, "Logical unit communication failure") }, /* DTL WROMAEBKVF */ { SST(0x08, 0x01, SS_RDEF, "Logical unit communication time-out") }, /* DTL WROMAEBKVF */ { SST(0x08, 0x02, SS_RDEF, "Logical unit communication parity error") }, /* DT ROM BK */ { SST(0x08, 0x03, SS_RDEF, "Logical unit communication CRC error (Ultra-DMA/32)") }, /* DTLPWRO K */ { SST(0x08, 0x04, SS_RDEF, /* XXX TBD */ "Unreachable copy target") }, /* DT WRO B */ { SST(0x09, 0x00, SS_RDEF, "Track following error") }, /* WRO K */ { SST(0x09, 0x01, SS_RDEF, "Tracking servo failure") }, /* WRO K */ { SST(0x09, 0x02, SS_RDEF, "Focus servo failure") }, /* WRO */ { SST(0x09, 0x03, SS_RDEF, "Spindle servo failure") }, /* DT WRO B */ { SST(0x09, 0x04, SS_RDEF, "Head select fault") }, /* DT RO B */ { SST(0x09, 0x05, SS_RDEF, "Vibration induced tracking error") }, /* DTLPWROMAEBKVF */ { SST(0x0A, 0x00, SS_FATAL | ENOSPC, "Error log overflow") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x00, SS_RDEF, "Warning") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x01, SS_RDEF, "Warning - specified temperature exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x02, SS_RDEF, "Warning - enclosure degraded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x03, SS_RDEF, /* XXX TBD */ "Warning - background self-test failed") }, /* DTLPWRO AEBKVF */ { SST(0x0B, 0x04, SS_RDEF, /* XXX TBD */ "Warning - background pre-scan detected medium error") }, /* DTLPWRO AEBKVF */ { SST(0x0B, 0x05, SS_RDEF, /* XXX TBD */ "Warning - background medium scan detected medium error") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x06, SS_RDEF, /* XXX TBD */ "Warning - non-volatile cache now volatile") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x07, SS_RDEF, /* XXX TBD */ "Warning - degraded power to non-volatile cache") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x08, SS_RDEF, /* XXX TBD */ "Warning - power loss expected") }, /* D */ { SST(0x0B, 0x09, SS_RDEF, /* XXX TBD */ "Warning - device statistics notification available") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0A, SS_RDEF, /* XXX TBD */ "Warning - High critical temperature limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0B, SS_RDEF, /* XXX TBD */ "Warning - Low critical temperature limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0C, SS_RDEF, /* XXX TBD */ "Warning - High operating temperature limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0D, SS_RDEF, /* XXX TBD */ "Warning - Low operating temperature limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0E, SS_RDEF, /* XXX TBD */ "Warning - High citical humidity limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x0F, SS_RDEF, /* XXX TBD */ "Warning - Low citical humidity limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x10, SS_RDEF, /* XXX TBD */ "Warning - High operating humidity limit exceeded") }, /* DTLPWROMAEBKVF */ { SST(0x0B, 0x11, SS_RDEF, /* XXX TBD */ "Warning - Low operating humidity limit exceeded") }, /* T R */ { SST(0x0C, 0x00, SS_RDEF, "Write error") }, /* K */ { SST(0x0C, 0x01, SS_NOP | SSQ_PRINT_SENSE, "Write error - recovered with auto reallocation") }, /* D W O BK */ { SST(0x0C, 0x02, SS_RDEF, "Write error - auto reallocation failed") }, /* D W O BK */ { SST(0x0C, 0x03, SS_RDEF, "Write error - recommend reassignment") }, /* DT W O B */ { SST(0x0C, 0x04, SS_RDEF, "Compression check miscompare error") }, /* DT W O B */ { SST(0x0C, 0x05, SS_RDEF, "Data expansion occurred during compression") }, /* DT W O B */ { SST(0x0C, 0x06, SS_RDEF, "Block not compressible") }, /* R */ { SST(0x0C, 0x07, SS_RDEF, "Write error - recovery needed") }, /* R */ { SST(0x0C, 0x08, SS_RDEF, "Write error - recovery failed") }, /* R */ { SST(0x0C, 0x09, SS_RDEF, "Write error - loss of streaming") }, /* R */ { SST(0x0C, 0x0A, SS_RDEF, "Write error - padding blocks added") }, /* DT WROM B */ { SST(0x0C, 0x0B, SS_RDEF, /* XXX TBD */ "Auxiliary memory write error") }, /* DTLPWRO AEBKVF */ { SST(0x0C, 0x0C, SS_RDEF, /* XXX TBD */ "Write error - unexpected unsolicited data") }, /* DTLPWRO AEBKVF */ { SST(0x0C, 0x0D, SS_RDEF, /* XXX TBD */ "Write error - not enough unsolicited data") }, /* DT W O BK */ { SST(0x0C, 0x0E, SS_RDEF, /* XXX TBD */ "Multiple write errors") }, /* R */ { SST(0x0C, 0x0F, SS_RDEF, /* XXX TBD */ "Defects in error window") }, /* D */ { SST(0x0C, 0x10, SS_RDEF, /* XXX TBD */ "Incomplete multiple atomic write operations") }, /* D */ { SST(0x0C, 0x11, SS_RDEF, /* XXX TBD */ "Write error - recovery scan needed") }, /* D */ { SST(0x0C, 0x12, SS_RDEF, /* XXX TBD */ "Write error - insufficient zone resources") }, /* DTLPWRO A K */ { SST(0x0D, 0x00, SS_RDEF, /* XXX TBD */ "Error detected by third party temporary initiator") }, /* DTLPWRO A K */ { SST(0x0D, 0x01, SS_RDEF, /* XXX TBD */ "Third party device failure") }, /* DTLPWRO A K */ { SST(0x0D, 0x02, SS_RDEF, /* XXX TBD */ "Copy target device not reachable") }, /* DTLPWRO A K */ { SST(0x0D, 0x03, SS_RDEF, /* XXX TBD */ "Incorrect copy target device type") }, /* DTLPWRO A K */ { SST(0x0D, 0x04, SS_RDEF, /* XXX TBD */ "Copy target device data underrun") }, /* DTLPWRO A K */ { SST(0x0D, 0x05, SS_RDEF, /* XXX TBD */ "Copy target device data overrun") }, /* DT PWROMAEBK F */ { SST(0x0E, 0x00, SS_RDEF, /* XXX TBD */ "Invalid information unit") }, /* DT PWROMAEBK F */ { SST(0x0E, 0x01, SS_RDEF, /* XXX TBD */ "Information unit too short") }, /* DT PWROMAEBK F */ { SST(0x0E, 0x02, SS_RDEF, /* XXX TBD */ "Information unit too long") }, /* DT P R MAEBK F */ { SST(0x0E, 0x03, SS_RDEF, /* XXX TBD */ "Invalid field in command information unit") }, /* D W O BK */ { SST(0x10, 0x00, SS_RDEF, "ID CRC or ECC error") }, /* DT W O */ { SST(0x10, 0x01, SS_RDEF, /* XXX TBD */ "Logical block guard check failed") }, /* DT W O */ { SST(0x10, 0x02, SS_RDEF, /* XXX TBD */ "Logical block application tag check failed") }, /* DT W O */ { SST(0x10, 0x03, SS_RDEF, /* XXX TBD */ "Logical block reference tag check failed") }, /* T */ { SST(0x10, 0x04, SS_RDEF, /* XXX TBD */ "Logical block protection error on recovered buffer data") }, /* T */ { SST(0x10, 0x05, SS_RDEF, /* XXX TBD */ "Logical block protection method error") }, /* DT WRO BK */ { SST(0x11, 0x00, SS_FATAL|EIO, "Unrecovered read error") }, /* DT WRO BK */ { SST(0x11, 0x01, SS_FATAL|EIO, "Read retries exhausted") }, /* DT WRO BK */ { SST(0x11, 0x02, SS_FATAL|EIO, "Error too long to correct") }, /* DT W O BK */ { SST(0x11, 0x03, SS_FATAL|EIO, "Multiple read errors") }, /* D W O BK */ { SST(0x11, 0x04, SS_FATAL|EIO, "Unrecovered read error - auto reallocate failed") }, /* WRO B */ { SST(0x11, 0x05, SS_FATAL|EIO, "L-EC uncorrectable error") }, /* WRO B */ { SST(0x11, 0x06, SS_FATAL|EIO, "CIRC unrecovered error") }, /* W O B */ { SST(0x11, 0x07, SS_RDEF, "Data re-synchronization error") }, /* T */ { SST(0x11, 0x08, SS_RDEF, "Incomplete block read") }, /* T */ { SST(0x11, 0x09, SS_RDEF, "No gap found") }, /* DT O BK */ { SST(0x11, 0x0A, SS_RDEF, "Miscorrected error") }, /* D W O BK */ { SST(0x11, 0x0B, SS_FATAL|EIO, "Unrecovered read error - recommend reassignment") }, /* D W O BK */ { SST(0x11, 0x0C, SS_FATAL|EIO, "Unrecovered read error - recommend rewrite the data") }, /* DT WRO B */ { SST(0x11, 0x0D, SS_RDEF, "De-compression CRC error") }, /* DT WRO B */ { SST(0x11, 0x0E, SS_RDEF, "Cannot decompress using declared algorithm") }, /* R */ { SST(0x11, 0x0F, SS_RDEF, "Error reading UPC/EAN number") }, /* R */ { SST(0x11, 0x10, SS_RDEF, "Error reading ISRC number") }, /* R */ { SST(0x11, 0x11, SS_RDEF, "Read error - loss of streaming") }, /* DT WROM B */ { SST(0x11, 0x12, SS_RDEF, /* XXX TBD */ "Auxiliary memory read error") }, /* DTLPWRO AEBKVF */ { SST(0x11, 0x13, SS_RDEF, /* XXX TBD */ "Read error - failed retransmission request") }, /* D */ { SST(0x11, 0x14, SS_RDEF, /* XXX TBD */ "Read error - LBA marked bad by application client") }, /* D */ { SST(0x11, 0x15, SS_RDEF, /* XXX TBD */ "Write after sanitize required") }, /* D W O BK */ { SST(0x12, 0x00, SS_RDEF, "Address mark not found for ID field") }, /* D W O BK */ { SST(0x13, 0x00, SS_RDEF, "Address mark not found for data field") }, /* DTL WRO BK */ { SST(0x14, 0x00, SS_RDEF, "Recorded entity not found") }, /* DT WRO BK */ { SST(0x14, 0x01, SS_RDEF, "Record not found") }, /* T */ { SST(0x14, 0x02, SS_RDEF, "Filemark or setmark not found") }, /* T */ { SST(0x14, 0x03, SS_RDEF, "End-of-data not found") }, /* T */ { SST(0x14, 0x04, SS_RDEF, "Block sequence error") }, /* DT W O BK */ { SST(0x14, 0x05, SS_RDEF, "Record not found - recommend reassignment") }, /* DT W O BK */ { SST(0x14, 0x06, SS_RDEF, "Record not found - data auto-reallocated") }, /* T */ { SST(0x14, 0x07, SS_RDEF, /* XXX TBD */ "Locate operation failure") }, /* DTL WROM BK */ { SST(0x15, 0x00, SS_RDEF, "Random positioning error") }, /* DTL WROM BK */ { SST(0x15, 0x01, SS_RDEF, "Mechanical positioning error") }, /* DT WRO BK */ { SST(0x15, 0x02, SS_RDEF, "Positioning error detected by read of medium") }, /* D W O BK */ { SST(0x16, 0x00, SS_RDEF, "Data synchronization mark error") }, /* D W O BK */ { SST(0x16, 0x01, SS_RDEF, "Data sync error - data rewritten") }, /* D W O BK */ { SST(0x16, 0x02, SS_RDEF, "Data sync error - recommend rewrite") }, /* D W O BK */ { SST(0x16, 0x03, SS_NOP | SSQ_PRINT_SENSE, "Data sync error - data auto-reallocated") }, /* D W O BK */ { SST(0x16, 0x04, SS_RDEF, "Data sync error - recommend reassignment") }, /* DT WRO BK */ { SST(0x17, 0x00, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with no error correction applied") }, /* DT WRO BK */ { SST(0x17, 0x01, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with retries") }, /* DT WRO BK */ { SST(0x17, 0x02, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with positive head offset") }, /* DT WRO BK */ { SST(0x17, 0x03, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with negative head offset") }, /* WRO B */ { SST(0x17, 0x04, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with retries and/or CIRC applied") }, /* D WRO BK */ { SST(0x17, 0x05, SS_NOP | SSQ_PRINT_SENSE, "Recovered data using previous sector ID") }, /* D W O BK */ { SST(0x17, 0x06, SS_NOP | SSQ_PRINT_SENSE, "Recovered data without ECC - data auto-reallocated") }, /* D WRO BK */ { SST(0x17, 0x07, SS_NOP | SSQ_PRINT_SENSE, "Recovered data without ECC - recommend reassignment") }, /* D WRO BK */ { SST(0x17, 0x08, SS_NOP | SSQ_PRINT_SENSE, "Recovered data without ECC - recommend rewrite") }, /* D WRO BK */ { SST(0x17, 0x09, SS_NOP | SSQ_PRINT_SENSE, "Recovered data without ECC - data rewritten") }, /* DT WRO BK */ { SST(0x18, 0x00, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with error correction applied") }, /* D WRO BK */ { SST(0x18, 0x01, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with error corr. & retries applied") }, /* D WRO BK */ { SST(0x18, 0x02, SS_NOP | SSQ_PRINT_SENSE, "Recovered data - data auto-reallocated") }, /* R */ { SST(0x18, 0x03, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with CIRC") }, /* R */ { SST(0x18, 0x04, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with L-EC") }, /* D WRO BK */ { SST(0x18, 0x05, SS_NOP | SSQ_PRINT_SENSE, "Recovered data - recommend reassignment") }, /* D WRO BK */ { SST(0x18, 0x06, SS_NOP | SSQ_PRINT_SENSE, "Recovered data - recommend rewrite") }, /* D W O BK */ { SST(0x18, 0x07, SS_NOP | SSQ_PRINT_SENSE, "Recovered data with ECC - data rewritten") }, /* R */ { SST(0x18, 0x08, SS_RDEF, /* XXX TBD */ "Recovered data with linking") }, /* D O K */ { SST(0x19, 0x00, SS_RDEF, "Defect list error") }, /* D O K */ { SST(0x19, 0x01, SS_RDEF, "Defect list not available") }, /* D O K */ { SST(0x19, 0x02, SS_RDEF, "Defect list error in primary list") }, /* D O K */ { SST(0x19, 0x03, SS_RDEF, "Defect list error in grown list") }, /* DTLPWROMAEBKVF */ { SST(0x1A, 0x00, SS_RDEF, "Parameter list length error") }, /* DTLPWROMAEBKVF */ { SST(0x1B, 0x00, SS_RDEF, "Synchronous data transfer error") }, /* D O BK */ { SST(0x1C, 0x00, SS_RDEF, "Defect list not found") }, /* D O BK */ { SST(0x1C, 0x01, SS_RDEF, "Primary defect list not found") }, /* D O BK */ { SST(0x1C, 0x02, SS_RDEF, "Grown defect list not found") }, /* DT WRO BK */ { SST(0x1D, 0x00, SS_FATAL, "Miscompare during verify operation") }, /* D B */ { SST(0x1D, 0x01, SS_RDEF, /* XXX TBD */ "Miscomparable verify of unmapped LBA") }, /* D W O BK */ { SST(0x1E, 0x00, SS_NOP | SSQ_PRINT_SENSE, "Recovered ID with ECC correction") }, /* D O K */ { SST(0x1F, 0x00, SS_RDEF, "Partial defect list transfer") }, /* DTLPWROMAEBKVF */ { SST(0x20, 0x00, SS_FATAL | EINVAL, "Invalid command operation code") }, /* DT PWROMAEBK */ { SST(0x20, 0x01, SS_RDEF, /* XXX TBD */ "Access denied - initiator pending-enrolled") }, /* DT PWROMAEBK */ { SST(0x20, 0x02, SS_RDEF, /* XXX TBD */ "Access denied - no access rights") }, /* DT PWROMAEBK */ { SST(0x20, 0x03, SS_RDEF, /* XXX TBD */ "Access denied - invalid mgmt ID key") }, /* T */ { SST(0x20, 0x04, SS_RDEF, /* XXX TBD */ "Illegal command while in write capable state") }, /* T */ { SST(0x20, 0x05, SS_RDEF, /* XXX TBD */ "Obsolete") }, /* T */ { SST(0x20, 0x06, SS_RDEF, /* XXX TBD */ "Illegal command while in explicit address mode") }, /* T */ { SST(0x20, 0x07, SS_RDEF, /* XXX TBD */ "Illegal command while in implicit address mode") }, /* DT PWROMAEBK */ { SST(0x20, 0x08, SS_RDEF, /* XXX TBD */ "Access denied - enrollment conflict") }, /* DT PWROMAEBK */ { SST(0x20, 0x09, SS_RDEF, /* XXX TBD */ "Access denied - invalid LU identifier") }, /* DT PWROMAEBK */ { SST(0x20, 0x0A, SS_RDEF, /* XXX TBD */ "Access denied - invalid proxy token") }, /* DT PWROMAEBK */ { SST(0x20, 0x0B, SS_RDEF, /* XXX TBD */ "Access denied - ACL LUN conflict") }, /* T */ { SST(0x20, 0x0C, SS_FATAL | EINVAL, "Illegal command when not in append-only mode") }, /* DT WRO BK */ { SST(0x21, 0x00, SS_FATAL | EINVAL, "Logical block address out of range") }, /* DT WROM BK */ { SST(0x21, 0x01, SS_FATAL | EINVAL, "Invalid element address") }, /* R */ { SST(0x21, 0x02, SS_RDEF, /* XXX TBD */ "Invalid address for write") }, /* R */ { SST(0x21, 0x03, SS_RDEF, /* XXX TBD */ "Invalid write crossing layer jump") }, /* D */ { SST(0x21, 0x04, SS_RDEF, /* XXX TBD */ "Unaligned write command") }, /* D */ { SST(0x21, 0x05, SS_RDEF, /* XXX TBD */ "Write boundary violation") }, /* D */ { SST(0x21, 0x06, SS_RDEF, /* XXX TBD */ "Attempt to read invalid data") }, /* D */ { SST(0x21, 0x07, SS_RDEF, /* XXX TBD */ "Read boundary violation") }, /* D */ { SST(0x22, 0x00, SS_FATAL | EINVAL, "Illegal function (use 20 00, 24 00, or 26 00)") }, /* DT P B */ { SST(0x23, 0x00, SS_FATAL | EINVAL, "Invalid token operation, cause not reportable") }, /* DT P B */ { SST(0x23, 0x01, SS_FATAL | EINVAL, "Invalid token operation, unsupported token type") }, /* DT P B */ { SST(0x23, 0x02, SS_FATAL | EINVAL, "Invalid token operation, remote token usage not supported") }, /* DT P B */ { SST(0x23, 0x03, SS_FATAL | EINVAL, "Invalid token operation, remote ROD token creation not supported") }, /* DT P B */ { SST(0x23, 0x04, SS_FATAL | EINVAL, "Invalid token operation, token unknown") }, /* DT P B */ { SST(0x23, 0x05, SS_FATAL | EINVAL, "Invalid token operation, token corrupt") }, /* DT P B */ { SST(0x23, 0x06, SS_FATAL | EINVAL, "Invalid token operation, token revoked") }, /* DT P B */ { SST(0x23, 0x07, SS_FATAL | EINVAL, "Invalid token operation, token expired") }, /* DT P B */ { SST(0x23, 0x08, SS_FATAL | EINVAL, "Invalid token operation, token cancelled") }, /* DT P B */ { SST(0x23, 0x09, SS_FATAL | EINVAL, "Invalid token operation, token deleted") }, /* DT P B */ { SST(0x23, 0x0A, SS_FATAL | EINVAL, "Invalid token operation, invalid token length") }, /* DTLPWROMAEBKVF */ { SST(0x24, 0x00, SS_FATAL | EINVAL, "Invalid field in CDB") }, /* DTLPWRO AEBKVF */ { SST(0x24, 0x01, SS_RDEF, /* XXX TBD */ "CDB decryption error") }, /* T */ { SST(0x24, 0x02, SS_RDEF, /* XXX TBD */ "Obsolete") }, /* T */ { SST(0x24, 0x03, SS_RDEF, /* XXX TBD */ "Obsolete") }, /* F */ { SST(0x24, 0x04, SS_RDEF, /* XXX TBD */ "Security audit value frozen") }, /* F */ { SST(0x24, 0x05, SS_RDEF, /* XXX TBD */ "Security working key frozen") }, /* F */ { SST(0x24, 0x06, SS_RDEF, /* XXX TBD */ "NONCE not unique") }, /* F */ { SST(0x24, 0x07, SS_RDEF, /* XXX TBD */ "NONCE timestamp out of range") }, /* DT R MAEBKV */ { SST(0x24, 0x08, SS_RDEF, /* XXX TBD */ "Invalid XCDB") }, /* DTLPWROMAEBKVF */ { SST(0x25, 0x00, SS_FATAL | ENXIO | SSQ_LOST, "Logical unit not supported") }, /* DTLPWROMAEBKVF */ { SST(0x26, 0x00, SS_FATAL | EINVAL, "Invalid field in parameter list") }, /* DTLPWROMAEBKVF */ { SST(0x26, 0x01, SS_FATAL | EINVAL, "Parameter not supported") }, /* DTLPWROMAEBKVF */ { SST(0x26, 0x02, SS_FATAL | EINVAL, "Parameter value invalid") }, /* DTLPWROMAE K */ { SST(0x26, 0x03, SS_FATAL | EINVAL, "Threshold parameters not supported") }, /* DTLPWROMAEBKVF */ { SST(0x26, 0x04, SS_FATAL | EINVAL, "Invalid release of persistent reservation") }, /* DTLPWRO A BK */ { SST(0x26, 0x05, SS_RDEF, /* XXX TBD */ "Data decryption error") }, /* DTLPWRO K */ { SST(0x26, 0x06, SS_FATAL | EINVAL, "Too many target descriptors") }, /* DTLPWRO K */ { SST(0x26, 0x07, SS_FATAL | EINVAL, "Unsupported target descriptor type code") }, /* DTLPWRO K */ { SST(0x26, 0x08, SS_FATAL | EINVAL, "Too many segment descriptors") }, /* DTLPWRO K */ { SST(0x26, 0x09, SS_FATAL | EINVAL, "Unsupported segment descriptor type code") }, /* DTLPWRO K */ { SST(0x26, 0x0A, SS_FATAL | EINVAL, "Unexpected inexact segment") }, /* DTLPWRO K */ { SST(0x26, 0x0B, SS_FATAL | EINVAL, "Inline data length exceeded") }, /* DTLPWRO K */ { SST(0x26, 0x0C, SS_FATAL | EINVAL, "Invalid operation for copy source or destination") }, /* DTLPWRO K */ { SST(0x26, 0x0D, SS_FATAL | EINVAL, "Copy segment granularity violation") }, /* DT PWROMAEBK */ { SST(0x26, 0x0E, SS_RDEF, /* XXX TBD */ "Invalid parameter while port is enabled") }, /* F */ { SST(0x26, 0x0F, SS_RDEF, /* XXX TBD */ "Invalid data-out buffer integrity check value") }, /* T */ { SST(0x26, 0x10, SS_RDEF, /* XXX TBD */ "Data decryption key fail limit reached") }, /* T */ { SST(0x26, 0x11, SS_RDEF, /* XXX TBD */ "Incomplete key-associated data set") }, /* T */ { SST(0x26, 0x12, SS_RDEF, /* XXX TBD */ "Vendor specific key reference not found") }, /* D */ { SST(0x26, 0x13, SS_RDEF, /* XXX TBD */ "Application tag mode page is invalid") }, /* DT WRO BK */ { SST(0x27, 0x00, SS_FATAL | EACCES, "Write protected") }, /* DT WRO BK */ { SST(0x27, 0x01, SS_FATAL | EACCES, "Hardware write protected") }, /* DT WRO BK */ { SST(0x27, 0x02, SS_FATAL | EACCES, "Logical unit software write protected") }, /* T R */ { SST(0x27, 0x03, SS_FATAL | EACCES, "Associated write protect") }, /* T R */ { SST(0x27, 0x04, SS_FATAL | EACCES, "Persistent write protect") }, /* T R */ { SST(0x27, 0x05, SS_FATAL | EACCES, "Permanent write protect") }, /* R F */ { SST(0x27, 0x06, SS_RDEF, /* XXX TBD */ "Conditional write protect") }, /* D B */ { SST(0x27, 0x07, SS_FATAL | ENOSPC, "Space allocation failed write protect") }, /* D */ { SST(0x27, 0x08, SS_FATAL | EACCES, "Zone is read only") }, /* DTLPWROMAEBKVF */ { SST(0x28, 0x00, SS_FATAL | ENXIO, "Not ready to ready change, medium may have changed") }, /* DT WROM B */ { SST(0x28, 0x01, SS_FATAL | ENXIO, "Import or export element accessed") }, /* R */ { SST(0x28, 0x02, SS_RDEF, /* XXX TBD */ "Format-layer may have changed") }, /* M */ { SST(0x28, 0x03, SS_RDEF, /* XXX TBD */ "Import/export element accessed, medium changed") }, /* * XXX JGibbs - All of these should use the same errno, but I don't * think ENXIO is the correct choice. Should we borrow from * the networking errnos? ECONNRESET anyone? */ /* DTLPWROMAEBKVF */ { SST(0x29, 0x00, SS_FATAL | ENXIO, "Power on, reset, or bus device reset occurred") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x01, SS_RDEF, "Power on occurred") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x02, SS_RDEF, "SCSI bus reset occurred") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x03, SS_RDEF, "Bus device reset function occurred") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x04, SS_RDEF, "Device internal reset") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x05, SS_RDEF, "Transceiver mode changed to single-ended") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x06, SS_RDEF, "Transceiver mode changed to LVD") }, /* DTLPWROMAEBKVF */ { SST(0x29, 0x07, SS_RDEF, /* XXX TBD */ "I_T nexus loss occurred") }, /* DTL WROMAEBKVF */ { SST(0x2A, 0x00, SS_RDEF, "Parameters changed") }, /* DTL WROMAEBKVF */ { SST(0x2A, 0x01, SS_RDEF, "Mode parameters changed") }, /* DTL WROMAE K */ { SST(0x2A, 0x02, SS_RDEF, "Log parameters changed") }, /* DTLPWROMAE K */ { SST(0x2A, 0x03, SS_RDEF, "Reservations preempted") }, /* DTLPWROMAE */ { SST(0x2A, 0x04, SS_RDEF, /* XXX TBD */ "Reservations released") }, /* DTLPWROMAE */ { SST(0x2A, 0x05, SS_RDEF, /* XXX TBD */ "Registrations preempted") }, /* DTLPWROMAEBKVF */ { SST(0x2A, 0x06, SS_RDEF, /* XXX TBD */ "Asymmetric access state changed") }, /* DTLPWROMAEBKVF */ { SST(0x2A, 0x07, SS_RDEF, /* XXX TBD */ "Implicit asymmetric access state transition failed") }, /* DT WROMAEBKVF */ { SST(0x2A, 0x08, SS_RDEF, /* XXX TBD */ "Priority changed") }, /* D */ { SST(0x2A, 0x09, SS_RDEF, /* XXX TBD */ "Capacity data has changed") }, /* DT */ { SST(0x2A, 0x0A, SS_RDEF, /* XXX TBD */ "Error history I_T nexus cleared") }, /* DT */ { SST(0x2A, 0x0B, SS_RDEF, /* XXX TBD */ "Error history snapshot released") }, /* F */ { SST(0x2A, 0x0C, SS_RDEF, /* XXX TBD */ "Error recovery attributes have changed") }, /* T */ { SST(0x2A, 0x0D, SS_RDEF, /* XXX TBD */ "Data encryption capabilities changed") }, /* DT M E V */ { SST(0x2A, 0x10, SS_RDEF, /* XXX TBD */ "Timestamp changed") }, /* T */ { SST(0x2A, 0x11, SS_RDEF, /* XXX TBD */ "Data encryption parameters changed by another I_T nexus") }, /* T */ { SST(0x2A, 0x12, SS_RDEF, /* XXX TBD */ "Data encryption parameters changed by vendor specific event") }, /* T */ { SST(0x2A, 0x13, SS_RDEF, /* XXX TBD */ "Data encryption key instance counter has changed") }, /* DT R MAEBKV */ { SST(0x2A, 0x14, SS_RDEF, /* XXX TBD */ "SA creation capabilities data has changed") }, /* T M V */ { SST(0x2A, 0x15, SS_RDEF, /* XXX TBD */ "Medium removal prevention preempted") }, /* DTLPWRO K */ { SST(0x2B, 0x00, SS_RDEF, "Copy cannot execute since host cannot disconnect") }, /* DTLPWROMAEBKVF */ { SST(0x2C, 0x00, SS_RDEF, "Command sequence error") }, /* */ { SST(0x2C, 0x01, SS_RDEF, "Too many windows specified") }, /* */ { SST(0x2C, 0x02, SS_RDEF, "Invalid combination of windows specified") }, /* R */ { SST(0x2C, 0x03, SS_RDEF, "Current program area is not empty") }, /* R */ { SST(0x2C, 0x04, SS_RDEF, "Current program area is empty") }, /* B */ { SST(0x2C, 0x05, SS_RDEF, /* XXX TBD */ "Illegal power condition request") }, /* R */ { SST(0x2C, 0x06, SS_RDEF, /* XXX TBD */ "Persistent prevent conflict") }, /* DTLPWROMAEBKVF */ { SST(0x2C, 0x07, SS_RDEF, /* XXX TBD */ "Previous busy status") }, /* DTLPWROMAEBKVF */ { SST(0x2C, 0x08, SS_RDEF, /* XXX TBD */ "Previous task set full status") }, /* DTLPWROM EBKVF */ { SST(0x2C, 0x09, SS_RDEF, /* XXX TBD */ "Previous reservation conflict status") }, /* F */ { SST(0x2C, 0x0A, SS_RDEF, /* XXX TBD */ "Partition or collection contains user objects") }, /* T */ { SST(0x2C, 0x0B, SS_RDEF, /* XXX TBD */ "Not reserved") }, /* D */ { SST(0x2C, 0x0C, SS_RDEF, /* XXX TBD */ "ORWRITE generation does not match") }, /* D */ { SST(0x2C, 0x0D, SS_RDEF, /* XXX TBD */ "Reset write pointer not allowed") }, /* D */ { SST(0x2C, 0x0E, SS_RDEF, /* XXX TBD */ "Zone is offline") }, /* D */ { SST(0x2C, 0x0F, SS_RDEF, /* XXX TBD */ "Stream not open") }, /* D */ { SST(0x2C, 0x10, SS_RDEF, /* XXX TBD */ "Unwritten data in zone") }, /* T */ { SST(0x2D, 0x00, SS_RDEF, "Overwrite error on update in place") }, /* R */ { SST(0x2E, 0x00, SS_RDEF, /* XXX TBD */ "Insufficient time for operation") }, /* D */ { SST(0x2E, 0x01, SS_RDEF, /* XXX TBD */ "Command timeout before processing") }, /* D */ { SST(0x2E, 0x02, SS_RDEF, /* XXX TBD */ "Command timeout during processing") }, /* D */ { SST(0x2E, 0x03, SS_RDEF, /* XXX TBD */ "Command timeout during processing due to error recovery") }, /* DTLPWROMAEBKVF */ { SST(0x2F, 0x00, SS_RDEF, "Commands cleared by another initiator") }, /* D */ { SST(0x2F, 0x01, SS_RDEF, /* XXX TBD */ "Commands cleared by power loss notification") }, /* DTLPWROMAEBKVF */ { SST(0x2F, 0x02, SS_RDEF, /* XXX TBD */ "Commands cleared by device server") }, /* DTLPWROMAEBKVF */ { SST(0x2F, 0x03, SS_RDEF, /* XXX TBD */ "Some commands cleared by queuing layer event") }, /* DT WROM BK */ { SST(0x30, 0x00, SS_RDEF, "Incompatible medium installed") }, /* DT WRO BK */ { SST(0x30, 0x01, SS_RDEF, "Cannot read medium - unknown format") }, /* DT WRO BK */ { SST(0x30, 0x02, SS_RDEF, "Cannot read medium - incompatible format") }, /* DT R K */ { SST(0x30, 0x03, SS_RDEF, "Cleaning cartridge installed") }, /* DT WRO BK */ { SST(0x30, 0x04, SS_RDEF, "Cannot write medium - unknown format") }, /* DT WRO BK */ { SST(0x30, 0x05, SS_RDEF, "Cannot write medium - incompatible format") }, /* DT WRO B */ { SST(0x30, 0x06, SS_RDEF, "Cannot format medium - incompatible medium") }, /* DTL WROMAEBKVF */ { SST(0x30, 0x07, SS_RDEF, "Cleaning failure") }, /* R */ { SST(0x30, 0x08, SS_RDEF, "Cannot write - application code mismatch") }, /* R */ { SST(0x30, 0x09, SS_RDEF, "Current session not fixated for append") }, /* DT WRO AEBK */ { SST(0x30, 0x0A, SS_RDEF, /* XXX TBD */ "Cleaning request rejected") }, /* T */ { SST(0x30, 0x0C, SS_RDEF, /* XXX TBD */ "WORM medium - overwrite attempted") }, /* T */ { SST(0x30, 0x0D, SS_RDEF, /* XXX TBD */ "WORM medium - integrity check") }, /* R */ { SST(0x30, 0x10, SS_RDEF, /* XXX TBD */ "Medium not formatted") }, /* M */ { SST(0x30, 0x11, SS_RDEF, /* XXX TBD */ "Incompatible volume type") }, /* M */ { SST(0x30, 0x12, SS_RDEF, /* XXX TBD */ "Incompatible volume qualifier") }, /* M */ { SST(0x30, 0x13, SS_RDEF, /* XXX TBD */ "Cleaning volume expired") }, /* DT WRO BK */ { SST(0x31, 0x00, SS_RDEF, "Medium format corrupted") }, /* D L RO B */ { SST(0x31, 0x01, SS_RDEF, "Format command failed") }, /* R */ { SST(0x31, 0x02, SS_RDEF, /* XXX TBD */ "Zoned formatting failed due to spare linking") }, /* D B */ { SST(0x31, 0x03, SS_RDEF, /* XXX TBD */ "SANITIZE command failed") }, /* D W O BK */ { SST(0x32, 0x00, SS_RDEF, "No defect spare location available") }, /* D W O BK */ { SST(0x32, 0x01, SS_RDEF, "Defect list update failure") }, /* T */ { SST(0x33, 0x00, SS_RDEF, "Tape length error") }, /* DTLPWROMAEBKVF */ { SST(0x34, 0x00, SS_RDEF, "Enclosure failure") }, /* DTLPWROMAEBKVF */ { SST(0x35, 0x00, SS_RDEF, "Enclosure services failure") }, /* DTLPWROMAEBKVF */ { SST(0x35, 0x01, SS_RDEF, "Unsupported enclosure function") }, /* DTLPWROMAEBKVF */ { SST(0x35, 0x02, SS_RDEF, "Enclosure services unavailable") }, /* DTLPWROMAEBKVF */ { SST(0x35, 0x03, SS_RDEF, "Enclosure services transfer failure") }, /* DTLPWROMAEBKVF */ { SST(0x35, 0x04, SS_RDEF, "Enclosure services transfer refused") }, /* DTL WROMAEBKVF */ { SST(0x35, 0x05, SS_RDEF, /* XXX TBD */ "Enclosure services checksum error") }, /* L */ { SST(0x36, 0x00, SS_RDEF, "Ribbon, ink, or toner failure") }, /* DTL WROMAEBKVF */ { SST(0x37, 0x00, SS_RDEF, "Rounded parameter") }, /* B */ { SST(0x38, 0x00, SS_RDEF, /* XXX TBD */ "Event status notification") }, /* B */ { SST(0x38, 0x02, SS_RDEF, /* XXX TBD */ "ESN - power management class event") }, /* B */ { SST(0x38, 0x04, SS_RDEF, /* XXX TBD */ "ESN - media class event") }, /* B */ { SST(0x38, 0x06, SS_RDEF, /* XXX TBD */ "ESN - device busy class event") }, /* D */ { SST(0x38, 0x07, SS_RDEF, /* XXX TBD */ "Thin provisioning soft threshold reached") }, /* DTL WROMAE K */ { SST(0x39, 0x00, SS_RDEF, "Saving parameters not supported") }, /* DTL WROM BK */ { SST(0x3A, 0x00, SS_FATAL | ENXIO, "Medium not present") }, /* DT WROM BK */ { SST(0x3A, 0x01, SS_FATAL | ENXIO, "Medium not present - tray closed") }, /* DT WROM BK */ { SST(0x3A, 0x02, SS_FATAL | ENXIO, "Medium not present - tray open") }, /* DT WROM B */ { SST(0x3A, 0x03, SS_RDEF, /* XXX TBD */ "Medium not present - loadable") }, /* DT WRO B */ { SST(0x3A, 0x04, SS_RDEF, /* XXX TBD */ "Medium not present - medium auxiliary memory accessible") }, /* TL */ { SST(0x3B, 0x00, SS_RDEF, "Sequential positioning error") }, /* T */ { SST(0x3B, 0x01, SS_RDEF, "Tape position error at beginning-of-medium") }, /* T */ { SST(0x3B, 0x02, SS_RDEF, "Tape position error at end-of-medium") }, /* L */ { SST(0x3B, 0x03, SS_RDEF, "Tape or electronic vertical forms unit not ready") }, /* L */ { SST(0x3B, 0x04, SS_RDEF, "Slew failure") }, /* L */ { SST(0x3B, 0x05, SS_RDEF, "Paper jam") }, /* L */ { SST(0x3B, 0x06, SS_RDEF, "Failed to sense top-of-form") }, /* L */ { SST(0x3B, 0x07, SS_RDEF, "Failed to sense bottom-of-form") }, /* T */ { SST(0x3B, 0x08, SS_RDEF, "Reposition error") }, /* */ { SST(0x3B, 0x09, SS_RDEF, "Read past end of medium") }, /* */ { SST(0x3B, 0x0A, SS_RDEF, "Read past beginning of medium") }, /* */ { SST(0x3B, 0x0B, SS_RDEF, "Position past end of medium") }, /* T */ { SST(0x3B, 0x0C, SS_RDEF, "Position past beginning of medium") }, /* DT WROM BK */ { SST(0x3B, 0x0D, SS_FATAL | ENOSPC, "Medium destination element full") }, /* DT WROM BK */ { SST(0x3B, 0x0E, SS_RDEF, "Medium source element empty") }, /* R */ { SST(0x3B, 0x0F, SS_RDEF, "End of medium reached") }, /* DT WROM BK */ { SST(0x3B, 0x11, SS_RDEF, "Medium magazine not accessible") }, /* DT WROM BK */ { SST(0x3B, 0x12, SS_RDEF, "Medium magazine removed") }, /* DT WROM BK */ { SST(0x3B, 0x13, SS_RDEF, "Medium magazine inserted") }, /* DT WROM BK */ { SST(0x3B, 0x14, SS_RDEF, "Medium magazine locked") }, /* DT WROM BK */ { SST(0x3B, 0x15, SS_RDEF, "Medium magazine unlocked") }, /* R */ { SST(0x3B, 0x16, SS_RDEF, /* XXX TBD */ "Mechanical positioning or changer error") }, /* F */ { SST(0x3B, 0x17, SS_RDEF, /* XXX TBD */ "Read past end of user object") }, /* M */ { SST(0x3B, 0x18, SS_RDEF, /* XXX TBD */ "Element disabled") }, /* M */ { SST(0x3B, 0x19, SS_RDEF, /* XXX TBD */ "Element enabled") }, /* M */ { SST(0x3B, 0x1A, SS_RDEF, /* XXX TBD */ "Data transfer device removed") }, /* M */ { SST(0x3B, 0x1B, SS_RDEF, /* XXX TBD */ "Data transfer device inserted") }, /* T */ { SST(0x3B, 0x1C, SS_RDEF, /* XXX TBD */ "Too many logical objects on partition to support operation") }, /* DTLPWROMAE K */ { SST(0x3D, 0x00, SS_RDEF, "Invalid bits in IDENTIFY message") }, /* DTLPWROMAEBKVF */ { SST(0x3E, 0x00, SS_RDEF, "Logical unit has not self-configured yet") }, /* DTLPWROMAEBKVF */ { SST(0x3E, 0x01, SS_RDEF, "Logical unit failure") }, /* DTLPWROMAEBKVF */ { SST(0x3E, 0x02, SS_RDEF, "Timeout on logical unit") }, /* DTLPWROMAEBKVF */ { SST(0x3E, 0x03, SS_RDEF, /* XXX TBD */ "Logical unit failed self-test") }, /* DTLPWROMAEBKVF */ { SST(0x3E, 0x04, SS_RDEF, /* XXX TBD */ "Logical unit unable to update self-test log") }, /* DTLPWROMAEBKVF */ { SST(0x3F, 0x00, SS_RDEF, "Target operating conditions have changed") }, /* DTLPWROMAEBKVF */ { SST(0x3F, 0x01, SS_RDEF, "Microcode has been changed") }, /* DTLPWROM BK */ { SST(0x3F, 0x02, SS_RDEF, "Changed operating definition") }, /* DTLPWROMAEBKVF */ { SST(0x3F, 0x03, SS_RDEF, "INQUIRY data has changed") }, /* DT WROMAEBK */ { SST(0x3F, 0x04, SS_RDEF, "Component device attached") }, /* DT WROMAEBK */ { SST(0x3F, 0x05, SS_RDEF, "Device identifier changed") }, /* DT WROMAEB */ { SST(0x3F, 0x06, SS_RDEF, "Redundancy group created or modified") }, /* DT WROMAEB */ { SST(0x3F, 0x07, SS_RDEF, "Redundancy group deleted") }, /* DT WROMAEB */ { SST(0x3F, 0x08, SS_RDEF, "Spare created or modified") }, /* DT WROMAEB */ { SST(0x3F, 0x09, SS_RDEF, "Spare deleted") }, /* DT WROMAEBK */ { SST(0x3F, 0x0A, SS_RDEF, "Volume set created or modified") }, /* DT WROMAEBK */ { SST(0x3F, 0x0B, SS_RDEF, "Volume set deleted") }, /* DT WROMAEBK */ { SST(0x3F, 0x0C, SS_RDEF, "Volume set deassigned") }, /* DT WROMAEBK */ { SST(0x3F, 0x0D, SS_RDEF, "Volume set reassigned") }, /* DTLPWROMAE */ { SST(0x3F, 0x0E, SS_RDEF | SSQ_RESCAN , "Reported LUNs data has changed") }, /* DTLPWROMAEBKVF */ { SST(0x3F, 0x0F, SS_RDEF, /* XXX TBD */ "Echo buffer overwritten") }, /* DT WROM B */ { SST(0x3F, 0x10, SS_RDEF, /* XXX TBD */ "Medium loadable") }, /* DT WROM B */ { SST(0x3F, 0x11, SS_RDEF, /* XXX TBD */ "Medium auxiliary memory accessible") }, /* DTLPWR MAEBK F */ { SST(0x3F, 0x12, SS_RDEF, /* XXX TBD */ "iSCSI IP address added") }, /* DTLPWR MAEBK F */ { SST(0x3F, 0x13, SS_RDEF, /* XXX TBD */ "iSCSI IP address removed") }, /* DTLPWR MAEBK F */ { SST(0x3F, 0x14, SS_RDEF, /* XXX TBD */ "iSCSI IP address changed") }, /* DTLPWR MAEBK */ { SST(0x3F, 0x15, SS_RDEF, /* XXX TBD */ "Inspect referrals sense descriptors") }, /* DTLPWROMAEBKVF */ { SST(0x3F, 0x16, SS_RDEF, /* XXX TBD */ "Microcode has been changed without reset") }, /* D */ { SST(0x3F, 0x17, SS_RDEF, /* XXX TBD */ "Zone transition to full") }, /* D */ { SST(0x40, 0x00, SS_RDEF, "RAM failure") }, /* deprecated - use 40 NN instead */ /* DTLPWROMAEBKVF */ { SST(0x40, 0x80, SS_RDEF, "Diagnostic failure: ASCQ = Component ID") }, /* DTLPWROMAEBKVF */ { SST(0x40, 0xFF, SS_RDEF | SSQ_RANGE, NULL) }, /* Range 0x80->0xFF */ /* D */ { SST(0x41, 0x00, SS_RDEF, "Data path failure") }, /* deprecated - use 40 NN instead */ /* D */ { SST(0x42, 0x00, SS_RDEF, "Power-on or self-test failure") }, /* deprecated - use 40 NN instead */ /* DTLPWROMAEBKVF */ { SST(0x43, 0x00, SS_RDEF, "Message error") }, /* DTLPWROMAEBKVF */ { SST(0x44, 0x00, SS_RDEF, "Internal target failure") }, /* DT P MAEBKVF */ { SST(0x44, 0x01, SS_RDEF, /* XXX TBD */ "Persistent reservation information lost") }, /* DT B */ { SST(0x44, 0x71, SS_RDEF, /* XXX TBD */ "ATA device failed set features") }, /* DTLPWROMAEBKVF */ { SST(0x45, 0x00, SS_RDEF, "Select or reselect failure") }, /* DTLPWROM BK */ { SST(0x46, 0x00, SS_RDEF, "Unsuccessful soft reset") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x00, SS_RDEF, "SCSI parity error") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x01, SS_RDEF, /* XXX TBD */ "Data phase CRC error detected") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x02, SS_RDEF, /* XXX TBD */ "SCSI parity error detected during ST data phase") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x03, SS_RDEF, /* XXX TBD */ "Information unit iuCRC error detected") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x04, SS_RDEF, /* XXX TBD */ "Asynchronous information protection error detected") }, /* DTLPWROMAEBKVF */ { SST(0x47, 0x05, SS_RDEF, /* XXX TBD */ "Protocol service CRC error") }, /* DT MAEBKVF */ { SST(0x47, 0x06, SS_RDEF, /* XXX TBD */ "PHY test function in progress") }, /* DT PWROMAEBK */ { SST(0x47, 0x7F, SS_RDEF, /* XXX TBD */ "Some commands cleared by iSCSI protocol event") }, /* DTLPWROMAEBKVF */ { SST(0x48, 0x00, SS_RDEF, "Initiator detected error message received") }, /* DTLPWROMAEBKVF */ { SST(0x49, 0x00, SS_RDEF, "Invalid message error") }, /* DTLPWROMAEBKVF */ { SST(0x4A, 0x00, SS_RDEF, "Command phase error") }, /* DTLPWROMAEBKVF */ { SST(0x4B, 0x00, SS_RDEF, "Data phase error") }, /* DT PWROMAEBK */ { SST(0x4B, 0x01, SS_RDEF, /* XXX TBD */ "Invalid target port transfer tag received") }, /* DT PWROMAEBK */ { SST(0x4B, 0x02, SS_RDEF, /* XXX TBD */ "Too much write data") }, /* DT PWROMAEBK */ { SST(0x4B, 0x03, SS_RDEF, /* XXX TBD */ "ACK/NAK timeout") }, /* DT PWROMAEBK */ { SST(0x4B, 0x04, SS_RDEF, /* XXX TBD */ "NAK received") }, /* DT PWROMAEBK */ { SST(0x4B, 0x05, SS_RDEF, /* XXX TBD */ "Data offset error") }, /* DT PWROMAEBK */ { SST(0x4B, 0x06, SS_RDEF, /* XXX TBD */ "Initiator response timeout") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x07, SS_RDEF, /* XXX TBD */ "Connection lost") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x08, SS_RDEF, /* XXX TBD */ "Data-in buffer overflow - data buffer size") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x09, SS_RDEF, /* XXX TBD */ "Data-in buffer overflow - data buffer descriptor area") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0A, SS_RDEF, /* XXX TBD */ "Data-in buffer error") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0B, SS_RDEF, /* XXX TBD */ "Data-out buffer overflow - data buffer size") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0C, SS_RDEF, /* XXX TBD */ "Data-out buffer overflow - data buffer descriptor area") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0D, SS_RDEF, /* XXX TBD */ "Data-out buffer error") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0E, SS_RDEF, /* XXX TBD */ "PCIe fabric error") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x0F, SS_RDEF, /* XXX TBD */ "PCIe completion timeout") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x10, SS_RDEF, /* XXX TBD */ "PCIe completer abort") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x11, SS_RDEF, /* XXX TBD */ "PCIe poisoned TLP received") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x12, SS_RDEF, /* XXX TBD */ "PCIe ECRC check failed") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x13, SS_RDEF, /* XXX TBD */ "PCIe unsupported request") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x14, SS_RDEF, /* XXX TBD */ "PCIe ACS violation") }, /* DT PWROMAEBK F */ { SST(0x4B, 0x15, SS_RDEF, /* XXX TBD */ "PCIe TLP prefix blocket") }, /* DTLPWROMAEBKVF */ { SST(0x4C, 0x00, SS_RDEF, "Logical unit failed self-configuration") }, /* DTLPWROMAEBKVF */ { SST(0x4D, 0x00, SS_RDEF, "Tagged overlapped commands: ASCQ = Queue tag ID") }, /* DTLPWROMAEBKVF */ { SST(0x4D, 0xFF, SS_RDEF | SSQ_RANGE, NULL) }, /* Range 0x00->0xFF */ /* DTLPWROMAEBKVF */ { SST(0x4E, 0x00, SS_RDEF, "Overlapped commands attempted") }, /* T */ { SST(0x50, 0x00, SS_RDEF, "Write append error") }, /* T */ { SST(0x50, 0x01, SS_RDEF, "Write append position error") }, /* T */ { SST(0x50, 0x02, SS_RDEF, "Position error related to timing") }, /* T RO */ { SST(0x51, 0x00, SS_RDEF, "Erase failure") }, /* R */ { SST(0x51, 0x01, SS_RDEF, /* XXX TBD */ "Erase failure - incomplete erase operation detected") }, /* T */ { SST(0x52, 0x00, SS_RDEF, "Cartridge fault") }, /* DTL WROM BK */ { SST(0x53, 0x00, SS_RDEF, "Media load or eject failed") }, /* T */ { SST(0x53, 0x01, SS_RDEF, "Unload tape failure") }, /* DT WROM BK */ { SST(0x53, 0x02, SS_RDEF, "Medium removal prevented") }, /* M */ { SST(0x53, 0x03, SS_RDEF, /* XXX TBD */ "Medium removal prevented by data transfer element") }, /* T */ { SST(0x53, 0x04, SS_RDEF, /* XXX TBD */ "Medium thread or unthread failure") }, /* M */ { SST(0x53, 0x05, SS_RDEF, /* XXX TBD */ "Volume identifier invalid") }, /* T */ { SST(0x53, 0x06, SS_RDEF, /* XXX TBD */ "Volume identifier missing") }, /* M */ { SST(0x53, 0x07, SS_RDEF, /* XXX TBD */ "Duplicate volume identifier") }, /* M */ { SST(0x53, 0x08, SS_RDEF, /* XXX TBD */ "Element status unknown") }, /* M */ { SST(0x53, 0x09, SS_RDEF, /* XXX TBD */ "Data transfer device error - load failed") }, /* M */ { SST(0x53, 0x0A, SS_RDEF, /* XXX TBD */ "Data transfer device error - unload failed") }, /* M */ { SST(0x53, 0x0B, SS_RDEF, /* XXX TBD */ "Data transfer device error - unload missing") }, /* M */ { SST(0x53, 0x0C, SS_RDEF, /* XXX TBD */ "Data transfer device error - eject failed") }, /* M */ { SST(0x53, 0x0D, SS_RDEF, /* XXX TBD */ "Data transfer device error - library communication failed") }, /* P */ { SST(0x54, 0x00, SS_RDEF, "SCSI to host system interface failure") }, /* P */ { SST(0x55, 0x00, SS_RDEF, "System resource failure") }, /* D O BK */ { SST(0x55, 0x01, SS_FATAL | ENOSPC, "System buffer full") }, /* DTLPWROMAE K */ { SST(0x55, 0x02, SS_RDEF, /* XXX TBD */ "Insufficient reservation resources") }, /* DTLPWROMAE K */ { SST(0x55, 0x03, SS_RDEF, /* XXX TBD */ "Insufficient resources") }, /* DTLPWROMAE K */ { SST(0x55, 0x04, SS_RDEF, /* XXX TBD */ "Insufficient registration resources") }, /* DT PWROMAEBK */ { SST(0x55, 0x05, SS_RDEF, /* XXX TBD */ "Insufficient access control resources") }, /* DT WROM B */ { SST(0x55, 0x06, SS_RDEF, /* XXX TBD */ "Auxiliary memory out of space") }, /* F */ { SST(0x55, 0x07, SS_RDEF, /* XXX TBD */ "Quota error") }, /* T */ { SST(0x55, 0x08, SS_RDEF, /* XXX TBD */ "Maximum number of supplemental decryption keys exceeded") }, /* M */ { SST(0x55, 0x09, SS_RDEF, /* XXX TBD */ "Medium auxiliary memory not accessible") }, /* M */ { SST(0x55, 0x0A, SS_RDEF, /* XXX TBD */ "Data currently unavailable") }, /* DTLPWROMAEBKVF */ { SST(0x55, 0x0B, SS_RDEF, /* XXX TBD */ "Insufficient power for operation") }, /* DT P B */ { SST(0x55, 0x0C, SS_RDEF, /* XXX TBD */ "Insufficient resources to create ROD") }, /* DT P B */ { SST(0x55, 0x0D, SS_RDEF, /* XXX TBD */ "Insufficient resources to create ROD token") }, /* D */ { SST(0x55, 0x0E, SS_RDEF, /* XXX TBD */ "Insufficient zone resources") }, /* D */ { SST(0x55, 0x0F, SS_RDEF, /* XXX TBD */ "Insufficient zone resources to complete write") }, /* D */ { SST(0x55, 0x10, SS_RDEF, /* XXX TBD */ "Maximum number of streams open") }, /* R */ { SST(0x57, 0x00, SS_RDEF, "Unable to recover table-of-contents") }, /* O */ { SST(0x58, 0x00, SS_RDEF, "Generation does not exist") }, /* O */ { SST(0x59, 0x00, SS_RDEF, "Updated block read") }, /* DTLPWRO BK */ { SST(0x5A, 0x00, SS_RDEF, "Operator request or state change input") }, /* DT WROM BK */ { SST(0x5A, 0x01, SS_RDEF, "Operator medium removal request") }, /* DT WRO A BK */ { SST(0x5A, 0x02, SS_RDEF, "Operator selected write protect") }, /* DT WRO A BK */ { SST(0x5A, 0x03, SS_RDEF, "Operator selected write permit") }, /* DTLPWROM K */ { SST(0x5B, 0x00, SS_RDEF, "Log exception") }, /* DTLPWROM K */ { SST(0x5B, 0x01, SS_RDEF, "Threshold condition met") }, /* DTLPWROM K */ { SST(0x5B, 0x02, SS_RDEF, "Log counter at maximum") }, /* DTLPWROM K */ { SST(0x5B, 0x03, SS_RDEF, "Log list codes exhausted") }, /* D O */ { SST(0x5C, 0x00, SS_RDEF, "RPL status change") }, /* D O */ { SST(0x5C, 0x01, SS_NOP | SSQ_PRINT_SENSE, "Spindles synchronized") }, /* D O */ { SST(0x5C, 0x02, SS_RDEF, "Spindles not synchronized") }, /* DTLPWROMAEBKVF */ { SST(0x5D, 0x00, SS_RDEF, "Failure prediction threshold exceeded") }, /* R B */ { SST(0x5D, 0x01, SS_RDEF, /* XXX TBD */ "Media failure prediction threshold exceeded") }, /* R */ { SST(0x5D, 0x02, SS_RDEF, /* XXX TBD */ "Logical unit failure prediction threshold exceeded") }, /* R */ { SST(0x5D, 0x03, SS_RDEF, /* XXX TBD */ "Spare area exhaustion prediction threshold exceeded") }, /* D B */ { SST(0x5D, 0x10, SS_RDEF, /* XXX TBD */ "Hardware impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x11, SS_RDEF, /* XXX TBD */ "Hardware impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x12, SS_RDEF, /* XXX TBD */ "Hardware impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x13, SS_RDEF, /* XXX TBD */ "Hardware impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x14, SS_RDEF, /* XXX TBD */ "Hardware impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x15, SS_RDEF, /* XXX TBD */ "Hardware impending failure access times too high") }, /* D B */ { SST(0x5D, 0x16, SS_RDEF, /* XXX TBD */ "Hardware impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x17, SS_RDEF, /* XXX TBD */ "Hardware impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x18, SS_RDEF, /* XXX TBD */ "Hardware impending failure controller detected") }, /* D B */ { SST(0x5D, 0x19, SS_RDEF, /* XXX TBD */ "Hardware impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x1A, SS_RDEF, /* XXX TBD */ "Hardware impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x1B, SS_RDEF, /* XXX TBD */ "Hardware impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x1C, SS_RDEF, /* XXX TBD */ "Hardware impending failure drive calibration retry count") }, /* D B */ { SST(0x5D, 0x20, SS_RDEF, /* XXX TBD */ "Controller impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x21, SS_RDEF, /* XXX TBD */ "Controller impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x22, SS_RDEF, /* XXX TBD */ "Controller impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x23, SS_RDEF, /* XXX TBD */ "Controller impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x24, SS_RDEF, /* XXX TBD */ "Controller impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x25, SS_RDEF, /* XXX TBD */ "Controller impending failure access times too high") }, /* D B */ { SST(0x5D, 0x26, SS_RDEF, /* XXX TBD */ "Controller impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x27, SS_RDEF, /* XXX TBD */ "Controller impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x28, SS_RDEF, /* XXX TBD */ "Controller impending failure controller detected") }, /* D B */ { SST(0x5D, 0x29, SS_RDEF, /* XXX TBD */ "Controller impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x2A, SS_RDEF, /* XXX TBD */ "Controller impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x2B, SS_RDEF, /* XXX TBD */ "Controller impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x2C, SS_RDEF, /* XXX TBD */ "Controller impending failure drive calibration retry count") }, /* D B */ { SST(0x5D, 0x30, SS_RDEF, /* XXX TBD */ "Data channel impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x31, SS_RDEF, /* XXX TBD */ "Data channel impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x32, SS_RDEF, /* XXX TBD */ "Data channel impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x33, SS_RDEF, /* XXX TBD */ "Data channel impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x34, SS_RDEF, /* XXX TBD */ "Data channel impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x35, SS_RDEF, /* XXX TBD */ "Data channel impending failure access times too high") }, /* D B */ { SST(0x5D, 0x36, SS_RDEF, /* XXX TBD */ "Data channel impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x37, SS_RDEF, /* XXX TBD */ "Data channel impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x38, SS_RDEF, /* XXX TBD */ "Data channel impending failure controller detected") }, /* D B */ { SST(0x5D, 0x39, SS_RDEF, /* XXX TBD */ "Data channel impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x3A, SS_RDEF, /* XXX TBD */ "Data channel impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x3B, SS_RDEF, /* XXX TBD */ "Data channel impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x3C, SS_RDEF, /* XXX TBD */ "Data channel impending failure drive calibration retry count") }, /* D B */ { SST(0x5D, 0x40, SS_RDEF, /* XXX TBD */ "Servo impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x41, SS_RDEF, /* XXX TBD */ "Servo impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x42, SS_RDEF, /* XXX TBD */ "Servo impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x43, SS_RDEF, /* XXX TBD */ "Servo impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x44, SS_RDEF, /* XXX TBD */ "Servo impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x45, SS_RDEF, /* XXX TBD */ "Servo impending failure access times too high") }, /* D B */ { SST(0x5D, 0x46, SS_RDEF, /* XXX TBD */ "Servo impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x47, SS_RDEF, /* XXX TBD */ "Servo impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x48, SS_RDEF, /* XXX TBD */ "Servo impending failure controller detected") }, /* D B */ { SST(0x5D, 0x49, SS_RDEF, /* XXX TBD */ "Servo impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x4A, SS_RDEF, /* XXX TBD */ "Servo impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x4B, SS_RDEF, /* XXX TBD */ "Servo impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x4C, SS_RDEF, /* XXX TBD */ "Servo impending failure drive calibration retry count") }, /* D B */ { SST(0x5D, 0x50, SS_RDEF, /* XXX TBD */ "Spindle impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x51, SS_RDEF, /* XXX TBD */ "Spindle impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x52, SS_RDEF, /* XXX TBD */ "Spindle impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x53, SS_RDEF, /* XXX TBD */ "Spindle impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x54, SS_RDEF, /* XXX TBD */ "Spindle impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x55, SS_RDEF, /* XXX TBD */ "Spindle impending failure access times too high") }, /* D B */ { SST(0x5D, 0x56, SS_RDEF, /* XXX TBD */ "Spindle impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x57, SS_RDEF, /* XXX TBD */ "Spindle impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x58, SS_RDEF, /* XXX TBD */ "Spindle impending failure controller detected") }, /* D B */ { SST(0x5D, 0x59, SS_RDEF, /* XXX TBD */ "Spindle impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x5A, SS_RDEF, /* XXX TBD */ "Spindle impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x5B, SS_RDEF, /* XXX TBD */ "Spindle impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x5C, SS_RDEF, /* XXX TBD */ "Spindle impending failure drive calibration retry count") }, /* D B */ { SST(0x5D, 0x60, SS_RDEF, /* XXX TBD */ "Firmware impending failure general hard drive failure") }, /* D B */ { SST(0x5D, 0x61, SS_RDEF, /* XXX TBD */ "Firmware impending failure drive error rate too high") }, /* D B */ { SST(0x5D, 0x62, SS_RDEF, /* XXX TBD */ "Firmware impending failure data error rate too high") }, /* D B */ { SST(0x5D, 0x63, SS_RDEF, /* XXX TBD */ "Firmware impending failure seek error rate too high") }, /* D B */ { SST(0x5D, 0x64, SS_RDEF, /* XXX TBD */ "Firmware impending failure too many block reassigns") }, /* D B */ { SST(0x5D, 0x65, SS_RDEF, /* XXX TBD */ "Firmware impending failure access times too high") }, /* D B */ { SST(0x5D, 0x66, SS_RDEF, /* XXX TBD */ "Firmware impending failure start unit times too high") }, /* D B */ { SST(0x5D, 0x67, SS_RDEF, /* XXX TBD */ "Firmware impending failure channel parametrics") }, /* D B */ { SST(0x5D, 0x68, SS_RDEF, /* XXX TBD */ "Firmware impending failure controller detected") }, /* D B */ { SST(0x5D, 0x69, SS_RDEF, /* XXX TBD */ "Firmware impending failure throughput performance") }, /* D B */ { SST(0x5D, 0x6A, SS_RDEF, /* XXX TBD */ "Firmware impending failure seek time performance") }, /* D B */ { SST(0x5D, 0x6B, SS_RDEF, /* XXX TBD */ "Firmware impending failure spin-up retry count") }, /* D B */ { SST(0x5D, 0x6C, SS_RDEF, /* XXX TBD */ "Firmware impending failure drive calibration retry count") }, /* DTLPWROMAEBKVF */ { SST(0x5D, 0xFF, SS_RDEF, "Failure prediction threshold exceeded (false)") }, /* DTLPWRO A K */ { SST(0x5E, 0x00, SS_RDEF, "Low power condition on") }, /* DTLPWRO A K */ { SST(0x5E, 0x01, SS_RDEF, "Idle condition activated by timer") }, /* DTLPWRO A K */ { SST(0x5E, 0x02, SS_RDEF, "Standby condition activated by timer") }, /* DTLPWRO A K */ { SST(0x5E, 0x03, SS_RDEF, "Idle condition activated by command") }, /* DTLPWRO A K */ { SST(0x5E, 0x04, SS_RDEF, "Standby condition activated by command") }, /* DTLPWRO A K */ { SST(0x5E, 0x05, SS_RDEF, "Idle-B condition activated by timer") }, /* DTLPWRO A K */ { SST(0x5E, 0x06, SS_RDEF, "Idle-B condition activated by command") }, /* DTLPWRO A K */ { SST(0x5E, 0x07, SS_RDEF, "Idle-C condition activated by timer") }, /* DTLPWRO A K */ { SST(0x5E, 0x08, SS_RDEF, "Idle-C condition activated by command") }, /* DTLPWRO A K */ { SST(0x5E, 0x09, SS_RDEF, "Standby-Y condition activated by timer") }, /* DTLPWRO A K */ { SST(0x5E, 0x0A, SS_RDEF, "Standby-Y condition activated by command") }, /* B */ { SST(0x5E, 0x41, SS_RDEF, /* XXX TBD */ "Power state change to active") }, /* B */ { SST(0x5E, 0x42, SS_RDEF, /* XXX TBD */ "Power state change to idle") }, /* B */ { SST(0x5E, 0x43, SS_RDEF, /* XXX TBD */ "Power state change to standby") }, /* B */ { SST(0x5E, 0x45, SS_RDEF, /* XXX TBD */ "Power state change to sleep") }, /* BK */ { SST(0x5E, 0x47, SS_RDEF, /* XXX TBD */ "Power state change to device control") }, /* */ { SST(0x60, 0x00, SS_RDEF, "Lamp failure") }, /* */ { SST(0x61, 0x00, SS_RDEF, "Video acquisition error") }, /* */ { SST(0x61, 0x01, SS_RDEF, "Unable to acquire video") }, /* */ { SST(0x61, 0x02, SS_RDEF, "Out of focus") }, /* */ { SST(0x62, 0x00, SS_RDEF, "Scan head positioning error") }, /* R */ { SST(0x63, 0x00, SS_RDEF, "End of user area encountered on this track") }, /* R */ { SST(0x63, 0x01, SS_FATAL | ENOSPC, "Packet does not fit in available space") }, /* R */ { SST(0x64, 0x00, SS_FATAL | ENXIO, "Illegal mode for this track") }, /* R */ { SST(0x64, 0x01, SS_RDEF, "Invalid packet size") }, /* DTLPWROMAEBKVF */ { SST(0x65, 0x00, SS_RDEF, "Voltage fault") }, /* */ { SST(0x66, 0x00, SS_RDEF, "Automatic document feeder cover up") }, /* */ { SST(0x66, 0x01, SS_RDEF, "Automatic document feeder lift up") }, /* */ { SST(0x66, 0x02, SS_RDEF, "Document jam in automatic document feeder") }, /* */ { SST(0x66, 0x03, SS_RDEF, "Document miss feed automatic in document feeder") }, /* A */ { SST(0x67, 0x00, SS_RDEF, "Configuration failure") }, /* A */ { SST(0x67, 0x01, SS_RDEF, "Configuration of incapable logical units failed") }, /* A */ { SST(0x67, 0x02, SS_RDEF, "Add logical unit failed") }, /* A */ { SST(0x67, 0x03, SS_RDEF, "Modification of logical unit failed") }, /* A */ { SST(0x67, 0x04, SS_RDEF, "Exchange of logical unit failed") }, /* A */ { SST(0x67, 0x05, SS_RDEF, "Remove of logical unit failed") }, /* A */ { SST(0x67, 0x06, SS_RDEF, "Attachment of logical unit failed") }, /* A */ { SST(0x67, 0x07, SS_RDEF, "Creation of logical unit failed") }, /* A */ { SST(0x67, 0x08, SS_RDEF, /* XXX TBD */ "Assign failure occurred") }, /* A */ { SST(0x67, 0x09, SS_RDEF, /* XXX TBD */ "Multiply assigned logical unit") }, /* DTLPWROMAEBKVF */ { SST(0x67, 0x0A, SS_RDEF, /* XXX TBD */ "Set target port groups command failed") }, /* DT B */ { SST(0x67, 0x0B, SS_RDEF, /* XXX TBD */ "ATA device feature not enabled") }, /* A */ { SST(0x68, 0x00, SS_RDEF, "Logical unit not configured") }, /* D */ { SST(0x68, 0x01, SS_RDEF, "Subsidiary logical unit not configured") }, /* A */ { SST(0x69, 0x00, SS_RDEF, "Data loss on logical unit") }, /* A */ { SST(0x69, 0x01, SS_RDEF, "Multiple logical unit failures") }, /* A */ { SST(0x69, 0x02, SS_RDEF, "Parity/data mismatch") }, /* A */ { SST(0x6A, 0x00, SS_RDEF, "Informational, refer to log") }, /* A */ { SST(0x6B, 0x00, SS_RDEF, "State change has occurred") }, /* A */ { SST(0x6B, 0x01, SS_RDEF, "Redundancy level got better") }, /* A */ { SST(0x6B, 0x02, SS_RDEF, "Redundancy level got worse") }, /* A */ { SST(0x6C, 0x00, SS_RDEF, "Rebuild failure occurred") }, /* A */ { SST(0x6D, 0x00, SS_RDEF, "Recalculate failure occurred") }, /* A */ { SST(0x6E, 0x00, SS_RDEF, "Command to logical unit failed") }, /* R */ { SST(0x6F, 0x00, SS_RDEF, /* XXX TBD */ "Copy protection key exchange failure - authentication failure") }, /* R */ { SST(0x6F, 0x01, SS_RDEF, /* XXX TBD */ "Copy protection key exchange failure - key not present") }, /* R */ { SST(0x6F, 0x02, SS_RDEF, /* XXX TBD */ "Copy protection key exchange failure - key not established") }, /* R */ { SST(0x6F, 0x03, SS_RDEF, /* XXX TBD */ "Read of scrambled sector without authentication") }, /* R */ { SST(0x6F, 0x04, SS_RDEF, /* XXX TBD */ "Media region code is mismatched to logical unit region") }, /* R */ { SST(0x6F, 0x05, SS_RDEF, /* XXX TBD */ "Drive region must be permanent/region reset count error") }, /* R */ { SST(0x6F, 0x06, SS_RDEF, /* XXX TBD */ "Insufficient block count for binding NONCE recording") }, /* R */ { SST(0x6F, 0x07, SS_RDEF, /* XXX TBD */ "Conflict in binding NONCE recording") }, /* T */ { SST(0x70, 0x00, SS_RDEF, "Decompression exception short: ASCQ = Algorithm ID") }, /* T */ { SST(0x70, 0xFF, SS_RDEF | SSQ_RANGE, NULL) }, /* Range 0x00 -> 0xFF */ /* T */ { SST(0x71, 0x00, SS_RDEF, "Decompression exception long: ASCQ = Algorithm ID") }, /* T */ { SST(0x71, 0xFF, SS_RDEF | SSQ_RANGE, NULL) }, /* Range 0x00 -> 0xFF */ /* R */ { SST(0x72, 0x00, SS_RDEF, "Session fixation error") }, /* R */ { SST(0x72, 0x01, SS_RDEF, "Session fixation error writing lead-in") }, /* R */ { SST(0x72, 0x02, SS_RDEF, "Session fixation error writing lead-out") }, /* R */ { SST(0x72, 0x03, SS_RDEF, "Session fixation error - incomplete track in session") }, /* R */ { SST(0x72, 0x04, SS_RDEF, "Empty or partially written reserved track") }, /* R */ { SST(0x72, 0x05, SS_RDEF, /* XXX TBD */ "No more track reservations allowed") }, /* R */ { SST(0x72, 0x06, SS_RDEF, /* XXX TBD */ "RMZ extension is not allowed") }, /* R */ { SST(0x72, 0x07, SS_RDEF, /* XXX TBD */ "No more test zone extensions are allowed") }, /* R */ { SST(0x73, 0x00, SS_RDEF, "CD control error") }, /* R */ { SST(0x73, 0x01, SS_RDEF, "Power calibration area almost full") }, /* R */ { SST(0x73, 0x02, SS_FATAL | ENOSPC, "Power calibration area is full") }, /* R */ { SST(0x73, 0x03, SS_RDEF, "Power calibration area error") }, /* R */ { SST(0x73, 0x04, SS_RDEF, "Program memory area update failure") }, /* R */ { SST(0x73, 0x05, SS_RDEF, "Program memory area is full") }, /* R */ { SST(0x73, 0x06, SS_RDEF, /* XXX TBD */ "RMA/PMA is almost full") }, /* R */ { SST(0x73, 0x10, SS_RDEF, /* XXX TBD */ "Current power calibration area almost full") }, /* R */ { SST(0x73, 0x11, SS_RDEF, /* XXX TBD */ "Current power calibration area is full") }, /* R */ { SST(0x73, 0x17, SS_RDEF, /* XXX TBD */ "RDZ is full") }, /* T */ { SST(0x74, 0x00, SS_RDEF, /* XXX TBD */ "Security error") }, /* T */ { SST(0x74, 0x01, SS_RDEF, /* XXX TBD */ "Unable to decrypt data") }, /* T */ { SST(0x74, 0x02, SS_RDEF, /* XXX TBD */ "Unencrypted data encountered while decrypting") }, /* T */ { SST(0x74, 0x03, SS_RDEF, /* XXX TBD */ "Incorrect data encryption key") }, /* T */ { SST(0x74, 0x04, SS_RDEF, /* XXX TBD */ "Cryptographic integrity validation failed") }, /* T */ { SST(0x74, 0x05, SS_RDEF, /* XXX TBD */ "Error decrypting data") }, /* T */ { SST(0x74, 0x06, SS_RDEF, /* XXX TBD */ "Unknown signature verification key") }, /* T */ { SST(0x74, 0x07, SS_RDEF, /* XXX TBD */ "Encryption parameters not useable") }, /* DT R M E VF */ { SST(0x74, 0x08, SS_RDEF, /* XXX TBD */ "Digital signature validation failure") }, /* T */ { SST(0x74, 0x09, SS_RDEF, /* XXX TBD */ "Encryption mode mismatch on read") }, /* T */ { SST(0x74, 0x0A, SS_RDEF, /* XXX TBD */ "Encrypted block not raw read enabled") }, /* T */ { SST(0x74, 0x0B, SS_RDEF, /* XXX TBD */ "Incorrect encryption parameters") }, /* DT R MAEBKV */ { SST(0x74, 0x0C, SS_RDEF, /* XXX TBD */ "Unable to decrypt parameter list") }, /* T */ { SST(0x74, 0x0D, SS_RDEF, /* XXX TBD */ "Encryption algorithm disabled") }, /* DT R MAEBKV */ { SST(0x74, 0x10, SS_RDEF, /* XXX TBD */ "SA creation parameter value invalid") }, /* DT R MAEBKV */ { SST(0x74, 0x11, SS_RDEF, /* XXX TBD */ "SA creation parameter value rejected") }, /* DT R MAEBKV */ { SST(0x74, 0x12, SS_RDEF, /* XXX TBD */ "Invalid SA usage") }, /* T */ { SST(0x74, 0x21, SS_RDEF, /* XXX TBD */ "Data encryption configuration prevented") }, /* DT R MAEBKV */ { SST(0x74, 0x30, SS_RDEF, /* XXX TBD */ "SA creation parameter not supported") }, /* DT R MAEBKV */ { SST(0x74, 0x40, SS_RDEF, /* XXX TBD */ "Authentication failed") }, /* V */ { SST(0x74, 0x61, SS_RDEF, /* XXX TBD */ "External data encryption key manager access error") }, /* V */ { SST(0x74, 0x62, SS_RDEF, /* XXX TBD */ "External data encryption key manager error") }, /* V */ { SST(0x74, 0x63, SS_RDEF, /* XXX TBD */ "External data encryption key not found") }, /* V */ { SST(0x74, 0x64, SS_RDEF, /* XXX TBD */ "External data encryption request not authorized") }, /* T */ { SST(0x74, 0x6E, SS_RDEF, /* XXX TBD */ "External data encryption control timeout") }, /* T */ { SST(0x74, 0x6F, SS_RDEF, /* XXX TBD */ "External data encryption control error") }, /* DT R M E V */ { SST(0x74, 0x71, SS_RDEF, /* XXX TBD */ "Logical unit access not authorized") }, /* D */ { SST(0x74, 0x79, SS_RDEF, /* XXX TBD */ "Security conflict in translated device") } }; const int asc_table_size = sizeof(asc_table)/sizeof(asc_table[0]); struct asc_key { int asc; int ascq; }; static int ascentrycomp(const void *key, const void *member) { int asc; int ascq; const struct asc_table_entry *table_entry; asc = ((const struct asc_key *)key)->asc; ascq = ((const struct asc_key *)key)->ascq; table_entry = (const struct asc_table_entry *)member; if (asc >= table_entry->asc) { if (asc > table_entry->asc) return (1); if (ascq <= table_entry->ascq) { /* Check for ranges */ if (ascq == table_entry->ascq || ((table_entry->action & SSQ_RANGE) != 0 && ascq >= (table_entry - 1)->ascq)) return (0); return (-1); } return (1); } return (-1); } static int senseentrycomp(const void *key, const void *member) { int sense_key; const struct sense_key_table_entry *table_entry; sense_key = *((const int *)key); table_entry = (const struct sense_key_table_entry *)member; if (sense_key >= table_entry->sense_key) { if (sense_key == table_entry->sense_key) return (0); return (1); } return (-1); } static void fetchtableentries(int sense_key, int asc, int ascq, struct scsi_inquiry_data *inq_data, const struct sense_key_table_entry **sense_entry, const struct asc_table_entry **asc_entry) { caddr_t match; const struct asc_table_entry *asc_tables[2]; const struct sense_key_table_entry *sense_tables[2]; struct asc_key asc_ascq; size_t asc_tables_size[2]; size_t sense_tables_size[2]; int num_asc_tables; int num_sense_tables; int i; /* Default to failure */ *sense_entry = NULL; *asc_entry = NULL; match = NULL; if (inq_data != NULL) match = cam_quirkmatch((caddr_t)inq_data, (caddr_t)sense_quirk_table, sense_quirk_table_size, sizeof(*sense_quirk_table), scsi_inquiry_match); if (match != NULL) { struct scsi_sense_quirk_entry *quirk; quirk = (struct scsi_sense_quirk_entry *)match; asc_tables[0] = quirk->asc_info; asc_tables_size[0] = quirk->num_ascs; asc_tables[1] = asc_table; asc_tables_size[1] = asc_table_size; num_asc_tables = 2; sense_tables[0] = quirk->sense_key_info; sense_tables_size[0] = quirk->num_sense_keys; sense_tables[1] = sense_key_table; - sense_tables_size[1] = sense_key_table_size; + sense_tables_size[1] = nitems(sense_key_table); num_sense_tables = 2; } else { asc_tables[0] = asc_table; asc_tables_size[0] = asc_table_size; num_asc_tables = 1; sense_tables[0] = sense_key_table; - sense_tables_size[0] = sense_key_table_size; + sense_tables_size[0] = nitems(sense_key_table); num_sense_tables = 1; } asc_ascq.asc = asc; asc_ascq.ascq = ascq; for (i = 0; i < num_asc_tables; i++) { void *found_entry; found_entry = bsearch(&asc_ascq, asc_tables[i], asc_tables_size[i], sizeof(**asc_tables), ascentrycomp); if (found_entry) { *asc_entry = (struct asc_table_entry *)found_entry; break; } } for (i = 0; i < num_sense_tables; i++) { void *found_entry; found_entry = bsearch(&sense_key, sense_tables[i], sense_tables_size[i], sizeof(**sense_tables), senseentrycomp); if (found_entry) { *sense_entry = (struct sense_key_table_entry *)found_entry; break; } } } void scsi_sense_desc(int sense_key, int asc, int ascq, struct scsi_inquiry_data *inq_data, const char **sense_key_desc, const char **asc_desc) { const struct asc_table_entry *asc_entry; const struct sense_key_table_entry *sense_entry; fetchtableentries(sense_key, asc, ascq, inq_data, &sense_entry, &asc_entry); if (sense_entry != NULL) *sense_key_desc = sense_entry->desc; else *sense_key_desc = "Invalid Sense Key"; if (asc_entry != NULL) *asc_desc = asc_entry->desc; else if (asc >= 0x80 && asc <= 0xff) *asc_desc = "Vendor Specific ASC"; else if (ascq >= 0x80 && ascq <= 0xff) *asc_desc = "Vendor Specific ASCQ"; else *asc_desc = "Reserved ASC/ASCQ pair"; } /* * Given sense and device type information, return the appropriate action. * If we do not understand the specific error as identified by the ASC/ASCQ * pair, fall back on the more generic actions derived from the sense key. */ scsi_sense_action scsi_error_action(struct ccb_scsiio *csio, struct scsi_inquiry_data *inq_data, u_int32_t sense_flags) { const struct asc_table_entry *asc_entry; const struct sense_key_table_entry *sense_entry; int error_code, sense_key, asc, ascq; scsi_sense_action action; if (!scsi_extract_sense_ccb((union ccb *)csio, &error_code, &sense_key, &asc, &ascq)) { action = SS_RETRY | SSQ_DECREMENT_COUNT | SSQ_PRINT_SENSE | EIO; } else if ((error_code == SSD_DEFERRED_ERROR) || (error_code == SSD_DESC_DEFERRED_ERROR)) { /* * XXX dufault@FreeBSD.org * This error doesn't relate to the command associated * with this request sense. A deferred error is an error * for a command that has already returned GOOD status * (see SCSI2 8.2.14.2). * * By my reading of that section, it looks like the current * command has been cancelled, we should now clean things up * (hopefully recovering any lost data) and then retry the * current command. There are two easy choices, both wrong: * * 1. Drop through (like we had been doing), thus treating * this as if the error were for the current command and * return and stop the current command. * * 2. Issue a retry (like I made it do) thus hopefully * recovering the current transfer, and ignoring the * fact that we've dropped a command. * * These should probably be handled in a device specific * sense handler or punted back up to a user mode daemon */ action = SS_RETRY|SSQ_DECREMENT_COUNT|SSQ_PRINT_SENSE; } else { fetchtableentries(sense_key, asc, ascq, inq_data, &sense_entry, &asc_entry); /* * Override the 'No additional Sense' entry (0,0) * with the error action of the sense key. */ if (asc_entry != NULL && (asc != 0 || ascq != 0)) action = asc_entry->action; else if (sense_entry != NULL) action = sense_entry->action; else action = SS_RETRY|SSQ_DECREMENT_COUNT|SSQ_PRINT_SENSE; if (sense_key == SSD_KEY_RECOVERED_ERROR) { /* * The action succeeded but the device wants * the user to know that some recovery action * was required. */ action &= ~(SS_MASK|SSQ_MASK|SS_ERRMASK); action |= SS_NOP|SSQ_PRINT_SENSE; } else if (sense_key == SSD_KEY_ILLEGAL_REQUEST) { if ((sense_flags & SF_QUIET_IR) != 0) action &= ~SSQ_PRINT_SENSE; } else if (sense_key == SSD_KEY_UNIT_ATTENTION) { if ((sense_flags & SF_RETRY_UA) != 0 && (action & SS_MASK) == SS_FAIL) { action &= ~(SS_MASK|SSQ_MASK); action |= SS_RETRY|SSQ_DECREMENT_COUNT| SSQ_PRINT_SENSE; } action |= SSQ_UA; } } if ((action & SS_MASK) >= SS_START && (sense_flags & SF_NO_RECOVERY)) { action &= ~SS_MASK; action |= SS_FAIL; } else if ((action & SS_MASK) == SS_RETRY && (sense_flags & SF_NO_RETRY)) { action &= ~SS_MASK; action |= SS_FAIL; } if ((sense_flags & SF_PRINT_ALWAYS) != 0) action |= SSQ_PRINT_SENSE; else if ((sense_flags & SF_NO_PRINT) != 0) action &= ~SSQ_PRINT_SENSE; return (action); } char * scsi_cdb_string(u_int8_t *cdb_ptr, char *cdb_string, size_t len) { struct sbuf sb; int error; if (len == 0) return (""); sbuf_new(&sb, cdb_string, len, SBUF_FIXEDLEN); scsi_cdb_sbuf(cdb_ptr, &sb); /* ENOMEM just means that the fixed buffer is full, OK to ignore */ error = sbuf_finish(&sb); if (error != 0 && error != ENOMEM) return (""); return(sbuf_data(&sb)); } void scsi_cdb_sbuf(u_int8_t *cdb_ptr, struct sbuf *sb) { u_int8_t cdb_len; int i; if (cdb_ptr == NULL) return; /* * This is taken from the SCSI-3 draft spec. * (T10/1157D revision 0.3) * The top 3 bits of an opcode are the group code. The next 5 bits * are the command code. * Group 0: six byte commands * Group 1: ten byte commands * Group 2: ten byte commands * Group 3: reserved * Group 4: sixteen byte commands * Group 5: twelve byte commands * Group 6: vendor specific * Group 7: vendor specific */ switch((*cdb_ptr >> 5) & 0x7) { case 0: cdb_len = 6; break; case 1: case 2: cdb_len = 10; break; case 3: case 6: case 7: /* in this case, just print out the opcode */ cdb_len = 1; break; case 4: cdb_len = 16; break; case 5: cdb_len = 12; break; } for (i = 0; i < cdb_len; i++) sbuf_printf(sb, "%02hhx ", cdb_ptr[i]); return; } const char * scsi_status_string(struct ccb_scsiio *csio) { switch(csio->scsi_status) { case SCSI_STATUS_OK: return("OK"); case SCSI_STATUS_CHECK_COND: return("Check Condition"); case SCSI_STATUS_BUSY: return("Busy"); case SCSI_STATUS_INTERMED: return("Intermediate"); case SCSI_STATUS_INTERMED_COND_MET: return("Intermediate-Condition Met"); case SCSI_STATUS_RESERV_CONFLICT: return("Reservation Conflict"); case SCSI_STATUS_CMD_TERMINATED: return("Command Terminated"); case SCSI_STATUS_QUEUE_FULL: return("Queue Full"); case SCSI_STATUS_ACA_ACTIVE: return("ACA Active"); case SCSI_STATUS_TASK_ABORTED: return("Task Aborted"); default: { static char unkstr[64]; snprintf(unkstr, sizeof(unkstr), "Unknown %#x", csio->scsi_status); return(unkstr); } } } /* * scsi_command_string() returns 0 for success and -1 for failure. */ #ifdef _KERNEL int scsi_command_string(struct ccb_scsiio *csio, struct sbuf *sb) #else /* !_KERNEL */ int scsi_command_string(struct cam_device *device, struct ccb_scsiio *csio, struct sbuf *sb) #endif /* _KERNEL/!_KERNEL */ { struct scsi_inquiry_data *inq_data; #ifdef _KERNEL struct ccb_getdev *cgd; #endif /* _KERNEL */ #ifdef _KERNEL if ((cgd = (struct ccb_getdev*)xpt_alloc_ccb_nowait()) == NULL) return(-1); /* * Get the device information. */ xpt_setup_ccb(&cgd->ccb_h, csio->ccb_h.path, CAM_PRIORITY_NORMAL); cgd->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)cgd); /* * If the device is unconfigured, just pretend that it is a hard * drive. scsi_op_desc() needs this. */ if (cgd->ccb_h.status == CAM_DEV_NOT_THERE) cgd->inq_data.device = T_DIRECT; inq_data = &cgd->inq_data; #else /* !_KERNEL */ inq_data = &device->inq_data; #endif /* _KERNEL/!_KERNEL */ if ((csio->ccb_h.flags & CAM_CDB_POINTER) != 0) { sbuf_printf(sb, "%s. CDB: ", scsi_op_desc(csio->cdb_io.cdb_ptr[0], inq_data)); scsi_cdb_sbuf(csio->cdb_io.cdb_ptr, sb); } else { sbuf_printf(sb, "%s. CDB: ", scsi_op_desc(csio->cdb_io.cdb_bytes[0], inq_data)); scsi_cdb_sbuf(csio->cdb_io.cdb_bytes, sb); } #ifdef _KERNEL xpt_free_ccb((union ccb *)cgd); #endif return(0); } /* * Iterate over sense descriptors. Each descriptor is passed into iter_func(). * If iter_func() returns 0, list traversal continues. If iter_func() * returns non-zero, list traversal is stopped. */ void scsi_desc_iterate(struct scsi_sense_data_desc *sense, u_int sense_len, int (*iter_func)(struct scsi_sense_data_desc *sense, u_int, struct scsi_sense_desc_header *, void *), void *arg) { int cur_pos; int desc_len; /* * First make sure the extra length field is present. */ if (SSD_DESC_IS_PRESENT(sense, sense_len, extra_len) == 0) return; /* * The length of data actually returned may be different than the * extra_len recorded in the sturcture. */ desc_len = sense_len -offsetof(struct scsi_sense_data_desc, sense_desc); /* * Limit this further by the extra length reported, and the maximum * allowed extra length. */ desc_len = MIN(desc_len, MIN(sense->extra_len, SSD_EXTRA_MAX)); /* * Subtract the size of the header from the descriptor length. * This is to ensure that we have at least the header left, so we * don't have to check that inside the loop. This can wind up * being a negative value. */ desc_len -= sizeof(struct scsi_sense_desc_header); for (cur_pos = 0; cur_pos < desc_len;) { struct scsi_sense_desc_header *header; header = (struct scsi_sense_desc_header *) &sense->sense_desc[cur_pos]; /* * Check to make sure we have the entire descriptor. We * don't call iter_func() unless we do. * * Note that although cur_pos is at the beginning of the * descriptor, desc_len already has the header length * subtracted. So the comparison of the length in the * header (which does not include the header itself) to * desc_len - cur_pos is correct. */ if (header->length > (desc_len - cur_pos)) break; if (iter_func(sense, sense_len, header, arg) != 0) break; cur_pos += sizeof(*header) + header->length; } } struct scsi_find_desc_info { uint8_t desc_type; struct scsi_sense_desc_header *header; }; static int scsi_find_desc_func(struct scsi_sense_data_desc *sense, u_int sense_len, struct scsi_sense_desc_header *header, void *arg) { struct scsi_find_desc_info *desc_info; desc_info = (struct scsi_find_desc_info *)arg; if (header->desc_type == desc_info->desc_type) { desc_info->header = header; /* We found the descriptor, tell the iterator to stop. */ return (1); } else return (0); } /* * Given a descriptor type, return a pointer to it if it is in the sense * data and not truncated. Avoiding truncating sense data will simplify * things significantly for the caller. */ uint8_t * scsi_find_desc(struct scsi_sense_data_desc *sense, u_int sense_len, uint8_t desc_type) { struct scsi_find_desc_info desc_info; desc_info.desc_type = desc_type; desc_info.header = NULL; scsi_desc_iterate(sense, sense_len, scsi_find_desc_func, &desc_info); return ((uint8_t *)desc_info.header); } /* * Fill in SCSI sense data with the specified parameters. This routine can * fill in either fixed or descriptor type sense data. */ void scsi_set_sense_data_va(struct scsi_sense_data *sense_data, scsi_sense_data_type sense_format, int current_error, int sense_key, int asc, int ascq, va_list ap) { int descriptor_sense; scsi_sense_elem_type elem_type; /* * Determine whether to return fixed or descriptor format sense * data. If the user specifies SSD_TYPE_NONE for some reason, * they'll just get fixed sense data. */ if (sense_format == SSD_TYPE_DESC) descriptor_sense = 1; else descriptor_sense = 0; /* * Zero the sense data, so that we don't pass back any garbage data * to the user. */ memset(sense_data, 0, sizeof(*sense_data)); if (descriptor_sense != 0) { struct scsi_sense_data_desc *sense; sense = (struct scsi_sense_data_desc *)sense_data; /* * The descriptor sense format eliminates the use of the * valid bit. */ if (current_error != 0) sense->error_code = SSD_DESC_CURRENT_ERROR; else sense->error_code = SSD_DESC_DEFERRED_ERROR; sense->sense_key = sense_key; sense->add_sense_code = asc; sense->add_sense_code_qual = ascq; /* * Start off with no extra length, since the above data * fits in the standard descriptor sense information. */ sense->extra_len = 0; while ((elem_type = (scsi_sense_elem_type)va_arg(ap, scsi_sense_elem_type)) != SSD_ELEM_NONE) { int sense_len, len_to_copy; uint8_t *data; if (elem_type >= SSD_ELEM_MAX) { printf("%s: invalid sense type %d\n", __func__, elem_type); break; } sense_len = (int)va_arg(ap, int); len_to_copy = MIN(sense_len, SSD_EXTRA_MAX - sense->extra_len); data = (uint8_t *)va_arg(ap, uint8_t *); /* * We've already consumed the arguments for this one. */ if (elem_type == SSD_ELEM_SKIP) continue; switch (elem_type) { case SSD_ELEM_DESC: { /* * This is a straight descriptor. All we * need to do is copy the data in. */ bcopy(data, &sense->sense_desc[ sense->extra_len], len_to_copy); sense->extra_len += len_to_copy; break; } case SSD_ELEM_SKS: { struct scsi_sense_sks sks; bzero(&sks, sizeof(sks)); /* * This is already-formatted sense key * specific data. We just need to fill out * the header and copy everything in. */ bcopy(data, &sks.sense_key_spec, MIN(len_to_copy, sizeof(sks.sense_key_spec))); sks.desc_type = SSD_DESC_SKS; sks.length = sizeof(sks) - offsetof(struct scsi_sense_sks, reserved1); bcopy(&sks,&sense->sense_desc[sense->extra_len], sizeof(sks)); sense->extra_len += sizeof(sks); break; } case SSD_ELEM_INFO: case SSD_ELEM_COMMAND: { struct scsi_sense_command cmd; struct scsi_sense_info info; uint8_t *data_dest; uint8_t *descriptor; int descriptor_size, i, copy_len; bzero(&cmd, sizeof(cmd)); bzero(&info, sizeof(info)); /* * Command or information data. The * operate in pretty much the same way. */ if (elem_type == SSD_ELEM_COMMAND) { len_to_copy = MIN(len_to_copy, sizeof(cmd.command_info)); descriptor = (uint8_t *)&cmd; descriptor_size = sizeof(cmd); data_dest =(uint8_t *)&cmd.command_info; cmd.desc_type = SSD_DESC_COMMAND; cmd.length = sizeof(cmd) - offsetof(struct scsi_sense_command, reserved); } else { len_to_copy = MIN(len_to_copy, sizeof(info.info)); descriptor = (uint8_t *)&info; descriptor_size = sizeof(cmd); data_dest = (uint8_t *)&info.info; info.desc_type = SSD_DESC_INFO; info.byte2 = SSD_INFO_VALID; info.length = sizeof(info) - offsetof(struct scsi_sense_info, byte2); } /* * Copy this in reverse because the spec * (SPC-4) says that when 4 byte quantities * are stored in this 8 byte field, the * first four bytes shall be 0. * * So we fill the bytes in from the end, and * if we have less than 8 bytes to copy, * the initial, most significant bytes will * be 0. */ for (i = sense_len - 1; i >= 0 && len_to_copy > 0; i--, len_to_copy--) data_dest[len_to_copy - 1] = data[i]; /* * This calculation looks much like the * initial len_to_copy calculation, but * we have to do it again here, because * we're looking at a larger amount that * may or may not fit. It's not only the * data the user passed in, but also the * rest of the descriptor. */ copy_len = MIN(descriptor_size, SSD_EXTRA_MAX - sense->extra_len); bcopy(descriptor, &sense->sense_desc[ sense->extra_len], copy_len); sense->extra_len += copy_len; break; } case SSD_ELEM_FRU: { struct scsi_sense_fru fru; int copy_len; bzero(&fru, sizeof(fru)); fru.desc_type = SSD_DESC_FRU; fru.length = sizeof(fru) - offsetof(struct scsi_sense_fru, reserved); fru.fru = *data; copy_len = MIN(sizeof(fru), SSD_EXTRA_MAX - sense->extra_len); bcopy(&fru, &sense->sense_desc[ sense->extra_len], copy_len); sense->extra_len += copy_len; break; } case SSD_ELEM_STREAM: { struct scsi_sense_stream stream_sense; int copy_len; bzero(&stream_sense, sizeof(stream_sense)); stream_sense.desc_type = SSD_DESC_STREAM; stream_sense.length = sizeof(stream_sense) - offsetof(struct scsi_sense_stream, reserved); stream_sense.byte3 = *data; copy_len = MIN(sizeof(stream_sense), SSD_EXTRA_MAX - sense->extra_len); bcopy(&stream_sense, &sense->sense_desc[ sense->extra_len], copy_len); sense->extra_len += copy_len; break; } default: /* * We shouldn't get here, but if we do, do * nothing. We've already consumed the * arguments above. */ break; } } } else { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; if (current_error != 0) sense->error_code = SSD_CURRENT_ERROR; else sense->error_code = SSD_DEFERRED_ERROR; sense->flags = sense_key; sense->add_sense_code = asc; sense->add_sense_code_qual = ascq; /* * We've set the ASC and ASCQ, so we have 6 more bytes of * valid data. If we wind up setting any of the other * fields, we'll bump this to 10 extra bytes. */ sense->extra_len = 6; while ((elem_type = (scsi_sense_elem_type)va_arg(ap, scsi_sense_elem_type)) != SSD_ELEM_NONE) { int sense_len, len_to_copy; uint8_t *data; if (elem_type >= SSD_ELEM_MAX) { printf("%s: invalid sense type %d\n", __func__, elem_type); break; } /* * If we get in here, just bump the extra length to * 10 bytes. That will encompass anything we're * going to set here. */ sense->extra_len = 10; sense_len = (int)va_arg(ap, int); data = (uint8_t *)va_arg(ap, uint8_t *); switch (elem_type) { case SSD_ELEM_SKS: /* * The user passed in pre-formatted sense * key specific data. */ bcopy(data, &sense->sense_key_spec[0], MIN(sizeof(sense->sense_key_spec), sense_len)); break; case SSD_ELEM_INFO: case SSD_ELEM_COMMAND: { uint8_t *data_dest; int i; if (elem_type == SSD_ELEM_COMMAND) { data_dest = &sense->cmd_spec_info[0]; len_to_copy = MIN(sense_len, sizeof(sense->cmd_spec_info)); } else { data_dest = &sense->info[0]; len_to_copy = MIN(sense_len, sizeof(sense->info)); /* * We're setting the info field, so * set the valid bit. */ sense->error_code |= SSD_ERRCODE_VALID; } /* * Copy this in reverse so that if we have * less than 4 bytes to fill, the least * significant bytes will be at the end. * If we have more than 4 bytes, only the * least significant bytes will be included. */ for (i = sense_len - 1; i >= 0 && len_to_copy > 0; i--, len_to_copy--) data_dest[len_to_copy - 1] = data[i]; break; } case SSD_ELEM_FRU: sense->fru = *data; break; case SSD_ELEM_STREAM: sense->flags |= *data; break; case SSD_ELEM_DESC: default: /* * If the user passes in descriptor sense, * we can't handle that in fixed format. * So just skip it, and any unknown argument * types. */ break; } } } } void scsi_set_sense_data(struct scsi_sense_data *sense_data, scsi_sense_data_type sense_format, int current_error, int sense_key, int asc, int ascq, ...) { va_list ap; va_start(ap, ascq); scsi_set_sense_data_va(sense_data, sense_format, current_error, sense_key, asc, ascq, ap); va_end(ap); } /* * Get sense information for three similar sense data types. */ int scsi_get_sense_info(struct scsi_sense_data *sense_data, u_int sense_len, uint8_t info_type, uint64_t *info, int64_t *signed_info) { scsi_sense_data_type sense_type; if (sense_len == 0) goto bailout; sense_type = scsi_sense_type(sense_data); switch (sense_type) { case SSD_TYPE_DESC: { struct scsi_sense_data_desc *sense; uint8_t *desc; sense = (struct scsi_sense_data_desc *)sense_data; desc = scsi_find_desc(sense, sense_len, info_type); if (desc == NULL) goto bailout; switch (info_type) { case SSD_DESC_INFO: { struct scsi_sense_info *info_desc; info_desc = (struct scsi_sense_info *)desc; *info = scsi_8btou64(info_desc->info); if (signed_info != NULL) *signed_info = *info; break; } case SSD_DESC_COMMAND: { struct scsi_sense_command *cmd_desc; cmd_desc = (struct scsi_sense_command *)desc; *info = scsi_8btou64(cmd_desc->command_info); if (signed_info != NULL) *signed_info = *info; break; } case SSD_DESC_FRU: { struct scsi_sense_fru *fru_desc; fru_desc = (struct scsi_sense_fru *)desc; *info = fru_desc->fru; if (signed_info != NULL) *signed_info = (int8_t)fru_desc->fru; break; } default: goto bailout; break; } break; } case SSD_TYPE_FIXED: { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; switch (info_type) { case SSD_DESC_INFO: { uint32_t info_val; if ((sense->error_code & SSD_ERRCODE_VALID) == 0) goto bailout; if (SSD_FIXED_IS_PRESENT(sense, sense_len, info) == 0) goto bailout; info_val = scsi_4btoul(sense->info); *info = info_val; if (signed_info != NULL) *signed_info = (int32_t)info_val; break; } case SSD_DESC_COMMAND: { uint32_t cmd_val; if ((SSD_FIXED_IS_PRESENT(sense, sense_len, cmd_spec_info) == 0) || (SSD_FIXED_IS_FILLED(sense, cmd_spec_info) == 0)) goto bailout; cmd_val = scsi_4btoul(sense->cmd_spec_info); if (cmd_val == 0) goto bailout; *info = cmd_val; if (signed_info != NULL) *signed_info = (int32_t)cmd_val; break; } case SSD_DESC_FRU: if ((SSD_FIXED_IS_PRESENT(sense, sense_len, fru) == 0) || (SSD_FIXED_IS_FILLED(sense, fru) == 0)) goto bailout; if (sense->fru == 0) goto bailout; *info = sense->fru; if (signed_info != NULL) *signed_info = (int8_t)sense->fru; break; default: goto bailout; break; } break; } default: goto bailout; break; } return (0); bailout: return (1); } int scsi_get_sks(struct scsi_sense_data *sense_data, u_int sense_len, uint8_t *sks) { scsi_sense_data_type sense_type; if (sense_len == 0) goto bailout; sense_type = scsi_sense_type(sense_data); switch (sense_type) { case SSD_TYPE_DESC: { struct scsi_sense_data_desc *sense; struct scsi_sense_sks *desc; sense = (struct scsi_sense_data_desc *)sense_data; desc = (struct scsi_sense_sks *)scsi_find_desc(sense, sense_len, SSD_DESC_SKS); if (desc == NULL) goto bailout; /* * No need to check the SKS valid bit for descriptor sense. * If the descriptor is present, it is valid. */ bcopy(desc->sense_key_spec, sks, sizeof(desc->sense_key_spec)); break; } case SSD_TYPE_FIXED: { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; if ((SSD_FIXED_IS_PRESENT(sense, sense_len, sense_key_spec)== 0) || (SSD_FIXED_IS_FILLED(sense, sense_key_spec) == 0)) goto bailout; if ((sense->sense_key_spec[0] & SSD_SCS_VALID) == 0) goto bailout; bcopy(sense->sense_key_spec, sks,sizeof(sense->sense_key_spec)); break; } default: goto bailout; break; } return (0); bailout: return (1); } /* * Provide a common interface for fixed and descriptor sense to detect * whether we have block-specific sense information. It is clear by the * presence of the block descriptor in descriptor mode, but we have to * infer from the inquiry data and ILI bit in fixed mode. */ int scsi_get_block_info(struct scsi_sense_data *sense_data, u_int sense_len, struct scsi_inquiry_data *inq_data, uint8_t *block_bits) { scsi_sense_data_type sense_type; if (inq_data != NULL) { switch (SID_TYPE(inq_data)) { case T_DIRECT: case T_RBC: break; default: goto bailout; break; } } sense_type = scsi_sense_type(sense_data); switch (sense_type) { case SSD_TYPE_DESC: { struct scsi_sense_data_desc *sense; struct scsi_sense_block *block; sense = (struct scsi_sense_data_desc *)sense_data; block = (struct scsi_sense_block *)scsi_find_desc(sense, sense_len, SSD_DESC_BLOCK); if (block == NULL) goto bailout; *block_bits = block->byte3; break; } case SSD_TYPE_FIXED: { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags) == 0) goto bailout; if ((sense->flags & SSD_ILI) == 0) goto bailout; *block_bits = sense->flags & SSD_ILI; break; } default: goto bailout; break; } return (0); bailout: return (1); } int scsi_get_stream_info(struct scsi_sense_data *sense_data, u_int sense_len, struct scsi_inquiry_data *inq_data, uint8_t *stream_bits) { scsi_sense_data_type sense_type; if (inq_data != NULL) { switch (SID_TYPE(inq_data)) { case T_SEQUENTIAL: break; default: goto bailout; break; } } sense_type = scsi_sense_type(sense_data); switch (sense_type) { case SSD_TYPE_DESC: { struct scsi_sense_data_desc *sense; struct scsi_sense_stream *stream; sense = (struct scsi_sense_data_desc *)sense_data; stream = (struct scsi_sense_stream *)scsi_find_desc(sense, sense_len, SSD_DESC_STREAM); if (stream == NULL) goto bailout; *stream_bits = stream->byte3; break; } case SSD_TYPE_FIXED: { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags) == 0) goto bailout; if ((sense->flags & (SSD_ILI|SSD_EOM|SSD_FILEMARK)) == 0) goto bailout; *stream_bits = sense->flags & (SSD_ILI|SSD_EOM|SSD_FILEMARK); break; } default: goto bailout; break; } return (0); bailout: return (1); } void scsi_info_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, uint64_t info) { sbuf_printf(sb, "Info: %#jx", info); } void scsi_command_sbuf(struct sbuf *sb, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, uint64_t csi) { sbuf_printf(sb, "Command Specific Info: %#jx", csi); } void scsi_progress_sbuf(struct sbuf *sb, uint16_t progress) { sbuf_printf(sb, "Progress: %d%% (%d/%d) complete", (progress * 100) / SSD_SKS_PROGRESS_DENOM, progress, SSD_SKS_PROGRESS_DENOM); } /* * Returns 1 for failure (i.e. SKS isn't valid) and 0 for success. */ int scsi_sks_sbuf(struct sbuf *sb, int sense_key, uint8_t *sks) { if ((sks[0] & SSD_SKS_VALID) == 0) return (1); switch (sense_key) { case SSD_KEY_ILLEGAL_REQUEST: { struct scsi_sense_sks_field *field; int bad_command; char tmpstr[40]; /*Field Pointer*/ field = (struct scsi_sense_sks_field *)sks; if (field->byte0 & SSD_SKS_FIELD_CMD) bad_command = 1; else bad_command = 0; tmpstr[0] = '\0'; /* Bit pointer is valid */ if (field->byte0 & SSD_SKS_BPV) snprintf(tmpstr, sizeof(tmpstr), "bit %d ", field->byte0 & SSD_SKS_BIT_VALUE); sbuf_printf(sb, "%s byte %d %sis invalid", bad_command ? "Command" : "Data", scsi_2btoul(field->field), tmpstr); break; } case SSD_KEY_UNIT_ATTENTION: { struct scsi_sense_sks_overflow *overflow; overflow = (struct scsi_sense_sks_overflow *)sks; /*UA Condition Queue Overflow*/ sbuf_printf(sb, "Unit Attention Condition Queue %s", (overflow->byte0 & SSD_SKS_OVERFLOW_SET) ? "Overflowed" : "Did Not Overflow??"); break; } case SSD_KEY_RECOVERED_ERROR: case SSD_KEY_HARDWARE_ERROR: case SSD_KEY_MEDIUM_ERROR: { struct scsi_sense_sks_retry *retry; /*Actual Retry Count*/ retry = (struct scsi_sense_sks_retry *)sks; sbuf_printf(sb, "Actual Retry Count: %d", scsi_2btoul(retry->actual_retry_count)); break; } case SSD_KEY_NO_SENSE: case SSD_KEY_NOT_READY: { struct scsi_sense_sks_progress *progress; int progress_val; /*Progress Indication*/ progress = (struct scsi_sense_sks_progress *)sks; progress_val = scsi_2btoul(progress->progress); scsi_progress_sbuf(sb, progress_val); break; } case SSD_KEY_COPY_ABORTED: { struct scsi_sense_sks_segment *segment; char tmpstr[40]; /*Segment Pointer*/ segment = (struct scsi_sense_sks_segment *)sks; tmpstr[0] = '\0'; if (segment->byte0 & SSD_SKS_SEGMENT_BPV) snprintf(tmpstr, sizeof(tmpstr), "bit %d ", segment->byte0 & SSD_SKS_SEGMENT_BITPTR); sbuf_printf(sb, "%s byte %d %sis invalid", (segment->byte0 & SSD_SKS_SEGMENT_SD) ? "Segment" : "Data", scsi_2btoul(segment->field), tmpstr); break; } default: sbuf_printf(sb, "Sense Key Specific: %#x,%#x", sks[0], scsi_2btoul(&sks[1])); break; } return (0); } void scsi_fru_sbuf(struct sbuf *sb, uint64_t fru) { sbuf_printf(sb, "Field Replaceable Unit: %d", (int)fru); } void scsi_stream_sbuf(struct sbuf *sb, uint8_t stream_bits, uint64_t info) { int need_comma; need_comma = 0; /* * XXX KDM this needs more descriptive decoding. */ if (stream_bits & SSD_DESC_STREAM_FM) { sbuf_printf(sb, "Filemark"); need_comma = 1; } if (stream_bits & SSD_DESC_STREAM_EOM) { sbuf_printf(sb, "%sEOM", (need_comma) ? "," : ""); need_comma = 1; } if (stream_bits & SSD_DESC_STREAM_ILI) sbuf_printf(sb, "%sILI", (need_comma) ? "," : ""); sbuf_printf(sb, ": Info: %#jx", (uintmax_t) info); } void scsi_block_sbuf(struct sbuf *sb, uint8_t block_bits, uint64_t info) { if (block_bits & SSD_DESC_BLOCK_ILI) sbuf_printf(sb, "ILI: residue %#jx", (uintmax_t) info); } void scsi_sense_info_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_info *info; info = (struct scsi_sense_info *)header; scsi_info_sbuf(sb, cdb, cdb_len, inq_data, scsi_8btou64(info->info)); } void scsi_sense_command_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_command *command; command = (struct scsi_sense_command *)header; scsi_command_sbuf(sb, cdb, cdb_len, inq_data, scsi_8btou64(command->command_info)); } void scsi_sense_sks_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_sks *sks; int error_code, sense_key, asc, ascq; sks = (struct scsi_sense_sks *)header; scsi_extract_sense_len(sense, sense_len, &error_code, &sense_key, &asc, &ascq, /*show_errors*/ 1); scsi_sks_sbuf(sb, sense_key, sks->sense_key_spec); } void scsi_sense_fru_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_fru *fru; fru = (struct scsi_sense_fru *)header; scsi_fru_sbuf(sb, (uint64_t)fru->fru); } void scsi_sense_stream_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_stream *stream; uint64_t info; stream = (struct scsi_sense_stream *)header; info = 0; scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &info, NULL); scsi_stream_sbuf(sb, stream->byte3, info); } void scsi_sense_block_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_block *block; uint64_t info; block = (struct scsi_sense_block *)header; info = 0; scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &info, NULL); scsi_block_sbuf(sb, block->byte3, info); } void scsi_sense_progress_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { struct scsi_sense_progress *progress; const char *sense_key_desc; const char *asc_desc; int progress_val; progress = (struct scsi_sense_progress *)header; /* * Get descriptions for the sense key, ASC, and ASCQ in the * progress descriptor. These could be different than the values * in the overall sense data. */ scsi_sense_desc(progress->sense_key, progress->add_sense_code, progress->add_sense_code_qual, inq_data, &sense_key_desc, &asc_desc); progress_val = scsi_2btoul(progress->progress); /* * The progress indicator is for the operation described by the * sense key, ASC, and ASCQ in the descriptor. */ sbuf_cat(sb, sense_key_desc); sbuf_printf(sb, " asc:%x,%x (%s): ", progress->add_sense_code, progress->add_sense_code_qual, asc_desc); scsi_progress_sbuf(sb, progress_val); } /* * Generic sense descriptor printing routine. This is used when we have * not yet implemented a specific printing routine for this descriptor. */ void scsi_sense_generic_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { int i; uint8_t *buf_ptr; sbuf_printf(sb, "Descriptor %#x:", header->desc_type); buf_ptr = (uint8_t *)&header[1]; for (i = 0; i < header->length; i++, buf_ptr++) sbuf_printf(sb, " %02x", *buf_ptr); } /* * Keep this list in numeric order. This speeds the array traversal. */ struct scsi_sense_desc_printer { uint8_t desc_type; /* * The function arguments here are the superset of what is needed * to print out various different descriptors. Command and * information descriptors need inquiry data and command type. * Sense key specific descriptors need the sense key. * * The sense, cdb, and inquiry data arguments may be NULL, but the * information printed may not be fully decoded as a result. */ void (*print_func)(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header); } scsi_sense_printers[] = { {SSD_DESC_INFO, scsi_sense_info_sbuf}, {SSD_DESC_COMMAND, scsi_sense_command_sbuf}, {SSD_DESC_SKS, scsi_sense_sks_sbuf}, {SSD_DESC_FRU, scsi_sense_fru_sbuf}, {SSD_DESC_STREAM, scsi_sense_stream_sbuf}, {SSD_DESC_BLOCK, scsi_sense_block_sbuf}, {SSD_DESC_PROGRESS, scsi_sense_progress_sbuf} }; void scsi_sense_desc_sbuf(struct sbuf *sb, struct scsi_sense_data *sense, u_int sense_len, uint8_t *cdb, int cdb_len, struct scsi_inquiry_data *inq_data, struct scsi_sense_desc_header *header) { int i; for (i = 0; i < (sizeof(scsi_sense_printers) / sizeof(scsi_sense_printers[0])); i++) { struct scsi_sense_desc_printer *printer; printer = &scsi_sense_printers[i]; /* * The list is sorted, so quit if we've passed our * descriptor number. */ if (printer->desc_type > header->desc_type) break; if (printer->desc_type != header->desc_type) continue; printer->print_func(sb, sense, sense_len, cdb, cdb_len, inq_data, header); return; } /* * No specific printing routine, so use the generic routine. */ scsi_sense_generic_sbuf(sb, sense, sense_len, cdb, cdb_len, inq_data, header); } scsi_sense_data_type scsi_sense_type(struct scsi_sense_data *sense_data) { switch (sense_data->error_code & SSD_ERRCODE) { case SSD_DESC_CURRENT_ERROR: case SSD_DESC_DEFERRED_ERROR: return (SSD_TYPE_DESC); break; case SSD_CURRENT_ERROR: case SSD_DEFERRED_ERROR: return (SSD_TYPE_FIXED); break; default: break; } return (SSD_TYPE_NONE); } struct scsi_print_sense_info { struct sbuf *sb; char *path_str; uint8_t *cdb; int cdb_len; struct scsi_inquiry_data *inq_data; }; static int scsi_print_desc_func(struct scsi_sense_data_desc *sense, u_int sense_len, struct scsi_sense_desc_header *header, void *arg) { struct scsi_print_sense_info *print_info; print_info = (struct scsi_print_sense_info *)arg; switch (header->desc_type) { case SSD_DESC_INFO: case SSD_DESC_FRU: case SSD_DESC_COMMAND: case SSD_DESC_SKS: case SSD_DESC_BLOCK: case SSD_DESC_STREAM: /* * We have already printed these descriptors, if they are * present. */ break; default: { sbuf_printf(print_info->sb, "%s", print_info->path_str); scsi_sense_desc_sbuf(print_info->sb, (struct scsi_sense_data *)sense, sense_len, print_info->cdb, print_info->cdb_len, print_info->inq_data, header); sbuf_printf(print_info->sb, "\n"); break; } } /* * Tell the iterator that we want to see more descriptors if they * are present. */ return (0); } void scsi_sense_only_sbuf(struct scsi_sense_data *sense, u_int sense_len, struct sbuf *sb, char *path_str, struct scsi_inquiry_data *inq_data, uint8_t *cdb, int cdb_len) { int error_code, sense_key, asc, ascq; sbuf_cat(sb, path_str); scsi_extract_sense_len(sense, sense_len, &error_code, &sense_key, &asc, &ascq, /*show_errors*/ 1); sbuf_printf(sb, "SCSI sense: "); switch (error_code) { case SSD_DEFERRED_ERROR: case SSD_DESC_DEFERRED_ERROR: sbuf_printf(sb, "Deferred error: "); /* FALLTHROUGH */ case SSD_CURRENT_ERROR: case SSD_DESC_CURRENT_ERROR: { struct scsi_sense_data_desc *desc_sense; struct scsi_print_sense_info print_info; const char *sense_key_desc; const char *asc_desc; uint8_t sks[3]; uint64_t val; int info_valid; /* * Get descriptions for the sense key, ASC, and ASCQ. If * these aren't present in the sense data (i.e. the sense * data isn't long enough), the -1 values that * scsi_extract_sense_len() returns will yield default * or error descriptions. */ scsi_sense_desc(sense_key, asc, ascq, inq_data, &sense_key_desc, &asc_desc); /* * We first print the sense key and ASC/ASCQ. */ sbuf_cat(sb, sense_key_desc); sbuf_printf(sb, " asc:%x,%x (%s)\n", asc, ascq, asc_desc); /* * Get the info field if it is valid. */ if (scsi_get_sense_info(sense, sense_len, SSD_DESC_INFO, &val, NULL) == 0) info_valid = 1; else info_valid = 0; if (info_valid != 0) { uint8_t bits; /* * Determine whether we have any block or stream * device-specific information. */ if (scsi_get_block_info(sense, sense_len, inq_data, &bits) == 0) { sbuf_cat(sb, path_str); scsi_block_sbuf(sb, bits, val); sbuf_printf(sb, "\n"); } else if (scsi_get_stream_info(sense, sense_len, inq_data, &bits) == 0) { sbuf_cat(sb, path_str); scsi_stream_sbuf(sb, bits, val); sbuf_printf(sb, "\n"); } else if (val != 0) { /* * The information field can be valid but 0. * If the block or stream bits aren't set, * and this is 0, it isn't terribly useful * to print it out. */ sbuf_cat(sb, path_str); scsi_info_sbuf(sb, cdb, cdb_len, inq_data, val); sbuf_printf(sb, "\n"); } } /* * Print the FRU. */ if (scsi_get_sense_info(sense, sense_len, SSD_DESC_FRU, &val, NULL) == 0) { sbuf_cat(sb, path_str); scsi_fru_sbuf(sb, val); sbuf_printf(sb, "\n"); } /* * Print any command-specific information. */ if (scsi_get_sense_info(sense, sense_len, SSD_DESC_COMMAND, &val, NULL) == 0) { sbuf_cat(sb, path_str); scsi_command_sbuf(sb, cdb, cdb_len, inq_data, val); sbuf_printf(sb, "\n"); } /* * Print out any sense-key-specific information. */ if (scsi_get_sks(sense, sense_len, sks) == 0) { sbuf_cat(sb, path_str); scsi_sks_sbuf(sb, sense_key, sks); sbuf_printf(sb, "\n"); } /* * If this is fixed sense, we're done. If we have * descriptor sense, we might have more information * available. */ if (scsi_sense_type(sense) != SSD_TYPE_DESC) break; desc_sense = (struct scsi_sense_data_desc *)sense; print_info.sb = sb; print_info.path_str = path_str; print_info.cdb = cdb; print_info.cdb_len = cdb_len; print_info.inq_data = inq_data; /* * Print any sense descriptors that we have not already printed. */ scsi_desc_iterate(desc_sense, sense_len, scsi_print_desc_func, &print_info); break; } case -1: /* * scsi_extract_sense_len() sets values to -1 if the * show_errors flag is set and they aren't present in the * sense data. This means that sense_len is 0. */ sbuf_printf(sb, "No sense data present\n"); break; default: { sbuf_printf(sb, "Error code 0x%x", error_code); if (sense->error_code & SSD_ERRCODE_VALID) { struct scsi_sense_data_fixed *fixed_sense; fixed_sense = (struct scsi_sense_data_fixed *)sense; if (SSD_FIXED_IS_PRESENT(fixed_sense, sense_len, info)){ uint32_t info; info = scsi_4btoul(fixed_sense->info); sbuf_printf(sb, " at block no. %d (decimal)", info); } } sbuf_printf(sb, "\n"); break; } } } /* * scsi_sense_sbuf() returns 0 for success and -1 for failure. */ #ifdef _KERNEL int scsi_sense_sbuf(struct ccb_scsiio *csio, struct sbuf *sb, scsi_sense_string_flags flags) #else /* !_KERNEL */ int scsi_sense_sbuf(struct cam_device *device, struct ccb_scsiio *csio, struct sbuf *sb, scsi_sense_string_flags flags) #endif /* _KERNEL/!_KERNEL */ { struct scsi_sense_data *sense; struct scsi_inquiry_data *inq_data; #ifdef _KERNEL struct ccb_getdev *cgd; #endif /* _KERNEL */ char path_str[64]; uint8_t *cdb; #ifndef _KERNEL if (device == NULL) return(-1); #endif /* !_KERNEL */ if ((csio == NULL) || (sb == NULL)) return(-1); /* * If the CDB is a physical address, we can't deal with it.. */ if ((csio->ccb_h.flags & CAM_CDB_PHYS) != 0) flags &= ~SSS_FLAG_PRINT_COMMAND; #ifdef _KERNEL xpt_path_string(csio->ccb_h.path, path_str, sizeof(path_str)); #else /* !_KERNEL */ cam_path_string(device, path_str, sizeof(path_str)); #endif /* _KERNEL/!_KERNEL */ #ifdef _KERNEL if ((cgd = (struct ccb_getdev*)xpt_alloc_ccb_nowait()) == NULL) return(-1); /* * Get the device information. */ xpt_setup_ccb(&cgd->ccb_h, csio->ccb_h.path, CAM_PRIORITY_NORMAL); cgd->ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)cgd); /* * If the device is unconfigured, just pretend that it is a hard * drive. scsi_op_desc() needs this. */ if (cgd->ccb_h.status == CAM_DEV_NOT_THERE) cgd->inq_data.device = T_DIRECT; inq_data = &cgd->inq_data; #else /* !_KERNEL */ inq_data = &device->inq_data; #endif /* _KERNEL/!_KERNEL */ sense = NULL; if (flags & SSS_FLAG_PRINT_COMMAND) { sbuf_cat(sb, path_str); #ifdef _KERNEL scsi_command_string(csio, sb); #else /* !_KERNEL */ scsi_command_string(device, csio, sb); #endif /* _KERNEL/!_KERNEL */ sbuf_printf(sb, "\n"); } /* * If the sense data is a physical pointer, forget it. */ if (csio->ccb_h.flags & CAM_SENSE_PTR) { if (csio->ccb_h.flags & CAM_SENSE_PHYS) { #ifdef _KERNEL xpt_free_ccb((union ccb*)cgd); #endif /* _KERNEL/!_KERNEL */ return(-1); } else { /* * bcopy the pointer to avoid unaligned access * errors on finicky architectures. We don't * ensure that the sense data is pointer aligned. */ bcopy(&csio->sense_data, &sense, sizeof(struct scsi_sense_data *)); } } else { /* * If the physical sense flag is set, but the sense pointer * is not also set, we assume that the user is an idiot and * return. (Well, okay, it could be that somehow, the * entire csio is physical, but we would have probably core * dumped on one of the bogus pointer deferences above * already.) */ if (csio->ccb_h.flags & CAM_SENSE_PHYS) { #ifdef _KERNEL xpt_free_ccb((union ccb*)cgd); #endif /* _KERNEL/!_KERNEL */ return(-1); } else sense = &csio->sense_data; } if (csio->ccb_h.flags & CAM_CDB_POINTER) cdb = csio->cdb_io.cdb_ptr; else cdb = csio->cdb_io.cdb_bytes; scsi_sense_only_sbuf(sense, csio->sense_len - csio->sense_resid, sb, path_str, inq_data, cdb, csio->cdb_len); #ifdef _KERNEL xpt_free_ccb((union ccb*)cgd); #endif /* _KERNEL/!_KERNEL */ return(0); } #ifdef _KERNEL char * scsi_sense_string(struct ccb_scsiio *csio, char *str, int str_len) #else /* !_KERNEL */ char * scsi_sense_string(struct cam_device *device, struct ccb_scsiio *csio, char *str, int str_len) #endif /* _KERNEL/!_KERNEL */ { struct sbuf sb; sbuf_new(&sb, str, str_len, 0); #ifdef _KERNEL scsi_sense_sbuf(csio, &sb, SSS_FLAG_PRINT_COMMAND); #else /* !_KERNEL */ scsi_sense_sbuf(device, csio, &sb, SSS_FLAG_PRINT_COMMAND); #endif /* _KERNEL/!_KERNEL */ sbuf_finish(&sb); return(sbuf_data(&sb)); } #ifdef _KERNEL void scsi_sense_print(struct ccb_scsiio *csio) { struct sbuf sb; char str[512]; sbuf_new(&sb, str, sizeof(str), 0); scsi_sense_sbuf(csio, &sb, SSS_FLAG_PRINT_COMMAND); sbuf_finish(&sb); printf("%s", sbuf_data(&sb)); } #else /* !_KERNEL */ void scsi_sense_print(struct cam_device *device, struct ccb_scsiio *csio, FILE *ofile) { struct sbuf sb; char str[512]; if ((device == NULL) || (csio == NULL) || (ofile == NULL)) return; sbuf_new(&sb, str, sizeof(str), 0); scsi_sense_sbuf(device, csio, &sb, SSS_FLAG_PRINT_COMMAND); sbuf_finish(&sb); fprintf(ofile, "%s", sbuf_data(&sb)); } #endif /* _KERNEL/!_KERNEL */ /* * Extract basic sense information. This is backward-compatible with the * previous implementation. For new implementations, * scsi_extract_sense_len() is recommended. */ void scsi_extract_sense(struct scsi_sense_data *sense_data, int *error_code, int *sense_key, int *asc, int *ascq) { scsi_extract_sense_len(sense_data, sizeof(*sense_data), error_code, sense_key, asc, ascq, /*show_errors*/ 0); } /* * Extract basic sense information from SCSI I/O CCB structure. */ int scsi_extract_sense_ccb(union ccb *ccb, int *error_code, int *sense_key, int *asc, int *ascq) { struct scsi_sense_data *sense_data; /* Make sure there are some sense data we can access. */ if (ccb->ccb_h.func_code != XPT_SCSI_IO || (ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_SCSI_STATUS_ERROR || (ccb->csio.scsi_status != SCSI_STATUS_CHECK_COND) || (ccb->ccb_h.status & CAM_AUTOSNS_VALID) == 0 || (ccb->ccb_h.flags & CAM_SENSE_PHYS)) return (0); if (ccb->ccb_h.flags & CAM_SENSE_PTR) bcopy(&ccb->csio.sense_data, &sense_data, sizeof(struct scsi_sense_data *)); else sense_data = &ccb->csio.sense_data; scsi_extract_sense_len(sense_data, ccb->csio.sense_len - ccb->csio.sense_resid, error_code, sense_key, asc, ascq, 1); if (*error_code == -1) return (0); return (1); } /* * Extract basic sense information. If show_errors is set, sense values * will be set to -1 if they are not present. */ void scsi_extract_sense_len(struct scsi_sense_data *sense_data, u_int sense_len, int *error_code, int *sense_key, int *asc, int *ascq, int show_errors) { /* * If we have no length, we have no sense. */ if (sense_len == 0) { if (show_errors == 0) { *error_code = 0; *sense_key = 0; *asc = 0; *ascq = 0; } else { *error_code = -1; *sense_key = -1; *asc = -1; *ascq = -1; } return; } *error_code = sense_data->error_code & SSD_ERRCODE; switch (*error_code) { case SSD_DESC_CURRENT_ERROR: case SSD_DESC_DEFERRED_ERROR: { struct scsi_sense_data_desc *sense; sense = (struct scsi_sense_data_desc *)sense_data; if (SSD_DESC_IS_PRESENT(sense, sense_len, sense_key)) *sense_key = sense->sense_key & SSD_KEY; else *sense_key = (show_errors) ? -1 : 0; if (SSD_DESC_IS_PRESENT(sense, sense_len, add_sense_code)) *asc = sense->add_sense_code; else *asc = (show_errors) ? -1 : 0; if (SSD_DESC_IS_PRESENT(sense, sense_len, add_sense_code_qual)) *ascq = sense->add_sense_code_qual; else *ascq = (show_errors) ? -1 : 0; break; } case SSD_CURRENT_ERROR: case SSD_DEFERRED_ERROR: default: { struct scsi_sense_data_fixed *sense; sense = (struct scsi_sense_data_fixed *)sense_data; if (SSD_FIXED_IS_PRESENT(sense, sense_len, flags)) *sense_key = sense->flags & SSD_KEY; else *sense_key = (show_errors) ? -1 : 0; if ((SSD_FIXED_IS_PRESENT(sense, sense_len, add_sense_code)) && (SSD_FIXED_IS_FILLED(sense, add_sense_code))) *asc = sense->add_sense_code; else *asc = (show_errors) ? -1 : 0; if ((SSD_FIXED_IS_PRESENT(sense, sense_len,add_sense_code_qual)) && (SSD_FIXED_IS_FILLED(sense, add_sense_code_qual))) *ascq = sense->add_sense_code_qual; else *ascq = (show_errors) ? -1 : 0; break; } } } int scsi_get_sense_key(struct scsi_sense_data *sense_data, u_int sense_len, int show_errors) { int error_code, sense_key, asc, ascq; scsi_extract_sense_len(sense_data, sense_len, &error_code, &sense_key, &asc, &ascq, show_errors); return (sense_key); } int scsi_get_asc(struct scsi_sense_data *sense_data, u_int sense_len, int show_errors) { int error_code, sense_key, asc, ascq; scsi_extract_sense_len(sense_data, sense_len, &error_code, &sense_key, &asc, &ascq, show_errors); return (asc); } int scsi_get_ascq(struct scsi_sense_data *sense_data, u_int sense_len, int show_errors) { int error_code, sense_key, asc, ascq; scsi_extract_sense_len(sense_data, sense_len, &error_code, &sense_key, &asc, &ascq, show_errors); return (ascq); } /* * This function currently requires at least 36 bytes, or * SHORT_INQUIRY_LENGTH, worth of data to function properly. If this * function needs more or less data in the future, another length should be * defined in scsi_all.h to indicate the minimum amount of data necessary * for this routine to function properly. */ void scsi_print_inquiry(struct scsi_inquiry_data *inq_data) { u_int8_t type; char *dtype, *qtype; char vendor[16], product[48], revision[16], rstr[12]; type = SID_TYPE(inq_data); /* * Figure out basic device type and qualifier. */ if (SID_QUAL_IS_VENDOR_UNIQUE(inq_data)) { qtype = " (vendor-unique qualifier)"; } else { switch (SID_QUAL(inq_data)) { case SID_QUAL_LU_CONNECTED: qtype = ""; break; case SID_QUAL_LU_OFFLINE: qtype = " (offline)"; break; case SID_QUAL_RSVD: qtype = " (reserved qualifier)"; break; default: case SID_QUAL_BAD_LU: qtype = " (LUN not supported)"; break; } } switch (type) { case T_DIRECT: dtype = "Direct Access"; break; case T_SEQUENTIAL: dtype = "Sequential Access"; break; case T_PRINTER: dtype = "Printer"; break; case T_PROCESSOR: dtype = "Processor"; break; case T_WORM: dtype = "WORM"; break; case T_CDROM: dtype = "CD-ROM"; break; case T_SCANNER: dtype = "Scanner"; break; case T_OPTICAL: dtype = "Optical"; break; case T_CHANGER: dtype = "Changer"; break; case T_COMM: dtype = "Communication"; break; case T_STORARRAY: dtype = "Storage Array"; break; case T_ENCLOSURE: dtype = "Enclosure Services"; break; case T_RBC: dtype = "Simplified Direct Access"; break; case T_OCRW: dtype = "Optical Card Read/Write"; break; case T_OSD: dtype = "Object-Based Storage"; break; case T_ADC: dtype = "Automation/Drive Interface"; break; case T_NODEVICE: dtype = "Uninstalled"; break; default: dtype = "unknown"; break; } cam_strvis(vendor, inq_data->vendor, sizeof(inq_data->vendor), sizeof(vendor)); cam_strvis(product, inq_data->product, sizeof(inq_data->product), sizeof(product)); cam_strvis(revision, inq_data->revision, sizeof(inq_data->revision), sizeof(revision)); if (SID_ANSI_REV(inq_data) == SCSI_REV_0) snprintf(rstr, sizeof(rstr), "SCSI"); else if (SID_ANSI_REV(inq_data) <= SCSI_REV_SPC) { snprintf(rstr, sizeof(rstr), "SCSI-%d", SID_ANSI_REV(inq_data)); } else { snprintf(rstr, sizeof(rstr), "SPC-%d SCSI", SID_ANSI_REV(inq_data) - 2); } printf("<%s %s %s> %s %s %s device%s\n", vendor, product, revision, SID_IS_REMOVABLE(inq_data) ? "Removable" : "Fixed", dtype, rstr, qtype); } void scsi_print_inquiry_short(struct scsi_inquiry_data *inq_data) { char vendor[16], product[48], revision[16]; cam_strvis(vendor, inq_data->vendor, sizeof(inq_data->vendor), sizeof(vendor)); cam_strvis(product, inq_data->product, sizeof(inq_data->product), sizeof(product)); cam_strvis(revision, inq_data->revision, sizeof(inq_data->revision), sizeof(revision)); printf("<%s %s %s>", vendor, product, revision); } /* * Table of syncrates that don't follow the "divisible by 4" * rule. This table will be expanded in future SCSI specs. */ static struct { u_int period_factor; u_int period; /* in 100ths of ns */ } scsi_syncrates[] = { { 0x08, 625 }, /* FAST-160 */ { 0x09, 1250 }, /* FAST-80 */ { 0x0a, 2500 }, /* FAST-40 40MHz */ { 0x0b, 3030 }, /* FAST-40 33MHz */ { 0x0c, 5000 } /* FAST-20 */ }; /* * Return the frequency in kHz corresponding to the given * sync period factor. */ u_int scsi_calc_syncsrate(u_int period_factor) { int i; int num_syncrates; /* * It's a bug if period is zero, but if it is anyway, don't * die with a divide fault- instead return something which * 'approximates' async */ if (period_factor == 0) { return (3300); } num_syncrates = sizeof(scsi_syncrates) / sizeof(scsi_syncrates[0]); /* See if the period is in the "exception" table */ for (i = 0; i < num_syncrates; i++) { if (period_factor == scsi_syncrates[i].period_factor) { /* Period in kHz */ return (100000000 / scsi_syncrates[i].period); } } /* * Wasn't in the table, so use the standard * 4 times conversion. */ return (10000000 / (period_factor * 4 * 10)); } /* * Return the SCSI sync parameter that corresponsd to * the passed in period in 10ths of ns. */ u_int scsi_calc_syncparam(u_int period) { int i; int num_syncrates; if (period == 0) return (~0); /* Async */ /* Adjust for exception table being in 100ths. */ period *= 10; num_syncrates = sizeof(scsi_syncrates) / sizeof(scsi_syncrates[0]); /* See if the period is in the "exception" table */ for (i = 0; i < num_syncrates; i++) { if (period <= scsi_syncrates[i].period) { /* Period in 100ths of ns */ return (scsi_syncrates[i].period_factor); } } /* * Wasn't in the table, so use the standard * 1/4 period in ns conversion. */ return (period/400); } int scsi_devid_is_naa_ieee_reg(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; struct scsi_vpd_id_naa_basic *naa; descr = (struct scsi_vpd_id_descriptor *)bufp; naa = (struct scsi_vpd_id_naa_basic *)descr->identifier; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_NAA) return 0; if (descr->length < sizeof(struct scsi_vpd_id_naa_ieee_reg)) return 0; if ((naa->naa >> SVPD_ID_NAA_NAA_SHIFT) != SVPD_ID_NAA_IEEE_REG) return 0; return 1; } int scsi_devid_is_sas_target(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if (!scsi_devid_is_naa_ieee_reg(bufp)) return 0; if ((descr->id_type & SVPD_ID_PIV) == 0) /* proto field reserved */ return 0; if ((descr->proto_codeset >> SVPD_ID_PROTO_SHIFT) != SCSI_PROTO_SAS) return 0; return 1; } int scsi_devid_is_lun_eui64(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if ((descr->id_type & SVPD_ID_ASSOC_MASK) != SVPD_ID_ASSOC_LUN) return 0; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_EUI64) return 0; return 1; } int scsi_devid_is_lun_naa(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if ((descr->id_type & SVPD_ID_ASSOC_MASK) != SVPD_ID_ASSOC_LUN) return 0; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_NAA) return 0; return 1; } int scsi_devid_is_lun_t10(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if ((descr->id_type & SVPD_ID_ASSOC_MASK) != SVPD_ID_ASSOC_LUN) return 0; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_T10) return 0; return 1; } int scsi_devid_is_lun_name(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if ((descr->id_type & SVPD_ID_ASSOC_MASK) != SVPD_ID_ASSOC_LUN) return 0; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_SCSI_NAME) return 0; return 1; } int scsi_devid_is_port_naa(uint8_t *bufp) { struct scsi_vpd_id_descriptor *descr; descr = (struct scsi_vpd_id_descriptor *)bufp; if ((descr->id_type & SVPD_ID_ASSOC_MASK) != SVPD_ID_ASSOC_PORT) return 0; if ((descr->id_type & SVPD_ID_TYPE_MASK) != SVPD_ID_TYPE_NAA) return 0; return 1; } struct scsi_vpd_id_descriptor * scsi_get_devid_desc(struct scsi_vpd_id_descriptor *desc, uint32_t len, scsi_devid_checkfn_t ck_fn) { uint8_t *desc_buf_end; desc_buf_end = (uint8_t *)desc + len; for (; desc->identifier <= desc_buf_end && desc->identifier + desc->length <= desc_buf_end; desc = (struct scsi_vpd_id_descriptor *)(desc->identifier + desc->length)) { if (ck_fn == NULL || ck_fn((uint8_t *)desc) != 0) return (desc); } return (NULL); } struct scsi_vpd_id_descriptor * scsi_get_devid(struct scsi_vpd_device_id *id, uint32_t page_len, scsi_devid_checkfn_t ck_fn) { uint32_t len; if (page_len < sizeof(*id)) return (NULL); len = MIN(scsi_2btoul(id->length), page_len - sizeof(*id)); return (scsi_get_devid_desc((struct scsi_vpd_id_descriptor *) id->desc_list, len, ck_fn)); } int scsi_transportid_sbuf(struct sbuf *sb, struct scsi_transportid_header *hdr, uint32_t valid_len) { switch (hdr->format_protocol & SCSI_TRN_PROTO_MASK) { case SCSI_PROTO_FC: { struct scsi_transportid_fcp *fcp; uint64_t n_port_name; fcp = (struct scsi_transportid_fcp *)hdr; n_port_name = scsi_8btou64(fcp->n_port_name); sbuf_printf(sb, "FCP address: 0x%.16jx",(uintmax_t)n_port_name); break; } case SCSI_PROTO_SPI: { struct scsi_transportid_spi *spi; spi = (struct scsi_transportid_spi *)hdr; sbuf_printf(sb, "SPI address: %u,%u", scsi_2btoul(spi->scsi_addr), scsi_2btoul(spi->rel_trgt_port_id)); break; } case SCSI_PROTO_SSA: /* * XXX KDM there is no transport ID defined in SPC-4 for * SSA. */ break; case SCSI_PROTO_1394: { struct scsi_transportid_1394 *sbp; uint64_t eui64; sbp = (struct scsi_transportid_1394 *)hdr; eui64 = scsi_8btou64(sbp->eui64); sbuf_printf(sb, "SBP address: 0x%.16jx", (uintmax_t)eui64); break; } case SCSI_PROTO_RDMA: { struct scsi_transportid_rdma *rdma; unsigned int i; rdma = (struct scsi_transportid_rdma *)hdr; sbuf_printf(sb, "RDMA address: 0x"); for (i = 0; i < sizeof(rdma->initiator_port_id); i++) sbuf_printf(sb, "%02x", rdma->initiator_port_id[i]); break; } case SCSI_PROTO_ISCSI: { uint32_t add_len, i; uint8_t *iscsi_name = NULL; int nul_found = 0; sbuf_printf(sb, "iSCSI address: "); if ((hdr->format_protocol & SCSI_TRN_FORMAT_MASK) == SCSI_TRN_ISCSI_FORMAT_DEVICE) { struct scsi_transportid_iscsi_device *dev; dev = (struct scsi_transportid_iscsi_device *)hdr; /* * Verify how much additional data we really have. */ add_len = scsi_2btoul(dev->additional_length); add_len = MIN(add_len, valid_len - __offsetof(struct scsi_transportid_iscsi_device, iscsi_name)); iscsi_name = &dev->iscsi_name[0]; } else if ((hdr->format_protocol & SCSI_TRN_FORMAT_MASK) == SCSI_TRN_ISCSI_FORMAT_PORT) { struct scsi_transportid_iscsi_port *port; port = (struct scsi_transportid_iscsi_port *)hdr; add_len = scsi_2btoul(port->additional_length); add_len = MIN(add_len, valid_len - __offsetof(struct scsi_transportid_iscsi_port, iscsi_name)); iscsi_name = &port->iscsi_name[0]; } else { sbuf_printf(sb, "unknown format %x", (hdr->format_protocol & SCSI_TRN_FORMAT_MASK) >> SCSI_TRN_FORMAT_SHIFT); break; } if (add_len == 0) { sbuf_printf(sb, "not enough data"); break; } /* * This is supposed to be a NUL-terminated ASCII * string, but you never know. So we're going to * check. We need to do this because there is no * sbuf equivalent of strncat(). */ for (i = 0; i < add_len; i++) { if (iscsi_name[i] == '\0') { nul_found = 1; break; } } /* * If there is a NUL in the name, we can just use * sbuf_cat(). Otherwise we need to use sbuf_bcat(). */ if (nul_found != 0) sbuf_cat(sb, iscsi_name); else sbuf_bcat(sb, iscsi_name, add_len); break; } case SCSI_PROTO_SAS: { struct scsi_transportid_sas *sas; uint64_t sas_addr; sas = (struct scsi_transportid_sas *)hdr; sas_addr = scsi_8btou64(sas->sas_address); sbuf_printf(sb, "SAS address: 0x%.16jx", (uintmax_t)sas_addr); break; } case SCSI_PROTO_ADITP: case SCSI_PROTO_ATA: case SCSI_PROTO_UAS: /* * No Transport ID format for ADI, ATA or USB is defined in * SPC-4. */ sbuf_printf(sb, "No known Transport ID format for protocol " "%#x", hdr->format_protocol & SCSI_TRN_PROTO_MASK); break; case SCSI_PROTO_SOP: { struct scsi_transportid_sop *sop; struct scsi_sop_routing_id_norm *rid; sop = (struct scsi_transportid_sop *)hdr; rid = (struct scsi_sop_routing_id_norm *)sop->routing_id; /* * Note that there is no alternate format specified in SPC-4 * for the PCIe routing ID, so we don't really have a way * to know whether the second byte of the routing ID is * a device and function or just a function. So we just * assume bus,device,function. */ sbuf_printf(sb, "SOP Routing ID: %u,%u,%u", rid->bus, rid->devfunc >> SCSI_TRN_SOP_DEV_SHIFT, rid->devfunc & SCSI_TRN_SOP_FUNC_NORM_MAX); break; } case SCSI_PROTO_NONE: default: sbuf_printf(sb, "Unknown protocol %#x", hdr->format_protocol & SCSI_TRN_PROTO_MASK); break; } return (0); } struct scsi_nv scsi_proto_map[] = { { "fcp", SCSI_PROTO_FC }, { "spi", SCSI_PROTO_SPI }, { "ssa", SCSI_PROTO_SSA }, { "sbp", SCSI_PROTO_1394 }, { "1394", SCSI_PROTO_1394 }, { "srp", SCSI_PROTO_RDMA }, { "rdma", SCSI_PROTO_RDMA }, { "iscsi", SCSI_PROTO_ISCSI }, { "iqn", SCSI_PROTO_ISCSI }, { "sas", SCSI_PROTO_SAS }, { "aditp", SCSI_PROTO_ADITP }, { "ata", SCSI_PROTO_ATA }, { "uas", SCSI_PROTO_UAS }, { "usb", SCSI_PROTO_UAS }, { "sop", SCSI_PROTO_SOP } }; const char * scsi_nv_to_str(struct scsi_nv *table, int num_table_entries, uint64_t value) { int i; for (i = 0; i < num_table_entries; i++) { if (table[i].value == value) return (table[i].name); } return (NULL); } /* * Given a name/value table, find a value matching the given name. * Return values: * SCSI_NV_FOUND - match found * SCSI_NV_AMBIGUOUS - more than one match, none of them exact * SCSI_NV_NOT_FOUND - no match found */ scsi_nv_status scsi_get_nv(struct scsi_nv *table, int num_table_entries, char *name, int *table_entry, scsi_nv_flags flags) { int i, num_matches = 0; for (i = 0; i < num_table_entries; i++) { size_t table_len, name_len; table_len = strlen(table[i].name); name_len = strlen(name); if ((((flags & SCSI_NV_FLAG_IG_CASE) != 0) && (strncasecmp(table[i].name, name, name_len) == 0)) || (((flags & SCSI_NV_FLAG_IG_CASE) == 0) && (strncmp(table[i].name, name, name_len) == 0))) { *table_entry = i; /* * Check for an exact match. If we have the same * number of characters in the table as the argument, * and we already know they're the same, we have * an exact match. */ if (table_len == name_len) return (SCSI_NV_FOUND); /* * Otherwise, bump up the number of matches. We'll * see later how many we have. */ num_matches++; } } if (num_matches > 1) return (SCSI_NV_AMBIGUOUS); else if (num_matches == 1) return (SCSI_NV_FOUND); else return (SCSI_NV_NOT_FOUND); } /* * Parse transport IDs for Fibre Channel, 1394 and SAS. Since these are * all 64-bit numbers, the code is similar. */ int scsi_parse_transportid_64bit(int proto_id, char *id_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { uint64_t value; char *endptr; int retval; size_t alloc_size; retval = 0; value = strtouq(id_str, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: error " "parsing ID %s, 64-bit number required", __func__, id_str); } retval = 1; goto bailout; } switch (proto_id) { case SCSI_PROTO_FC: alloc_size = sizeof(struct scsi_transportid_fcp); break; case SCSI_PROTO_1394: alloc_size = sizeof(struct scsi_transportid_1394); break; case SCSI_PROTO_SAS: alloc_size = sizeof(struct scsi_transportid_sas); break; default: if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unsupoprted " "protocol %d", __func__, proto_id); } retval = 1; goto bailout; break; /* NOTREACHED */ } #ifdef _KERNEL *hdr = malloc(alloc_size, type, flags); #else /* _KERNEL */ *hdr = malloc(alloc_size); #endif /*_KERNEL */ if (*hdr == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unable to " "allocate %zu bytes", __func__, alloc_size); } retval = 1; goto bailout; } *alloc_len = alloc_size; bzero(*hdr, alloc_size); switch (proto_id) { case SCSI_PROTO_FC: { struct scsi_transportid_fcp *fcp; fcp = (struct scsi_transportid_fcp *)(*hdr); fcp->format_protocol = SCSI_PROTO_FC | SCSI_TRN_FCP_FORMAT_DEFAULT; scsi_u64to8b(value, fcp->n_port_name); break; } case SCSI_PROTO_1394: { struct scsi_transportid_1394 *sbp; sbp = (struct scsi_transportid_1394 *)(*hdr); sbp->format_protocol = SCSI_PROTO_1394 | SCSI_TRN_1394_FORMAT_DEFAULT; scsi_u64to8b(value, sbp->eui64); break; } case SCSI_PROTO_SAS: { struct scsi_transportid_sas *sas; sas = (struct scsi_transportid_sas *)(*hdr); sas->format_protocol = SCSI_PROTO_SAS | SCSI_TRN_SAS_FORMAT_DEFAULT; scsi_u64to8b(value, sas->sas_address); break; } default: break; } bailout: return (retval); } /* * Parse a SPI (Parallel SCSI) address of the form: id,rel_tgt_port */ int scsi_parse_transportid_spi(char *id_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { unsigned long scsi_addr, target_port; struct scsi_transportid_spi *spi; char *tmpstr, *endptr; int retval; retval = 0; tmpstr = strsep(&id_str, ","); if (tmpstr == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no ID found", __func__); } retval = 1; goto bailout; } scsi_addr = strtoul(tmpstr, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: error " "parsing SCSI ID %s, number required", __func__, tmpstr); } retval = 1; goto bailout; } if (id_str == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no relative " "target port found", __func__); } retval = 1; goto bailout; } target_port = strtoul(id_str, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: error " "parsing relative target port %s, number " "required", __func__, id_str); } retval = 1; goto bailout; } #ifdef _KERNEL spi = malloc(sizeof(*spi), type, flags); #else spi = malloc(sizeof(*spi)); #endif if (spi == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unable to " "allocate %zu bytes", __func__, sizeof(*spi)); } retval = 1; goto bailout; } *alloc_len = sizeof(*spi); bzero(spi, sizeof(*spi)); spi->format_protocol = SCSI_PROTO_SPI | SCSI_TRN_SPI_FORMAT_DEFAULT; scsi_ulto2b(scsi_addr, spi->scsi_addr); scsi_ulto2b(target_port, spi->rel_trgt_port_id); *hdr = (struct scsi_transportid_header *)spi; bailout: return (retval); } /* * Parse an RDMA/SRP Initiator Port ID string. This is 32 hexadecimal digits, * optionally prefixed by "0x" or "0X". */ int scsi_parse_transportid_rdma(char *id_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { struct scsi_transportid_rdma *rdma; int retval; size_t id_len, rdma_id_size; uint8_t rdma_id[SCSI_TRN_RDMA_PORT_LEN]; char *tmpstr; unsigned int i, j; retval = 0; id_len = strlen(id_str); rdma_id_size = SCSI_TRN_RDMA_PORT_LEN; /* * Check the size. It needs to be either 32 or 34 characters long. */ if ((id_len != (rdma_id_size * 2)) && (id_len != ((rdma_id_size * 2) + 2))) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: RDMA ID " "must be 32 hex digits (0x prefix " "optional), only %zu seen", __func__, id_len); } retval = 1; goto bailout; } tmpstr = id_str; /* * If the user gave us 34 characters, the string needs to start * with '0x'. */ if (id_len == ((rdma_id_size * 2) + 2)) { if ((tmpstr[0] == '0') && ((tmpstr[1] == 'x') || (tmpstr[1] == 'X'))) { tmpstr += 2; } else { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: RDMA " "ID prefix, if used, must be \"0x\", " "got %s", __func__, tmpstr); } retval = 1; goto bailout; } } bzero(rdma_id, sizeof(rdma_id)); /* * Convert ASCII hex into binary bytes. There is no standard * 128-bit integer type, and so no strtou128t() routine to convert * from hex into a large integer. In the end, we're not going to * an integer, but rather to a byte array, so that and the fact * that we require the user to give us 32 hex digits simplifies the * logic. */ for (i = 0; i < (rdma_id_size * 2); i++) { int cur_shift; unsigned char c; /* Increment the byte array one for every 2 hex digits */ j = i >> 1; /* * The first digit in every pair is the most significant * 4 bits. The second is the least significant 4 bits. */ if ((i % 2) == 0) cur_shift = 4; else cur_shift = 0; c = tmpstr[i]; /* Convert the ASCII hex character into a number */ if (isdigit(c)) c -= '0'; else if (isalpha(c)) c -= isupper(c) ? 'A' - 10 : 'a' - 10; else { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: " "RDMA ID must be hex digits, got " "invalid character %c", __func__, tmpstr[i]); } retval = 1; goto bailout; } /* * The converted number can't be less than 0; the type is * unsigned, and the subtraction logic will not give us * a negative number. So we only need to make sure that * the value is not greater than 0xf. (i.e. make sure the * user didn't give us a value like "0x12jklmno"). */ if (c > 0xf) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: " "RDMA ID must be hex digits, got " "invalid character %c", __func__, tmpstr[i]); } retval = 1; goto bailout; } rdma_id[j] |= c << cur_shift; } #ifdef _KERNEL rdma = malloc(sizeof(*rdma), type, flags); #else rdma = malloc(sizeof(*rdma)); #endif if (rdma == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unable to " "allocate %zu bytes", __func__, sizeof(*rdma)); } retval = 1; goto bailout; } *alloc_len = sizeof(*rdma); bzero(rdma, *alloc_len); rdma->format_protocol = SCSI_PROTO_RDMA | SCSI_TRN_RDMA_FORMAT_DEFAULT; bcopy(rdma_id, rdma->initiator_port_id, SCSI_TRN_RDMA_PORT_LEN); *hdr = (struct scsi_transportid_header *)rdma; bailout: return (retval); } /* * Parse an iSCSI name. The format is either just the name: * * iqn.2012-06.com.example:target0 * or the name, separator and initiator session ID: * * iqn.2012-06.com.example:target0,i,0x123 * * The separator format is exact. */ int scsi_parse_transportid_iscsi(char *id_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { size_t id_len, sep_len, id_size, name_len; int retval; unsigned int i, sep_pos, sep_found; const char *sep_template = ",i,0x"; const char *iqn_prefix = "iqn."; struct scsi_transportid_iscsi_device *iscsi; retval = 0; sep_found = 0; id_len = strlen(id_str); sep_len = strlen(sep_template); /* * The separator is defined as exactly ',i,0x'. Any other commas, * or any other form, is an error. So look for a comma, and once * we find that, the next few characters must match the separator * exactly. Once we get through the separator, there should be at * least one character. */ for (i = 0, sep_pos = 0; i < id_len; i++) { if (sep_pos == 0) { if (id_str[i] == sep_template[sep_pos]) sep_pos++; continue; } if (sep_pos < sep_len) { if (id_str[i] == sep_template[sep_pos]) { sep_pos++; continue; } if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: " "invalid separator in iSCSI name " "\"%s\"", __func__, id_str); } retval = 1; goto bailout; } else { sep_found = 1; break; } } /* * Check to see whether we have a separator but no digits after it. */ if ((sep_pos != 0) && (sep_found == 0)) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no digits " "found after separator in iSCSI name \"%s\"", __func__, id_str); } retval = 1; goto bailout; } /* * The incoming ID string has the "iqn." prefix stripped off. We * need enough space for the base structure (the structures are the * same for the two iSCSI forms), the prefix, the ID string and a * terminating NUL. */ id_size = sizeof(*iscsi) + strlen(iqn_prefix) + id_len + 1; #ifdef _KERNEL iscsi = malloc(id_size, type, flags); #else iscsi = malloc(id_size); #endif if (iscsi == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unable to " "allocate %zu bytes", __func__, id_size); } retval = 1; goto bailout; } *alloc_len = id_size; bzero(iscsi, id_size); iscsi->format_protocol = SCSI_PROTO_ISCSI; if (sep_found == 0) iscsi->format_protocol |= SCSI_TRN_ISCSI_FORMAT_DEVICE; else iscsi->format_protocol |= SCSI_TRN_ISCSI_FORMAT_PORT; name_len = id_size - sizeof(*iscsi); scsi_ulto2b(name_len, iscsi->additional_length); snprintf(iscsi->iscsi_name, name_len, "%s%s", iqn_prefix, id_str); *hdr = (struct scsi_transportid_header *)iscsi; bailout: return (retval); } /* * Parse a SCSI over PCIe (SOP) identifier. The Routing ID can either be * of the form 'bus,device,function' or 'bus,function'. */ int scsi_parse_transportid_sop(char *id_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { struct scsi_transportid_sop *sop; unsigned long bus, device, function; char *tmpstr, *endptr; int retval, device_spec; retval = 0; device_spec = 0; device = 0; tmpstr = strsep(&id_str, ","); if ((tmpstr == NULL) || (*tmpstr == '\0')) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no ID found", __func__); } retval = 1; goto bailout; } bus = strtoul(tmpstr, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: error " "parsing PCIe bus %s, number required", __func__, tmpstr); } retval = 1; goto bailout; } if ((id_str == NULL) || (*id_str == '\0')) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no PCIe " "device or function found", __func__); } retval = 1; goto bailout; } tmpstr = strsep(&id_str, ","); function = strtoul(tmpstr, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: error " "parsing PCIe device/function %s, number " "required", __func__, tmpstr); } retval = 1; goto bailout; } /* * Check to see whether the user specified a third value. If so, * the second is the device. */ if (id_str != NULL) { if (*id_str == '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: " "no PCIe function found", __func__); } retval = 1; goto bailout; } device = function; device_spec = 1; function = strtoul(id_str, &endptr, 0); if (*endptr != '\0') { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: " "error parsing PCIe function %s, " "number required", __func__, id_str); } retval = 1; goto bailout; } } if (bus > SCSI_TRN_SOP_BUS_MAX) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: bus value " "%lu greater than maximum %u", __func__, bus, SCSI_TRN_SOP_BUS_MAX); } retval = 1; goto bailout; } if ((device_spec != 0) && (device > SCSI_TRN_SOP_DEV_MASK)) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: device value " "%lu greater than maximum %u", __func__, device, SCSI_TRN_SOP_DEV_MAX); } retval = 1; goto bailout; } if (((device_spec != 0) && (function > SCSI_TRN_SOP_FUNC_NORM_MAX)) || ((device_spec == 0) && (function > SCSI_TRN_SOP_FUNC_ALT_MAX))) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: function value " "%lu greater than maximum %u", __func__, function, (device_spec == 0) ? SCSI_TRN_SOP_FUNC_ALT_MAX : SCSI_TRN_SOP_FUNC_NORM_MAX); } retval = 1; goto bailout; } #ifdef _KERNEL sop = malloc(sizeof(*sop), type, flags); #else sop = malloc(sizeof(*sop)); #endif if (sop == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: unable to " "allocate %zu bytes", __func__, sizeof(*sop)); } retval = 1; goto bailout; } *alloc_len = sizeof(*sop); bzero(sop, sizeof(*sop)); sop->format_protocol = SCSI_PROTO_SOP | SCSI_TRN_SOP_FORMAT_DEFAULT; if (device_spec != 0) { struct scsi_sop_routing_id_norm rid; rid.bus = bus; rid.devfunc = (device << SCSI_TRN_SOP_DEV_SHIFT) | function; bcopy(&rid, sop->routing_id, MIN(sizeof(rid), sizeof(sop->routing_id))); } else { struct scsi_sop_routing_id_alt rid; rid.bus = bus; rid.function = function; bcopy(&rid, sop->routing_id, MIN(sizeof(rid), sizeof(sop->routing_id))); } *hdr = (struct scsi_transportid_header *)sop; bailout: return (retval); } /* * transportid_str: NUL-terminated string with format: protcol,id * The ID is protocol specific. * hdr: Storage will be allocated for the transport ID. * alloc_len: The amount of memory allocated is returned here. * type: Malloc bucket (kernel only). * flags: Malloc flags (kernel only). * error_str: If non-NULL, it will contain error information (without * a terminating newline) if an error is returned. * error_str_len: Allocated length of the error string. * * Returns 0 for success, non-zero for failure. */ int scsi_parse_transportid(char *transportid_str, struct scsi_transportid_header **hdr, unsigned int *alloc_len, #ifdef _KERNEL struct malloc_type *type, int flags, #endif char *error_str, int error_str_len) { char *tmpstr; scsi_nv_status status; int retval, num_proto_entries, table_entry; retval = 0; table_entry = 0; /* * We do allow a period as well as a comma to separate the protocol * from the ID string. This is to accommodate iSCSI names, which * start with "iqn.". */ tmpstr = strsep(&transportid_str, ",."); if (tmpstr == NULL) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: transportid_str is NULL", __func__); } retval = 1; goto bailout; } num_proto_entries = sizeof(scsi_proto_map) / sizeof(scsi_proto_map[0]); status = scsi_get_nv(scsi_proto_map, num_proto_entries, tmpstr, &table_entry, SCSI_NV_FLAG_IG_CASE); if (status != SCSI_NV_FOUND) { if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: %s protocol " "name %s", __func__, (status == SCSI_NV_AMBIGUOUS) ? "ambiguous" : "invalid", tmpstr); } retval = 1; goto bailout; } switch (scsi_proto_map[table_entry].value) { case SCSI_PROTO_FC: case SCSI_PROTO_1394: case SCSI_PROTO_SAS: retval = scsi_parse_transportid_64bit( scsi_proto_map[table_entry].value, transportid_str, hdr, alloc_len, #ifdef _KERNEL type, flags, #endif error_str, error_str_len); break; case SCSI_PROTO_SPI: retval = scsi_parse_transportid_spi(transportid_str, hdr, alloc_len, #ifdef _KERNEL type, flags, #endif error_str, error_str_len); break; case SCSI_PROTO_RDMA: retval = scsi_parse_transportid_rdma(transportid_str, hdr, alloc_len, #ifdef _KERNEL type, flags, #endif error_str, error_str_len); break; case SCSI_PROTO_ISCSI: retval = scsi_parse_transportid_iscsi(transportid_str, hdr, alloc_len, #ifdef _KERNEL type, flags, #endif error_str, error_str_len); break; case SCSI_PROTO_SOP: retval = scsi_parse_transportid_sop(transportid_str, hdr, alloc_len, #ifdef _KERNEL type, flags, #endif error_str, error_str_len); break; case SCSI_PROTO_SSA: case SCSI_PROTO_ADITP: case SCSI_PROTO_ATA: case SCSI_PROTO_UAS: case SCSI_PROTO_NONE: default: /* * There is no format defined for a Transport ID for these * protocols. So even if the user gives us something, we * have no way to turn it into a standard SCSI Transport ID. */ retval = 1; if (error_str != NULL) { snprintf(error_str, error_str_len, "%s: no Transport " "ID format exists for protocol %s", __func__, tmpstr); } goto bailout; break; /* NOTREACHED */ } bailout: return (retval); } struct scsi_attrib_table_entry scsi_mam_attr_table[] = { { SMA_ATTR_REM_CAP_PARTITION, SCSI_ATTR_FLAG_NONE, "Remaining Capacity in Partition", /*suffix*/ "MB", /*to_str*/ scsi_attrib_int_sbuf,/*parse_str*/ NULL }, { SMA_ATTR_MAX_CAP_PARTITION, SCSI_ATTR_FLAG_NONE, "Maximum Capacity in Partition", /*suffix*/"MB", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TAPEALERT_FLAGS, SCSI_ATTR_FLAG_HEX, "TapeAlert Flags", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_LOAD_COUNT, SCSI_ATTR_FLAG_NONE, "Load Count", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MAM_SPACE_REMAINING, SCSI_ATTR_FLAG_NONE, "MAM Space Remaining", /*suffix*/"bytes", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_DEV_ASSIGNING_ORG, SCSI_ATTR_FLAG_NONE, "Assigning Organization", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_FORMAT_DENSITY_CODE, SCSI_ATTR_FLAG_HEX, "Format Density Code", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_INITIALIZATION_COUNT, SCSI_ATTR_FLAG_NONE, "Initialization Count", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_VOLUME_ID, SCSI_ATTR_FLAG_NONE, "Volume Identifier", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_VOLUME_CHANGE_REF, SCSI_ATTR_FLAG_HEX, "Volume Change Reference", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_DEV_SERIAL_LAST_LOAD, SCSI_ATTR_FLAG_NONE, "Device Vendor/Serial at Last Load", /*suffix*/NULL, /*to_str*/ scsi_attrib_vendser_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_DEV_SERIAL_LAST_LOAD_1, SCSI_ATTR_FLAG_NONE, "Device Vendor/Serial at Last Load - 1", /*suffix*/NULL, /*to_str*/ scsi_attrib_vendser_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_DEV_SERIAL_LAST_LOAD_2, SCSI_ATTR_FLAG_NONE, "Device Vendor/Serial at Last Load - 2", /*suffix*/NULL, /*to_str*/ scsi_attrib_vendser_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_DEV_SERIAL_LAST_LOAD_3, SCSI_ATTR_FLAG_NONE, "Device Vendor/Serial at Last Load - 3", /*suffix*/NULL, /*to_str*/ scsi_attrib_vendser_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TOTAL_MB_WRITTEN_LT, SCSI_ATTR_FLAG_NONE, "Total MB Written in Medium Life", /*suffix*/ "MB", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TOTAL_MB_READ_LT, SCSI_ATTR_FLAG_NONE, "Total MB Read in Medium Life", /*suffix*/ "MB", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TOTAL_MB_WRITTEN_CUR, SCSI_ATTR_FLAG_NONE, "Total MB Written in Current/Last Load", /*suffix*/ "MB", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TOTAL_MB_READ_CUR, SCSI_ATTR_FLAG_NONE, "Total MB Read in Current/Last Load", /*suffix*/ "MB", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_FIRST_ENC_BLOCK, SCSI_ATTR_FLAG_NONE, "Logical Position of First Encrypted Block", /*suffix*/ NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_NEXT_UNENC_BLOCK, SCSI_ATTR_FLAG_NONE, "Logical Position of First Unencrypted Block after First " "Encrypted Block", /*suffix*/ NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MEDIUM_USAGE_HIST, SCSI_ATTR_FLAG_NONE, "Medium Usage History", /*suffix*/ NULL, /*to_str*/ NULL, /*parse_str*/ NULL }, { SMA_ATTR_PART_USAGE_HIST, SCSI_ATTR_FLAG_NONE, "Partition Usage History", /*suffix*/ NULL, /*to_str*/ NULL, /*parse_str*/ NULL }, { SMA_ATTR_MED_MANUF, SCSI_ATTR_FLAG_NONE, "Medium Manufacturer", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_SERIAL, SCSI_ATTR_FLAG_NONE, "Medium Serial Number", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_LENGTH, SCSI_ATTR_FLAG_NONE, "Medium Length", /*suffix*/"m", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_WIDTH, SCSI_ATTR_FLAG_FP | SCSI_ATTR_FLAG_DIV_10 | SCSI_ATTR_FLAG_FP_1DIGIT, "Medium Width", /*suffix*/"mm", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_ASSIGNING_ORG, SCSI_ATTR_FLAG_NONE, "Assigning Organization", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_DENSITY_CODE, SCSI_ATTR_FLAG_HEX, "Medium Density Code", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_MANUF_DATE, SCSI_ATTR_FLAG_NONE, "Medium Manufacture Date", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MAM_CAPACITY, SCSI_ATTR_FLAG_NONE, "MAM Capacity", /*suffix*/"bytes", /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_TYPE, SCSI_ATTR_FLAG_HEX, "Medium Type", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_TYPE_INFO, SCSI_ATTR_FLAG_HEX, "Medium Type Information", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MED_SERIAL_NUM, SCSI_ATTR_FLAG_NONE, "Medium Serial Number", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_APP_VENDOR, SCSI_ATTR_FLAG_NONE, "Application Vendor", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_APP_NAME, SCSI_ATTR_FLAG_NONE, "Application Name", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_APP_VERSION, SCSI_ATTR_FLAG_NONE, "Application Version", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_USER_MED_TEXT_LABEL, SCSI_ATTR_FLAG_NONE, "User Medium Text Label", /*suffix*/NULL, /*to_str*/ scsi_attrib_text_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_LAST_WRITTEN_TIME, SCSI_ATTR_FLAG_NONE, "Date and Time Last Written", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_TEXT_LOCAL_ID, SCSI_ATTR_FLAG_HEX, "Text Localization Identifier", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_BARCODE, SCSI_ATTR_FLAG_NONE, "Barcode", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_HOST_OWNER_NAME, SCSI_ATTR_FLAG_NONE, "Owning Host Textual Name", /*suffix*/NULL, /*to_str*/ scsi_attrib_text_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_MEDIA_POOL, SCSI_ATTR_FLAG_NONE, "Media Pool", /*suffix*/NULL, /*to_str*/ scsi_attrib_text_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_PART_USER_LABEL, SCSI_ATTR_FLAG_NONE, "Partition User Text Label", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_LOAD_UNLOAD_AT_PART, SCSI_ATTR_FLAG_NONE, "Load/Unload at Partition", /*suffix*/NULL, /*to_str*/ scsi_attrib_int_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_APP_FORMAT_VERSION, SCSI_ATTR_FLAG_NONE, "Application Format Version", /*suffix*/NULL, /*to_str*/ scsi_attrib_ascii_sbuf, /*parse_str*/ NULL }, { SMA_ATTR_VOL_COHERENCY_INFO, SCSI_ATTR_FLAG_NONE, "Volume Coherency Information", /*suffix*/NULL, /*to_str*/ scsi_attrib_volcoh_sbuf, /*parse_str*/ NULL }, { 0x0ff1, SCSI_ATTR_FLAG_NONE, "Spectra MLM Creation", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x0ff2, SCSI_ATTR_FLAG_NONE, "Spectra MLM C3", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x0ff3, SCSI_ATTR_FLAG_NONE, "Spectra MLM RW", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x0ff4, SCSI_ATTR_FLAG_NONE, "Spectra MLM SDC List", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x0ff7, SCSI_ATTR_FLAG_NONE, "Spectra MLM Post Scan", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x0ffe, SCSI_ATTR_FLAG_NONE, "Spectra MLM Checksum", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17f1, SCSI_ATTR_FLAG_NONE, "Spectra MLM Creation", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17f2, SCSI_ATTR_FLAG_NONE, "Spectra MLM C3", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17f3, SCSI_ATTR_FLAG_NONE, "Spectra MLM RW", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17f4, SCSI_ATTR_FLAG_NONE, "Spectra MLM SDC List", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17f7, SCSI_ATTR_FLAG_NONE, "Spectra MLM Post Scan", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, { 0x17ff, SCSI_ATTR_FLAG_NONE, "Spectra MLM Checksum", /*suffix*/NULL, /*to_str*/ scsi_attrib_hexdump_sbuf, /*parse_str*/ NULL }, }; /* * Print out Volume Coherency Information (Attribute 0x080c). * This field has two variable length members, including one at the * beginning, so it isn't practical to have a fixed structure definition. * This is current as of SSC4r03 (see section 4.2.21.3), dated March 25, * 2013. */ int scsi_attrib_volcoh_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { size_t avail_len; uint32_t field_size; uint64_t tmp_val; uint8_t *cur_ptr; int retval; int vcr_len, as_len; retval = 0; tmp_val = 0; field_size = scsi_2btoul(hdr->length); avail_len = valid_len - sizeof(*hdr); if (field_size > avail_len) { if (error_str != NULL) { snprintf(error_str, error_str_len, "Available " "length of attribute ID 0x%.4x %zu < field " "length %u", scsi_2btoul(hdr->id), avail_len, field_size); } retval = 1; goto bailout; } else if (field_size == 0) { /* * It isn't clear from the spec whether a field length of * 0 is invalid here. It probably is, but be lenient here * to avoid inconveniencing the user. */ goto bailout; } cur_ptr = hdr->attribute; vcr_len = *cur_ptr; cur_ptr++; sbuf_printf(sb, "\n\tVolume Change Reference Value:"); switch (vcr_len) { case 0: if (error_str != NULL) { snprintf(error_str, error_str_len, "Volume Change " "Reference value has length of 0"); } retval = 1; goto bailout; break; /*NOTREACHED*/ case 1: tmp_val = *cur_ptr; break; case 2: tmp_val = scsi_2btoul(cur_ptr); break; case 3: tmp_val = scsi_3btoul(cur_ptr); break; case 4: tmp_val = scsi_4btoul(cur_ptr); break; case 8: tmp_val = scsi_8btou64(cur_ptr); break; default: sbuf_printf(sb, "\n"); sbuf_hexdump(sb, cur_ptr, vcr_len, NULL, 0); break; } if (vcr_len <= 8) sbuf_printf(sb, " 0x%jx\n", (uintmax_t)tmp_val); cur_ptr += vcr_len; tmp_val = scsi_8btou64(cur_ptr); sbuf_printf(sb, "\tVolume Coherency Count: %ju\n", (uintmax_t)tmp_val); cur_ptr += sizeof(tmp_val); tmp_val = scsi_8btou64(cur_ptr); sbuf_printf(sb, "\tVolume Coherency Set Identifier: 0x%jx\n", (uintmax_t)tmp_val); /* * Figure out how long the Application Client Specific Information * is and produce a hexdump. */ cur_ptr += sizeof(tmp_val); as_len = scsi_2btoul(cur_ptr); cur_ptr += sizeof(uint16_t); sbuf_printf(sb, "\tApplication Client Specific Information: "); if (((as_len == SCSI_LTFS_VER0_LEN) || (as_len == SCSI_LTFS_VER1_LEN)) && (strncmp(cur_ptr, SCSI_LTFS_STR_NAME, SCSI_LTFS_STR_LEN) == 0)) { sbuf_printf(sb, "LTFS\n"); cur_ptr += SCSI_LTFS_STR_LEN + 1; if (cur_ptr[SCSI_LTFS_UUID_LEN] != '\0') cur_ptr[SCSI_LTFS_UUID_LEN] = '\0'; sbuf_printf(sb, "\tLTFS UUID: %s\n", cur_ptr); cur_ptr += SCSI_LTFS_UUID_LEN + 1; /* XXX KDM check the length */ sbuf_printf(sb, "\tLTFS Version: %d\n", *cur_ptr); } else { sbuf_printf(sb, "Unknown\n"); sbuf_hexdump(sb, cur_ptr, as_len, NULL, 0); } bailout: return (retval); } int scsi_attrib_vendser_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { size_t avail_len; uint32_t field_size; struct scsi_attrib_vendser *vendser; cam_strvis_flags strvis_flags; int retval = 0; field_size = scsi_2btoul(hdr->length); avail_len = valid_len - sizeof(*hdr); if (field_size > avail_len) { if (error_str != NULL) { snprintf(error_str, error_str_len, "Available " "length of attribute ID 0x%.4x %zu < field " "length %u", scsi_2btoul(hdr->id), avail_len, field_size); } retval = 1; goto bailout; } else if (field_size == 0) { /* * A field size of 0 doesn't make sense here. The device * can at least give you the vendor ID, even if it can't * give you the serial number. */ if (error_str != NULL) { snprintf(error_str, error_str_len, "The length of " "attribute ID 0x%.4x is 0", scsi_2btoul(hdr->id)); } retval = 1; goto bailout; } vendser = (struct scsi_attrib_vendser *)hdr->attribute; switch (output_flags & SCSI_ATTR_OUTPUT_NONASCII_MASK) { case SCSI_ATTR_OUTPUT_NONASCII_TRIM: strvis_flags = CAM_STRVIS_FLAG_NONASCII_TRIM; break; case SCSI_ATTR_OUTPUT_NONASCII_RAW: strvis_flags = CAM_STRVIS_FLAG_NONASCII_RAW; break; case SCSI_ATTR_OUTPUT_NONASCII_ESC: default: strvis_flags = CAM_STRVIS_FLAG_NONASCII_ESC; break;; } cam_strvis_sbuf(sb, vendser->vendor, sizeof(vendser->vendor), strvis_flags); sbuf_putc(sb, ' '); cam_strvis_sbuf(sb, vendser->serial_num, sizeof(vendser->serial_num), strvis_flags); bailout: return (retval); } int scsi_attrib_hexdump_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { uint32_t field_size; ssize_t avail_len; uint32_t print_len; uint8_t *num_ptr; int retval = 0; field_size = scsi_2btoul(hdr->length); avail_len = valid_len - sizeof(*hdr); print_len = MIN(avail_len, field_size); num_ptr = hdr->attribute; if (print_len > 0) { sbuf_printf(sb, "\n"); sbuf_hexdump(sb, num_ptr, print_len, NULL, 0); } return (retval); } int scsi_attrib_int_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { uint64_t print_number; size_t avail_len; uint32_t number_size; int retval = 0; number_size = scsi_2btoul(hdr->length); avail_len = valid_len - sizeof(*hdr); if (avail_len < number_size) { if (error_str != NULL) { snprintf(error_str, error_str_len, "Available " "length of attribute ID 0x%.4x %zu < field " "length %u", scsi_2btoul(hdr->id), avail_len, number_size); } retval = 1; goto bailout; } switch (number_size) { case 0: /* * We don't treat this as an error, since there may be * scenarios where a device reports a field but then gives * a length of 0. See the note in scsi_attrib_ascii_sbuf(). */ goto bailout; break; /*NOTREACHED*/ case 1: print_number = hdr->attribute[0]; break; case 2: print_number = scsi_2btoul(hdr->attribute); break; case 3: print_number = scsi_3btoul(hdr->attribute); break; case 4: print_number = scsi_4btoul(hdr->attribute); break; case 8: print_number = scsi_8btou64(hdr->attribute); break; default: /* * If we wind up here, the number is too big to print * normally, so just do a hexdump. */ retval = scsi_attrib_hexdump_sbuf(sb, hdr, valid_len, flags, output_flags, error_str, error_str_len); goto bailout; break; } if (flags & SCSI_ATTR_FLAG_FP) { #ifndef _KERNEL long double num_float; num_float = (long double)print_number; if (flags & SCSI_ATTR_FLAG_DIV_10) num_float /= 10; sbuf_printf(sb, "%.*Lf", (flags & SCSI_ATTR_FLAG_FP_1DIGIT) ? 1 : 0, num_float); #else /* _KERNEL */ sbuf_printf(sb, "%ju", (flags & SCSI_ATTR_FLAG_DIV_10) ? (print_number / 10) : print_number); #endif /* _KERNEL */ } else if (flags & SCSI_ATTR_FLAG_HEX) { sbuf_printf(sb, "0x%jx", (uintmax_t)print_number); } else sbuf_printf(sb, "%ju", (uintmax_t)print_number); bailout: return (retval); } int scsi_attrib_ascii_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { size_t avail_len; uint32_t field_size, print_size; int retval = 0; avail_len = valid_len - sizeof(*hdr); field_size = scsi_2btoul(hdr->length); print_size = MIN(avail_len, field_size); if (print_size > 0) { cam_strvis_flags strvis_flags; switch (output_flags & SCSI_ATTR_OUTPUT_NONASCII_MASK) { case SCSI_ATTR_OUTPUT_NONASCII_TRIM: strvis_flags = CAM_STRVIS_FLAG_NONASCII_TRIM; break; case SCSI_ATTR_OUTPUT_NONASCII_RAW: strvis_flags = CAM_STRVIS_FLAG_NONASCII_RAW; break; case SCSI_ATTR_OUTPUT_NONASCII_ESC: default: strvis_flags = CAM_STRVIS_FLAG_NONASCII_ESC; break; } cam_strvis_sbuf(sb, hdr->attribute, print_size, strvis_flags); } else if (avail_len < field_size) { /* * We only report an error if the user didn't allocate * enough space to hold the full value of this field. If * the field length is 0, that is allowed by the spec. * e.g. in SPC-4r37, section 7.4.2.2.5, VOLUME IDENTIFIER * "This attribute indicates the current volume identifier * (see SMC-3) of the medium. If the device server supports * this attribute but does not have access to the volume * identifier, the device server shall report this attribute * with an attribute length value of zero." */ if (error_str != NULL) { snprintf(error_str, error_str_len, "Available " "length of attribute ID 0x%.4x %zu < field " "length %u", scsi_2btoul(hdr->id), avail_len, field_size); } retval = 1; } return (retval); } int scsi_attrib_text_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, uint32_t flags, uint32_t output_flags, char *error_str, int error_str_len) { size_t avail_len; uint32_t field_size, print_size; int retval = 0; int esc_text = 1; avail_len = valid_len - sizeof(*hdr); field_size = scsi_2btoul(hdr->length); print_size = MIN(avail_len, field_size); if ((output_flags & SCSI_ATTR_OUTPUT_TEXT_MASK) == SCSI_ATTR_OUTPUT_TEXT_RAW) esc_text = 0; if (print_size > 0) { uint32_t i; for (i = 0; i < print_size; i++) { if (hdr->attribute[i] == '\0') continue; else if (((unsigned char)hdr->attribute[i] < 0x80) || (esc_text == 0)) sbuf_putc(sb, hdr->attribute[i]); else sbuf_printf(sb, "%%%02x", (unsigned char)hdr->attribute[i]); } } else if (avail_len < field_size) { /* * We only report an error if the user didn't allocate * enough space to hold the full value of this field. */ if (error_str != NULL) { snprintf(error_str, error_str_len, "Available " "length of attribute ID 0x%.4x %zu < field " "length %u", scsi_2btoul(hdr->id), avail_len, field_size); } retval = 1; } return (retval); } struct scsi_attrib_table_entry * scsi_find_attrib_entry(struct scsi_attrib_table_entry *table, size_t num_table_entries, uint32_t id) { uint32_t i; for (i = 0; i < num_table_entries; i++) { if (table[i].id == id) return (&table[i]); } return (NULL); } struct scsi_attrib_table_entry * scsi_get_attrib_entry(uint32_t id) { return (scsi_find_attrib_entry(scsi_mam_attr_table, sizeof(scsi_mam_attr_table) / sizeof(scsi_mam_attr_table[0]), id)); } int scsi_attrib_value_sbuf(struct sbuf *sb, uint32_t valid_len, struct scsi_mam_attribute_header *hdr, uint32_t output_flags, char *error_str, size_t error_str_len) { int retval; switch (hdr->byte2 & SMA_FORMAT_MASK) { case SMA_FORMAT_ASCII: retval = scsi_attrib_ascii_sbuf(sb, hdr, valid_len, SCSI_ATTR_FLAG_NONE, output_flags, error_str,error_str_len); break; case SMA_FORMAT_BINARY: if (scsi_2btoul(hdr->length) <= 8) retval = scsi_attrib_int_sbuf(sb, hdr, valid_len, SCSI_ATTR_FLAG_NONE, output_flags, error_str, error_str_len); else retval = scsi_attrib_hexdump_sbuf(sb, hdr, valid_len, SCSI_ATTR_FLAG_NONE, output_flags, error_str, error_str_len); break; case SMA_FORMAT_TEXT: retval = scsi_attrib_text_sbuf(sb, hdr, valid_len, SCSI_ATTR_FLAG_NONE, output_flags, error_str, error_str_len); break; default: if (error_str != NULL) { snprintf(error_str, error_str_len, "Unknown attribute " "format 0x%x", hdr->byte2 & SMA_FORMAT_MASK); } retval = 1; goto bailout; break; /*NOTREACHED*/ } sbuf_trim(sb); bailout: return (retval); } void scsi_attrib_prefix_sbuf(struct sbuf *sb, uint32_t output_flags, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, const char *desc) { int need_space = 0; uint32_t len; uint32_t id; /* * We can't do anything if we don't have enough valid data for the * header. */ if (valid_len < sizeof(*hdr)) return; id = scsi_2btoul(hdr->id); /* * Note that we print out the value of the attribute listed in the * header, regardless of whether we actually got that many bytes * back from the device through the controller. A truncated result * could be the result of a failure to ask for enough data; the * header indicates how many bytes are allocated for this attribute * in the MAM. */ len = scsi_2btoul(hdr->length); if ((output_flags & SCSI_ATTR_OUTPUT_FIELD_MASK) == SCSI_ATTR_OUTPUT_FIELD_NONE) return; if ((output_flags & SCSI_ATTR_OUTPUT_FIELD_DESC) && (desc != NULL)) { sbuf_printf(sb, "%s", desc); need_space = 1; } if (output_flags & SCSI_ATTR_OUTPUT_FIELD_NUM) { sbuf_printf(sb, "%s(0x%.4x)", (need_space) ? " " : "", id); need_space = 0; } if (output_flags & SCSI_ATTR_OUTPUT_FIELD_SIZE) { sbuf_printf(sb, "%s[%d]", (need_space) ? " " : "", len); need_space = 0; } if (output_flags & SCSI_ATTR_OUTPUT_FIELD_RW) { sbuf_printf(sb, "%s(%s)", (need_space) ? " " : "", (hdr->byte2 & SMA_READ_ONLY) ? "RO" : "RW"); } sbuf_printf(sb, ": "); } int scsi_attrib_sbuf(struct sbuf *sb, struct scsi_mam_attribute_header *hdr, uint32_t valid_len, struct scsi_attrib_table_entry *user_table, size_t num_user_entries, int prefer_user_table, uint32_t output_flags, char *error_str, int error_str_len) { int retval; struct scsi_attrib_table_entry *table1 = NULL, *table2 = NULL; struct scsi_attrib_table_entry *entry = NULL; size_t table1_size = 0, table2_size = 0; uint32_t id; retval = 0; if (valid_len < sizeof(*hdr)) { retval = 1; goto bailout; } id = scsi_2btoul(hdr->id); if (user_table != NULL) { if (prefer_user_table != 0) { table1 = user_table; table1_size = num_user_entries; table2 = scsi_mam_attr_table; table2_size = sizeof(scsi_mam_attr_table) / sizeof(scsi_mam_attr_table[0]); } else { table1 = scsi_mam_attr_table; table1_size = sizeof(scsi_mam_attr_table) / sizeof(scsi_mam_attr_table[0]); table2 = user_table; table2_size = num_user_entries; } } else { table1 = scsi_mam_attr_table; table1_size = sizeof(scsi_mam_attr_table) / sizeof(scsi_mam_attr_table[0]); } entry = scsi_find_attrib_entry(table1, table1_size, id); if (entry != NULL) { scsi_attrib_prefix_sbuf(sb, output_flags, hdr, valid_len, entry->desc); if (entry->to_str == NULL) goto print_default; retval = entry->to_str(sb, hdr, valid_len, entry->flags, output_flags, error_str, error_str_len); goto bailout; } if (table2 != NULL) { entry = scsi_find_attrib_entry(table2, table2_size, id); if (entry != NULL) { if (entry->to_str == NULL) goto print_default; scsi_attrib_prefix_sbuf(sb, output_flags, hdr, valid_len, entry->desc); retval = entry->to_str(sb, hdr, valid_len, entry->flags, output_flags, error_str, error_str_len); goto bailout; } } scsi_attrib_prefix_sbuf(sb, output_flags, hdr, valid_len, NULL); print_default: retval = scsi_attrib_value_sbuf(sb, valid_len, hdr, output_flags, error_str, error_str_len); bailout: if (retval == 0) { if ((entry != NULL) && (entry->suffix != NULL)) sbuf_printf(sb, " %s", entry->suffix); sbuf_trim(sb); sbuf_printf(sb, "\n"); } return (retval); } void scsi_test_unit_ready(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t sense_len, u_int32_t timeout) { struct scsi_test_unit_ready *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, CAM_DIR_NONE, tag_action, /*data_ptr*/NULL, /*dxfer_len*/0, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_test_unit_ready *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = TEST_UNIT_READY; } void scsi_request_sense(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), void *data_ptr, u_int8_t dxfer_len, u_int8_t tag_action, u_int8_t sense_len, u_int32_t timeout) { struct scsi_request_sense *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, CAM_DIR_IN, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_request_sense *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = REQUEST_SENSE; scsi_cmd->length = dxfer_len; } void scsi_inquiry(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t *inq_buf, u_int32_t inq_len, int evpd, u_int8_t page_code, u_int8_t sense_len, u_int32_t timeout) { struct scsi_inquiry *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/inq_buf, /*dxfer_len*/inq_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_inquiry *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = INQUIRY; if (evpd) { scsi_cmd->byte2 |= SI_EVPD; scsi_cmd->page_code = page_code; } scsi_ulto2b(inq_len, scsi_cmd->length); } void scsi_mode_sense(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int dbd, u_int8_t page_code, u_int8_t page, u_int8_t *param_buf, u_int32_t param_len, u_int8_t sense_len, u_int32_t timeout) { scsi_mode_sense_len(csio, retries, cbfcnp, tag_action, dbd, page_code, page, param_buf, param_len, 0, sense_len, timeout); } void scsi_mode_sense_len(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int dbd, u_int8_t page_code, u_int8_t page, u_int8_t *param_buf, u_int32_t param_len, int minimum_cmd_size, u_int8_t sense_len, u_int32_t timeout) { u_int8_t cdb_len; /* * Use the smallest possible command to perform the operation. */ if ((param_len < 256) && (minimum_cmd_size < 10)) { /* * We can fit in a 6 byte cdb. */ struct scsi_mode_sense_6 *scsi_cmd; scsi_cmd = (struct scsi_mode_sense_6 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MODE_SENSE_6; if (dbd != 0) scsi_cmd->byte2 |= SMS_DBD; scsi_cmd->page = page_code | page; scsi_cmd->length = param_len; cdb_len = sizeof(*scsi_cmd); } else { /* * Need a 10 byte cdb. */ struct scsi_mode_sense_10 *scsi_cmd; scsi_cmd = (struct scsi_mode_sense_10 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MODE_SENSE_10; if (dbd != 0) scsi_cmd->byte2 |= SMS_DBD; scsi_cmd->page = page_code | page; scsi_ulto2b(param_len, scsi_cmd->length); cdb_len = sizeof(*scsi_cmd); } cam_fill_csio(csio, retries, cbfcnp, CAM_DIR_IN, tag_action, param_buf, param_len, sense_len, cdb_len, timeout); } void scsi_mode_select(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int scsi_page_fmt, int save_pages, u_int8_t *param_buf, u_int32_t param_len, u_int8_t sense_len, u_int32_t timeout) { scsi_mode_select_len(csio, retries, cbfcnp, tag_action, scsi_page_fmt, save_pages, param_buf, param_len, 0, sense_len, timeout); } void scsi_mode_select_len(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int scsi_page_fmt, int save_pages, u_int8_t *param_buf, u_int32_t param_len, int minimum_cmd_size, u_int8_t sense_len, u_int32_t timeout) { u_int8_t cdb_len; /* * Use the smallest possible command to perform the operation. */ if ((param_len < 256) && (minimum_cmd_size < 10)) { /* * We can fit in a 6 byte cdb. */ struct scsi_mode_select_6 *scsi_cmd; scsi_cmd = (struct scsi_mode_select_6 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MODE_SELECT_6; if (scsi_page_fmt != 0) scsi_cmd->byte2 |= SMS_PF; if (save_pages != 0) scsi_cmd->byte2 |= SMS_SP; scsi_cmd->length = param_len; cdb_len = sizeof(*scsi_cmd); } else { /* * Need a 10 byte cdb. */ struct scsi_mode_select_10 *scsi_cmd; scsi_cmd = (struct scsi_mode_select_10 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MODE_SELECT_10; if (scsi_page_fmt != 0) scsi_cmd->byte2 |= SMS_PF; if (save_pages != 0) scsi_cmd->byte2 |= SMS_SP; scsi_ulto2b(param_len, scsi_cmd->length); cdb_len = sizeof(*scsi_cmd); } cam_fill_csio(csio, retries, cbfcnp, CAM_DIR_OUT, tag_action, param_buf, param_len, sense_len, cdb_len, timeout); } void scsi_log_sense(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t page_code, u_int8_t page, int save_pages, int ppc, u_int32_t paramptr, u_int8_t *param_buf, u_int32_t param_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_log_sense *scsi_cmd; u_int8_t cdb_len; scsi_cmd = (struct scsi_log_sense *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = LOG_SENSE; scsi_cmd->page = page_code | page; if (save_pages != 0) scsi_cmd->byte2 |= SLS_SP; if (ppc != 0) scsi_cmd->byte2 |= SLS_PPC; scsi_ulto2b(paramptr, scsi_cmd->paramptr); scsi_ulto2b(param_len, scsi_cmd->length); cdb_len = sizeof(*scsi_cmd); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/param_buf, /*dxfer_len*/param_len, sense_len, cdb_len, timeout); } void scsi_log_select(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t page_code, int save_pages, int pc_reset, u_int8_t *param_buf, u_int32_t param_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_log_select *scsi_cmd; u_int8_t cdb_len; scsi_cmd = (struct scsi_log_select *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = LOG_SELECT; scsi_cmd->page = page_code & SLS_PAGE_CODE; if (save_pages != 0) scsi_cmd->byte2 |= SLS_SP; if (pc_reset != 0) scsi_cmd->byte2 |= SLS_PCR; scsi_ulto2b(param_len, scsi_cmd->length); cdb_len = sizeof(*scsi_cmd); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, /*data_ptr*/param_buf, /*dxfer_len*/param_len, sense_len, cdb_len, timeout); } /* * Prevent or allow the user to remove the media */ void scsi_prevent(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t action, u_int8_t sense_len, u_int32_t timeout) { struct scsi_prevent *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_NONE, tag_action, /*data_ptr*/NULL, /*dxfer_len*/0, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_prevent *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = PREVENT_ALLOW; scsi_cmd->how = action; } /* XXX allow specification of address and PMI bit and LBA */ void scsi_read_capacity(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, struct scsi_read_capacity_data *rcap_buf, u_int8_t sense_len, u_int32_t timeout) { struct scsi_read_capacity *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/(u_int8_t *)rcap_buf, /*dxfer_len*/sizeof(*rcap_buf), sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_read_capacity *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = READ_CAPACITY; } void scsi_read_capacity_16(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint64_t lba, int reladr, int pmi, uint8_t *rcap_buf, int rcap_buf_len, uint8_t sense_len, uint32_t timeout) { struct scsi_read_capacity_16 *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/(u_int8_t *)rcap_buf, /*dxfer_len*/rcap_buf_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = SERVICE_ACTION_IN; scsi_cmd->service_action = SRC16_SERVICE_ACTION; scsi_u64to8b(lba, scsi_cmd->addr); scsi_ulto4b(rcap_buf_len, scsi_cmd->alloc_len); if (pmi) reladr |= SRC16_PMI; if (reladr) reladr |= SRC16_RELADR; } void scsi_report_luns(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t select_report, struct scsi_report_luns_data *rpl_buf, u_int32_t alloc_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_report_luns *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/(u_int8_t *)rpl_buf, /*dxfer_len*/alloc_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_report_luns *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = REPORT_LUNS; scsi_cmd->select_report = select_report; scsi_ulto4b(alloc_len, scsi_cmd->length); } void scsi_report_target_group(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t pdf, void *buf, u_int32_t alloc_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_target_group *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/(u_int8_t *)buf, /*dxfer_len*/alloc_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_target_group *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MAINTENANCE_IN; scsi_cmd->service_action = REPORT_TARGET_PORT_GROUPS | pdf; scsi_ulto4b(alloc_len, scsi_cmd->length); } void scsi_set_target_group(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, void *buf, u_int32_t alloc_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_target_group *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, /*data_ptr*/(u_int8_t *)buf, /*dxfer_len*/alloc_len, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_target_group *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MAINTENANCE_OUT; scsi_cmd->service_action = SET_TARGET_PORT_GROUPS; scsi_ulto4b(alloc_len, scsi_cmd->length); } /* * Syncronize the media to the contents of the cache for * the given lba/count pair. Specifying 0/0 means sync * the whole cache. */ void scsi_synchronize_cache(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int32_t begin_lba, u_int16_t lb_count, u_int8_t sense_len, u_int32_t timeout) { struct scsi_sync_cache *scsi_cmd; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_NONE, tag_action, /*data_ptr*/NULL, /*dxfer_len*/0, sense_len, sizeof(*scsi_cmd), timeout); scsi_cmd = (struct scsi_sync_cache *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = SYNCHRONIZE_CACHE; scsi_ulto4b(begin_lba, scsi_cmd->begin_lba); scsi_ulto2b(lb_count, scsi_cmd->lb_count); } void scsi_read_write(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int readop, u_int8_t byte2, int minimum_cmd_size, u_int64_t lba, u_int32_t block_count, u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { int read; u_int8_t cdb_len; read = (readop & SCSI_RW_DIRMASK) == SCSI_RW_READ; /* * Use the smallest possible command to perform the operation * as some legacy hardware does not support the 10 byte commands. * If any of the bits in byte2 is set, we have to go with a larger * command. */ if ((minimum_cmd_size < 10) && ((lba & 0x1fffff) == lba) && ((block_count & 0xff) == block_count) && (byte2 == 0)) { /* * We can fit in a 6 byte cdb. */ struct scsi_rw_6 *scsi_cmd; scsi_cmd = (struct scsi_rw_6 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = read ? READ_6 : WRITE_6; scsi_ulto3b(lba, scsi_cmd->addr); scsi_cmd->length = block_count & 0xff; scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); CAM_DEBUG(csio->ccb_h.path, CAM_DEBUG_SUBTRACE, ("6byte: %x%x%x:%d:%d\n", scsi_cmd->addr[0], scsi_cmd->addr[1], scsi_cmd->addr[2], scsi_cmd->length, dxfer_len)); } else if ((minimum_cmd_size < 12) && ((block_count & 0xffff) == block_count) && ((lba & 0xffffffff) == lba)) { /* * Need a 10 byte cdb. */ struct scsi_rw_10 *scsi_cmd; scsi_cmd = (struct scsi_rw_10 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = read ? READ_10 : WRITE_10; scsi_cmd->byte2 = byte2; scsi_ulto4b(lba, scsi_cmd->addr); scsi_cmd->reserved = 0; scsi_ulto2b(block_count, scsi_cmd->length); scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); CAM_DEBUG(csio->ccb_h.path, CAM_DEBUG_SUBTRACE, ("10byte: %x%x%x%x:%x%x: %d\n", scsi_cmd->addr[0], scsi_cmd->addr[1], scsi_cmd->addr[2], scsi_cmd->addr[3], scsi_cmd->length[0], scsi_cmd->length[1], dxfer_len)); } else if ((minimum_cmd_size < 16) && ((block_count & 0xffffffff) == block_count) && ((lba & 0xffffffff) == lba)) { /* * The block count is too big for a 10 byte CDB, use a 12 * byte CDB. */ struct scsi_rw_12 *scsi_cmd; scsi_cmd = (struct scsi_rw_12 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = read ? READ_12 : WRITE_12; scsi_cmd->byte2 = byte2; scsi_ulto4b(lba, scsi_cmd->addr); scsi_cmd->reserved = 0; scsi_ulto4b(block_count, scsi_cmd->length); scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); CAM_DEBUG(csio->ccb_h.path, CAM_DEBUG_SUBTRACE, ("12byte: %x%x%x%x:%x%x%x%x: %d\n", scsi_cmd->addr[0], scsi_cmd->addr[1], scsi_cmd->addr[2], scsi_cmd->addr[3], scsi_cmd->length[0], scsi_cmd->length[1], scsi_cmd->length[2], scsi_cmd->length[3], dxfer_len)); } else { /* * 16 byte CDB. We'll only get here if the LBA is larger * than 2^32, or if the user asks for a 16 byte command. */ struct scsi_rw_16 *scsi_cmd; scsi_cmd = (struct scsi_rw_16 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = read ? READ_16 : WRITE_16; scsi_cmd->byte2 = byte2; scsi_u64to8b(lba, scsi_cmd->addr); scsi_cmd->reserved = 0; scsi_ulto4b(block_count, scsi_cmd->length); scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); } cam_fill_csio(csio, retries, cbfcnp, (read ? CAM_DIR_IN : CAM_DIR_OUT) | ((readop & SCSI_RW_BIO) != 0 ? CAM_DATA_BIO : 0), tag_action, data_ptr, dxfer_len, sense_len, cdb_len, timeout); } void scsi_write_same(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t byte2, int minimum_cmd_size, u_int64_t lba, u_int32_t block_count, u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { u_int8_t cdb_len; if ((minimum_cmd_size < 16) && ((block_count & 0xffff) == block_count) && ((lba & 0xffffffff) == lba)) { /* * Need a 10 byte cdb. */ struct scsi_write_same_10 *scsi_cmd; scsi_cmd = (struct scsi_write_same_10 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = WRITE_SAME_10; scsi_cmd->byte2 = byte2; scsi_ulto4b(lba, scsi_cmd->addr); scsi_cmd->group = 0; scsi_ulto2b(block_count, scsi_cmd->length); scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); CAM_DEBUG(csio->ccb_h.path, CAM_DEBUG_SUBTRACE, ("10byte: %x%x%x%x:%x%x: %d\n", scsi_cmd->addr[0], scsi_cmd->addr[1], scsi_cmd->addr[2], scsi_cmd->addr[3], scsi_cmd->length[0], scsi_cmd->length[1], dxfer_len)); } else { /* * 16 byte CDB. We'll only get here if the LBA is larger * than 2^32, or if the user asks for a 16 byte command. */ struct scsi_write_same_16 *scsi_cmd; scsi_cmd = (struct scsi_write_same_16 *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = WRITE_SAME_16; scsi_cmd->byte2 = byte2; scsi_u64to8b(lba, scsi_cmd->addr); scsi_ulto4b(block_count, scsi_cmd->length); scsi_cmd->group = 0; scsi_cmd->control = 0; cdb_len = sizeof(*scsi_cmd); CAM_DEBUG(csio->ccb_h.path, CAM_DEBUG_SUBTRACE, ("16byte: %x%x%x%x%x%x%x%x:%x%x%x%x: %d\n", scsi_cmd->addr[0], scsi_cmd->addr[1], scsi_cmd->addr[2], scsi_cmd->addr[3], scsi_cmd->addr[4], scsi_cmd->addr[5], scsi_cmd->addr[6], scsi_cmd->addr[7], scsi_cmd->length[0], scsi_cmd->length[1], scsi_cmd->length[2], scsi_cmd->length[3], dxfer_len)); } cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, data_ptr, dxfer_len, sense_len, cdb_len, timeout); } void scsi_ata_identify(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t *data_ptr, u_int16_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { scsi_ata_pass_16(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*protocol*/AP_PROTO_PIO_IN, /*ata_flags*/AP_FLAG_TDIR_FROM_DEV| AP_FLAG_BYT_BLOK_BYTES|AP_FLAG_TLEN_SECT_CNT, /*features*/0, /*sector_count*/dxfer_len, /*lba*/0, /*command*/ATA_ATA_IDENTIFY, /*control*/0, data_ptr, dxfer_len, sense_len, timeout); } void scsi_ata_trim(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int16_t block_count, u_int8_t *data_ptr, u_int16_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { scsi_ata_pass_16(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, /*protocol*/AP_EXTEND|AP_PROTO_DMA, /*ata_flags*/AP_FLAG_TLEN_SECT_CNT|AP_FLAG_BYT_BLOK_BLOCKS, /*features*/ATA_DSM_TRIM, /*sector_count*/block_count, /*lba*/0, /*command*/ATA_DATA_SET_MANAGEMENT, /*control*/0, data_ptr, dxfer_len, sense_len, timeout); } void scsi_ata_pass_16(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int32_t flags, u_int8_t tag_action, u_int8_t protocol, u_int8_t ata_flags, u_int16_t features, u_int16_t sector_count, uint64_t lba, u_int8_t command, u_int8_t control, u_int8_t *data_ptr, u_int16_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { struct ata_pass_16 *ata_cmd; ata_cmd = (struct ata_pass_16 *)&csio->cdb_io.cdb_bytes; ata_cmd->opcode = ATA_PASS_16; ata_cmd->protocol = protocol; ata_cmd->flags = ata_flags; ata_cmd->features_ext = features >> 8; ata_cmd->features = features; ata_cmd->sector_count_ext = sector_count >> 8; ata_cmd->sector_count = sector_count; ata_cmd->lba_low = lba; ata_cmd->lba_mid = lba >> 8; ata_cmd->lba_high = lba >> 16; ata_cmd->device = ATA_DEV_LBA; if (protocol & AP_EXTEND) { ata_cmd->lba_low_ext = lba >> 24; ata_cmd->lba_mid_ext = lba >> 32; ata_cmd->lba_high_ext = lba >> 40; } else ata_cmd->device |= (lba >> 24) & 0x0f; ata_cmd->command = command; ata_cmd->control = control; cam_fill_csio(csio, retries, cbfcnp, flags, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*ata_cmd), timeout); } void scsi_unmap(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t byte2, u_int8_t *data_ptr, u_int16_t dxfer_len, u_int8_t sense_len, u_int32_t timeout) { struct scsi_unmap *scsi_cmd; scsi_cmd = (struct scsi_unmap *)&csio->cdb_io.cdb_bytes; scsi_cmd->opcode = UNMAP; scsi_cmd->byte2 = byte2; scsi_ulto4b(0, scsi_cmd->reserved); scsi_cmd->group = 0; scsi_ulto2b(dxfer_len, scsi_cmd->length); scsi_cmd->control = 0; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_receive_diagnostic_results(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb*), uint8_t tag_action, int pcv, uint8_t page_code, uint8_t *data_ptr, uint16_t allocation_length, uint8_t sense_len, uint32_t timeout) { struct scsi_receive_diag *scsi_cmd; scsi_cmd = (struct scsi_receive_diag *)&csio->cdb_io.cdb_bytes; memset(scsi_cmd, 0, sizeof(*scsi_cmd)); scsi_cmd->opcode = RECEIVE_DIAGNOSTIC; if (pcv) { scsi_cmd->byte2 |= SRD_PCV; scsi_cmd->page_code = page_code; } scsi_ulto2b(allocation_length, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, data_ptr, allocation_length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_send_diagnostic(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int unit_offline, int device_offline, int self_test, int page_format, int self_test_code, uint8_t *data_ptr, uint16_t param_list_length, uint8_t sense_len, uint32_t timeout) { struct scsi_send_diag *scsi_cmd; scsi_cmd = (struct scsi_send_diag *)&csio->cdb_io.cdb_bytes; memset(scsi_cmd, 0, sizeof(*scsi_cmd)); scsi_cmd->opcode = SEND_DIAGNOSTIC; /* * The default self-test mode control and specific test * control are mutually exclusive. */ if (self_test) self_test_code = SSD_SELF_TEST_CODE_NONE; scsi_cmd->byte2 = ((self_test_code << SSD_SELF_TEST_CODE_SHIFT) & SSD_SELF_TEST_CODE_MASK) | (unit_offline ? SSD_UNITOFFL : 0) | (device_offline ? SSD_DEVOFFL : 0) | (self_test ? SSD_SELFTEST : 0) | (page_format ? SSD_PF : 0); scsi_ulto2b(param_list_length, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/param_list_length ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, data_ptr, param_list_length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_read_buffer(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb*), uint8_t tag_action, int mode, uint8_t buffer_id, u_int32_t offset, uint8_t *data_ptr, uint32_t allocation_length, uint8_t sense_len, uint32_t timeout) { struct scsi_read_buffer *scsi_cmd; scsi_cmd = (struct scsi_read_buffer *)&csio->cdb_io.cdb_bytes; memset(scsi_cmd, 0, sizeof(*scsi_cmd)); scsi_cmd->opcode = READ_BUFFER; scsi_cmd->byte2 = mode; scsi_cmd->buffer_id = buffer_id; scsi_ulto3b(offset, scsi_cmd->offset); scsi_ulto3b(allocation_length, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, data_ptr, allocation_length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_write_buffer(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int mode, uint8_t buffer_id, u_int32_t offset, uint8_t *data_ptr, uint32_t param_list_length, uint8_t sense_len, uint32_t timeout) { struct scsi_write_buffer *scsi_cmd; scsi_cmd = (struct scsi_write_buffer *)&csio->cdb_io.cdb_bytes; memset(scsi_cmd, 0, sizeof(*scsi_cmd)); scsi_cmd->opcode = WRITE_BUFFER; scsi_cmd->byte2 = mode; scsi_cmd->buffer_id = buffer_id; scsi_ulto3b(offset, scsi_cmd->offset); scsi_ulto3b(param_list_length, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/param_list_length ? CAM_DIR_OUT : CAM_DIR_NONE, tag_action, data_ptr, param_list_length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_start_stop(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, int start, int load_eject, int immediate, u_int8_t sense_len, u_int32_t timeout) { struct scsi_start_stop_unit *scsi_cmd; int extra_flags = 0; scsi_cmd = (struct scsi_start_stop_unit *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = START_STOP_UNIT; if (start != 0) { scsi_cmd->how |= SSS_START; /* it takes a lot of power to start a drive */ extra_flags |= CAM_HIGH_POWER; } if (load_eject != 0) scsi_cmd->how |= SSS_LOEJ; if (immediate != 0) scsi_cmd->byte2 |= SSS_IMMED; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_NONE | extra_flags, tag_action, /*data_ptr*/NULL, /*dxfer_len*/0, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_read_attribute(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, u_int8_t service_action, uint32_t element, u_int8_t elem_type, int logical_volume, int partition, u_int32_t first_attribute, int cache, u_int8_t *data_ptr, u_int32_t length, int sense_len, u_int32_t timeout) { struct scsi_read_attribute *scsi_cmd; scsi_cmd = (struct scsi_read_attribute *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = READ_ATTRIBUTE; scsi_cmd->service_action = service_action, scsi_ulto2b(element, scsi_cmd->element); scsi_cmd->elem_type = elem_type; scsi_cmd->logical_volume = logical_volume; scsi_cmd->partition = partition; scsi_ulto2b(first_attribute, scsi_cmd->first_attribute); scsi_ulto4b(length, scsi_cmd->length); if (cache != 0) scsi_cmd->cache |= SRA_CACHE; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, /*data_ptr*/data_ptr, /*dxfer_len*/length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_write_attribute(struct ccb_scsiio *csio, u_int32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), u_int8_t tag_action, uint32_t element, int logical_volume, int partition, int wtc, u_int8_t *data_ptr, u_int32_t length, int sense_len, u_int32_t timeout) { struct scsi_write_attribute *scsi_cmd; scsi_cmd = (struct scsi_write_attribute *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = WRITE_ATTRIBUTE; if (wtc != 0) scsi_cmd->byte2 = SWA_WTC; scsi_ulto3b(element, scsi_cmd->element); scsi_cmd->logical_volume = logical_volume; scsi_cmd->partition = partition; scsi_ulto4b(length, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, /*data_ptr*/data_ptr, /*dxfer_len*/length, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_persistent_reserve_in(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int service_action, uint8_t *data_ptr, uint32_t dxfer_len, int sense_len, int timeout) { struct scsi_per_res_in *scsi_cmd; scsi_cmd = (struct scsi_per_res_in *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = PERSISTENT_RES_IN; scsi_cmd->action = service_action; scsi_ulto2b(dxfer_len, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_persistent_reserve_out(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int service_action, int scope, int res_type, uint8_t *data_ptr, uint32_t dxfer_len, int sense_len, int timeout) { struct scsi_per_res_out *scsi_cmd; scsi_cmd = (struct scsi_per_res_out *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = PERSISTENT_RES_OUT; scsi_cmd->action = service_action; scsi_cmd->scope_type = scope | res_type; cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, /*data_ptr*/data_ptr, /*dxfer_len*/dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_security_protocol_in(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint32_t security_protocol, uint32_t security_protocol_specific, int byte4, uint8_t *data_ptr, uint32_t dxfer_len, int sense_len, int timeout) { struct scsi_security_protocol_in *scsi_cmd; scsi_cmd = (struct scsi_security_protocol_in *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = SECURITY_PROTOCOL_IN; scsi_cmd->security_protocol = security_protocol; scsi_ulto2b(security_protocol_specific, scsi_cmd->security_protocol_specific); scsi_cmd->byte4 = byte4; scsi_ulto4b(dxfer_len, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_security_protocol_out(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, uint32_t security_protocol, uint32_t security_protocol_specific, int byte4, uint8_t *data_ptr, uint32_t dxfer_len, int sense_len, int timeout) { struct scsi_security_protocol_out *scsi_cmd; scsi_cmd = (struct scsi_security_protocol_out *)&csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = SECURITY_PROTOCOL_OUT; scsi_cmd->security_protocol = security_protocol; scsi_ulto2b(security_protocol_specific, scsi_cmd->security_protocol_specific); scsi_cmd->byte4 = byte4; scsi_ulto4b(dxfer_len, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_OUT, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } void scsi_report_supported_opcodes(struct ccb_scsiio *csio, uint32_t retries, void (*cbfcnp)(struct cam_periph *, union ccb *), uint8_t tag_action, int options, int req_opcode, int req_service_action, uint8_t *data_ptr, uint32_t dxfer_len, int sense_len, int timeout) { struct scsi_report_supported_opcodes *scsi_cmd; scsi_cmd = (struct scsi_report_supported_opcodes *) &csio->cdb_io.cdb_bytes; bzero(scsi_cmd, sizeof(*scsi_cmd)); scsi_cmd->opcode = MAINTENANCE_IN; scsi_cmd->service_action = REPORT_SUPPORTED_OPERATION_CODES; scsi_cmd->options = options; scsi_cmd->requested_opcode = req_opcode; scsi_ulto2b(req_service_action, scsi_cmd->requested_service_action); scsi_ulto4b(dxfer_len, scsi_cmd->length); cam_fill_csio(csio, retries, cbfcnp, /*flags*/CAM_DIR_IN, tag_action, data_ptr, dxfer_len, sense_len, sizeof(*scsi_cmd), timeout); } /* * Try make as good a match as possible with * available sub drivers */ int scsi_inquiry_match(caddr_t inqbuffer, caddr_t table_entry) { struct scsi_inquiry_pattern *entry; struct scsi_inquiry_data *inq; entry = (struct scsi_inquiry_pattern *)table_entry; inq = (struct scsi_inquiry_data *)inqbuffer; if (((SID_TYPE(inq) == entry->type) || (entry->type == T_ANY)) && (SID_IS_REMOVABLE(inq) ? entry->media_type & SIP_MEDIA_REMOVABLE : entry->media_type & SIP_MEDIA_FIXED) && (cam_strmatch(inq->vendor, entry->vendor, sizeof(inq->vendor)) == 0) && (cam_strmatch(inq->product, entry->product, sizeof(inq->product)) == 0) && (cam_strmatch(inq->revision, entry->revision, sizeof(inq->revision)) == 0)) { return (0); } return (-1); } /* * Try make as good a match as possible with * available sub drivers */ int scsi_static_inquiry_match(caddr_t inqbuffer, caddr_t table_entry) { struct scsi_static_inquiry_pattern *entry; struct scsi_inquiry_data *inq; entry = (struct scsi_static_inquiry_pattern *)table_entry; inq = (struct scsi_inquiry_data *)inqbuffer; if (((SID_TYPE(inq) == entry->type) || (entry->type == T_ANY)) && (SID_IS_REMOVABLE(inq) ? entry->media_type & SIP_MEDIA_REMOVABLE : entry->media_type & SIP_MEDIA_FIXED) && (cam_strmatch(inq->vendor, entry->vendor, sizeof(inq->vendor)) == 0) && (cam_strmatch(inq->product, entry->product, sizeof(inq->product)) == 0) && (cam_strmatch(inq->revision, entry->revision, sizeof(inq->revision)) == 0)) { return (0); } return (-1); } /** * Compare two buffers of vpd device descriptors for a match. * * \param lhs Pointer to first buffer of descriptors to compare. * \param lhs_len The length of the first buffer. * \param rhs Pointer to second buffer of descriptors to compare. * \param rhs_len The length of the second buffer. * * \return 0 on a match, -1 otherwise. * * Treat rhs and lhs as arrays of vpd device id descriptors. Walk lhs matching * agains each element in rhs until all data are exhausted or we have found * a match. */ int scsi_devid_match(uint8_t *lhs, size_t lhs_len, uint8_t *rhs, size_t rhs_len) { struct scsi_vpd_id_descriptor *lhs_id; struct scsi_vpd_id_descriptor *lhs_last; struct scsi_vpd_id_descriptor *rhs_last; uint8_t *lhs_end; uint8_t *rhs_end; lhs_end = lhs + lhs_len; rhs_end = rhs + rhs_len; /* * rhs_last and lhs_last are the last posible position of a valid * descriptor assuming it had a zero length identifier. We use * these variables to insure we can safely dereference the length * field in our loop termination tests. */ lhs_last = (struct scsi_vpd_id_descriptor *) (lhs_end - __offsetof(struct scsi_vpd_id_descriptor, identifier)); rhs_last = (struct scsi_vpd_id_descriptor *) (rhs_end - __offsetof(struct scsi_vpd_id_descriptor, identifier)); lhs_id = (struct scsi_vpd_id_descriptor *)lhs; while (lhs_id <= lhs_last && (lhs_id->identifier + lhs_id->length) <= lhs_end) { struct scsi_vpd_id_descriptor *rhs_id; rhs_id = (struct scsi_vpd_id_descriptor *)rhs; while (rhs_id <= rhs_last && (rhs_id->identifier + rhs_id->length) <= rhs_end) { if ((rhs_id->id_type & (SVPD_ID_ASSOC_MASK | SVPD_ID_TYPE_MASK)) == (lhs_id->id_type & (SVPD_ID_ASSOC_MASK | SVPD_ID_TYPE_MASK)) && rhs_id->length == lhs_id->length && memcmp(rhs_id->identifier, lhs_id->identifier, rhs_id->length) == 0) return (0); rhs_id = (struct scsi_vpd_id_descriptor *) (rhs_id->identifier + rhs_id->length); } lhs_id = (struct scsi_vpd_id_descriptor *) (lhs_id->identifier + lhs_id->length); } return (-1); } #ifdef _KERNEL int scsi_vpd_supported_page(struct cam_periph *periph, uint8_t page_id) { struct cam_ed *device; struct scsi_vpd_supported_pages *vpds; int i, num_pages; device = periph->path->device; vpds = (struct scsi_vpd_supported_pages *)device->supported_vpds; if (vpds != NULL) { num_pages = device->supported_vpds_len - SVPD_SUPPORTED_PAGES_HDR_LEN; for (i = 0; i < num_pages; i++) { if (vpds->page_list[i] == page_id) return (1); } } return (0); } static void init_scsi_delay(void) { int delay; delay = SCSI_DELAY; TUNABLE_INT_FETCH("kern.cam.scsi_delay", &delay); if (set_scsi_delay(delay) != 0) { printf("cam: invalid value for tunable kern.cam.scsi_delay\n"); set_scsi_delay(SCSI_DELAY); } } SYSINIT(scsi_delay, SI_SUB_TUNABLES, SI_ORDER_ANY, init_scsi_delay, NULL); static int sysctl_scsi_delay(SYSCTL_HANDLER_ARGS) { int error, delay; delay = scsi_delay; error = sysctl_handle_int(oidp, &delay, 0, req); if (error != 0 || req->newptr == NULL) return (error); return (set_scsi_delay(delay)); } SYSCTL_PROC(_kern_cam, OID_AUTO, scsi_delay, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_scsi_delay, "I", "Delay to allow devices to settle after a SCSI bus reset (ms)"); static int set_scsi_delay(int delay) { /* * If someone sets this to 0, we assume that they want the * minimum allowable bus settle delay. */ if (delay == 0) { printf("cam: using minimum scsi_delay (%dms)\n", SCSI_MIN_DELAY); delay = SCSI_MIN_DELAY; } if (delay < SCSI_MIN_DELAY) return (EINVAL); scsi_delay = delay; return (0); } #endif /* _KERNEL */ Index: head/sys/dev/advansys/adw_pci.c =================================================================== --- head/sys/dev/advansys/adw_pci.c (revision 298410) +++ head/sys/dev/advansys/adw_pci.c (revision 298411) @@ -1,396 +1,393 @@ /*- * Device probe and attach routines for the following * Advanced Systems Inc. SCSI controllers: * * ABP[3]940UW - Bus-Master PCI Ultra-Wide (253 CDB) * ABP950UW - Dual Channel Bus-Master PCI Ultra-Wide (253 CDB/Channel) * ABP970UW - Bus-Master PCI Ultra-Wide (253 CDB) * ABP3940U2W - Bus-Master PCI LVD/Ultra2-Wide (253 CDB) * ABP3950U2W - Bus-Master PCI LVD/Ultra2-Wide (253 CDB) * * Copyright (c) 1998, 1999, 2000 Justin Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification. * 2. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define ADW_PCI_IOBASE PCIR_BAR(0) /* I/O Address */ #define ADW_PCI_MEMBASE PCIR_BAR(1) /* Mem I/O Address */ #define PCI_ID_ADVANSYS_3550 0x230010CD00000000ull #define PCI_ID_ADVANSYS_38C0800_REV1 0x250010CD00000000ull #define PCI_ID_ADVANSYS_38C1600_REV1 0x270010CD00000000ull #define PCI_ID_ALL_MASK 0xFFFFFFFFFFFFFFFFull #define PCI_ID_DEV_VENDOR_MASK 0xFFFFFFFF00000000ull struct adw_pci_identity; typedef int (adw_device_setup_t)(device_t, struct adw_pci_identity *, struct adw_softc *adw); struct adw_pci_identity { u_int64_t full_id; u_int64_t id_mask; char *name; adw_device_setup_t *setup; const struct adw_mcode *mcode_data; const struct adw_eeprom *default_eeprom; }; static adw_device_setup_t adw_asc3550_setup; static adw_device_setup_t adw_asc38C0800_setup; #ifdef NOTYET static adw_device_setup_t adw_asc38C1600_setup; #endif struct adw_pci_identity adw_pci_ident_table[] = { /* asc3550 based controllers */ { PCI_ID_ADVANSYS_3550, PCI_ID_DEV_VENDOR_MASK, "AdvanSys 3550 Ultra SCSI Adapter", adw_asc3550_setup, &adw_asc3550_mcode_data, &adw_asc3550_default_eeprom }, /* asc38C0800 based controllers */ { PCI_ID_ADVANSYS_38C0800_REV1, PCI_ID_DEV_VENDOR_MASK, "AdvanSys 38C0800 Ultra2 SCSI Adapter", adw_asc38C0800_setup, &adw_asc38C0800_mcode_data, &adw_asc38C0800_default_eeprom }, #ifdef NOTYET /* XXX Disabled until I have hardware to test with */ /* asc38C1600 based controllers */ { PCI_ID_ADVANSYS_38C1600_REV1, PCI_ID_DEV_VENDOR_MASK, "AdvanSys 38C1600 Ultra160 SCSI Adapter", adw_asc38C1600_setup, NULL, /* None provided by vendor thus far */ NULL /* None provided by vendor thus far */ } #endif }; -static const int adw_num_pci_devs = - sizeof(adw_pci_ident_table) / sizeof(*adw_pci_ident_table); - #define ADW_PCI_MAX_DMA_ADDR (0xFFFFFFFFUL) #define ADW_PCI_MAX_DMA_COUNT (0xFFFFFFFFUL) static int adw_pci_probe(device_t dev); static int adw_pci_attach(device_t dev); static device_method_t adw_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, adw_pci_probe), DEVMETHOD(device_attach, adw_pci_attach), { 0, 0 } }; static driver_t adw_pci_driver = { "adw", adw_pci_methods, sizeof(struct adw_softc) }; static devclass_t adw_devclass; DRIVER_MODULE(adw, pci, adw_pci_driver, adw_devclass, 0, 0); MODULE_DEPEND(adw, pci, 1, 1, 1); static __inline u_int64_t adw_compose_id(u_int device, u_int vendor, u_int subdevice, u_int subvendor) { u_int64_t id; id = subvendor | (subdevice << 16) | ((u_int64_t)vendor << 32) | ((u_int64_t)device << 48); return (id); } static struct adw_pci_identity * adw_find_pci_device(device_t dev) { u_int64_t full_id; struct adw_pci_identity *entry; u_int i; full_id = adw_compose_id(pci_get_device(dev), pci_get_vendor(dev), pci_get_subdevice(dev), pci_get_subvendor(dev)); - for (i = 0; i < adw_num_pci_devs; i++) { + for (i = 0; i < nitems(adw_pci_ident_table); i++) { entry = &adw_pci_ident_table[i]; if (entry->full_id == (full_id & entry->id_mask)) return (entry); } return (NULL); } static int adw_pci_probe(device_t dev) { struct adw_pci_identity *entry; entry = adw_find_pci_device(dev); if (entry != NULL) { device_set_desc(dev, entry->name); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int adw_pci_attach(device_t dev) { struct adw_softc *adw; struct adw_pci_identity *entry; u_int16_t command; struct resource *regs; int regs_type; int regs_id; int error; int zero; entry = adw_find_pci_device(dev); if (entry == NULL) return (ENXIO); regs = NULL; regs_type = 0; regs_id = 0; #ifdef ADW_ALLOW_MEMIO regs_type = SYS_RES_MEMORY; regs_id = ADW_PCI_MEMBASE; regs = bus_alloc_resource_any(dev, regs_type, ®s_id, RF_ACTIVE); #endif if (regs == NULL) { regs_type = SYS_RES_IOPORT; regs_id = ADW_PCI_IOBASE; regs = bus_alloc_resource_any(dev, regs_type, ®s_id, RF_ACTIVE); } if (regs == NULL) { device_printf(dev, "can't allocate register resources\n"); return (ENOMEM); } adw = adw_alloc(dev, regs, regs_type, regs_id); if (adw == NULL) return(ENOMEM); /* * Now that we have access to our registers, just verify that * this really is an AdvanSys device. */ if (adw_find_signature(adw) == 0) { adw_free(adw); return (ENXIO); } adw_reset_chip(adw); error = entry->setup(dev, entry, adw); if (error != 0) return (error); /* Ensure busmastering is enabled */ pci_enable_busmaster(dev); /* Allocate a dmatag for our transfer DMA maps */ error = bus_dma_tag_create( /* parent */ bus_get_dma_tag(dev), /* alignment */ 1, /* boundary */ 0, /* lowaddr */ ADW_PCI_MAX_DMA_ADDR, /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ BUS_SPACE_MAXSIZE_32BIT, /* nsegments */ ~0, /* maxsegsz */ ADW_PCI_MAX_DMA_COUNT, /* flags */ 0, /* lockfunc */ NULL, /* lockarg */ NULL, &adw->parent_dmat); adw->init_level++; if (error != 0) { device_printf(dev, "Could not allocate DMA tag - error %d\n", error); adw_free(adw); return (error); } adw->init_level++; error = adw_init(adw); if (error != 0) { adw_free(adw); return (error); } /* * If the PCI Configuration Command Register "Parity Error Response * Control" Bit was clear (0), then set the microcode variable * 'control_flag' CONTROL_FLAG_IGNORE_PERR flag to tell the microcode * to ignore DMA parity errors. */ command = pci_read_config(dev, PCIR_COMMAND, /*bytes*/2); if ((command & PCIM_CMD_PERRESPEN) == 0) adw_lram_write_16(adw, ADW_MC_CONTROL_FLAG, adw_lram_read_16(adw, ADW_MC_CONTROL_FLAG) | ADW_MC_CONTROL_IGN_PERR); zero = 0; adw->irq_res_type = SYS_RES_IRQ; adw->irq = bus_alloc_resource_any(dev, adw->irq_res_type, &zero, RF_ACTIVE | RF_SHAREABLE); if (adw->irq == NULL) { adw_free(adw); return (ENOMEM); } error = adw_attach(adw); if (error != 0) adw_free(adw); return (error); } static int adw_generic_setup(device_t dev, struct adw_pci_identity *entry, struct adw_softc *adw) { adw->channel = pci_get_function(dev) == 1 ? 'B' : 'A'; adw->chip = ADW_CHIP_NONE; adw->features = ADW_FENONE; adw->flags = ADW_FNONE; adw->mcode_data = entry->mcode_data; adw->default_eeprom = entry->default_eeprom; return (0); } static int adw_asc3550_setup(device_t dev, struct adw_pci_identity *entry, struct adw_softc *adw) { int error; error = adw_generic_setup(dev, entry, adw); if (error != 0) return (error); adw->chip = ADW_CHIP_ASC3550; adw->features = ADW_ASC3550_FE; adw->memsize = ADW_3550_MEMSIZE; /* * For ASC-3550, setting the START_CTL_EMFU [3:2] bits * sets a FIFO threshold of 128 bytes. This register is * only accessible to the host. */ adw_outb(adw, ADW_DMA_CFG0, ADW_DMA_CFG0_START_CTL_EM_FU|ADW_DMA_CFG0_READ_CMD_MRM); adw_outb(adw, ADW_MEM_CFG, adw_inb(adw, ADW_MEM_CFG) | ADW_MEM_CFG_RAM_SZ_8KB); return (0); } static int adw_asc38C0800_setup(device_t dev, struct adw_pci_identity *entry, struct adw_softc *adw) { int error; error = adw_generic_setup(dev, entry, adw); if (error != 0) return (error); /* * For ASC-38C0800, set FIFO_THRESH_80B [6:4] bits and * START_CTL_TH [3:2] bits for the default FIFO threshold. * * Note: ASC-38C0800 FIFO threshold has been changed to 256 bytes. * * For DMA Errata #4 set the BC_THRESH_ENB bit. */ adw_outb(adw, ADW_DMA_CFG0, ADW_DMA_CFG0_BC_THRESH_ENB|ADW_DMA_CFG0_FIFO_THRESH_80B |ADW_DMA_CFG0_START_CTL_TH|ADW_DMA_CFG0_READ_CMD_MRM); adw_outb(adw, ADW_MEM_CFG, adw_inb(adw, ADW_MEM_CFG) | ADW_MEM_CFG_RAM_SZ_16KB); adw->chip = ADW_CHIP_ASC38C0800; adw->features = ADW_ASC38C0800_FE; adw->memsize = ADW_38C0800_MEMSIZE; return (error); } #ifdef NOTYET static int adw_asc38C1600_setup(device_t dev, struct adw_pci_identity *entry, struct adw_softc *adw) { int error; error = adw_generic_setup(dev, entry, adw); if (error != 0) return (error); adw->chip = ADW_CHIP_ASC38C1600; adw->features = ADW_ASC38C1600_FE; adw->memsize = ADW_38C1600_MEMSIZE; return (error); } #endif Index: head/sys/dev/advansys/adwlib.c =================================================================== --- head/sys/dev/advansys/adwlib.c (revision 298410) +++ head/sys/dev/advansys/adwlib.c (revision 298411) @@ -1,899 +1,897 @@ /*- * Low level routines for Second Generation * Advanced Systems Inc. SCSI controllers chips * * Copyright (c) 1998, 1999, 2000 Justin Gibbs. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Ported from: * advansys.c - Linux Host Driver for AdvanSys SCSI Adapters * * Copyright (c) 1995-1998 Advanced System Products, Inc. * All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that redistributions of source * code retain the above copyright notice and this comment without * modification. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include const struct adw_eeprom adw_asc3550_default_eeprom = { ADW_EEPROM_BIOS_ENABLE, /* cfg_lsw */ 0x0000, /* cfg_msw */ 0xFFFF, /* disc_enable */ 0xFFFF, /* wdtr_able */ { 0xFFFF }, /* sdtr_able */ 0xFFFF, /* start_motor */ 0xFFFF, /* tagqng_able */ 0xFFFF, /* bios_scan */ 0, /* scam_tolerant */ 7, /* adapter_scsi_id */ 0, /* bios_boot_delay */ 3, /* scsi_reset_delay */ 0, /* bios_id_lun */ 0, /* termination */ 0, /* reserved1 */ 0xFFE7, /* bios_ctrl */ { 0xFFFF }, /* ultra_able */ { 0 }, /* reserved2 */ ADW_DEF_MAX_HOST_QNG, /* max_host_qng */ ADW_DEF_MAX_DVC_QNG, /* max_dvc_qng */ 0, /* dvc_cntl */ { 0 }, /* bug_fix */ { 0, 0, 0 }, /* serial_number */ 0, /* check_sum */ { /* oem_name[16] */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, /* dvc_err_code */ 0, /* adv_err_code */ 0, /* adv_err_addr */ 0, /* saved_dvc_err_code */ 0, /* saved_adv_err_code */ 0 /* saved_adv_err_addr */ }; const struct adw_eeprom adw_asc38C0800_default_eeprom = { ADW_EEPROM_BIOS_ENABLE, /* 00 cfg_lsw */ 0x0000, /* 01 cfg_msw */ 0xFFFF, /* 02 disc_enable */ 0xFFFF, /* 03 wdtr_able */ { 0x4444 }, /* 04 sdtr_speed1 */ 0xFFFF, /* 05 start_motor */ 0xFFFF, /* 06 tagqng_able */ 0xFFFF, /* 07 bios_scan */ 0, /* 08 scam_tolerant */ 7, /* 09 adapter_scsi_id */ 0, /* bios_boot_delay */ 3, /* 10 scsi_reset_delay */ 0, /* bios_id_lun */ 0, /* 11 termination_se */ 0, /* termination_lvd */ 0xFFE7, /* 12 bios_ctrl */ { 0x4444 }, /* 13 sdtr_speed2 */ { 0x4444 }, /* 14 sdtr_speed3 */ ADW_DEF_MAX_HOST_QNG, /* 15 max_host_qng */ ADW_DEF_MAX_DVC_QNG, /* max_dvc_qng */ 0, /* 16 dvc_cntl */ { 0x4444 } , /* 17 sdtr_speed4 */ { 0, 0, 0 }, /* 18-20 serial_number */ 0, /* 21 check_sum */ { /* 22-29 oem_name[16] */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, /* 30 dvc_err_code */ 0, /* 31 adv_err_code */ 0, /* 32 adv_err_addr */ 0, /* 33 saved_dvc_err_code */ 0, /* 34 saved_adv_err_code */ 0, /* 35 saved_adv_err_addr */ { /* 36 - 55 reserved */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, 0, /* 56 cisptr_lsw */ 0, /* 57 cisprt_msw */ /* 58-59 sub-id */ (PCI_ID_ADVANSYS_38C0800_REV1 & PCI_ID_DEV_VENDOR_MASK) >> 32, }; #define ADW_MC_SDTR_OFFSET_ULTRA2_DT 0 #define ADW_MC_SDTR_OFFSET_ULTRA2 1 #define ADW_MC_SDTR_OFFSET_ULTRA 2 const struct adw_syncrate adw_syncrates[] = { /* mc_sdtr period rate */ { ADW_MC_SDTR_80, 9, "80.0" }, { ADW_MC_SDTR_40, 10, "40.0" }, { ADW_MC_SDTR_20, 12, "20.0" }, { ADW_MC_SDTR_10, 25, "10.0" }, { ADW_MC_SDTR_5, 50, "5.0" }, { ADW_MC_SDTR_ASYNC, 0, "async" } }; -const int adw_num_syncrates = sizeof(adw_syncrates) / sizeof(adw_syncrates[0]); - static u_int16_t adw_eeprom_read_16(struct adw_softc *adw, int addr); static void adw_eeprom_write_16(struct adw_softc *adw, int addr, u_int data); static void adw_eeprom_wait(struct adw_softc *adw); int adw_find_signature(struct adw_softc *adw) { if (adw_inb(adw, ADW_SIGNATURE_BYTE) == ADW_CHIP_ID_BYTE && adw_inw(adw, ADW_SIGNATURE_WORD) == ADW_CHIP_ID_WORD) return (1); return (0); } /* * Reset Chip. */ void adw_reset_chip(struct adw_softc *adw) { adw_outw(adw, ADW_CTRL_REG, ADW_CTRL_REG_CMD_RESET); DELAY(1000 * 100); adw_outw(adw, ADW_CTRL_REG, ADW_CTRL_REG_CMD_WR_IO_REG); /* * Initialize Chip registers. */ adw_outw(adw, ADW_SCSI_CFG1, adw_inw(adw, ADW_SCSI_CFG1) & ~ADW_SCSI_CFG1_BIG_ENDIAN); } /* * Reset the SCSI bus. */ int adw_reset_bus(struct adw_softc *adw) { adw_idle_cmd_status_t status; if (!dumping) mtx_assert(&adw->lock, MA_OWNED); status = adw_idle_cmd_send(adw, ADW_IDLE_CMD_SCSI_RESET_START, /*param*/0); if (status != ADW_IDLE_CMD_SUCCESS) { xpt_print_path(adw->path); printf("Bus Reset start attempt failed\n"); return (1); } DELAY(ADW_BUS_RESET_HOLD_DELAY_US); status = adw_idle_cmd_send(adw, ADW_IDLE_CMD_SCSI_RESET_END, /*param*/0); if (status != ADW_IDLE_CMD_SUCCESS) { xpt_print_path(adw->path); printf("Bus Reset end attempt failed\n"); return (1); } return (0); } /* * Read the specified EEPROM location */ static u_int16_t adw_eeprom_read_16(struct adw_softc *adw, int addr) { adw_outw(adw, ADW_EEP_CMD, ADW_EEP_CMD_READ | addr); adw_eeprom_wait(adw); return (adw_inw(adw, ADW_EEP_DATA)); } static void adw_eeprom_write_16(struct adw_softc *adw, int addr, u_int data) { adw_outw(adw, ADW_EEP_DATA, data); adw_outw(adw, ADW_EEP_CMD, ADW_EEP_CMD_WRITE | addr); adw_eeprom_wait(adw); } /* * Wait for and EEPROM command to complete */ static void adw_eeprom_wait(struct adw_softc *adw) { int i; for (i = 0; i < ADW_EEP_DELAY_MS; i++) { if ((adw_inw(adw, ADW_EEP_CMD) & ADW_EEP_CMD_DONE) != 0) break; DELAY(1000); } if (i == ADW_EEP_DELAY_MS) panic("%s: Timedout Reading EEPROM", device_get_nameunit(adw->device)); } /* * Read EEPROM configuration into the specified buffer. * * Return a checksum based on the EEPROM configuration read. */ u_int16_t adw_eeprom_read(struct adw_softc *adw, struct adw_eeprom *eep_buf) { u_int16_t *wbuf; u_int16_t wval; u_int16_t chksum; int eep_addr; wbuf = (u_int16_t *)eep_buf; chksum = 0; for (eep_addr = ADW_EEP_DVC_CFG_BEGIN; eep_addr < ADW_EEP_DVC_CFG_END; eep_addr++, wbuf++) { wval = adw_eeprom_read_16(adw, eep_addr); chksum += wval; *wbuf = wval; } /* checksum field is not counted in the checksum */ *wbuf = adw_eeprom_read_16(adw, eep_addr); wbuf++; /* Driver seeprom variables are not included in the checksum */ for (eep_addr = ADW_EEP_DVC_CTL_BEGIN; eep_addr < ADW_EEP_MAX_WORD_ADDR; eep_addr++, wbuf++) *wbuf = adw_eeprom_read_16(adw, eep_addr); return (chksum); } void adw_eeprom_write(struct adw_softc *adw, struct adw_eeprom *eep_buf) { u_int16_t *wbuf; u_int16_t addr; u_int16_t chksum; wbuf = (u_int16_t *)eep_buf; chksum = 0; adw_outw(adw, ADW_EEP_CMD, ADW_EEP_CMD_WRITE_ABLE); adw_eeprom_wait(adw); /* * Write EEPROM until checksum. */ for (addr = ADW_EEP_DVC_CFG_BEGIN; addr < ADW_EEP_DVC_CFG_END; addr++, wbuf++) { chksum += *wbuf; adw_eeprom_write_16(adw, addr, *wbuf); } /* * Write calculated EEPROM checksum */ adw_eeprom_write_16(adw, addr, chksum); /* skip over buffer's checksum */ wbuf++; /* * Write the rest. */ for (addr = ADW_EEP_DVC_CTL_BEGIN; addr < ADW_EEP_MAX_WORD_ADDR; addr++, wbuf++) adw_eeprom_write_16(adw, addr, *wbuf); adw_outw(adw, ADW_EEP_CMD, ADW_EEP_CMD_WRITE_DISABLE); adw_eeprom_wait(adw); } int adw_init_chip(struct adw_softc *adw, u_int term_scsicfg1) { u_int8_t biosmem[ADW_MC_BIOSLEN]; const u_int16_t *word_table; const u_int8_t *byte_codes; const u_int8_t *byte_codes_end; u_int bios_sig; u_int bytes_downloaded; u_int addr; u_int end_addr; u_int checksum; u_int scsicfg1; u_int tid; /* * Save the RISC memory BIOS region before writing the microcode. * The BIOS may already be loaded and using its RISC LRAM region * so its region must be saved and restored. */ for (addr = 0; addr < ADW_MC_BIOSLEN; addr++) biosmem[addr] = adw_lram_read_8(adw, ADW_MC_BIOSMEM + addr); /* * Save current per TID negotiated values if the BIOS has been * loaded (BIOS signature is present). These will be used if * we cannot get information from the EEPROM. */ addr = ADW_MC_BIOS_SIGNATURE - ADW_MC_BIOSMEM; bios_sig = biosmem[addr] | (biosmem[addr + 1] << 8); if (bios_sig == 0x55AA && (adw->flags & ADW_EEPROM_FAILED) != 0) { u_int major_ver; u_int minor_ver; u_int sdtr_able; addr = ADW_MC_BIOS_VERSION - ADW_MC_BIOSMEM; minor_ver = biosmem[addr + 1] & 0xF; major_ver = (biosmem[addr + 1] >> 4) & 0xF; if ((adw->chip == ADW_CHIP_ASC3550) && (major_ver <= 3 || (major_ver == 3 && minor_ver <= 1))) { /* * BIOS 3.1 and earlier location of * 'wdtr_able' variable. */ adw->user_wdtr = adw_lram_read_16(adw, ADW_MC_WDTR_ABLE_BIOS_31); } else { adw->user_wdtr = adw_lram_read_16(adw, ADW_MC_WDTR_ABLE); } sdtr_able = adw_lram_read_16(adw, ADW_MC_SDTR_ABLE); for (tid = 0; tid < ADW_MAX_TID; tid++) { u_int tid_mask; u_int mc_sdtr; tid_mask = 0x1 << tid; if ((sdtr_able & tid_mask) == 0) mc_sdtr = ADW_MC_SDTR_ASYNC; else if ((adw->features & ADW_DT) != 0) mc_sdtr = ADW_MC_SDTR_80; else if ((adw->features & ADW_ULTRA2) != 0) mc_sdtr = ADW_MC_SDTR_40; else mc_sdtr = ADW_MC_SDTR_20; adw_set_user_sdtr(adw, tid, mc_sdtr); } adw->user_tagenb = adw_lram_read_16(adw, ADW_MC_TAGQNG_ABLE); } /* * Load the Microcode. * * Assume the following compressed format of the microcode buffer: * * 253 word (506 byte) table indexed by byte code followed * by the following byte codes: * * 1-Byte Code: * 00: Emit word 0 in table. * 01: Emit word 1 in table. * . * FD: Emit word 253 in table. * * Multi-Byte Code: * FD RESEVED * * FE WW WW: (3 byte code) * Word to emit is the next word WW WW. * FF BB WW WW: (4 byte code) * Emit BB count times next word WW WW. * */ bytes_downloaded = 0; word_table = (const u_int16_t *)adw->mcode_data->mcode_buf; byte_codes = (const u_int8_t *)&word_table[253]; byte_codes_end = adw->mcode_data->mcode_buf + adw->mcode_data->mcode_size; adw_outw(adw, ADW_RAM_ADDR, 0); while (byte_codes < byte_codes_end) { if (*byte_codes == 0xFF) { u_int16_t value; value = byte_codes[2] | byte_codes[3] << 8; adw_set_multi_2(adw, ADW_RAM_DATA, value, byte_codes[1]); bytes_downloaded += byte_codes[1]; byte_codes += 4; } else if (*byte_codes == 0xFE) { u_int16_t value; value = byte_codes[1] | byte_codes[2] << 8; adw_outw(adw, ADW_RAM_DATA, value); bytes_downloaded++; byte_codes += 3; } else { adw_outw(adw, ADW_RAM_DATA, word_table[*byte_codes]); bytes_downloaded++; byte_codes++; } } /* Convert from words to bytes */ bytes_downloaded *= 2; /* * Clear the rest of LRAM. */ for (addr = bytes_downloaded; addr < adw->memsize; addr += 2) adw_outw(adw, ADW_RAM_DATA, 0); /* * Verify the microcode checksum. */ checksum = 0; adw_outw(adw, ADW_RAM_ADDR, 0); for (addr = 0; addr < bytes_downloaded; addr += 2) checksum += adw_inw(adw, ADW_RAM_DATA); if (checksum != adw->mcode_data->mcode_chksum) { device_printf(adw->device, "Firmware load failed!\n"); return (EIO); } /* * Restore the RISC memory BIOS region. */ for (addr = 0; addr < ADW_MC_BIOSLEN; addr++) adw_lram_write_8(adw, addr + ADW_MC_BIOSLEN, biosmem[addr]); /* * Calculate and write the microcode code checksum to * the microcode code checksum location. */ addr = adw_lram_read_16(adw, ADW_MC_CODE_BEGIN_ADDR); end_addr = adw_lram_read_16(adw, ADW_MC_CODE_END_ADDR); checksum = 0; adw_outw(adw, ADW_RAM_ADDR, addr); for (; addr < end_addr; addr += 2) checksum += adw_inw(adw, ADW_RAM_DATA); adw_lram_write_16(adw, ADW_MC_CODE_CHK_SUM, checksum); /* * Tell the microcode what kind of chip it's running on. */ adw_lram_write_16(adw, ADW_MC_CHIP_TYPE, adw->chip); /* * Leave WDTR and SDTR negotiation disabled until the XPT has * informed us of device capabilities, but do set the desired * user rates in case we receive an SDTR request from the target * before we negotiate. We turn on tagged queuing at the microcode * level for all devices, and modulate this on a per command basis. */ adw_lram_write_16(adw, ADW_MC_SDTR_SPEED1, adw->user_sdtr[0]); adw_lram_write_16(adw, ADW_MC_SDTR_SPEED2, adw->user_sdtr[1]); adw_lram_write_16(adw, ADW_MC_SDTR_SPEED3, adw->user_sdtr[2]); adw_lram_write_16(adw, ADW_MC_SDTR_SPEED4, adw->user_sdtr[3]); adw_lram_write_16(adw, ADW_MC_DISC_ENABLE, adw->user_discenb); for (tid = 0; tid < ADW_MAX_TID; tid++) { /* Cam limits the maximum number of commands for us */ adw_lram_write_8(adw, ADW_MC_NUMBER_OF_MAX_CMD + tid, adw->max_acbs); } adw_lram_write_16(adw, ADW_MC_TAGQNG_ABLE, ~0); /* * Set SCSI_CFG0 Microcode Default Value. * * The microcode will set the SCSI_CFG0 register using this value * after it is started. */ adw_lram_write_16(adw, ADW_MC_DEFAULT_SCSI_CFG0, ADW_SCSI_CFG0_PARITY_EN|ADW_SCSI_CFG0_SEL_TMO_LONG| ADW_SCSI_CFG0_OUR_ID_EN|adw->initiator_id); /* * Tell the MC about the memory size that * was setup by the probe code. */ adw_lram_write_16(adw, ADW_MC_DEFAULT_MEM_CFG, adw_inb(adw, ADW_MEM_CFG) & ADW_MEM_CFG_RAM_SZ_MASK); /* * Determine SCSI_CFG1 Microcode Default Value. * * The microcode will set the SCSI_CFG1 register using this value * after it is started below. */ scsicfg1 = adw_inw(adw, ADW_SCSI_CFG1); /* * If the internal narrow cable is reversed all of the SCSI_CTRL * register signals will be set. Check for and return an error if * this condition is found. */ if ((adw_inw(adw, ADW_SCSI_CTRL) & 0x3F07) == 0x3F07) { device_printf(adw->device, "Illegal Cable Config!\n"); device_printf(adw->device, "Internal cable is reversed!\n"); return (EIO); } /* * If this is a differential board and a single-ended device * is attached to one of the connectors, return an error. */ if ((adw->features & ADW_ULTRA) != 0) { if ((scsicfg1 & ADW_SCSI_CFG1_DIFF_MODE) != 0 && (scsicfg1 & ADW_SCSI_CFG1_DIFF_SENSE) == 0) { device_printf(adw->device, "A Single Ended Device is " "attached to our differential bus!\n"); return (EIO); } } else { if ((scsicfg1 & ADW2_SCSI_CFG1_DEV_DETECT_HVD) != 0) { device_printf(adw->device, "A High Voltage Differential Device " "is attached to this controller.\n"); device_printf(adw->device, "HVD devices are not supported.\n"); return (EIO); } } /* * Perform automatic termination control if desired. */ if ((adw->features & ADW_ULTRA2) != 0) { u_int cable_det; /* * Ultra2 Chips require termination disabled to * detect cable presence. */ adw_outw(adw, ADW_SCSI_CFG1, scsicfg1 | ADW2_SCSI_CFG1_DIS_TERM_DRV); cable_det = adw_inw(adw, ADW_SCSI_CFG1); adw_outw(adw, ADW_SCSI_CFG1, scsicfg1); /* SE Termination first if auto-term has been specified */ if ((term_scsicfg1 & ADW_SCSI_CFG1_TERM_CTL_MASK) == 0) { /* * For all SE cable configurations, high byte * termination is enabled. */ term_scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_H; if ((cable_det & ADW_SCSI_CFG1_INT8_MASK) != 0 || (cable_det & ADW_SCSI_CFG1_INT16_MASK) != 0) { /* * If either cable is not present, the * low byte must be terminated as well. */ term_scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_L; } } /* LVD auto-term */ if ((term_scsicfg1 & ADW2_SCSI_CFG1_TERM_CTL_LVD) == 0 && (term_scsicfg1 & ADW2_SCSI_CFG1_DIS_TERM_DRV) == 0) { /* * If both cables are installed, termination * is disabled. Otherwise it is enabled. */ if ((cable_det & ADW2_SCSI_CFG1_EXTLVD_MASK) != 0 || (cable_det & ADW2_SCSI_CFG1_INTLVD_MASK) != 0) { term_scsicfg1 |= ADW2_SCSI_CFG1_TERM_CTL_LVD; } } term_scsicfg1 &= ~ADW2_SCSI_CFG1_DIS_TERM_DRV; } else { /* Ultra Controller Termination */ if ((term_scsicfg1 & ADW_SCSI_CFG1_TERM_CTL_MASK) == 0) { int cable_count; int wide_cable_count; cable_count = 0; wide_cable_count = 0; if ((scsicfg1 & ADW_SCSI_CFG1_INT16_MASK) == 0) { cable_count++; wide_cable_count++; } if ((scsicfg1 & ADW_SCSI_CFG1_INT8_MASK) == 0) cable_count++; /* There is only one external port */ if ((scsicfg1 & ADW_SCSI_CFG1_EXT16_MASK) == 0) { cable_count++; wide_cable_count++; } else if ((scsicfg1 & ADW_SCSI_CFG1_EXT8_MASK) == 0) cable_count++; if (cable_count == 3) { device_printf(adw->device, "Illegal Cable Config!\n"); device_printf(adw->device, "Only Two Ports may be used at a time!\n"); } else if (cable_count <= 1) { /* * At least two out of three cables missing. * Terminate both bytes. */ term_scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_H | ADW_SCSI_CFG1_TERM_CTL_L; } else if (wide_cable_count <= 1) { /* No two 16bit cables present. High on. */ term_scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_H; } } } /* Tell the user about our decission */ switch (term_scsicfg1 & ADW_SCSI_CFG1_TERM_CTL_MASK) { case ADW_SCSI_CFG1_TERM_CTL_MASK: printf("High & Low SE Term Enabled, "); break; case ADW_SCSI_CFG1_TERM_CTL_H: printf("High SE Termination Enabled, "); break; case ADW_SCSI_CFG1_TERM_CTL_L: printf("Low SE Term Enabled, "); break; default: break; } if ((adw->features & ADW_ULTRA2) != 0 && (term_scsicfg1 & ADW2_SCSI_CFG1_TERM_CTL_LVD) != 0) printf("LVD Term Enabled, "); /* * Invert the TERM_CTL_H and TERM_CTL_L bits and then * set 'scsicfg1'. The TERM_POL bit does not need to be * referenced, because the hardware internally inverts * the Termination High and Low bits if TERM_POL is set. */ if ((adw->features & ADW_ULTRA2) != 0) { term_scsicfg1 = ~term_scsicfg1; term_scsicfg1 &= ADW_SCSI_CFG1_TERM_CTL_MASK | ADW2_SCSI_CFG1_TERM_CTL_LVD; scsicfg1 &= ~(ADW_SCSI_CFG1_TERM_CTL_MASK |ADW2_SCSI_CFG1_TERM_CTL_LVD |ADW_SCSI_CFG1_BIG_ENDIAN |ADW_SCSI_CFG1_TERM_POL |ADW2_SCSI_CFG1_DEV_DETECT); scsicfg1 |= term_scsicfg1; } else { term_scsicfg1 = ~term_scsicfg1 & ADW_SCSI_CFG1_TERM_CTL_MASK; scsicfg1 &= ~ADW_SCSI_CFG1_TERM_CTL_MASK; scsicfg1 |= term_scsicfg1 | ADW_SCSI_CFG1_TERM_CTL_MANUAL; scsicfg1 |= ADW_SCSI_CFG1_FLTR_DISABLE; } /* * Set SCSI_CFG1 Microcode Default Value * * The microcode will set the SCSI_CFG1 register using this value * after it is started below. */ adw_lram_write_16(adw, ADW_MC_DEFAULT_SCSI_CFG1, scsicfg1); /* * Only accept selections on our initiator target id. * This may change in target mode scenarios... */ adw_lram_write_16(adw, ADW_MC_DEFAULT_SEL_MASK, (0x01 << adw->initiator_id)); /* * Tell the microcode where it can find our * Initiator Command Queue (ICQ). It is * currently empty hence the "stopper" address. */ adw->commandq = adw->free_carriers; adw->free_carriers = carrierbotov(adw, adw->commandq->next_ba); adw->commandq->next_ba = ADW_CQ_STOPPER; adw_lram_write_32(adw, ADW_MC_ICQ, adw->commandq->carr_ba); /* * Tell the microcode where it can find our * Initiator Response Queue (IRQ). It too * is currently empty. */ adw->responseq = adw->free_carriers; adw->free_carriers = carrierbotov(adw, adw->responseq->next_ba); adw->responseq->next_ba = ADW_CQ_STOPPER; adw_lram_write_32(adw, ADW_MC_IRQ, adw->responseq->carr_ba); adw_outb(adw, ADW_INTR_ENABLES, ADW_INTR_ENABLE_HOST_INTR|ADW_INTR_ENABLE_GLOBAL_INTR); adw_outw(adw, ADW_PC, adw_lram_read_16(adw, ADW_MC_CODE_BEGIN_ADDR)); return (0); } void adw_set_user_sdtr(struct adw_softc *adw, u_int tid, u_int mc_sdtr) { adw->user_sdtr[ADW_TARGET_GROUP(tid)] &= ~ADW_TARGET_GROUP_MASK(tid); adw->user_sdtr[ADW_TARGET_GROUP(tid)] |= mc_sdtr << ADW_TARGET_GROUP_SHIFT(tid); } u_int adw_get_user_sdtr(struct adw_softc *adw, u_int tid) { u_int mc_sdtr; mc_sdtr = adw->user_sdtr[ADW_TARGET_GROUP(tid)]; mc_sdtr &= ADW_TARGET_GROUP_MASK(tid); mc_sdtr >>= ADW_TARGET_GROUP_SHIFT(tid); return (mc_sdtr); } void adw_set_chip_sdtr(struct adw_softc *adw, u_int tid, u_int sdtr) { u_int mc_sdtr_offset; u_int mc_sdtr; mc_sdtr_offset = ADW_MC_SDTR_SPEED1; mc_sdtr_offset += ADW_TARGET_GROUP(tid) * 2; mc_sdtr = adw_lram_read_16(adw, mc_sdtr_offset); mc_sdtr &= ~ADW_TARGET_GROUP_MASK(tid); mc_sdtr |= sdtr << ADW_TARGET_GROUP_SHIFT(tid); adw_lram_write_16(adw, mc_sdtr_offset, mc_sdtr); } u_int adw_get_chip_sdtr(struct adw_softc *adw, u_int tid) { u_int mc_sdtr_offset; u_int mc_sdtr; mc_sdtr_offset = ADW_MC_SDTR_SPEED1; mc_sdtr_offset += ADW_TARGET_GROUP(tid) * 2; mc_sdtr = adw_lram_read_16(adw, mc_sdtr_offset); mc_sdtr &= ADW_TARGET_GROUP_MASK(tid); mc_sdtr >>= ADW_TARGET_GROUP_SHIFT(tid); return (mc_sdtr); } u_int adw_find_sdtr(struct adw_softc *adw, u_int period) { int i; i = 0; if ((adw->features & ADW_DT) == 0) i = ADW_MC_SDTR_OFFSET_ULTRA2; if ((adw->features & ADW_ULTRA2) == 0) i = ADW_MC_SDTR_OFFSET_ULTRA; if (period == 0) return ADW_MC_SDTR_ASYNC; - for (; i < adw_num_syncrates; i++) { + for (; i < nitems(adw_syncrates); i++) { if (period <= adw_syncrates[i].period) return (adw_syncrates[i].mc_sdtr); } return ADW_MC_SDTR_ASYNC; } u_int adw_find_period(struct adw_softc *adw, u_int mc_sdtr) { int i; - for (i = 0; i < adw_num_syncrates; i++) { + for (i = 0; i < nitems(adw_syncrates); i++) { if (mc_sdtr == adw_syncrates[i].mc_sdtr) break; } return (adw_syncrates[i].period); } u_int adw_hshk_cfg_period_factor(u_int tinfo) { tinfo &= ADW_HSHK_CFG_RATE_MASK; tinfo >>= ADW_HSHK_CFG_RATE_SHIFT; if (tinfo == 0x11) /* 80MHz/DT */ return (9); else if (tinfo == 0x10) /* 40MHz */ return (10); else return (((tinfo * 25) + 50) / 4); } /* * Send an idle command to the chip and wait for completion. */ adw_idle_cmd_status_t adw_idle_cmd_send(struct adw_softc *adw, adw_idle_cmd_t cmd, u_int parameter) { u_int timeout; adw_idle_cmd_status_t status; if (!dumping) mtx_assert(&adw->lock, MA_OWNED); /* * Clear the idle command status which is set by the microcode * to a non-zero value to indicate when the command is completed. */ adw_lram_write_16(adw, ADW_MC_IDLE_CMD_STATUS, 0); /* * Write the idle command value after the idle command parameter * has been written to avoid a race condition. If the order is not * followed, the microcode may process the idle command before the * parameters have been written to LRAM. */ adw_lram_write_32(adw, ADW_MC_IDLE_CMD_PARAMETER, parameter); adw_lram_write_16(adw, ADW_MC_IDLE_CMD, cmd); /* * Tickle the RISC to tell it to process the idle command. */ adw_tickle_risc(adw, ADW_TICKLE_B); /* Wait for up to 10 seconds for the command to complete */ timeout = 5000000; while (--timeout) { status = adw_lram_read_16(adw, ADW_MC_IDLE_CMD_STATUS); if (status != 0) break; DELAY(20); } if (timeout == 0) panic("%s: Idle Command Timed Out!", device_get_nameunit(adw->device)); return (status); } Index: head/sys/dev/hwpmc/hwpmc_core.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_core.c (revision 298410) +++ head/sys/dev/hwpmc/hwpmc_core.c (revision 298411) @@ -1,2936 +1,2934 @@ /*- * Copyright (c) 2008 Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Core PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if (__FreeBSD_version >= 1100000) #include #else #include #endif #include #include #include #include #define CORE_CPUID_REQUEST 0xA #define CORE_CPUID_REQUEST_SIZE 0x4 #define CORE_CPUID_EAX 0x0 #define CORE_CPUID_EBX 0x1 #define CORE_CPUID_ECX 0x2 #define CORE_CPUID_EDX 0x3 #define IAF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ PMC_CAP_USER | PMC_CAP_SYSTEM) #define IAF_RI_TO_MSR(RI) ((RI) + (1 << 30)) #define IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define EV_IS_NOTARCH 0 #define EV_IS_ARCH_SUPP 1 #define EV_IS_ARCH_NOTSUPP -1 /* * "Architectural" events defined by Intel. The values of these * symbols correspond to positions in the bitmask returned by * the CPUID.0AH instruction. */ enum core_arch_events { CORE_AE_BRANCH_INSTRUCTION_RETIRED = 5, CORE_AE_BRANCH_MISSES_RETIRED = 6, CORE_AE_INSTRUCTION_RETIRED = 1, CORE_AE_LLC_MISSES = 4, CORE_AE_LLC_REFERENCE = 3, CORE_AE_UNHALTED_REFERENCE_CYCLES = 2, CORE_AE_UNHALTED_CORE_CYCLES = 0 }; static enum pmc_cputype core_cputype; struct core_cpu { volatile uint32_t pc_resync; volatile uint32_t pc_iafctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_corepmcs[]; }; static struct core_cpu **core_pcpu; static uint32_t core_architectural_events; static uint64_t core_pmcmask; static int core_iaf_ri; /* relative index of fixed counters */ static int core_iaf_width; static int core_iaf_npmc; static int core_iap_width; static int core_iap_npmc; static int core_iap_wroffset; static int core_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int core_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct core_cpu *cc; struct pmc_hw *phw; int core_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[iaf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu); core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; if (core_cputype != PMC_CPU_INTEL_CORE) npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); core_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + core_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + core_ri] = phw; } return (0); } static int core_pcpu_fini(struct pmc_mdep *md, int cpu) { int core_ri, n, npmc; struct pmc_cpu *pc; struct core_cpu *cc; uint64_t msr = 0; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu); if ((cc = core_pcpu[cpu]) == NULL) return (0); core_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num; core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri; for (n = 0; n < npmc; n++) { msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + n, msr); } if (core_cputype != PMC_CPU_INTEL_CORE) { msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num; } for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + core_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t iaf_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iaf_width - 1))) == 0) return (0); v &= (1ULL << core_iaf_width) - 1; return (1ULL << core_iaf_width) - v; } static pmc_value_t iaf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iaf_width) - rlc; } static int iaf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event ev; uint32_t caps, flags, validflags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > core_iaf_npmc) return (EINVAL); caps = a->pm_caps; if (a->pm_class != PMC_CLASS_IAF || (caps & IAF_PMC_CAPS) != caps) return (EINVAL); ev = pm->pm_event; if (ev < PMC_EV_IAF_FIRST || ev > PMC_EV_IAF_LAST) return (EINVAL); if (ev == PMC_EV_IAF_INSTR_RETIRED_ANY && ri != 0) return (EINVAL); if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_CORE && ri != 1) return (EINVAL); if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_REF && ri != 2) return (EINVAL); flags = a->pm_md.pm_iaf.pm_iaf_flags; validflags = IAF_MASK; if (core_cputype != PMC_CPU_INTEL_ATOM && core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT) validflags &= ~IAF_ANY; if ((flags & ~validflags) != 0) return (EINVAL); if (caps & PMC_CAP_INTERRUPT) flags |= IAF_PMI; if (caps & PMC_CAP_SYSTEM) flags |= IAF_OS; if (caps & PMC_CAP_USER) flags |= IAF_USR; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) flags |= (IAF_OS | IAF_USR); pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl); return (0); } static int iaf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm; return (0); } static int iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iaf_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri]; (void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri); if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iaf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; return (0); } static int iaf_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[iaf,%d] ri %d out of range", __LINE__, ri)); *msr = IAF_RI_TO_MSR(ri); return (0); } static int iaf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + core_iaf_ri)); tmp = rdpmc(IAF_RI_TO_MSR(ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iaf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAF_RI_TO_MSR(ri), *v); return (0); } static int iaf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iaf_start_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *iafc; uint64_t msr = 0; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri); iafc = core_pcpu[cpu]; pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl; msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); do { iafc->pc_resync = 0; iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET)); msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & IAF_GLOBAL_CTRL_MASK)); } while (iafc->pc_resync != 0); PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_stop_pmc(int cpu, int ri) { uint32_t fc; struct core_cpu *iafc; uint64_t msr = 0; PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri); iafc = core_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); fc = (IAF_MASK << (ri * 4)); if (core_cputype != PMC_CPU_INTEL_ATOM && core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT) fc &= ~IAF_ANY; iafc->pc_iafctrl &= ~fc; PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl); msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK)); do { iafc->pc_resync = 0; iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET)); msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl & IAF_GLOBAL_CTRL_MASK)); } while (iafc->pc_resync != 0); PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)", iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL), iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL)); return (0); } static int iaf_write_pmc(int cpu, int ri, pmc_value_t v) { struct core_cpu *cc; struct pmc *pm; uint64_t msr; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iaf_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iaf_reload_count_to_perfctr_value(v); /* Turn off fixed counters */ msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr); wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1)); /* Turn on fixed counters */ msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK; wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK)); PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx " "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri))); return (0); } static void iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iaf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF]; pcd->pcd_caps = IAF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iaf_allocate_pmc; pcd->pcd_config_pmc = iaf_config_pmc; pcd->pcd_describe = iaf_describe; pcd->pcd_get_config = iaf_get_config; pcd->pcd_get_msr = iaf_get_msr; pcd->pcd_pcpu_fini = core_pcpu_noop; pcd->pcd_pcpu_init = core_pcpu_noop; pcd->pcd_read_pmc = iaf_read_pmc; pcd->pcd_release_pmc = iaf_release_pmc; pcd->pcd_start_pmc = iaf_start_pmc; pcd->pcd_stop_pmc = iaf_stop_pmc; pcd->pcd_write_pmc = iaf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct iap_event_descr { enum pmc_event iap_ev; unsigned char iap_evcode; unsigned char iap_umask; unsigned int iap_flags; }; #define IAP_F_CC (1 << 0) /* CPU: Core */ #define IAP_F_CC2 (1 << 1) /* CPU: Core2 family */ #define IAP_F_CC2E (1 << 2) /* CPU: Core2 Extreme only */ #define IAP_F_CA (1 << 3) /* CPU: Atom */ #define IAP_F_I7 (1 << 4) /* CPU: Core i7 */ #define IAP_F_I7O (1 << 4) /* CPU: Core i7 (old) */ #define IAP_F_WM (1 << 5) /* CPU: Westmere */ #define IAP_F_SB (1 << 6) /* CPU: Sandy Bridge */ #define IAP_F_IB (1 << 7) /* CPU: Ivy Bridge */ #define IAP_F_SBX (1 << 8) /* CPU: Sandy Bridge Xeon */ #define IAP_F_IBX (1 << 9) /* CPU: Ivy Bridge Xeon */ #define IAP_F_HW (1 << 10) /* CPU: Haswell */ #define IAP_F_CAS (1 << 11) /* CPU: Atom Silvermont */ #define IAP_F_HWX (1 << 12) /* CPU: Haswell Xeon */ #define IAP_F_BW (1 << 13) /* CPU: Broadwell */ #define IAP_F_BWX (1 << 14) /* CPU: Broadwell Xeon */ #define IAP_F_SL (1 << 15) /* CPU: Skylake */ #define IAP_F_FM (1 << 18) /* Fixed mask */ #define IAP_F_ALLCPUSCORE2 \ (IAP_F_CC | IAP_F_CC2 | IAP_F_CC2E | IAP_F_CA) /* Sub fields of UMASK that this event supports. */ #define IAP_M_CORE (1 << 0) /* Core specificity */ #define IAP_M_AGENT (1 << 1) /* Agent specificity */ #define IAP_M_PREFETCH (1 << 2) /* Prefetch */ #define IAP_M_MESI (1 << 3) /* MESI */ #define IAP_M_SNOOPRESPONSE (1 << 4) /* Snoop response */ #define IAP_M_SNOOPTYPE (1 << 5) /* Snoop type */ #define IAP_M_TRANSITION (1 << 6) /* Transition */ #define IAP_F_CORE (0x3 << 14) /* Core specificity */ #define IAP_F_AGENT (0x1 << 13) /* Agent specificity */ #define IAP_F_PREFETCH (0x3 << 12) /* Prefetch */ #define IAP_F_MESI (0xF << 8) /* MESI */ #define IAP_F_SNOOPRESPONSE (0xB << 8) /* Snoop response */ #define IAP_F_SNOOPTYPE (0x3 << 8) /* Snoop type */ #define IAP_F_TRANSITION (0x1 << 12) /* Transition */ #define IAP_PREFETCH_RESERVED (0x2 << 12) #define IAP_CORE_THIS (0x1 << 14) #define IAP_CORE_ALL (0x3 << 14) #define IAP_F_CMASK 0xFF000000 static struct iap_event_descr iap_events[] = { #undef IAPDESCR #define IAPDESCR(N,EV,UM,FLAGS) { \ .iap_ev = PMC_EV_IAP_EVENT_##N, \ .iap_evcode = (EV), \ .iap_umask = (UM), \ .iap_flags = (FLAGS) \ } IAPDESCR(02H_01H, 0x02, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(02H_81H, 0x02, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(03H_00H, 0x03, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(03H_01H, 0x03, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_SBX | IAP_F_CAS), IAPDESCR(03H_02H, 0x03, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(03H_04H, 0x03, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O | IAP_F_CAS), IAPDESCR(03H_08H, 0x03, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_SBX | IAP_F_CAS | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(03H_10H, 0x03, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_SBX | IAP_F_CAS), IAPDESCR(03H_20H, 0x03, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS), IAPDESCR(03H_40H, 0x03, 0x40, IAP_F_FM | IAP_F_CAS), IAPDESCR(03H_80H, 0x03, 0x80, IAP_F_FM | IAP_F_CAS), IAPDESCR(04H_00H, 0x04, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CAS), IAPDESCR(04H_01H, 0x04, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O | IAP_F_CAS), IAPDESCR(04H_02H, 0x04, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS), IAPDESCR(04H_04H, 0x04, 0x04, IAP_F_FM | IAP_F_CAS), IAPDESCR(04H_07H, 0x04, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(04H_08H, 0x04, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS), IAPDESCR(04H_10H, 0x04, 0x10, IAP_F_FM | IAP_F_CAS), IAPDESCR(04H_20H, 0x04, 0x20, IAP_F_FM | IAP_F_CAS), IAPDESCR(04H_40H, 0x04, 0x40, IAP_F_FM | IAP_F_CAS), IAPDESCR(04H_80H, 0x04, 0x80, IAP_F_FM | IAP_F_CAS), IAPDESCR(05H_00H, 0x05, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(05H_01H, 0x05, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(05H_02H, 0x05, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(05H_03H, 0x05, 0x03, IAP_F_FM | IAP_F_I7O | IAP_F_CAS), IAPDESCR(06H_00H, 0x06, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2 | IAP_F_CC2E | IAP_F_CA), IAPDESCR(06H_01H, 0x06, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(06H_02H, 0x06, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(06H_04H, 0x06, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(06H_08H, 0x06, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(06H_0FH, 0x06, 0x0F, IAP_F_FM | IAP_F_I7O), IAPDESCR(07H_00H, 0x07, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2), IAPDESCR(07H_01H, 0x07, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(07H_02H, 0x07, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(07H_03H, 0x07, 0x03, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(07H_06H, 0x07, 0x06, IAP_F_FM | IAP_F_CA), IAPDESCR(07H_08H, 0x07, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_SB | IAP_F_SBX), IAPDESCR(08H_01H, 0x08, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(08H_02H, 0x08, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(08H_04H, 0x08, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(08H_05H, 0x08, 0x05, IAP_F_FM | IAP_F_CA), IAPDESCR(08H_06H, 0x08, 0x06, IAP_F_FM | IAP_F_CA), IAPDESCR(08H_07H, 0x08, 0x07, IAP_F_FM | IAP_F_CA), IAPDESCR(08H_08H, 0x08, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(08H_09H, 0x08, 0x09, IAP_F_FM | IAP_F_CA), IAPDESCR(08H_0EH, 0x08, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(08H_10H, 0x08, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(08H_40H, 0x08, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX), IAPDESCR(08H_60H, 0x08, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX), IAPDESCR(08H_80H, 0x08, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX), IAPDESCR(08H_81H, 0x08, 0x81, IAP_F_FM | IAP_F_IB | IAP_F_IBX), IAPDESCR(08H_82H, 0x08, 0x82, IAP_F_FM | IAP_F_IB | IAP_F_IBX), IAPDESCR(08H_84H, 0x08, 0x84, IAP_F_FM | IAP_F_IB | IAP_F_IBX), IAPDESCR(08H_88H, 0x08, 0x88, IAP_F_FM | IAP_F_IB | IAP_F_IBX), IAPDESCR(09H_01H, 0x09, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O), IAPDESCR(09H_02H, 0x09, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O), IAPDESCR(09H_04H, 0x09, 0x04, IAP_F_FM | IAP_F_I7O), IAPDESCR(09H_08H, 0x09, 0x08, IAP_F_FM | IAP_F_I7O), IAPDESCR(0BH_01H, 0x0B, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0BH_02H, 0x0B, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0BH_10H, 0x0B, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0CH_01H, 0x0C, 0x01, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(0CH_02H, 0x0C, 0x02, IAP_F_FM | IAP_F_CC2), IAPDESCR(0CH_03H, 0x0C, 0x03, IAP_F_FM | IAP_F_CA), IAPDESCR(0DH_03H, 0x0D, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_IB | IAP_F_IBX | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(0DH_40H, 0x0D, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(0DH_80H, 0x0D, 0x80, IAP_F_FM | IAP_F_SL), IAPDESCR(0EH_01H, 0x0E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(0EH_02H, 0x0E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(0FH_01H, 0x0F, 0x01, IAP_F_FM | IAP_F_I7), IAPDESCR(0FH_02H, 0x0F, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0FH_08H, 0x0F, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0FH_10H, 0x0F, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0FH_20H, 0x0F, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(0FH_80H, 0x0F, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(10H_00H, 0x10, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(10H_01H, 0x10, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX ), IAPDESCR(10H_02H, 0x10, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(10H_04H, 0x10, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(10H_08H, 0x10, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(10H_10H, 0x10, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(10H_20H, 0x10, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(10H_40H, 0x10, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(10H_80H, 0x10, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(10H_81H, 0x10, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(11H_00H, 0x11, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2), IAPDESCR(11H_01H, 0x11, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(11H_02H, 0x11, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX), IAPDESCR(11H_81H, 0x11, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(12H_00H, 0x12, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(12H_01H, 0x12, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_02H, 0x12, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_04H, 0x12, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_08H, 0x12, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_10H, 0x12, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_20H, 0x12, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_40H, 0x12, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(12H_81H, 0x12, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(13H_00H, 0x13, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(13H_01H, 0x13, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM), IAPDESCR(13H_02H, 0x13, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(13H_04H, 0x13, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(13H_07H, 0x13, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(13H_81H, 0x13, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(14H_00H, 0x14, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2), IAPDESCR(14H_01H, 0x14, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(14H_02H, 0x14, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(17H_01H, 0x17, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(18H_00H, 0x18, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(18H_01H, 0x18, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(19H_00H, 0x19, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(19H_01H, 0x19, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(19H_02H, 0x19, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(1DH_01H, 0x1D, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(1DH_02H, 0x1D, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(1DH_04H, 0x1D, 0x04, IAP_F_FM | IAP_F_I7O), IAPDESCR(1EH_01H, 0x1E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(20H_01H, 0x20, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(21H, 0x21, IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(22H, 0x22, IAP_M_CORE, IAP_F_CC2), IAPDESCR(23H, 0x23, IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(24H, 0x24, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2), IAPDESCR(24H_01H, 0x24, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX ), IAPDESCR(24H_02H, 0x24, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(24H_03H, 0x24, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_04H, 0x24, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_08H, 0x24, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_0CH, 0x24, 0x0C, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_10H, 0x24, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_20H, 0x24, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_21H, 0x24, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_22H, 0x24, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_24H, 0x24, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_27H, 0x24, 0x27, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_30H, 0x24, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(24H_38H, 0x24, 0x38, IAP_F_FM | IAP_F_SL), IAPDESCR(24H_3FH, 0x24, 0x3F, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_40H, 0x24, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_41H, 0x24, 0x41, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_42H, 0x24, 0x42, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_44H, 0x24, 0x44, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_50H, 0x24, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(24H_80H, 0x24, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_AAH, 0x24, 0xAA, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(24H_C0H, 0x24, 0xC0, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(24H_D8H, 0x24, 0xD8, IAP_F_FM | IAP_F_SL), IAPDESCR(24H_E1H, 0x24, 0xE1, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_E2H, 0x24, 0xE2, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_E4H, 0x24, 0xE4, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_E7H, 0x24, 0xE7, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(24H_EFH, 0x24, 0xEF, IAP_F_FM | IAP_F_SL), IAPDESCR(24H_F8H, 0x24, 0xF8, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW | IAP_F_HWX), IAPDESCR(25H, 0x25, IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(26H, 0x26, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2), IAPDESCR(26H_01H, 0x26, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_02H, 0x26, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_04H, 0x26, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_08H, 0x26, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_0FH, 0x26, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_10H, 0x26, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_20H, 0x26, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_40H, 0x26, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_80H, 0x26, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_F0H, 0x26, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(26H_FFH, 0x26, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H, 0x27, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2), IAPDESCR(27H_01H, 0x27, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(27H_02H, 0x27, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_04H, 0x27, 0x04, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_SBX), IAPDESCR(27H_08H, 0x27, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(27H_0EH, 0x27, 0x0E, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_0FH, 0x27, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(27H_10H, 0x27, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_20H, 0x27, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_40H, 0x27, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_50H, 0x27, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(27H_80H, 0x27, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_E0H, 0x27, 0xE0, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(27H_F0H, 0x27, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(28H, 0x28, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2), IAPDESCR(28H_01H, 0x28, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(28H_02H, 0x28, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SBX), IAPDESCR(28H_04H, 0x28, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(28H_08H, 0x28, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(28H_0FH, 0x28, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI, IAP_F_CC), IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_CA | IAP_F_CC2), IAPDESCR(2AH, 0x2A, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2), IAPDESCR(2BH, 0x2B, IAP_M_CORE | IAP_M_MESI, IAP_F_CA | IAP_F_CC2), IAPDESCR(2EH, 0x2E, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2), IAPDESCR(2EH_01H, 0x2E, 0x01, IAP_F_FM | IAP_F_WM), IAPDESCR(2EH_02H, 0x2E, 0x02, IAP_F_FM | IAP_F_WM), IAPDESCR(2EH_41H, 0x2E, 0x41, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(2EH_4FH, 0x2E, 0x4F, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(30H, 0x30, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2), IAPDESCR(30H_00H, 0x30, 0x00, IAP_F_FM | IAP_F_CAS), IAPDESCR(31H_00H, 0x31, 0x00, IAP_F_FM | IAP_F_CAS), IAPDESCR(32H, 0x32, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_CC), IAPDESCR(32H, 0x32, IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(3AH, 0x3A, IAP_M_TRANSITION, IAP_F_CC), IAPDESCR(3AH_00H, 0x3A, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(3BH_C0H, 0x3B, 0xC0, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(3CH_00H, 0x3C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(3CH_01H, 0x3C, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(3CH_02H, 0x3C, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_SL), IAPDESCR(3DH_01H, 0x3D, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(40H, 0x40, IAP_M_MESI, IAP_F_CC | IAP_F_CC2), IAPDESCR(40H_01H, 0x40, 0x01, IAP_F_FM | IAP_F_I7), IAPDESCR(40H_02H, 0x40, 0x02, IAP_F_FM | IAP_F_I7), IAPDESCR(40H_04H, 0x40, 0x04, IAP_F_FM | IAP_F_I7), IAPDESCR(40H_08H, 0x40, 0x08, IAP_F_FM | IAP_F_I7), IAPDESCR(40H_0FH, 0x40, 0x0F, IAP_F_FM | IAP_F_I7), IAPDESCR(40H_21H, 0x40, 0x21, IAP_F_FM | IAP_F_CA), IAPDESCR(41H, 0x41, IAP_M_MESI, IAP_F_CC | IAP_F_CC2), IAPDESCR(41H_01H, 0x41, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(41H_02H, 0x41, 0x02, IAP_F_FM | IAP_F_I7), IAPDESCR(41H_04H, 0x41, 0x04, IAP_F_FM | IAP_F_I7), IAPDESCR(41H_08H, 0x41, 0x08, IAP_F_FM | IAP_F_I7), IAPDESCR(41H_0FH, 0x41, 0x0F, IAP_F_FM | IAP_F_I7O), IAPDESCR(41H_22H, 0x41, 0x22, IAP_F_FM | IAP_F_CA), IAPDESCR(42H, 0x42, IAP_M_MESI, IAP_F_ALLCPUSCORE2), IAPDESCR(42H_01H, 0x42, 0x01, IAP_F_FM | IAP_F_I7), IAPDESCR(42H_02H, 0x42, 0x02, IAP_F_FM | IAP_F_I7), IAPDESCR(42H_04H, 0x42, 0x04, IAP_F_FM | IAP_F_I7), IAPDESCR(42H_08H, 0x42, 0x08, IAP_F_FM | IAP_F_I7), IAPDESCR(42H_10H, 0x42, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(43H_01H, 0x43, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7), IAPDESCR(43H_02H, 0x43, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7), IAPDESCR(44H_02H, 0x44, 0x02, IAP_F_FM | IAP_F_CC), IAPDESCR(45H_0FH, 0x45, 0x0F, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(46H_00H, 0x46, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(47H_00H, 0x47, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(48H_00H, 0x48, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(48H_01H, 0x48, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(48H_02H, 0x48, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_SL), IAPDESCR(49H_00H, 0x49, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(49H_01H, 0x49, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(49H_02H, 0x49, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(49H_04H, 0x49, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(49H_0EH, 0x49, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(49H_10H, 0x49, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(49H_20H, 0x49, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(49H_40H, 0x49, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX), IAPDESCR(49H_60H, 0x49, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX), IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX), IAPDESCR(4BH_00H, 0x4B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(4BH_01H, 0x4B, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7O), IAPDESCR(4BH_02H, 0x4B, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(4BH_03H, 0x4B, 0x03, IAP_F_FM | IAP_F_CC), IAPDESCR(4BH_08H, 0x4B, 0x08, IAP_F_FM | IAP_F_I7O), IAPDESCR(4CH_00H, 0x4C, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(4CH_01H, 0x4C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(4CH_02H, 0x4C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(4DH_01H, 0x4D, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(4EH_01H, 0x4E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(4EH_02H, 0x4E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(4EH_04H, 0x4E, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(4EH_10H, 0x4E, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(4FH_00H, 0x4F, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(4FH_02H, 0x4F, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(4FH_04H, 0x4F, 0x04, IAP_F_FM | IAP_F_I7O), IAPDESCR(4FH_08H, 0x4F, 0x08, IAP_F_FM | IAP_F_I7O), IAPDESCR(4FH_10H, 0x4F, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(51H_01H, 0x51, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(51H_02H, 0x51, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(51H_04H, 0x51, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(51H_08H, 0x51, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(52H_01H, 0x52, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(53H_01H, 0x53, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(59H_20H, 0x59, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(59H_40H, 0x59, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(59H_80H, 0x59, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(5BH_0CH, 0x5B, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(5BH_0FH, 0x5B, 0x0F, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(5BH_40H, 0x5B, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(5BH_4FH, 0x5B, 0x4F, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(5CH_01H, 0x5C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(5CH_02H, 0x5C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(5EH_01H, 0x5E, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(5FH_01H, 0x5F, 0x01, IAP_F_FM | IAP_F_IB ), /* IB not in manual */ IAPDESCR(5FH_04H, 0x5F, 0x04, IAP_F_FM | IAP_F_IBX | IAP_F_IB), IAPDESCR(60H, 0x60, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(60H_01H, 0x60, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(60H_02H, 0x60, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(60H_04H, 0x60, 0x04, IAP_F_FM |IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(60H_08H, 0x60, 0x08, IAP_F_FM |IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(60H_10H, 0x60, 0x10, IAP_F_FM | IAP_F_SL), IAPDESCR(61H, 0x61, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2), IAPDESCR(61H_00H, 0x61, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(62H, 0x62, IAP_M_AGENT, IAP_F_ALLCPUSCORE2), IAPDESCR(62H_00H, 0x62, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(63H, 0x63, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(63H, 0x63, IAP_M_CORE, IAP_F_CC), IAPDESCR(63H_01H, 0x63, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX ), IAPDESCR(63H_02H, 0x63, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(64H, 0x64, IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(64H_40H, 0x64, 0x40, IAP_F_FM | IAP_F_CC), IAPDESCR(65H, 0x65, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(65H, 0x65, IAP_M_CORE, IAP_F_CC), IAPDESCR(66H, 0x66, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(67H, 0x67, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(67H, 0x67, IAP_M_AGENT, IAP_F_CC), IAPDESCR(68H, 0x68, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(69H, 0x69, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(6AH, 0x6A, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(6BH, 0x6B, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(6CH, 0x6C, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(6CH_01H, 0x6C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(6DH, 0x6D, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(6DH, 0x6D, IAP_M_CORE, IAP_F_CC), IAPDESCR(6EH, 0x6E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(6EH, 0x6E, IAP_M_CORE, IAP_F_CC), IAPDESCR(6FH, 0x6F, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(6FH, 0x6F, IAP_M_CORE, IAP_F_CC), IAPDESCR(70H, 0x70, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(70H, 0x70, IAP_M_CORE, IAP_F_CC), IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_SNOOPRESPONSE, IAP_F_CA | IAP_F_CC2), IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_MESI, IAP_F_CC), IAPDESCR(78H, 0x78, IAP_M_CORE, IAP_F_CC), IAPDESCR(78H, 0x78, IAP_M_CORE | IAP_M_SNOOPTYPE, IAP_F_CA | IAP_F_CC2), IAPDESCR(79H_02H, 0x79, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(79H_04H, 0x79, 0x04, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_08H, 0x79, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL | IAP_F_BW | IAP_F_BWX), IAPDESCR(79H_10H, 0x79, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_20H, 0x79, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_24H, 0x79, 0x24, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_30H, 0x79, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(7AH, 0x7A, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2), IAPDESCR(7BH, 0x7B, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2), IAPDESCR(7DH, 0x7D, IAP_M_CORE, IAP_F_ALLCPUSCORE2), IAPDESCR(7EH, 0x7E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(7EH_00H, 0x7E, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(7FH, 0x7F, IAP_M_CORE, IAP_F_CA | IAP_F_CC2), IAPDESCR(80H_00H, 0x80, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(80H_01H, 0x80, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_CAS), IAPDESCR(80H_02H, 0x80, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(80H_03H, 0x80, 0x03, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM | IAP_F_CAS), IAPDESCR(80H_04H, 0x80, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_IBX | IAP_F_SL), /* SL may have a spec bug two with same entry no cmask */ IAPDESCR(81H_00H, 0x81, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(81H_01H, 0x81, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(81H_02H, 0x81, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(82H_01H, 0x82, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(82H_02H, 0x82, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(82H_04H, 0x82, 0x04, IAP_F_FM | IAP_F_CA), IAPDESCR(82H_10H, 0x82, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(82H_12H, 0x82, 0x12, IAP_F_FM | IAP_F_CC2), IAPDESCR(82H_40H, 0x82, 0x40, IAP_F_FM | IAP_F_CC2), IAPDESCR(83H_01H, 0x83, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SL), IAPDESCR(83H_02H, 0x83, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SL), IAPDESCR(85H_00H, 0x85, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(85H_01H, 0x85, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(85H_02H, 0x85, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(85H_04H, 0x85, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(85H_0EH, 0x85, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(85H_10H, 0x85, 0x10, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(85H_20H, 0x85, 0x20, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(85H_40H, 0x85, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX), IAPDESCR(85H_60H, 0x85, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX), IAPDESCR(85H_80H, 0x85, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7O), IAPDESCR(86H_00H, 0x86, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(87H_00H, 0x87, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(87H_01H, 0x87, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(87H_02H, 0x87, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(87H_04H, 0x87, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(87H_08H, 0x87, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(87H_0FH, 0x87, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(88H_00H, 0x88, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(88H_01H, 0x88, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_02H, 0x88, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_04H, 0x88, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_07H, 0x88, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(88H_08H, 0x88, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_10H, 0x88, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_20H, 0x88, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_30H, 0x88, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(88H_40H, 0x88, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_41H, 0x88, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_7FH, 0x88, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(88H_80H, 0x88, 0x80, IAP_F_FM | IAP_F_BW | IAP_F_BWX), IAPDESCR(88H_81H, 0x88, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_82H, 0x88, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_84H, 0x88, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_88H, 0x88, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_90H, 0x88, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_A0H, 0x88, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(88H_FFH, 0x88, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_00H, 0x89, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(89H_01H, 0x89, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_02H, 0x89, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(89H_04H, 0x89, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_07H, 0x89, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(89H_08H, 0x89, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_10H, 0x89, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_20H, 0x89, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_30H, 0x89, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(89H_40H, 0x89, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_41H, 0x89, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_7FH, 0x89, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(89H_80H, 0x89, 0x80, IAP_F_FM | IAP_F_BW | IAP_F_BWX), IAPDESCR(89H_81H, 0x89, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_82H, 0x89, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_84H, 0x89, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_88H, 0x89, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_90H, 0x89, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_A0H, 0x89, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(89H_FFH, 0x89, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(8AH_00H, 0x8A, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(8BH_00H, 0x8B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(8CH_00H, 0x8C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(8DH_00H, 0x8D, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(8EH_00H, 0x8E, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(8FH_00H, 0x8F, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(90H_00H, 0x90, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(91H_00H, 0x91, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(92H_00H, 0x92, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(93H_00H, 0x93, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(94H_00H, 0x94, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(97H_00H, 0x97, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(98H_00H, 0x98, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(9CH_01H, 0x9C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A0H_00H, 0xA0, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(A1H_01H, 0xA1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_02H, 0xA1, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_04H, 0xA1, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/ IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_08H, 0xA1, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/ IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_0CH, 0xA1, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(A1H_10H, 0xA1, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/ IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_20H, 0xA1, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | /* No desc in IB for this*/ IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_30H, 0xA1, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(A1H_40H, 0xA1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A1H_80H, 0xA1, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A2H_00H, 0xA2, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(A2H_01H, 0xA2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A2H_02H, 0xA2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(A2H_04H, 0xA2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(A2H_08H, 0xA2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(A2H_10H, 0xA2, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(A2H_20H, 0xA2, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(A2H_40H, 0xA2, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(A2H_80H, 0xA2, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(A3H_01H, 0xA3, 0x01, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(A3H_02H, 0xA3, 0x02, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(A3H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_SL), IAPDESCR(A3H_05H, 0xA3, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(A3H_06H, 0xA3, 0x06, IAP_F_FM | IAP_F_SL), IAPDESCR(A3H_08H, 0xA3, 0x08, IAP_F_FM | IAP_F_IBX | IAP_F_HW | IAP_F_IB | IAP_F_HWX | IAP_F_SL), IAPDESCR(A3H_0CH, 0xA3, 0x0C, IAP_F_FM | IAP_F_HW | IAP_F_HW | IAP_F_SL), IAPDESCR(A3H_10H, 0xA3, 0x10, IAP_F_FM | IAP_F_SL), IAPDESCR(A3H_14H, 0xA3, 0x14, IAP_F_FM | IAP_F_SL), IAPDESCR(A6H_01H, 0xA6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(A6H_02H, 0xA3, 0x02, IAP_F_FM | IAP_F_SL), IAPDESCR(A6H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SL), IAPDESCR(A6H_08H, 0xA3, 0x08, IAP_F_FM | IAP_F_SL), IAPDESCR(A6H_10H, 0xA3, 0x10, IAP_F_FM | IAP_F_SL), IAPDESCR(A6H_40H, 0xA3, 0x40, IAP_F_FM | IAP_F_SL), IAPDESCR(A7H_01H, 0xA7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM ), IAPDESCR(A8H_01H, 0xA8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IBX | IAP_F_IB |IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(AAH_01H, 0xAA, 0x01, IAP_F_FM | IAP_F_CC2), IAPDESCR(AAH_02H, 0xAA, 0x02, IAP_F_FM | IAP_F_CA), IAPDESCR(AAH_03H, 0xAA, 0x03, IAP_F_FM | IAP_F_CA), IAPDESCR(AAH_08H, 0xAA, 0x08, IAP_F_FM | IAP_F_CC2), IAPDESCR(ABH_01H, 0xAB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(ABH_02H, 0xAB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_BW | IAP_F_BWX), IAPDESCR(ACH_02H, 0xAC, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_SL), IAPDESCR(ACH_08H, 0xAC, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(ACH_0AH, 0xAC, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(AEH_01H, 0xAE, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B0H_00H, 0xB0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(B0H_01H, 0xB0, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B0H_02H, 0xB0, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B0H_04H, 0xB0, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B0H_08H, 0xB0, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B0H_10H, 0xB0, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SL), IAPDESCR(B0H_20H, 0xB0, 0x20, IAP_F_FM | IAP_F_I7O), IAPDESCR(B0H_40H, 0xB0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B0H_80H, 0xB0, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_WM | IAP_F_I7O | IAP_F_SL), IAPDESCR(B1H_00H, 0xB1, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(B1H_01H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B1H_02H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B1H_04H, 0xB1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_08H, 0xB1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_10H, 0xB1, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_1FH, 0xB1, 0x1F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_20H, 0xB1, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_3FH, 0xB1, 0x3F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_40H, 0xB1, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B1H_80H, 0xB1, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM), IAPDESCR(B2H_01H, 0xB2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(B3H_01H, 0xB3, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_WM | IAP_F_I7O), IAPDESCR(B3H_02H, 0xB3, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_WM | IAP_F_I7O), IAPDESCR(B3H_04H, 0xB3, 0x04, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_WM | IAP_F_I7O), IAPDESCR(B3H_08H, 0xB3, 0x08, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(B3H_10H, 0xB3, 0x10, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(B3H_20H, 0xB3, 0x20, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(B3H_81H, 0xB3, 0x81, IAP_F_FM | IAP_F_CA), IAPDESCR(B3H_82H, 0xB3, 0x82, IAP_F_FM | IAP_F_CA), IAPDESCR(B3H_84H, 0xB3, 0x84, IAP_F_FM | IAP_F_CA), IAPDESCR(B3H_88H, 0xB3, 0x88, IAP_F_FM | IAP_F_CA), IAPDESCR(B3H_90H, 0xB3, 0x90, IAP_F_FM | IAP_F_CA), IAPDESCR(B3H_A0H, 0xB3, 0xA0, IAP_F_FM | IAP_F_CA), IAPDESCR(B4H_01H, 0xB4, 0x01, IAP_F_FM | IAP_F_WM), IAPDESCR(B4H_02H, 0xB4, 0x02, IAP_F_FM | IAP_F_WM), IAPDESCR(B4H_04H, 0xB4, 0x04, IAP_F_FM | IAP_F_WM), IAPDESCR(B6H_01H, 0xB6, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(B6H_04H, 0xB6, 0x04, IAP_F_FM | IAP_F_CAS), IAPDESCR(B7H_01H, 0xB7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX |IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(B7H_02H, 0xB7, 0x02, IAP_F_CAS), IAPDESCR(B8H_01H, 0xB8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B8H_02H, 0xB8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(B8H_04H, 0xB8, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(BAH_01H, 0xBA, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(BAH_02H, 0xBA, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(BBH_01H, 0xBB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(BCH_11H, 0xBC, 0x11, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_12H, 0xBC, 0x12, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_14H, 0xBC, 0x14, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_18H, 0xBC, 0x18, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_21H, 0xBC, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_22H, 0xBC, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_24H, 0xBC, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(BCH_28H, 0xBC, 0x28, IAP_F_FM | IAP_F_HW | IAP_F_HWX), IAPDESCR(BDH_01H, 0xBD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL), /* spec bug SL? */ IAPDESCR(BDH_20H, 0xBD, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(BFH_05H, 0xBF, 0x05, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(C0H_00H, 0xC0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C0H_01H, 0xC0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C0H_02H, 0xC0, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_BW | IAP_F_BWX), IAPDESCR(C0H_04H, 0xC0, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(C0H_08H, 0xC0, 0x08, IAP_F_FM | IAP_F_CC2E), IAPDESCR(C1H_00H, 0xC1, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(C1H_01H, 0xC1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C1H_02H, 0xC1, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX), IAPDESCR(C1H_08H, 0xC1, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(C1H_10H, 0xC1, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(C1H_20H, 0xC1, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(C1H_3FH, 0xC1, 0x3F, IAP_F_FM | IAP_F_SL), IAPDESCR(C1H_40H, 0xC1, 0x40, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(C1H_80H, 0xC1, 0x80, IAP_F_FM |IAP_F_IB | IAP_F_IBX), IAPDESCR(C1H_FEH, 0xC1, 0xFE, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C2H_00H, 0xC2, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(C2H_01H, 0xC2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C2H_02H, 0xC2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C2H_04H, 0xC2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(C2H_07H, 0xC2, 0x07, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C2H_08H, 0xC2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C2H_0FH, 0xC2, 0x0F, IAP_F_FM | IAP_F_CC2), IAPDESCR(C2H_10H, 0xC2, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CAS), IAPDESCR(C3H_00H, 0xC3, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(C3H_01H, 0xC3, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_CAS | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C3H_02H, 0xC3, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C3H_04H, 0xC3, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C3H_08H, 0xC3, 0x08, IAP_F_FM | IAP_F_CAS), IAPDESCR(C3H_10H, 0xC3, 0x10, IAP_F_FM | IAP_F_I7O), IAPDESCR(C3H_20H, 0xC3, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(C4H_00H, 0xC4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_01H, 0xC4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_02H, 0xC4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_04H, 0xC4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_08H, 0xC4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_0CH, 0xC4, 0x0C, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C4H_0FH, 0xC4, 0x0F, IAP_F_FM | IAP_F_CA), IAPDESCR(C4H_10H, 0xC4, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_20H, 0xC4, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_40H, 0xC4, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C4H_7EH, 0xC4, 0x7E, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_BFH, 0xC4, 0xBF, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_EBH, 0xC4, 0xEB, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_F7H, 0xC4, 0xF7, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_F9H, 0xC4, 0xF9, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_FBH, 0xC4, 0xFB, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_FDH, 0xC4, 0xFD, IAP_F_FM | IAP_F_CAS), IAPDESCR(C4H_FEH, 0xC4, 0xFE, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_00H, 0xC5, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C5H_01H, 0xC5, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(C5H_02H, 0xC5, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_SL), IAPDESCR(C5H_04H, 0xC5, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(C5H_10H, 0xC5, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(C5H_20H, 0xC5, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL), IAPDESCR(C5H_7EH, 0xC5, 0x7E, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_BFH, 0xC5, 0xBF, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_EBH, 0xC5, 0xEB, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_F7H, 0xC5, 0xF7, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_F9H, 0xC5, 0xF9, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_FBH, 0xC5, 0xFB, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_FDH, 0xC5, 0xFD, IAP_F_FM | IAP_F_CAS), IAPDESCR(C5H_FEH, 0xC5, 0xFE, IAP_F_FM | IAP_F_CAS), IAPDESCR(C6H_00H, 0xC6, 0x00, IAP_F_FM | IAP_F_CC), /* For SL C6_01 needs EV_SEL? 0x11, 0x12, 0x13, 0x14, 0x15? */ IAPDESCR(C6H_01H, 0xC6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SL), IAPDESCR(C6H_02H, 0xC6, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C7H_00H, 0xC7, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(C7H_01H, 0xC7, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(C7H_02H, 0xC7, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(C7H_04H, 0xC7, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(C7H_08H, 0xC7, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(C7H_10H, 0xC7, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SL), IAPDESCR(C7H_1FH, 0xC7, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(C7H_20H, 0xC7, 0x20, IAP_F_FM | IAP_F_SL), IAPDESCR(C8H_00H, 0xC8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(C8H_20H, 0xC8, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(C9H_00H, 0xC9, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(CAH_00H, 0xCA, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(CAH_01H, 0xCA, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS), IAPDESCR(CAH_02H, 0xCA, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(CAH_04H, 0xCA, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(CAH_08H, 0xCA, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(CAH_10H, 0xCA, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(CAH_1EH, 0xCA, 0x1E, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(CAH_20H, 0xCA, 0x20, IAP_F_FM | IAP_F_CAS | IAP_F_BW | IAP_F_BWX), IAPDESCR(CAH_3FH, 0xCA, 0x3F, IAP_F_FM | IAP_F_CAS), IAPDESCR(CAH_50H, 0xCA, 0x50, IAP_F_FM | IAP_F_CAS), IAPDESCR(CBH_01H, 0xCB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_CAS | IAP_F_SL), IAPDESCR(CBH_02H, 0xCB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CBH_04H, 0xCB, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CBH_08H, 0xCB, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CBH_10H, 0xCB, 0x10, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CBH_1FH, 0xCB, 0x1F, IAP_F_FM | IAP_F_CAS), IAPDESCR(CBH_40H, 0xCB, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(CBH_80H, 0xCB, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(CCH_00H, 0xCC, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(CCH_01H, 0xCC, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CCH_02H, 0xCC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(CCH_03H, 0xCC, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(CCH_20H, 0xCC, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(CDH_00H, 0xCD, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(CDH_01H, 0xCD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(CDH_02H, 0xCD, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(CEH_00H, 0xCE, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(CFH_00H, 0xCF, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2), /* Sandy Bridge / Sandy Bridge Xeon - 11, 12, 21, 41, 42, 81, 82 */ IAPDESCR(D0H_00H, 0xD0, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(D0H_01H, 0xD0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(D0H_11H, 0xD0, 0x11, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_12H, 0xD0, 0x12, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_21H, 0xD0, 0x21, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_41H, 0xD0, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_42H, 0xD0, 0x42, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_81H, 0xD0, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D0H_82H, 0xD0, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_01H, 0xD1, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_02H, 0xD1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_04H, 0xD1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_08H, 0xD1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_10H, 0xD1, 0x10, IAP_F_HW | IAP_F_IB | IAP_F_IBX | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_20H, 0xD1, 0x20, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D1H_40H, 0xD1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D2H_01H, 0xD2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D2H_02H, 0xD2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D2H_04H, 0xD2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D2H_08H, 0xD2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(D2H_0FH, 0xD2, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(D2H_10H, 0xD2, 0x10, IAP_F_FM | IAP_F_CC2E), IAPDESCR(D3H_01H, 0xD3, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(D3H_03H, 0xD3, 0x03, IAP_F_FM | IAP_F_IBX), IAPDESCR(D3H_04H, 0xD3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX), /* Not defined for IBX */ IAPDESCR(D3H_0CH, 0xD3, 0x0C, IAP_F_FM | IAP_F_IBX), IAPDESCR(D3H_10H, 0xD3, 0x10, IAP_F_FM | IAP_F_IBX ), IAPDESCR(D3H_20H, 0xD3, 0x20, IAP_F_FM | IAP_F_IBX ), IAPDESCR(D4H_01H, 0xD4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(D4H_02H, 0xD4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB | IAP_F_SBX), IAPDESCR(D4H_04H, 0xD4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D4H_08H, 0xD4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D4H_0FH, 0xD4, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D5H_01H, 0xD5, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7 | IAP_F_WM), IAPDESCR(D5H_02H, 0xD5, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D5H_04H, 0xD5, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D5H_08H, 0xD5, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D5H_0FH, 0xD5, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(D7H_00H, 0xD7, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(D8H_00H, 0xD8, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(D8H_01H, 0xD8, 0x01, IAP_F_FM | IAP_F_CC), IAPDESCR(D8H_02H, 0xD8, 0x02, IAP_F_FM | IAP_F_CC), IAPDESCR(D8H_03H, 0xD8, 0x03, IAP_F_FM | IAP_F_CC), IAPDESCR(D8H_04H, 0xD8, 0x04, IAP_F_FM | IAP_F_CC), IAPDESCR(D9H_00H, 0xD9, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(D9H_01H, 0xD9, 0x01, IAP_F_FM | IAP_F_CC), IAPDESCR(D9H_02H, 0xD9, 0x02, IAP_F_FM | IAP_F_CC), IAPDESCR(D9H_03H, 0xD9, 0x03, IAP_F_FM | IAP_F_CC), IAPDESCR(DAH_00H, 0xDA, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(DAH_01H, 0xDA, 0x01, IAP_F_FM | IAP_F_CC), IAPDESCR(DAH_02H, 0xDA, 0x02, IAP_F_FM | IAP_F_CC), IAPDESCR(DBH_00H, 0xDB, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(DBH_01H, 0xDB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(DCH_01H, 0xDC, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(DCH_02H, 0xDC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(DCH_04H, 0xDC, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(DCH_08H, 0xDC, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(DCH_10H, 0xDC, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(DCH_1FH, 0xDC, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2), IAPDESCR(E0H_00H, 0xE0, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2), IAPDESCR(E0H_01H, 0xE0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM), IAPDESCR(E2H_00H, 0xE2, 0x00, IAP_F_FM | IAP_F_CC), IAPDESCR(E4H_00H, 0xE4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(E4H_01H, 0xE4, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(E5H_01H, 0xE5, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(E6H_00H, 0xE6, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2), IAPDESCR(E6H_01H, 0xE6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM | IAP_F_SBX | IAP_F_CAS | IAP_F_SL), IAPDESCR(E6H_02H, 0xE6, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(E6H_08H, 0xE6, 0x08, IAP_F_FM | IAP_F_CAS), IAPDESCR(E6H_10H, 0xE6, 0x10, IAP_F_FM | IAP_F_CAS), IAPDESCR(E6H_1FH, 0xE6, 0x1F, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX), IAPDESCR(E7H_01H, 0xE7, 0x01, IAP_F_FM | IAP_F_CAS), IAPDESCR(E8H_01H, 0xE8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(E8H_02H, 0xE8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(E8H_03H, 0xE8, 0x03, IAP_F_FM | IAP_F_I7O), IAPDESCR(ECH_01H, 0xEC, 0x01, IAP_F_FM | IAP_F_WM), IAPDESCR(F0H_00H, 0xF0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(F0H_01H, 0xF0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_02H, 0xF0, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_04H, 0xF0, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_08H, 0xF0, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_10H, 0xF0, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_20H, 0xF0, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F0H_40H, 0xF0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(F0H_80H, 0xF0, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F1H_01H, 0xF1, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F1H_02H, 0xF1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F1H_04H, 0xF1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX ), IAPDESCR(F1H_07H, 0xF1, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL), IAPDESCR(F2H_01H, 0xF2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(F2H_02H, 0xF2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(F2H_04H, 0xF2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(F2H_05H, 0xF2, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX), IAPDESCR(F2H_06H, 0xF2, 0x06, IAP_F_FM | IAP_F_HW | IAP_F_HWX), IAPDESCR(F2H_08H, 0xF2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(F2H_0AH, 0xF2, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IBX), IAPDESCR(F2H_0FH, 0xF2, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(F3H_01H, 0xF3, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(F3H_02H, 0xF3, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(F3H_04H, 0xF3, 0x04, IAP_F_FM | IAP_F_I7O), IAPDESCR(F3H_08H, 0xF3, 0x08, IAP_F_FM | IAP_F_I7O), IAPDESCR(F3H_10H, 0xF3, 0x10, IAP_F_FM | IAP_F_I7O), IAPDESCR(F3H_20H, 0xF3, 0x20, IAP_F_FM | IAP_F_I7O), IAPDESCR(F4H_01H, 0xF4, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(F4H_02H, 0xF4, 0x02, IAP_F_FM | IAP_F_I7O), IAPDESCR(F4H_04H, 0xF4, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O), IAPDESCR(F4H_08H, 0xF4, 0x08, IAP_F_FM | IAP_F_I7O), IAPDESCR(F4H_10H, 0xF4, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX), IAPDESCR(F6H_01H, 0xF6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM), IAPDESCR(F7H_01H, 0xF7, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(F7H_02H, 0xF7, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(F7H_04H, 0xF7, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(F8H_00H, 0xF8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2), IAPDESCR(F8H_01H, 0xF8, 0x01, IAP_F_FM | IAP_F_I7O), IAPDESCR(FDH_01H, 0xFD, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_02H, 0xFD, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_04H, 0xFD, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_08H, 0xFD, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_10H, 0xFD, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_20H, 0xFD, 0x20, IAP_F_FM | IAP_F_WM | IAP_F_I7), IAPDESCR(FDH_40H, 0xFD, 0x40, IAP_F_FM | IAP_F_WM | IAP_F_I7), }; -static const int niap_events = sizeof(iap_events) / sizeof(iap_events[0]); - static pmc_value_t iap_perfctr_value_to_reload_count(pmc_value_t v) { /* If the PMC has overflowed, return a reload count of zero. */ if ((v & (1ULL << (core_iap_width - 1))) == 0) return (0); v &= (1ULL << core_iap_width) - 1; return (1ULL << core_iap_width) - v; } static pmc_value_t iap_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << core_iap_width) - rlc; } static int iap_pmc_has_overflowed(int ri) { uint64_t v; /* * We treat a Core (i.e., Intel architecture v1) PMC as has * having overflowed if its MSB is zero. */ v = rdpmc(ri); return ((v & (1ULL << (core_iap_width - 1))) == 0); } /* * Check an event against the set of supported architectural events. * * If the event is not architectural EV_IS_NOTARCH is returned. * If the event is architectural and supported on this CPU, the correct * event+umask mapping is returned in map, and EV_IS_ARCH_SUPP is returned. * Otherwise, the function returns EV_IS_ARCH_NOTSUPP. */ static int iap_is_event_architectural(enum pmc_event pe, enum pmc_event *map) { enum core_arch_events ae; switch (pe) { case PMC_EV_IAP_ARCH_UNH_COR_CYC: ae = CORE_AE_UNHALTED_CORE_CYCLES; *map = PMC_EV_IAP_EVENT_3CH_00H; break; case PMC_EV_IAP_ARCH_INS_RET: ae = CORE_AE_INSTRUCTION_RETIRED; *map = PMC_EV_IAP_EVENT_C0H_00H; break; case PMC_EV_IAP_ARCH_UNH_REF_CYC: ae = CORE_AE_UNHALTED_REFERENCE_CYCLES; *map = PMC_EV_IAP_EVENT_3CH_01H; break; case PMC_EV_IAP_ARCH_LLC_REF: ae = CORE_AE_LLC_REFERENCE; *map = PMC_EV_IAP_EVENT_2EH_4FH; break; case PMC_EV_IAP_ARCH_LLC_MIS: ae = CORE_AE_LLC_MISSES; *map = PMC_EV_IAP_EVENT_2EH_41H; break; case PMC_EV_IAP_ARCH_BR_INS_RET: ae = CORE_AE_BRANCH_INSTRUCTION_RETIRED; *map = PMC_EV_IAP_EVENT_C4H_00H; break; case PMC_EV_IAP_ARCH_BR_MIS_RET: ae = CORE_AE_BRANCH_MISSES_RETIRED; *map = PMC_EV_IAP_EVENT_C5H_00H; break; default: /* Non architectural event. */ return (EV_IS_NOTARCH); } return (((core_architectural_events & (1 << ae)) == 0) ? EV_IS_ARCH_NOTSUPP : EV_IS_ARCH_SUPP); } static int iap_event_corei7_ok_on_counter(enum pmc_event pe, int ri) { uint32_t mask; switch (pe) { /* * Events valid only on counter 0, 1. */ case PMC_EV_IAP_EVENT_40H_01H: case PMC_EV_IAP_EVENT_40H_02H: case PMC_EV_IAP_EVENT_40H_04H: case PMC_EV_IAP_EVENT_40H_08H: case PMC_EV_IAP_EVENT_40H_0FH: case PMC_EV_IAP_EVENT_41H_02H: case PMC_EV_IAP_EVENT_41H_04H: case PMC_EV_IAP_EVENT_41H_08H: case PMC_EV_IAP_EVENT_42H_01H: case PMC_EV_IAP_EVENT_42H_02H: case PMC_EV_IAP_EVENT_42H_04H: case PMC_EV_IAP_EVENT_42H_08H: case PMC_EV_IAP_EVENT_43H_01H: case PMC_EV_IAP_EVENT_43H_02H: case PMC_EV_IAP_EVENT_51H_01H: case PMC_EV_IAP_EVENT_51H_02H: case PMC_EV_IAP_EVENT_51H_04H: case PMC_EV_IAP_EVENT_51H_08H: case PMC_EV_IAP_EVENT_63H_01H: case PMC_EV_IAP_EVENT_63H_02H: mask = 0x3; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_westmere_ok_on_counter(enum pmc_event pe, int ri) { uint32_t mask; switch (pe) { /* * Events valid only on counter 0. */ case PMC_EV_IAP_EVENT_60H_01H: case PMC_EV_IAP_EVENT_60H_02H: case PMC_EV_IAP_EVENT_60H_04H: case PMC_EV_IAP_EVENT_60H_08H: case PMC_EV_IAP_EVENT_B3H_01H: case PMC_EV_IAP_EVENT_B3H_02H: case PMC_EV_IAP_EVENT_B3H_04H: mask = 0x1; break; /* * Events valid only on counter 0, 1. */ case PMC_EV_IAP_EVENT_4CH_01H: case PMC_EV_IAP_EVENT_4EH_01H: case PMC_EV_IAP_EVENT_4EH_02H: case PMC_EV_IAP_EVENT_4EH_04H: case PMC_EV_IAP_EVENT_51H_01H: case PMC_EV_IAP_EVENT_51H_02H: case PMC_EV_IAP_EVENT_51H_04H: case PMC_EV_IAP_EVENT_51H_08H: case PMC_EV_IAP_EVENT_63H_01H: case PMC_EV_IAP_EVENT_63H_02H: mask = 0x3; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_sb_sbx_ib_ibx_ok_on_counter(enum pmc_event pe, int ri) { uint32_t mask; switch (pe) { /* Events valid only on counter 0. */ case PMC_EV_IAP_EVENT_B7H_01H: mask = 0x1; break; /* Events valid only on counter 1. */ case PMC_EV_IAP_EVENT_C0H_01H: mask = 0x2; break; /* Events valid only on counter 2. */ case PMC_EV_IAP_EVENT_48H_01H: case PMC_EV_IAP_EVENT_A2H_02H: case PMC_EV_IAP_EVENT_A3H_08H: mask = 0x4; break; /* Events valid only on counter 3. */ case PMC_EV_IAP_EVENT_BBH_01H: case PMC_EV_IAP_EVENT_CDH_01H: case PMC_EV_IAP_EVENT_CDH_02H: mask = 0x8; break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_event_ok_on_counter(enum pmc_event pe, int ri) { uint32_t mask; switch (pe) { /* * Events valid only on counter 0. */ case PMC_EV_IAP_EVENT_10H_00H: case PMC_EV_IAP_EVENT_14H_00H: case PMC_EV_IAP_EVENT_18H_00H: case PMC_EV_IAP_EVENT_B3H_01H: case PMC_EV_IAP_EVENT_B3H_02H: case PMC_EV_IAP_EVENT_B3H_04H: case PMC_EV_IAP_EVENT_C1H_00H: case PMC_EV_IAP_EVENT_CBH_01H: case PMC_EV_IAP_EVENT_CBH_02H: mask = (1 << 0); break; /* * Events valid only on counter 1. */ case PMC_EV_IAP_EVENT_11H_00H: case PMC_EV_IAP_EVENT_12H_00H: case PMC_EV_IAP_EVENT_13H_00H: mask = (1 << 1); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int iap_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int arch, n, model; enum pmc_event ev, map; struct iap_event_descr *ie; uint32_t c, caps, config, cpuflag, evsel, mask; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index value %d", __LINE__, ri)); /* check requested capabilities */ caps = a->pm_caps; if ((IAP_PMC_CAPS & caps) != caps) return (EPERM); map = 0; /* XXX: silent GCC warning */ arch = iap_is_event_architectural(pm->pm_event, &map); if (arch == EV_IS_ARCH_NOTSUPP) return (EOPNOTSUPP); else if (arch == EV_IS_ARCH_SUPP) ev = map; else ev = pm->pm_event; /* * A small number of events are not supported in all the * processors based on a given microarchitecture. */ if (ev == PMC_EV_IAP_EVENT_0FH_01H || ev == PMC_EV_IAP_EVENT_0FH_80H) { model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4); if (core_cputype == PMC_CPU_INTEL_COREI7 && model != 0x2E) return (EINVAL); } switch (core_cputype) { case PMC_CPU_INTEL_COREI7: case PMC_CPU_INTEL_NEHALEM_EX: if (iap_event_corei7_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_SKYLAKE: case PMC_CPU_INTEL_BROADWELL: case PMC_CPU_INTEL_BROADWELL_XEON: case PMC_CPU_INTEL_SANDYBRIDGE: case PMC_CPU_INTEL_SANDYBRIDGE_XEON: case PMC_CPU_INTEL_IVYBRIDGE: case PMC_CPU_INTEL_IVYBRIDGE_XEON: case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_HASWELL_XEON: if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0) return (EINVAL); break; case PMC_CPU_INTEL_WESTMERE: case PMC_CPU_INTEL_WESTMERE_EX: if (iap_event_westmere_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: if (iap_event_ok_on_counter(ev, ri) == 0) return (EINVAL); } /* * Look for an event descriptor with matching CPU and event id * fields. */ switch (core_cputype) { default: case PMC_CPU_INTEL_ATOM: cpuflag = IAP_F_CA; break; case PMC_CPU_INTEL_ATOM_SILVERMONT: cpuflag = IAP_F_CAS; break; case PMC_CPU_INTEL_SKYLAKE: cpuflag = IAP_F_SL; break; case PMC_CPU_INTEL_BROADWELL_XEON: cpuflag = IAP_F_BWX; break; case PMC_CPU_INTEL_BROADWELL: cpuflag = IAP_F_BW; break; case PMC_CPU_INTEL_CORE: cpuflag = IAP_F_CC; break; case PMC_CPU_INTEL_CORE2: cpuflag = IAP_F_CC2; break; case PMC_CPU_INTEL_CORE2EXTREME: cpuflag = IAP_F_CC2 | IAP_F_CC2E; break; case PMC_CPU_INTEL_COREI7: cpuflag = IAP_F_I7; break; case PMC_CPU_INTEL_HASWELL: cpuflag = IAP_F_HW; break; case PMC_CPU_INTEL_HASWELL_XEON: cpuflag = IAP_F_HWX; break; case PMC_CPU_INTEL_IVYBRIDGE: cpuflag = IAP_F_IB; break; case PMC_CPU_INTEL_IVYBRIDGE_XEON: cpuflag = IAP_F_IBX; break; case PMC_CPU_INTEL_SANDYBRIDGE: cpuflag = IAP_F_SB; break; case PMC_CPU_INTEL_SANDYBRIDGE_XEON: cpuflag = IAP_F_SBX; break; case PMC_CPU_INTEL_WESTMERE: cpuflag = IAP_F_WM; break; } - for (n = 0, ie = iap_events; n < niap_events; n++, ie++) + for (n = 0, ie = iap_events; n < nitems(iap_events); n++, ie++) if (ie->iap_ev == ev && ie->iap_flags & cpuflag) break; - if (n == niap_events) + if (n == nitems(iap_events)) return (EINVAL); /* * A matching event descriptor has been found, so start * assembling the contents of the event select register. */ evsel = ie->iap_evcode; config = a->pm_md.pm_iap.pm_iap_config & ~IAP_F_CMASK; /* * If the event uses a fixed umask value, reject any umask * bits set by the user. */ if (ie->iap_flags & IAP_F_FM) { if (IAP_UMASK(config) != 0) return (EINVAL); evsel |= (ie->iap_umask << 8); } else { /* * Otherwise, the UMASK value needs to be taken from * the MD fields of the allocation request. Reject * requests that specify reserved bits. */ mask = 0; if (ie->iap_umask & IAP_M_CORE) { if ((c = (config & IAP_F_CORE)) != IAP_CORE_ALL && c != IAP_CORE_THIS) return (EINVAL); mask |= IAP_F_CORE; } if (ie->iap_umask & IAP_M_AGENT) mask |= IAP_F_AGENT; if (ie->iap_umask & IAP_M_PREFETCH) { if ((c = (config & IAP_F_PREFETCH)) == IAP_PREFETCH_RESERVED) return (EINVAL); mask |= IAP_F_PREFETCH; } if (ie->iap_umask & IAP_M_MESI) mask |= IAP_F_MESI; if (ie->iap_umask & IAP_M_SNOOPRESPONSE) mask |= IAP_F_SNOOPRESPONSE; if (ie->iap_umask & IAP_M_SNOOPTYPE) mask |= IAP_F_SNOOPTYPE; if (ie->iap_umask & IAP_M_TRANSITION) mask |= IAP_F_TRANSITION; /* * If bits outside of the allowed set of umask bits * are set, reject the request. */ if (config & ~mask) return (EINVAL); evsel |= (config & mask); } /* * Only Atom and SandyBridge CPUs support the 'ANY' qualifier. */ if (core_cputype == PMC_CPU_INTEL_ATOM || core_cputype == PMC_CPU_INTEL_ATOM_SILVERMONT || core_cputype == PMC_CPU_INTEL_SANDYBRIDGE || core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON) evsel |= (config & IAP_ANY); else if (config & IAP_ANY) return (EINVAL); /* * Check offcore response configuration. */ if (a->pm_md.pm_iap.pm_iap_rsp != 0) { if (ev != PMC_EV_IAP_EVENT_B7H_01H && ev != PMC_EV_IAP_EVENT_BBH_01H) return (EINVAL); if (core_cputype == PMC_CPU_INTEL_COREI7 && ev == PMC_EV_IAP_EVENT_BBH_01H) return (EINVAL); if ((core_cputype == PMC_CPU_INTEL_COREI7 || core_cputype == PMC_CPU_INTEL_WESTMERE || core_cputype == PMC_CPU_INTEL_NEHALEM_EX || core_cputype == PMC_CPU_INTEL_WESTMERE_EX) && a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_I7WM) return (EINVAL); else if ((core_cputype == PMC_CPU_INTEL_SANDYBRIDGE || core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON || core_cputype == PMC_CPU_INTEL_IVYBRIDGE || core_cputype == PMC_CPU_INTEL_IVYBRIDGE_XEON) && a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_SBIB) return (EINVAL); pm->pm_md.pm_iap.pm_iap_rsp = a->pm_md.pm_iap.pm_iap_rsp; } if (caps & PMC_CAP_THRESHOLD) evsel |= (a->pm_md.pm_iap.pm_iap_config & IAP_F_CMASK); if (caps & PMC_CAP_USER) evsel |= IAP_USR; if (caps & PMC_CAP_SYSTEM) evsel |= IAP_OS; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) evsel |= (IAP_OS | IAP_USR); if (caps & PMC_CAP_EDGE) evsel |= IAP_EDGE; if (caps & PMC_CAP_INVERT) evsel |= IAP_INV; if (caps & PMC_CAP_INTERRUPT) evsel |= IAP_INT; pm->pm_md.pm_iap.pm_iap_evsel = evsel; return (0); } static int iap_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__, cpu)); core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm; return (0); } static int iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char iap_name[PMC_NAME_MAX]; phw = &core_pcpu[cpu]->pc_corepmcs[ri]; (void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri); if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_IAP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int iap_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; return (0); } static int iap_get_msr(int ri, uint32_t *msr) { KASSERT(ri >= 0 && ri < core_iap_npmc, ("[iap,%d] ri %d out of range", __LINE__, ri)); *msr = ri; return (0); } static int iap_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdpmc(ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = iap_perfctr_value_to_reload_count(tmp); else *v = tmp & ((1ULL << core_iap_width) - 1); PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, IAP_PMC0 + ri, *v); return (0); } static int iap_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int iap_start_pmc(int cpu, int ri) { struct pmc *pm; uint32_t evsel; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row-index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_iap.pm_iap_evsel; PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, IAP_EVSEL0 + ri, evsel); /* Event specific configuration. */ switch (pm->pm_event) { case PMC_EV_IAP_EVENT_B7H_01H: wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp); break; case PMC_EV_IAP_EVENT_BBH_01H: wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp); break; default: break; } wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN); if (core_cputype == PMC_CPU_INTEL_CORE) return (0); do { cc->pc_resync = 0; cc->pc_globalctrl |= (1ULL << ri); wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int iap_stop_pmc(int cpu, int ri) { struct pmc *pm; struct core_cpu *cc; uint64_t msr; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri); msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */ if (core_cputype == PMC_CPU_INTEL_CORE) return (0); msr = 0; do { cc->pc_resync = 0; cc->pc_globalctrl &= ~(1ULL << ri); msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int iap_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct core_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[core,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < core_iap_npmc, ("[core,%d] illegal row index %d", __LINE__, ri)); cc = core_pcpu[cpu]; pm = cc->pc_corepmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = iap_reload_count_to_perfctr_value(v); v &= (1ULL << core_iap_width) - 1; PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, IAP_PMC0 + ri, v); /* * Write the new value to the counter (or it's alias). The * counter will be in a stopped state when the pcd_write() * entry point is called. */ wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); return (0); } static void iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth, int flags) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "iap-initialize"); /* Remember the set of architectural events supported. */ core_architectural_events = ~flags; pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP]; pcd->pcd_caps = IAP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_IAP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = iap_allocate_pmc; pcd->pcd_config_pmc = iap_config_pmc; pcd->pcd_describe = iap_describe; pcd->pcd_get_config = iap_get_config; pcd->pcd_get_msr = iap_get_msr; pcd->pcd_pcpu_fini = core_pcpu_fini; pcd->pcd_pcpu_init = core_pcpu_init; pcd->pcd_read_pmc = iap_read_pmc; pcd->pcd_release_pmc = iap_release_pmc; pcd->pcd_start_pmc = iap_start_pmc; pcd->pcd_stop_pmc = iap_stop_pmc; pcd->pcd_write_pmc = iap_write_pmc; md->pmd_npmc += npmc; } static int core_intr(int cpu, struct trapframe *tf) { pmc_value_t v; struct pmc *pm; struct core_cpu *cc; int error, found_interrupt, ri; uint64_t msr; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); found_interrupt = 0; cc = core_pcpu[cpu]; for (ri = 0; ri < core_iap_npmc; ri++) { if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; if (!iap_pmc_has_overflowed(ri)) continue; found_interrupt = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); v = pm->pm_sc.pm_reloadcount; v = iap_reload_count_to_perfctr_value(v); /* * Stop the counter, reload it but only restart it if * the PMC is not stalled. */ msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK; wrmsr(IAP_EVSEL0 + ri, msr); wrmsr(core_iap_wroffset + IAP_PMC0 + ri, v); if (error) continue; wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN)); } if (found_interrupt) lapic_reenable_pmc(); atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed : &pmc_stats.pm_intr_ignored, 1); return (found_interrupt); } static int core2_intr(int cpu, struct trapframe *tf) { int error, found_interrupt, n; uint64_t flag, intrstatus, intrenable, msr; struct pmc *pm; struct core_cpu *cc; pmc_value_t v; PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); /* * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which * PMCs have a pending PMI interrupt. We take a 'snapshot' of * the current set of interrupting PMCs and process these * after stopping them. */ intrstatus = rdmsr(IA_GLOBAL_STATUS); intrenable = intrstatus & core_pmcmask; PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu, (uintmax_t) intrstatus); found_interrupt = 0; cc = core_pcpu[cpu]; KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__)); cc->pc_globalctrl &= ~intrenable; cc->pc_resync = 1; /* MSRs now potentially out of sync. */ /* * Stop PMCs and clear overflow status bits. */ msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK; wrmsr(IA_GLOBAL_CTRL, msr); wrmsr(IA_GLOBAL_OVF_CTRL, intrenable | IA_GLOBAL_STATUS_FLAG_OVFBUF | IA_GLOBAL_STATUS_FLAG_CONDCHG); /* * Look for interrupts from fixed function PMCs. */ for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) intrenable &= ~flag; v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); /* Reload sampling count. */ wrmsr(IAF_CTR0 + n, v); PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu, error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n))); } /* * Process interrupts from the programmable counters. */ for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) { if ((intrstatus & flag) == 0) continue; found_interrupt = 1; pm = cc->pc_corepmcs[n].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) intrenable &= ~flag; v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount); PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error, (uintmax_t) v); /* Reload sampling count. */ wrmsr(core_iap_wroffset + IAP_PMC0 + n, v); } /* * Reenable all non-stalled PMCs. */ PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu, (uintmax_t) intrenable); cc->pc_globalctrl |= intrenable; wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK); PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx " "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_CTRL), (uintmax_t) rdmsr(IA_GLOBAL_STATUS), (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL)); if (found_interrupt) lapic_reenable_pmc(); atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed : &pmc_stats.pm_intr_ignored, 1); return (found_interrupt); } int pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override) { int cpuid[CORE_CPUID_REQUEST_SIZE]; int ipa_version, flags, nflags; do_cpuid(CORE_CPUID_REQUEST, cpuid); ipa_version = (version_override > 0) ? version_override : cpuid[CORE_CPUID_EAX] & 0xFF; core_cputype = md->pmd_cputype; PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d", core_cputype, maxcpu, ipa_version); if (ipa_version < 1 || ipa_version > 3 || (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) { /* Unknown PMC architecture. */ printf("hwpc_core: unknown PMC architecture: %d\n", ipa_version); return (EPROGMISMATCH); } core_iap_wroffset = 0; if (cpu_feature2 & CPUID2_PDCM) { if (rdmsr(IA32_PERF_CAPABILITIES) & PERFCAP_FW_WRITE) { PMCDBG0(MDP, INI, 1, "core-init full-width write supported"); core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0; } else PMCDBG0(MDP, INI, 1, "core-init full-width write NOT supported"); } else PMCDBG0(MDP, INI, 1, "core-init pdcm not supported"); core_pmcmask = 0; /* * Initialize programmable counters. */ core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF; core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF; core_pmcmask |= ((1ULL << core_iap_npmc) - 1); nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF; flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1); iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags); /* * Initialize fixed function counters, if present. */ if (core_cputype != PMC_CPU_INTEL_CORE) { core_iaf_ri = core_iap_npmc; core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F; core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF; iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width); core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET; } PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask, core_iaf_ri); core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); /* * Choose the appropriate interrupt handler. */ if (ipa_version == 1) md->pmd_intr = core_intr; else md->pmd_intr = core2_intr; md->pmd_pcpu_fini = NULL; md->pmd_pcpu_init = NULL; return (0); } void pmc_core_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "core-finalize"); free(core_pcpu, M_PMC); core_pcpu = NULL; } Index: head/sys/dev/hwpmc/hwpmc_mpc7xxx.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_mpc7xxx.c (revision 298410) +++ head/sys/dev/hwpmc/hwpmc_mpc7xxx.c (revision 298411) @@ -1,755 +1,752 @@ /*- * Copyright (c) 2011 Justin Hibbits * Copyright (c) 2005, Joseph Koshy * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include "hwpmc_powerpc.h" #define POWERPC_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ PMC_CAP_SYSTEM | PMC_CAP_EDGE | \ PMC_CAP_THRESHOLD | PMC_CAP_READ | \ PMC_CAP_WRITE | PMC_CAP_INVERT | \ PMC_CAP_QUALIFIER) #define PPC_SET_PMC1SEL(r, x) ((r & ~(SPR_MMCR0_PMC1SEL(0x3f))) | SPR_MMCR0_PMC1SEL(x)) #define PPC_SET_PMC2SEL(r, x) ((r & ~(SPR_MMCR0_PMC2SEL(0x3f))) | SPR_MMCR0_PMC2SEL(x)) #define PPC_SET_PMC3SEL(r, x) ((r & ~(SPR_MMCR1_PMC3SEL(0x1f))) | SPR_MMCR1_PMC3SEL(x)) #define PPC_SET_PMC4SEL(r, x) ((r & ~(SPR_MMCR1_PMC4SEL(0x1f))) | SPR_MMCR1_PMC4SEL(x)) #define PPC_SET_PMC5SEL(r, x) ((r & ~(SPR_MMCR1_PMC5SEL(0x1f))) | SPR_MMCR1_PMC5SEL(x)) #define PPC_SET_PMC6SEL(r, x) ((r & ~(SPR_MMCR1_PMC6SEL(0x3f))) | SPR_MMCR1_PMC6SEL(x)) /* Change this when we support more than just the 7450. */ #define MPC7XXX_MAX_PMCS 6 #define MPC7XXX_PMC_HAS_OVERFLOWED(x) (mpc7xxx_pmcn_read(x) & (0x1 << 31)) /* * Things to improve on this: * - It stops (clears to 0) the PMC and resets it at every context switch * currently. */ /* * This should work for every 32-bit PowerPC implementation I know of (G3 and G4 * specifically). */ struct mpc7xxx_event_code_map { enum pmc_event pe_ev; /* enum value */ uint8_t pe_counter_mask; /* Which counter this can be counted in. */ uint8_t pe_code; /* numeric code */ }; #define PPC_PMC_MASK1 0 #define PPC_PMC_MASK2 1 #define PPC_PMC_MASK3 2 #define PPC_PMC_MASK4 3 #define PPC_PMC_MASK5 4 #define PPC_PMC_MASK6 5 #define PPC_PMC_MASK_ALL 0x3f #define PMC_POWERPC_EVENT(id, mask, number) \ { .pe_ev = PMC_EV_PPC7450_##id, .pe_counter_mask = mask, .pe_code = number } static struct mpc7xxx_event_code_map mpc7xxx_event_codes[] = { PMC_POWERPC_EVENT(CYCLE,PPC_PMC_MASK_ALL, 1), PMC_POWERPC_EVENT(INSTR_COMPLETED, 0x0f, 2), PMC_POWERPC_EVENT(TLB_BIT_TRANSITIONS, 0x0f, 3), PMC_POWERPC_EVENT(INSTR_DISPATCHED, 0x0f, 4), PMC_POWERPC_EVENT(PMON_EXCEPT, 0x0f, 5), PMC_POWERPC_EVENT(PMON_SIG, 0x0f, 7), PMC_POWERPC_EVENT(VPU_INSTR_COMPLETED, 0x03, 8), PMC_POWERPC_EVENT(VFPU_INSTR_COMPLETED, 0x03, 9), PMC_POWERPC_EVENT(VIU1_INSTR_COMPLETED, 0x03, 10), PMC_POWERPC_EVENT(VIU2_INSTR_COMPLETED, 0x03, 11), PMC_POWERPC_EVENT(MTVSCR_INSTR_COMPLETED, 0x03, 12), PMC_POWERPC_EVENT(MTVRSAVE_INSTR_COMPLETED, 0x03, 13), PMC_POWERPC_EVENT(VPU_INSTR_WAIT_CYCLES, 0x03, 14), PMC_POWERPC_EVENT(VFPU_INSTR_WAIT_CYCLES, 0x03, 15), PMC_POWERPC_EVENT(VIU1_INSTR_WAIT_CYCLES, 0x03, 16), PMC_POWERPC_EVENT(VIU2_INSTR_WAIT_CYCLES, 0x03, 17), PMC_POWERPC_EVENT(MFVSCR_SYNC_CYCLES, 0x03, 18), PMC_POWERPC_EVENT(VSCR_SAT_SET, 0x03, 19), PMC_POWERPC_EVENT(STORE_INSTR_COMPLETED, 0x03, 20), PMC_POWERPC_EVENT(L1_INSTR_CACHE_MISSES, 0x03, 21), PMC_POWERPC_EVENT(L1_DATA_SNOOPS, 0x03, 22), PMC_POWERPC_EVENT(UNRESOLVED_BRANCHES, 0x01, 23), PMC_POWERPC_EVENT(SPEC_BUFFER_CYCLES, 0x01, 24), PMC_POWERPC_EVENT(BRANCH_UNIT_STALL_CYCLES, 0x01, 25), PMC_POWERPC_EVENT(TRUE_BRANCH_TARGET_HITS, 0x01, 26), PMC_POWERPC_EVENT(BRANCH_LINK_STAC_PREDICTED, 0x01, 27), PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_DISPATCHES, 0x01, 28), PMC_POWERPC_EVENT(CYCLES_THREE_INSTR_DISPATCHED, 0x01, 29), PMC_POWERPC_EVENT(THRESHOLD_INSTR_QUEUE_ENTRIES_CYCLES, 0x01, 30), PMC_POWERPC_EVENT(THRESHOLD_VEC_INSTR_QUEUE_ENTRIES_CYCLES, 0x01, 31), PMC_POWERPC_EVENT(CYCLES_NO_COMPLETED_INSTRS, 0x01, 32), PMC_POWERPC_EVENT(IU2_INSTR_COMPLETED, 0x01, 33), PMC_POWERPC_EVENT(BRANCHES_COMPLETED, 0x01, 34), PMC_POWERPC_EVENT(EIEIO_INSTR_COMPLETED, 0x01, 35), PMC_POWERPC_EVENT(MTSPR_INSTR_COMPLETED, 0x01, 36), PMC_POWERPC_EVENT(SC_INSTR_COMPLETED, 0x01, 37), PMC_POWERPC_EVENT(LS_LM_COMPLETED, 0x01, 38), PMC_POWERPC_EVENT(ITLB_HW_TABLE_SEARCH_CYCLES, 0x01, 39), PMC_POWERPC_EVENT(DTLB_HW_SEARCH_CYCLES_OVER_THRESHOLD, 0x01, 40), PMC_POWERPC_EVENT(L1_INSTR_CACHE_ACCESSES, 0x01, 41), PMC_POWERPC_EVENT(INSTR_BKPT_MATCHES, 0x01, 42), PMC_POWERPC_EVENT(L1_DATA_CACHE_LOAD_MISS_CYCLES_OVER_THRESHOLD, 0x01, 43), PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_ON_MODIFIED, 0x01, 44), PMC_POWERPC_EVENT(LOAD_MISS_ALIAS, 0x01, 45), PMC_POWERPC_EVENT(LOAD_MISS_ALIAS_ON_TOUCH, 0x01, 46), PMC_POWERPC_EVENT(TOUCH_ALIAS, 0x01, 47), PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_CASTOUT_QUEUE, 0x01, 48), PMC_POWERPC_EVENT(L1_DATA_SNOOP_HIT_CASTOUT, 0x01, 49), PMC_POWERPC_EVENT(L1_DATA_SNOOP_HITS, 0x01, 50), PMC_POWERPC_EVENT(WRITE_THROUGH_STORES, 0x01, 51), PMC_POWERPC_EVENT(CACHE_INHIBITED_STORES, 0x01, 52), PMC_POWERPC_EVENT(L1_DATA_LOAD_HIT, 0x01, 53), PMC_POWERPC_EVENT(L1_DATA_TOUCH_HIT, 0x01, 54), PMC_POWERPC_EVENT(L1_DATA_STORE_HIT, 0x01, 55), PMC_POWERPC_EVENT(L1_DATA_TOTAL_HITS, 0x01, 56), PMC_POWERPC_EVENT(DST_INSTR_DISPATCHED, 0x01, 57), PMC_POWERPC_EVENT(REFRESHED_DSTS, 0x01, 58), PMC_POWERPC_EVENT(SUCCESSFUL_DST_TABLE_SEARCHES, 0x01, 59), PMC_POWERPC_EVENT(DSS_INSTR_COMPLETED, 0x01, 60), PMC_POWERPC_EVENT(DST_STREAM_0_CACHE_LINE_FETCHES, 0x01, 61), PMC_POWERPC_EVENT(VTQ_SUSPENDS_DUE_TO_CTX_CHANGE, 0x01, 62), PMC_POWERPC_EVENT(VTQ_LINE_FETCH_HIT, 0x01, 63), PMC_POWERPC_EVENT(VEC_LOAD_INSTR_COMPLETED, 0x01, 64), PMC_POWERPC_EVENT(FP_STORE_INSTR_COMPLETED_IN_LSU, 0x01, 65), PMC_POWERPC_EVENT(FPU_RENORMALIZATION, 0x01, 66), PMC_POWERPC_EVENT(FPU_DENORMALIZATION, 0x01, 67), PMC_POWERPC_EVENT(FP_STORE_CAUSES_STALL_IN_LSU, 0x01, 68), PMC_POWERPC_EVENT(LD_ST_TRUE_ALIAS_STALL, 0x01, 70), PMC_POWERPC_EVENT(LSU_INDEXED_ALIAS_STALL, 0x01, 71), PMC_POWERPC_EVENT(LSU_ALIAS_VS_FSQ_WB0_WB1, 0x01, 72), PMC_POWERPC_EVENT(LSU_ALIAS_VS_CSQ, 0x01, 73), PMC_POWERPC_EVENT(LSU_LOAD_HIT_LINE_ALIAS_VS_CSQ0, 0x01, 74), PMC_POWERPC_EVENT(LSU_LOAD_MISS_LINE_ALIAS_VS_CSQ0, 0x01, 75), PMC_POWERPC_EVENT(LSU_TOUCH_LINE_ALIAS_VS_FSQ_WB0_WB1, 0x01, 76), PMC_POWERPC_EVENT(LSU_TOUCH_ALIAS_VS_CSQ, 0x01, 77), PMC_POWERPC_EVENT(LSU_LMQ_FULL_STALL, 0x01, 78), PMC_POWERPC_EVENT(FP_LOAD_INSTR_COMPLETED_IN_LSU, 0x01, 79), PMC_POWERPC_EVENT(FP_LOAD_SINGLE_INSTR_COMPLETED_IN_LSU, 0x01, 80), PMC_POWERPC_EVENT(FP_LOAD_DOUBLE_COMPLETED_IN_LSU, 0x01, 81), PMC_POWERPC_EVENT(LSU_RA_LATCH_STALL, 0x01, 82), PMC_POWERPC_EVENT(LSU_LOAD_VS_STORE_QUEUE_ALIAS_STALL, 0x01, 83), PMC_POWERPC_EVENT(LSU_LMQ_INDEX_ALIAS, 0x01, 84), PMC_POWERPC_EVENT(LSU_STORE_QUEUE_INDEX_ALIAS, 0x01, 85), PMC_POWERPC_EVENT(LSU_CSQ_FORWARDING, 0x01, 86), PMC_POWERPC_EVENT(LSU_MISALIGNED_LOAD_FINISH, 0x01, 87), PMC_POWERPC_EVENT(LSU_MISALIGN_STORE_COMPLETED, 0x01, 88), PMC_POWERPC_EVENT(LSU_MISALIGN_STALL, 0x01, 89), PMC_POWERPC_EVENT(FP_ONE_QUARTER_FPSCR_RENAMES_BUSY, 0x01, 90), PMC_POWERPC_EVENT(FP_ONE_HALF_FPSCR_RENAMES_BUSY, 0x01, 91), PMC_POWERPC_EVENT(FP_THREE_QUARTERS_FPSCR_RENAMES_BUSY, 0x01, 92), PMC_POWERPC_EVENT(FP_ALL_FPSCR_RENAMES_BUSY, 0x01, 93), PMC_POWERPC_EVENT(FP_DENORMALIZED_RESULT, 0x01, 94), PMC_POWERPC_EVENT(L1_DATA_TOTAL_MISSES, 0x02, 23), PMC_POWERPC_EVENT(DISPATCHES_TO_FPR_ISSUE_QUEUE, 0x02, 24), PMC_POWERPC_EVENT(LSU_INSTR_COMPLETED, 0x02, 25), PMC_POWERPC_EVENT(LOAD_INSTR_COMPLETED, 0x02, 26), PMC_POWERPC_EVENT(SS_SM_INSTR_COMPLETED, 0x02, 27), PMC_POWERPC_EVENT(TLBIE_INSTR_COMPLETED, 0x02, 28), PMC_POWERPC_EVENT(LWARX_INSTR_COMPLETED, 0x02, 29), PMC_POWERPC_EVENT(MFSPR_INSTR_COMPLETED, 0x02, 30), PMC_POWERPC_EVENT(REFETCH_SERIALIZATION, 0x02, 31), PMC_POWERPC_EVENT(COMPLETION_QUEUE_ENTRIES_OVER_THRESHOLD, 0x02, 32), PMC_POWERPC_EVENT(CYCLES_ONE_INSTR_DISPATCHED, 0x02, 33), PMC_POWERPC_EVENT(CYCLES_TWO_INSTR_COMPLETED, 0x02, 34), PMC_POWERPC_EVENT(ITLB_NON_SPECULATIVE_MISSES, 0x02, 35), PMC_POWERPC_EVENT(CYCLES_WAITING_FROM_L1_INSTR_CACHE_MISS, 0x02, 36), PMC_POWERPC_EVENT(L1_DATA_LOAD_ACCESS_MISS, 0x02, 37), PMC_POWERPC_EVENT(L1_DATA_TOUCH_MISS, 0x02, 38), PMC_POWERPC_EVENT(L1_DATA_STORE_MISS, 0x02, 39), PMC_POWERPC_EVENT(L1_DATA_TOUCH_MISS_CYCLES, 0x02, 40), PMC_POWERPC_EVENT(L1_DATA_CYCLES_USED, 0x02, 41), PMC_POWERPC_EVENT(DST_STREAM_1_CACHE_LINE_FETCHES, 0x02, 42), PMC_POWERPC_EVENT(VTQ_STREAM_CANCELED_PREMATURELY, 0x02, 43), PMC_POWERPC_EVENT(VTQ_RESUMES_DUE_TO_CTX_CHANGE, 0x02, 44), PMC_POWERPC_EVENT(VTQ_LINE_FETCH_MISS, 0x02, 45), PMC_POWERPC_EVENT(VTQ_LINE_FETCH, 0x02, 46), PMC_POWERPC_EVENT(TLBIE_SNOOPS, 0x02, 47), PMC_POWERPC_EVENT(L1_INSTR_CACHE_RELOADS, 0x02, 48), PMC_POWERPC_EVENT(L1_DATA_CACHE_RELOADS, 0x02, 49), PMC_POWERPC_EVENT(L1_DATA_CACHE_CASTOUTS_TO_L2, 0x02, 50), PMC_POWERPC_EVENT(STORE_MERGE_GATHER, 0x02, 51), PMC_POWERPC_EVENT(CACHEABLE_STORE_MERGE_TO_32_BYTES, 0x02, 52), PMC_POWERPC_EVENT(DATA_BKPT_MATCHES, 0x02, 53), PMC_POWERPC_EVENT(FALL_THROUGH_BRANCHES_PROCESSED, 0x02, 54), PMC_POWERPC_EVENT(FIRST_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x02, 55), PMC_POWERPC_EVENT(SECOND_SPECULATION_BUFFER_ACTIVE, 0x02, 56), PMC_POWERPC_EVENT(BPU_STALL_ON_LR_DEPENDENCY, 0x02, 57), PMC_POWERPC_EVENT(BTIC_MISS, 0x02, 58), PMC_POWERPC_EVENT(BRANCH_LINK_STACK_CORRECTLY_RESOLVED, 0x02, 59), PMC_POWERPC_EVENT(FPR_ISSUE_STALLED, 0x02, 60), PMC_POWERPC_EVENT(SWITCHES_BETWEEN_PRIV_USER, 0x02, 61), PMC_POWERPC_EVENT(LSU_COMPLETES_FP_STORE_SINGLE, 0x02, 62), PMC_POWERPC_EVENT(CYCLES_TWO_INSTR_COMPLETED, 0x04, 8), PMC_POWERPC_EVENT(CYCLES_ONE_INSTR_DISPATCHED, 0x04, 9), PMC_POWERPC_EVENT(VR_ISSUE_QUEUE_DISPATCHES, 0x04, 10), PMC_POWERPC_EVENT(VR_STALLS, 0x04, 11), PMC_POWERPC_EVENT(GPR_RENAME_BUFFER_ENTRIES_OVER_THRESHOLD, 0x04, 12), PMC_POWERPC_EVENT(FPR_ISSUE_QUEUE_ENTRIES, 0x04, 13), PMC_POWERPC_EVENT(FPU_INSTR_COMPLETED, 0x04, 14), PMC_POWERPC_EVENT(STWCX_INSTR_COMPLETED, 0x04, 15), PMC_POWERPC_EVENT(LS_LM_INSTR_PIECES, 0x04, 16), PMC_POWERPC_EVENT(ITLB_HW_SEARCH_CYCLES_OVER_THRESHOLD, 0x04, 17), PMC_POWERPC_EVENT(DTLB_MISSES, 0x04, 18), PMC_POWERPC_EVENT(CANCELLED_L1_INSTR_CACHE_MISSES, 0x04, 19), PMC_POWERPC_EVENT(L1_DATA_CACHE_OP_HIT, 0x04, 20), PMC_POWERPC_EVENT(L1_DATA_LOAD_MISS_CYCLES, 0x04, 21), PMC_POWERPC_EVENT(L1_DATA_PUSHES, 0x04, 22), PMC_POWERPC_EVENT(L1_DATA_TOTAL_MISS, 0x04, 23), PMC_POWERPC_EVENT(VT2_FETCHES, 0x04, 24), PMC_POWERPC_EVENT(TAKEN_BRANCHES_PROCESSED, 0x04, 25), PMC_POWERPC_EVENT(BRANCH_FLUSHES, 0x04, 26), PMC_POWERPC_EVENT(SECOND_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x04, 27), PMC_POWERPC_EVENT(THIRD_SPECULATION_BUFFER_ACTIVE, 0x04, 28), PMC_POWERPC_EVENT(BRANCH_UNIT_STALL_ON_CTR_DEPENDENCY, 0x04, 29), PMC_POWERPC_EVENT(FAST_BTIC_HIT, 0x04, 30), PMC_POWERPC_EVENT(BRANCH_LINK_STACK_MISPREDICTED, 0x04, 31), PMC_POWERPC_EVENT(CYCLES_THREE_INSTR_COMPLETED, 0x08, 14), PMC_POWERPC_EVENT(CYCLES_NO_INSTR_DISPATCHED, 0x08, 15), PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_ENTRIES_OVER_THRESHOLD, 0x08, 16), PMC_POWERPC_EVENT(GPR_ISSUE_QUEUE_STALLED, 0x08, 17), PMC_POWERPC_EVENT(IU1_INSTR_COMPLETED, 0x08, 18), PMC_POWERPC_EVENT(DSSALL_INSTR_COMPLETED, 0x08, 19), PMC_POWERPC_EVENT(TLBSYNC_INSTR_COMPLETED, 0x08, 20), PMC_POWERPC_EVENT(SYNC_INSTR_COMPLETED, 0x08, 21), PMC_POWERPC_EVENT(SS_SM_INSTR_PIECES, 0x08, 22), PMC_POWERPC_EVENT(DTLB_HW_SEARCH_CYCLES, 0x08, 23), PMC_POWERPC_EVENT(SNOOP_RETRIES, 0x08, 24), PMC_POWERPC_EVENT(SUCCESSFUL_STWCX, 0x08, 25), PMC_POWERPC_EVENT(DST_STREAM_3_CACHE_LINE_FETCHES, 0x08, 26), PMC_POWERPC_EVENT(THIRD_SPECULATIVE_BRANCH_BUFFER_RESOLVED_CORRECTLY, 0x08, 27), PMC_POWERPC_EVENT(MISPREDICTED_BRANCHES, 0x08, 28), PMC_POWERPC_EVENT(FOLDED_BRANCHES, 0x08, 29), PMC_POWERPC_EVENT(FP_STORE_DOUBLE_COMPLETES_IN_LSU, 0x08, 30), PMC_POWERPC_EVENT(L2_CACHE_HITS, 0x30, 2), PMC_POWERPC_EVENT(L3_CACHE_HITS, 0x30, 3), PMC_POWERPC_EVENT(L2_INSTR_CACHE_MISSES, 0x30, 4), PMC_POWERPC_EVENT(L3_INSTR_CACHE_MISSES, 0x30, 5), PMC_POWERPC_EVENT(L2_DATA_CACHE_MISSES, 0x30, 6), PMC_POWERPC_EVENT(L3_DATA_CACHE_MISSES, 0x30, 7), PMC_POWERPC_EVENT(L2_LOAD_HITS, 0x10, 8), PMC_POWERPC_EVENT(L2_STORE_HITS, 0x10, 9), PMC_POWERPC_EVENT(L3_LOAD_HITS, 0x10, 10), PMC_POWERPC_EVENT(L3_STORE_HITS, 0x10, 11), PMC_POWERPC_EVENT(L2_TOUCH_HITS, 0x30, 13), PMC_POWERPC_EVENT(L3_TOUCH_HITS, 0x30, 14), PMC_POWERPC_EVENT(SNOOP_RETRIES, 0x30, 15), PMC_POWERPC_EVENT(SNOOP_MODIFIED, 0x10, 16), PMC_POWERPC_EVENT(SNOOP_VALID, 0x10, 17), PMC_POWERPC_EVENT(INTERVENTION, 0x30, 18), PMC_POWERPC_EVENT(L2_CACHE_MISSES, 0x10, 19), PMC_POWERPC_EVENT(L3_CACHE_MISSES, 0x10, 20), PMC_POWERPC_EVENT(L2_CACHE_CASTOUTS, 0x20, 8), PMC_POWERPC_EVENT(L3_CACHE_CASTOUTS, 0x20, 9), PMC_POWERPC_EVENT(L2SQ_FULL_CYCLES, 0x20, 10), PMC_POWERPC_EVENT(L3SQ_FULL_CYCLES, 0x20, 11), PMC_POWERPC_EVENT(RAQ_FULL_CYCLES, 0x20, 16), PMC_POWERPC_EVENT(WAQ_FULL_CYCLES, 0x20, 17), PMC_POWERPC_EVENT(L1_EXTERNAL_INTERVENTIONS, 0x20, 19), PMC_POWERPC_EVENT(L2_EXTERNAL_INTERVENTIONS, 0x20, 20), PMC_POWERPC_EVENT(L3_EXTERNAL_INTERVENTIONS, 0x20, 21), PMC_POWERPC_EVENT(EXTERNAL_INTERVENTIONS, 0x20, 22), PMC_POWERPC_EVENT(EXTERNAL_PUSHES, 0x20, 23), PMC_POWERPC_EVENT(EXTERNAL_SNOOP_RETRY, 0x20, 24), PMC_POWERPC_EVENT(DTQ_FULL_CYCLES, 0x20, 25), PMC_POWERPC_EVENT(BUS_RETRY, 0x20, 26), PMC_POWERPC_EVENT(L2_VALID_REQUEST, 0x20, 27), PMC_POWERPC_EVENT(BORDQ_FULL, 0x20, 28), PMC_POWERPC_EVENT(BUS_TAS_FOR_READS, 0x20, 42), PMC_POWERPC_EVENT(BUS_TAS_FOR_WRITES, 0x20, 43), PMC_POWERPC_EVENT(BUS_READS_NOT_RETRIED, 0x20, 44), PMC_POWERPC_EVENT(BUS_WRITES_NOT_RETRIED, 0x20, 45), PMC_POWERPC_EVENT(BUS_READS_WRITES_NOT_RETRIED, 0x20, 46), PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_L1_RETRY, 0x20, 47), PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_PREVIOUS_ADJACENT, 0x20, 48), PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_COLLISION, 0x20, 49), PMC_POWERPC_EVENT(BUS_RETRY_DUE_TO_INTERVENTION_ORDERING, 0x20, 50), PMC_POWERPC_EVENT(SNOOP_REQUESTS, 0x20, 51), PMC_POWERPC_EVENT(PREFETCH_ENGINE_REQUEST, 0x20, 52), PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_LOAD, 0x20, 53), PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_STORE, 0x20, 54), PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_INSTR_FETCH, 0x20, 55), PMC_POWERPC_EVENT(PREFETCH_ENGINE_COLLISION_VS_LOAD_STORE_INSTR_FETCH, 0x20, 56), PMC_POWERPC_EVENT(PREFETCH_ENGINE_FULL, 0x20, 57) }; -const size_t mpc7xxx_event_codes_size = - sizeof(mpc7xxx_event_codes) / sizeof(mpc7xxx_event_codes[0]); - static pmc_value_t mpc7xxx_pmcn_read(unsigned int pmc) { switch (pmc) { case 0: return mfspr(SPR_PMC1); break; case 1: return mfspr(SPR_PMC2); break; case 2: return mfspr(SPR_PMC3); break; case 3: return mfspr(SPR_PMC4); break; case 4: return mfspr(SPR_PMC5); break; case 5: return mfspr(SPR_PMC6); default: panic("Invalid PMC number: %d\n", pmc); } } static void mpc7xxx_pmcn_write(unsigned int pmc, uint32_t val) { switch (pmc) { case 0: mtspr(SPR_PMC1, val); break; case 1: mtspr(SPR_PMC2, val); break; case 2: mtspr(SPR_PMC3, val); break; case 3: mtspr(SPR_PMC4, val); break; case 4: mtspr(SPR_PMC5, val); break; case 5: mtspr(SPR_PMC6, val); break; default: panic("Invalid PMC number: %d\n", pmc); } } static int mpc7xxx_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; KASSERT(pm, ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = mpc7xxx_pmcn_read(ri); PMCDBG2(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); else *v = tmp; return 0; } static int mpc7xxx_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v); mpc7xxx_pmcn_write(ri, v); return 0; } static int mpc7xxx_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int mpc7xxx_start_pmc(int cpu, int ri) { uint32_t config; struct pmc *pm; struct pmc_hw *phw; register_t pmc_mmcr; phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_powerpc.pm_powerpc_evsel & ~POWERPC_PMC_ENABLE; /* Enable the PMC. */ switch (ri) { case 0: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC1SEL(pmc_mmcr, config); mtspr(SPR_MMCR0, pmc_mmcr); break; case 1: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC2SEL(pmc_mmcr, config); mtspr(SPR_MMCR0, pmc_mmcr); break; case 2: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC3SEL(pmc_mmcr, config); mtspr(SPR_MMCR1, pmc_mmcr); break; case 3: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC4SEL(pmc_mmcr, config); mtspr(SPR_MMCR0, pmc_mmcr); break; case 4: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC5SEL(pmc_mmcr, config); mtspr(SPR_MMCR1, pmc_mmcr); break; case 5: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC6SEL(pmc_mmcr, config); mtspr(SPR_MMCR1, pmc_mmcr); break; default: break; } /* The mask is inverted (enable is 1) compared to the flags in MMCR0, which * are Freeze flags. */ config = ~pm->pm_md.pm_powerpc.pm_powerpc_evsel & POWERPC_PMC_ENABLE; pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr &= ~SPR_MMCR0_FC; pmc_mmcr |= config; mtspr(SPR_MMCR0, pmc_mmcr); return 0; } static int mpc7xxx_stop_pmc(int cpu, int ri) { struct pmc *pm; struct pmc_hw *phw; register_t pmc_mmcr; phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. */ switch (ri) { case 0: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC1SEL(pmc_mmcr, 0); mtspr(SPR_MMCR0, pmc_mmcr); break; case 1: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC2SEL(pmc_mmcr, 0); mtspr(SPR_MMCR0, pmc_mmcr); break; case 2: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC3SEL(pmc_mmcr, 0); mtspr(SPR_MMCR1, pmc_mmcr); break; case 3: pmc_mmcr = mfspr(SPR_MMCR0); pmc_mmcr = PPC_SET_PMC4SEL(pmc_mmcr, 0); mtspr(SPR_MMCR0, pmc_mmcr); break; case 4: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC5SEL(pmc_mmcr, 0); mtspr(SPR_MMCR1, pmc_mmcr); break; case 5: pmc_mmcr = mfspr(SPR_MMCR1); pmc_mmcr = PPC_SET_PMC6SEL(pmc_mmcr, 0); mtspr(SPR_MMCR1, pmc_mmcr); break; default: break; } return 0; } static int mpc7xxx_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, i; struct pmc_cpu *pc; struct powerpc_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu); powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * MPC7XXX_MAX_PMCS, M_PMC, M_WAITOK|M_ZERO); pac->pc_class = PMC_CLASS_PPC7450; pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri; KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_ppcpmcs; i < MPC7XXX_MAX_PMCS; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* Clear the MMCRs, and set FC, to disable all PMCs. */ mtspr(SPR_MMCR0, SPR_MMCR0_FC | SPR_MMCR0_PMXE | SPR_MMCR0_FCECE | SPR_MMCR0_PMC1CE | SPR_MMCR0_PMCNCE); mtspr(SPR_MMCR1, 0); return 0; } static int mpc7xxx_pcpu_fini(struct pmc_mdep *md, int cpu) { uint32_t mmcr0 = mfspr(SPR_MMCR0); mtmsr(mfmsr() & ~PSL_PMM); mmcr0 |= SPR_MMCR0_FC; mtspr(SPR_MMCR0, mmcr0); free(powerpc_pcpu[cpu]->pc_ppcpmcs, M_PMC); free(powerpc_pcpu[cpu], M_PMC); return 0; } static int mpc7xxx_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t caps, config, counter; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS, ("[powerpc,%d] illegal row index %d", __LINE__, ri)); caps = a->pm_caps; pe = a->pm_ev; - for (i = 0; i < mpc7xxx_event_codes_size; i++) { + for (i = 0; i < nitems(mpc7xxx_event_codes); i++) { if (mpc7xxx_event_codes[i].pe_ev == pe) { config = mpc7xxx_event_codes[i].pe_code; counter = mpc7xxx_event_codes[i].pe_counter_mask; break; } } - if (i == mpc7xxx_event_codes_size) + if (i == nitems(mpc7xxx_event_codes)) return (EINVAL); if ((counter & (1 << ri)) == 0) return (EINVAL); if (caps & PMC_CAP_SYSTEM) config |= POWERPC_PMC_KERNEL_ENABLE; if (caps & PMC_CAP_USER) config |= POWERPC_PMC_USER_ENABLE; if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0) config |= POWERPC_PMC_ENABLE; pm->pm_md.pm_powerpc.pm_powerpc_evsel = config; PMCDBG2(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int mpc7xxx_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS, ("[powerpc,%d] illegal row-index %d", __LINE__, ri)); phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int mpc7xxx_intr(int cpu, struct trapframe *tf) { int i, error, retval; uint32_t config; struct pmc *pm; struct powerpc_cpu *pac; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[powerpc,%d] out of range CPU %d", __LINE__, cpu)); PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); retval = 0; pac = powerpc_pcpu[cpu]; config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC; /* * look for all PMCs that have interrupted: * - look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association * * If found, we call a helper to process the interrupt. */ for (i = 0; i < MPC7XXX_MAX_PMCS; i++) { if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; } if (!MPC7XXX_PMC_HAS_OVERFLOWED(i)) continue; retval = 1; /* Found an interrupting PMC. */ if (pm->pm_state != PMC_STATE_RUNNING) continue; /* Stop the counter if logging fails. */ error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error != 0) mpc7xxx_stop_pmc(cpu, i); /* reload count. */ mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount); } atomic_add_int(retval ? &pmc_stats.pm_intr_processed : &pmc_stats.pm_intr_ignored, 1); /* Re-enable PERF exceptions. */ if (retval) mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE); return (retval); } int pmc_mpc7xxx_initialize(struct pmc_mdep *pmc_mdep) { struct pmc_classdep *pcd; pmc_mdep->pmd_cputype = PMC_CPU_PPC_7450; pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC]; pcd->pcd_caps = POWERPC_PMC_CAPS; pcd->pcd_class = PMC_CLASS_PPC7450; pcd->pcd_num = MPC7XXX_MAX_PMCS; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; /* All PMCs, even in ppc970, are 32-bit */ pcd->pcd_allocate_pmc = mpc7xxx_allocate_pmc; pcd->pcd_config_pmc = mpc7xxx_config_pmc; pcd->pcd_pcpu_fini = mpc7xxx_pcpu_fini; pcd->pcd_pcpu_init = mpc7xxx_pcpu_init; pcd->pcd_describe = powerpc_describe; pcd->pcd_get_config = powerpc_get_config; pcd->pcd_read_pmc = mpc7xxx_read_pmc; pcd->pcd_release_pmc = mpc7xxx_release_pmc; pcd->pcd_start_pmc = mpc7xxx_start_pmc; pcd->pcd_stop_pmc = mpc7xxx_stop_pmc; pcd->pcd_write_pmc = mpc7xxx_write_pmc; pmc_mdep->pmd_npmc += MPC7XXX_MAX_PMCS; pmc_mdep->pmd_intr = mpc7xxx_intr; return (0); } Index: head/sys/dev/hwpmc/hwpmc_uncore.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_uncore.c (revision 298410) +++ head/sys/dev/hwpmc/hwpmc_uncore.c (revision 298411) @@ -1,1232 +1,1230 @@ /*- * Copyright (c) 2010 Fabien Thomas * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Intel Uncore PMCs. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if (__FreeBSD_version >= 1100000) #include #else #include #endif #include #include #include #define UCF_PMC_CAPS \ (PMC_CAP_READ | PMC_CAP_WRITE) #define UCP_PMC_CAPS \ (PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE) #define SELECTSEL(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCP_CB0_EVSEL0 : UCP_EVSEL0) #define SELECTOFF(x) \ (((x) == PMC_CPU_INTEL_SANDYBRIDGE || (x) == PMC_CPU_INTEL_HASWELL) ? \ UCF_OFFSET_SB : UCF_OFFSET) static enum pmc_cputype uncore_cputype; struct uncore_cpu { volatile uint32_t pc_resync; volatile uint32_t pc_ucfctrl; /* Fixed function control. */ volatile uint64_t pc_globalctrl; /* Global control register. */ struct pmc_hw pc_uncorepmcs[]; }; static struct uncore_cpu **uncore_pcpu; static uint64_t uncore_pmcmask; static int uncore_ucf_ri; /* relative index of fixed counters */ static int uncore_ucf_width; static int uncore_ucf_npmc; static int uncore_ucp_width; static int uncore_ucp_npmc; static int uncore_pcpu_noop(struct pmc_mdep *md, int cpu) { (void) md; (void) cpu; return (0); } static int uncore_pcpu_init(struct pmc_mdep *md, int cpu) { struct pmc_cpu *pc; struct uncore_cpu *cc; struct pmc_hw *phw; int uncore_ri, n, npmc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[ucf,%d] insane cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu); uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw), M_PMC, M_WAITOK | M_ZERO); uncore_pcpu[cpu] = cc; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL && cc != NULL, ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu)); for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n + uncore_ri); phw->phw_pmc = NULL; pc->pc_hwpmcs[n + uncore_ri] = phw; } return (0); } static int uncore_pcpu_fini(struct pmc_mdep *md, int cpu) { int uncore_ri, n, npmc; struct pmc_cpu *pc; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu); if ((cc = uncore_pcpu[cpu]) == NULL) return (0); uncore_pcpu[cpu] = NULL; pc = pmc_pcpu[cpu]; KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__, cpu)); npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num; uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri; for (n = 0; n < npmc; n++) wrmsr(SELECTSEL(uncore_cputype) + n, 0); wrmsr(UCF_CTRL, 0); npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num; for (n = 0; n < npmc; n++) pc->pc_hwpmcs[n + uncore_ri] = NULL; free(cc, M_PMC); return (0); } /* * Fixed function counters. */ static pmc_value_t ucf_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucf_width) - 1; return (1ULL << uncore_ucf_width) - v; } static pmc_value_t ucf_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucf_width) - rlc; } static int ucf_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event ev; uint32_t caps, flags; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps); if (ri < 0 || ri > uncore_ucf_npmc) return (EINVAL); caps = a->pm_caps; if (a->pm_class != PMC_CLASS_UCF || (caps & UCF_PMC_CAPS) != caps) return (EINVAL); ev = pm->pm_event; if (ev < PMC_EV_UCF_FIRST || ev > PMC_EV_UCF_LAST) return (EINVAL); flags = UCF_EN; pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4)); PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx", (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl); return (0); } static int ucf_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm; return (0); } static int ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucf_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri]; (void) snprintf(ucf_name, sizeof(ucf_name), "UCF-%d", ri); if ((error = copystr(ucf_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCF; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucf_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; return (0); } static int ucf_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu, ri, ri + uncore_ucf_ri)); tmp = rdmsr(UCF_CTR0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucf_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v); return (0); } static int ucf_release_pmc(int cpu, int ri, struct pmc *pmc) { PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); } static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); } static void ucf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucf,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucf-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF]; pcd->pcd_caps = UCF_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCF; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucf_allocate_pmc; pcd->pcd_config_pmc = ucf_config_pmc; pcd->pcd_describe = ucf_describe; pcd->pcd_get_config = ucf_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_noop; pcd->pcd_pcpu_init = uncore_pcpu_noop; pcd->pcd_read_pmc = ucf_read_pmc; pcd->pcd_release_pmc = ucf_release_pmc; pcd->pcd_start_pmc = ucf_start_pmc; pcd->pcd_stop_pmc = ucf_stop_pmc; pcd->pcd_write_pmc = ucf_write_pmc; md->pmd_npmc += npmc; } /* * Intel programmable PMCs. */ /* * Event descriptor tables. * * For each event id, we track: * * 1. The CPUs that the event is valid for. * * 2. If the event uses a fixed UMASK, the value of the umask field. * If the event doesn't use a fixed UMASK, a mask of legal bits * to check against. */ struct ucp_event_descr { enum pmc_event ucp_ev; unsigned char ucp_evcode; unsigned char ucp_umask; unsigned char ucp_flags; }; #define UCP_F_I7 (1 << 0) /* CPU: Core i7 */ #define UCP_F_WM (1 << 1) /* CPU: Westmere */ #define UCP_F_SB (1 << 2) /* CPU: Sandy Bridge */ #define UCP_F_HW (1 << 3) /* CPU: Haswell */ #define UCP_F_FM (1 << 4) /* Fixed mask */ #define UCP_F_ALLCPUS \ (UCP_F_I7 | UCP_F_WM) #define UCP_F_CMASK 0xFF000000 static struct ucp_event_descr ucp_events[] = { #undef UCPDESCR #define UCPDESCR(N,EV,UM,FLAGS) { \ .ucp_ev = PMC_EV_UCP_EVENT_##N, \ .ucp_evcode = (EV), \ .ucp_umask = (UM), \ .ucp_flags = (FLAGS) \ } UCPDESCR(00H_01H, 0x00, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(00H_02H, 0x00, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(00H_04H, 0x00, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(01H_01H, 0x01, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(01H_02H, 0x01, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(01H_04H, 0x01, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(02H_01H, 0x02, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_01H, 0x03, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_02H, 0x03, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_04H, 0x03, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_08H, 0x03, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_10H, 0x03, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_20H, 0x03, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(03H_40H, 0x03, 0x40, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(04H_01H, 0x04, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(04H_02H, 0x04, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(04H_04H, 0x04, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(04H_08H, 0x04, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(04H_10H, 0x04, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(05H_01H, 0x05, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(05H_02H, 0x05, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(05H_04H, 0x05, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_01H, 0x06, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_02H, 0x06, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_04H, 0x06, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_08H, 0x06, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_10H, 0x06, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(06H_20H, 0x06, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_01H, 0x07, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_02H, 0x07, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_04H, 0x07, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_08H, 0x07, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_10H, 0x07, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_20H, 0x07, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(07H_24H, 0x07, 0x24, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(08H_01H, 0x08, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(08H_02H, 0x08, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(08H_04H, 0x08, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(08H_03H, 0x08, 0x03, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(09H_01H, 0x09, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(09H_02H, 0x09, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(09H_04H, 0x09, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(09H_03H, 0x09, 0x03, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0AH_01H, 0x0A, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0AH_02H, 0x0A, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0AH_04H, 0x0A, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0AH_08H, 0x0A, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0AH_0FH, 0x0A, 0x0F, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_01H, 0x0B, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_02H, 0x0B, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_04H, 0x0B, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_08H, 0x0B, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_10H, 0x0B, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0BH_1FH, 0x0B, 0x1F, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(0CH_01H, 0x0C, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_02H, 0x0C, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_04H_E, 0x0C, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_04H_F, 0x0C, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_04H_M, 0x0C, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_04H_S, 0x0C, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_08H_E, 0x0C, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_08H_F, 0x0C, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_08H_M, 0x0C, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(0CH_08H_S, 0x0C, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(20H_01H, 0x20, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(20H_02H, 0x20, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(20H_04H, 0x20, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(20H_08H, 0x20, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(20H_10H, 0x20, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(20H_20H, 0x20, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(21H_01H, 0x21, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(21H_02H, 0x21, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(21H_04H, 0x21, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(22H_01H, 0x22, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_02H, 0x22, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_04H, 0x22, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_08H, 0x22, 0x08, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_10H, 0x22, 0x10, UCP_F_FM | UCP_F_HW), UCPDESCR(22H_20H, 0x22, 0x20, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_40H, 0x22, 0x40, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(22H_80H, 0x22, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(23H_01H, 0x23, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(23H_02H, 0x23, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(23H_04H, 0x23, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(24H_02H, 0x24, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(24H_04H, 0x24, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(25H_01H, 0x25, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(25H_02H, 0x25, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(25H_04H, 0x25, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(26H_01H, 0x26, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(27H_01H, 0x27, 0x01, UCP_F_FM | UCP_F_I7), UCPDESCR(27H_02H, 0x27, 0x02, UCP_F_FM | UCP_F_I7), UCPDESCR(27H_04H, 0x27, 0x04, UCP_F_FM | UCP_F_I7), UCPDESCR(27H_08H, 0x27, 0x08, UCP_F_FM | UCP_F_I7), UCPDESCR(27H_10H, 0x27, 0x10, UCP_F_FM | UCP_F_I7), UCPDESCR(27H_20H, 0x27, 0x20, UCP_F_FM | UCP_F_I7), UCPDESCR(28H_01H, 0x28, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(28H_02H, 0x28, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(28H_04H, 0x28, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(28H_08H, 0x28, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(28H_10H, 0x28, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(28H_20H, 0x28, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_01H, 0x29, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_02H, 0x29, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_04H, 0x29, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_08H, 0x29, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_10H, 0x29, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(29H_20H, 0x29, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2AH_01H, 0x2A, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2AH_02H, 0x2A, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2AH_04H, 0x2A, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2AH_07H, 0x2A, 0x07, UCP_F_FM | UCP_F_WM), UCPDESCR(2BH_01H, 0x2B, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2BH_02H, 0x2B, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2BH_04H, 0x2B, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2BH_07H, 0x2B, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2CH_01H, 0x2C, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2CH_02H, 0x2C, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2CH_04H, 0x2C, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2CH_07H, 0x2C, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2DH_01H, 0x2D, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2DH_02H, 0x2D, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2DH_04H, 0x2D, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2DH_07H, 0x2D, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2EH_01H, 0x2E, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2EH_02H, 0x2E, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2EH_04H, 0x2E, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2EH_07H, 0x2E, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_01H, 0x2F, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_02H, 0x2F, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_04H, 0x2F, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_07H, 0x2F, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_08H, 0x2F, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_10H, 0x2F, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_20H, 0x2F, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(2FH_38H, 0x2F, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(30H_01H, 0x30, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(30H_02H, 0x30, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(30H_04H, 0x30, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(30H_07H, 0x30, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(31H_01H, 0x31, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(31H_02H, 0x31, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(31H_04H, 0x31, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(31H_07H, 0x31, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(32H_01H, 0x32, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(32H_02H, 0x32, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(32H_04H, 0x32, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(32H_07H, 0x32, 0x07, UCP_F_FM | UCP_F_WM), UCPDESCR(33H_01H, 0x33, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(33H_02H, 0x33, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(33H_04H, 0x33, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(33H_07H, 0x33, 0x07, UCP_F_FM | UCP_F_WM), UCPDESCR(34H_01H, 0x34, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(34H_02H, 0x34, 0x02, UCP_F_FM | UCP_F_WM | UCP_F_SB), UCPDESCR(34H_04H, 0x34, 0x04, UCP_F_FM | UCP_F_WM | UCP_F_SB), UCPDESCR(34H_06H, 0x34, 0x06, UCP_F_FM | UCP_F_HW), UCPDESCR(34H_08H, 0x34, 0x08, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(34H_10H, 0x34, 0x10, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(34H_20H, 0x34, 0x20, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(34H_40H, 0x34, 0x40, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(34H_80H, 0x34, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(35H_01H, 0x35, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(35H_02H, 0x35, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(35H_04H, 0x35, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(40H_01H, 0x40, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_02H, 0x40, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_04H, 0x40, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_08H, 0x40, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_10H, 0x40, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_20H, 0x40, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_07H, 0x40, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(40H_38H, 0x40, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_01H, 0x41, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_02H, 0x41, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_04H, 0x41, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_08H, 0x41, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_10H, 0x41, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_20H, 0x41, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_07H, 0x41, 0x07, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(41H_38H, 0x41, 0x38, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(42H_01H, 0x42, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(42H_02H, 0x42, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(42H_04H, 0x42, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(42H_08H, 0x42, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(43H_01H, 0x43, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(43H_02H, 0x43, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(60H_01H, 0x60, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(60H_02H, 0x60, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(60H_04H, 0x60, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(61H_01H, 0x61, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(61H_02H, 0x61, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(61H_04H, 0x61, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(62H_01H, 0x62, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(62H_02H, 0x62, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(62H_04H, 0x62, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_01H, 0x63, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_02H, 0x63, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_04H, 0x63, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_08H, 0x63, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_10H, 0x63, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(63H_20H, 0x63, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_01H, 0x64, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_02H, 0x64, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_04H, 0x64, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_08H, 0x64, 0x08, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_10H, 0x64, 0x10, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(64H_20H, 0x64, 0x20, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(65H_01H, 0x65, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(65H_02H, 0x65, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(65H_04H, 0x65, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(66H_01H, 0x66, 0x01, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(66H_02H, 0x66, 0x02, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(66H_04H, 0x66, 0x04, UCP_F_FM | UCP_F_I7 | UCP_F_WM), UCPDESCR(67H_01H, 0x67, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(80H_01H, 0x80, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(80H_02H, 0x80, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(80H_04H, 0x80, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(80H_08H, 0x80, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(81H_01H, 0x81, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(81H_02H, 0x81, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(81H_04H, 0x81, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(81H_08H, 0x81, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(81H_20H, 0x81, 0x20, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(81H_80H, 0x81, 0x80, UCP_F_FM | UCP_F_SB | UCP_F_HW), UCPDESCR(82H_01H, 0x82, 0x01, UCP_F_FM | UCP_F_WM), UCPDESCR(83H_01H, 0x83, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(83H_02H, 0x83, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(83H_04H, 0x83, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(83H_08H, 0x83, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(84H_01H, 0x84, 0x01, UCP_F_FM | UCP_F_WM | UCP_F_SB | UCP_F_HW), UCPDESCR(84H_02H, 0x84, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(84H_04H, 0x84, 0x04, UCP_F_FM | UCP_F_WM), UCPDESCR(84H_08H, 0x84, 0x08, UCP_F_FM | UCP_F_WM), UCPDESCR(85H_02H, 0x85, 0x02, UCP_F_FM | UCP_F_WM), UCPDESCR(86H_01H, 0x86, 0x01, UCP_F_FM | UCP_F_WM) }; -static const int nucp_events = sizeof(ucp_events) / sizeof(ucp_events[0]); - static pmc_value_t ucp_perfctr_value_to_reload_count(pmc_value_t v) { v &= (1ULL << uncore_ucp_width) - 1; return (1ULL << uncore_ucp_width) - v; } static pmc_value_t ucp_reload_count_to_perfctr_value(pmc_value_t rlc) { return (1ULL << uncore_ucp_width) - rlc; } /* * Counter specific event information for Sandybridge and Haswell */ static int ucp_event_sb_hw_ok_on_counter(enum pmc_event pe, int ri) { uint32_t mask; switch (pe) { /* * Events valid only on counter 0. */ case PMC_EV_UCP_EVENT_80H_01H: case PMC_EV_UCP_EVENT_83H_01H: mask = (1 << 0); break; default: mask = ~0; /* Any row index is ok. */ } return (mask & (1 << ri)); } static int ucp_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int n; enum pmc_event ev; struct ucp_event_descr *ie; uint32_t caps, config, cpuflag, evsel; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index value %d", __LINE__, ri)); /* check requested capabilities */ caps = a->pm_caps; if ((UCP_PMC_CAPS & caps) != caps) return (EPERM); ev = pm->pm_event; switch (uncore_cputype) { case PMC_CPU_INTEL_HASWELL: case PMC_CPU_INTEL_SANDYBRIDGE: if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0) return (EINVAL); break; default: break; } /* * Look for an event descriptor with matching CPU and event id * fields. */ switch (uncore_cputype) { case PMC_CPU_INTEL_COREI7: cpuflag = UCP_F_I7; break; case PMC_CPU_INTEL_HASWELL: cpuflag = UCP_F_HW; break; case PMC_CPU_INTEL_SANDYBRIDGE: cpuflag = UCP_F_SB; break; case PMC_CPU_INTEL_WESTMERE: cpuflag = UCP_F_WM; break; default: return (EINVAL); } - for (n = 0, ie = ucp_events; n < nucp_events; n++, ie++) + for (n = 0, ie = ucp_events; n < nitems(ucp_events); n++, ie++) if (ie->ucp_ev == ev && ie->ucp_flags & cpuflag) break; - if (n == nucp_events) + if (n == nitems(ucp_events)) return (EINVAL); /* * A matching event descriptor has been found, so start * assembling the contents of the event select register. */ evsel = ie->ucp_evcode | UCP_EN; config = a->pm_md.pm_ucp.pm_ucp_config & ~UCP_F_CMASK; /* * If the event uses a fixed umask value, reject any umask * bits set by the user. */ if (ie->ucp_flags & UCP_F_FM) { if (UCP_UMASK(config) != 0) return (EINVAL); evsel |= (ie->ucp_umask << 8); } else return (EINVAL); if (caps & PMC_CAP_THRESHOLD) evsel |= (a->pm_md.pm_ucp.pm_ucp_config & UCP_F_CMASK); if (caps & PMC_CAP_EDGE) evsel |= UCP_EDGE; if (caps & PMC_CAP_INVERT) evsel |= UCP_INV; pm->pm_md.pm_ucp.pm_ucp_evsel = evsel; return (0); } static int ucp_config_pmc(int cpu, int ri, struct pmc *pm) { KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__, cpu)); uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm; return (0); } static int ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char ucp_name[PMC_NAME_MAX]; phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri]; (void) snprintf(ucp_name, sizeof(ucp_name), "UCP-%d", ri); if ((error = copystr(ucp_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return (error); pi->pm_class = PMC_CLASS_UCP; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int ucp_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; return (0); } static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); } static int ucp_release_pmc(int cpu, int ri, struct pmc *pm) { (void) pm; PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__)); return (0); } static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; uint32_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); /* Event specific configuration. */ switch (pm->pm_event) { case PMC_EV_UCP_EVENT_0CH_04H_E: case PMC_EV_UCP_EVENT_0CH_08H_E: wrmsr(MSR_GQ_SNOOP_MESF,0x2); break; case PMC_EV_UCP_EVENT_0CH_04H_F: case PMC_EV_UCP_EVENT_0CH_08H_F: wrmsr(MSR_GQ_SNOOP_MESF,0x8); break; case PMC_EV_UCP_EVENT_0CH_04H_M: case PMC_EV_UCP_EVENT_0CH_08H_M: wrmsr(MSR_GQ_SNOOP_MESF,0x1); break; case PMC_EV_UCP_EVENT_0CH_04H_S: case PMC_EV_UCP_EVENT_0CH_08H_S: wrmsr(MSR_GQ_SNOOP_MESF,0x4); break; default: break; } wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); do { cc->pc_resync = 0; cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int ucp_stop_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri); /* stop hw. */ wrmsr(SELECTSEL(uncore_cputype) + ri, 0); do { cc->pc_resync = 0; cc->pc_globalctrl &= ~(1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); } static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); } static void ucp_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth) { struct pmc_classdep *pcd; KASSERT(md != NULL, ("[ucp,%d] md is NULL", __LINE__)); PMCDBG0(MDP,INI,1, "ucp-initialize"); pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP]; pcd->pcd_caps = UCP_PMC_CAPS; pcd->pcd_class = PMC_CLASS_UCP; pcd->pcd_num = npmc; pcd->pcd_ri = md->pmd_npmc; pcd->pcd_width = pmcwidth; pcd->pcd_allocate_pmc = ucp_allocate_pmc; pcd->pcd_config_pmc = ucp_config_pmc; pcd->pcd_describe = ucp_describe; pcd->pcd_get_config = ucp_get_config; pcd->pcd_get_msr = NULL; pcd->pcd_pcpu_fini = uncore_pcpu_fini; pcd->pcd_pcpu_init = uncore_pcpu_init; pcd->pcd_read_pmc = ucp_read_pmc; pcd->pcd_release_pmc = ucp_release_pmc; pcd->pcd_start_pmc = ucp_start_pmc; pcd->pcd_stop_pmc = ucp_stop_pmc; pcd->pcd_write_pmc = ucp_write_pmc; md->pmd_npmc += npmc; } int pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu) { uncore_cputype = md->pmd_cputype; uncore_pmcmask = 0; /* * Initialize programmable counters. */ uncore_ucp_npmc = 8; uncore_ucp_width = 48; uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1); ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width); /* * Initialize fixed function counters, if present. */ uncore_ucf_ri = uncore_ucp_npmc; uncore_ucf_npmc = 1; uncore_ucf_width = 48; ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width); uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype); PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask, uncore_ucf_ri); uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC, M_ZERO | M_WAITOK); return (0); } void pmc_uncore_finalize(struct pmc_mdep *md) { PMCDBG0(MDP,INI,1, "uncore-finalize"); free(uncore_pcpu, M_PMC); uncore_pcpu = NULL; } Index: head/sys/dev/hwpmc/hwpmc_xscale.c =================================================================== --- head/sys/dev/hwpmc/hwpmc_xscale.c (revision 298410) +++ head/sys/dev/hwpmc/hwpmc_xscale.c (revision 298411) @@ -1,676 +1,673 @@ /*- * Copyright (c) 2009 Rui Paulo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * Support for the Intel XScale network processors * * XScale processors have up to now three generations. * * The first generation has two PMC; the event selection, interrupt config * and overflow flag setup are done by writing to the PMNC register. * It also has less monitoring events than the latter generations. * * The second and third generatiosn have four PMCs, one register for the event * selection, one register for the interrupt config and one register for * the overflow flags. */ static int xscale_npmcs; static int xscale_gen; /* XScale Core generation */ struct xscale_event_code_map { enum pmc_event pe_ev; uint8_t pe_code; }; const struct xscale_event_code_map xscale_event_codes[] = { /* 1st and 2nd Generation XScale cores */ { PMC_EV_XSCALE_IC_FETCH, 0x00 }, { PMC_EV_XSCALE_IC_MISS, 0x01 }, { PMC_EV_XSCALE_DATA_DEPENDENCY_STALLED,0x02 }, { PMC_EV_XSCALE_ITLB_MISS, 0x03 }, { PMC_EV_XSCALE_DTLB_MISS, 0x04 }, { PMC_EV_XSCALE_BRANCH_RETIRED, 0x05 }, { PMC_EV_XSCALE_BRANCH_MISPRED, 0x06 }, { PMC_EV_XSCALE_INSTR_RETIRED, 0x07 }, { PMC_EV_XSCALE_DC_FULL_CYCLE, 0x08 }, { PMC_EV_XSCALE_DC_FULL_CONTIG, 0x09 }, { PMC_EV_XSCALE_DC_ACCESS, 0x0a }, { PMC_EV_XSCALE_DC_MISS, 0x0b }, { PMC_EV_XSCALE_DC_WRITEBACK, 0x0c }, { PMC_EV_XSCALE_PC_CHANGE, 0x0d }, /* 3rd Generation XScale cores */ { PMC_EV_XSCALE_BRANCH_RETIRED_ALL, 0x0e }, { PMC_EV_XSCALE_INSTR_CYCLE, 0x0f }, { PMC_EV_XSCALE_CP_STALL, 0x17 }, { PMC_EV_XSCALE_PC_CHANGE_ALL, 0x18 }, { PMC_EV_XSCALE_PIPELINE_FLUSH, 0x19 }, { PMC_EV_XSCALE_BACKEND_STALL, 0x1a }, { PMC_EV_XSCALE_MULTIPLIER_USE, 0x1b }, { PMC_EV_XSCALE_MULTIPLIER_STALLED, 0x1c }, { PMC_EV_XSCALE_DATA_CACHE_STALLED, 0x1e }, { PMC_EV_XSCALE_L2_CACHE_REQ, 0x20 }, { PMC_EV_XSCALE_L2_CACHE_MISS, 0x23 }, { PMC_EV_XSCALE_ADDRESS_BUS_TRANS, 0x40 }, { PMC_EV_XSCALE_SELF_ADDRESS_BUS_TRANS, 0x41 }, { PMC_EV_XSCALE_DATA_BUS_TRANS, 0x48 }, }; -const int xscale_event_codes_size = - sizeof(xscale_event_codes) / sizeof(xscale_event_codes[0]); - /* * Per-processor information. */ struct xscale_cpu { struct pmc_hw *pc_xscalepmcs; }; static struct xscale_cpu **xscale_pcpu; /* * Performance Monitor Control Register */ static __inline uint32_t xscale_pmnc_read(void) { uint32_t reg; __asm __volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (reg)); return (reg); } static __inline void xscale_pmnc_write(uint32_t reg) { __asm __volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (reg)); } /* * Clock Counter Register */ static __inline uint32_t xscale_ccnt_read(void) { uint32_t reg; __asm __volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (reg)); return (reg); } static __inline void xscale_ccnt_write(uint32_t reg) { __asm __volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (reg)); } /* * Interrupt Enable Register */ static __inline uint32_t xscale_inten_read(void) { uint32_t reg; __asm __volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (reg)); return (reg); } static __inline void xscale_inten_write(uint32_t reg) { __asm __volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (reg)); } /* * Overflow Flag Register */ static __inline uint32_t xscale_flag_read(void) { uint32_t reg; __asm __volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (reg)); return (reg); } static __inline void xscale_flag_write(uint32_t reg) { __asm __volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (reg)); } /* * Event Selection Register */ static __inline uint32_t xscale_evtsel_read(void) { uint32_t reg; __asm __volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (reg)); return (reg); } static __inline void xscale_evtsel_write(uint32_t reg) { __asm __volatile("mcr p14, 0, %0, c8, c1, 0" : : "r" (reg)); } /* * Performance Count Register N */ static uint32_t xscale_pmcn_read(unsigned int pmc) { uint32_t reg = 0; KASSERT(pmc < 4, ("[xscale,%d] illegal PMC number %d", __LINE__, pmc)); switch (pmc) { case 0: __asm __volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (reg)); break; case 1: __asm __volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (reg)); break; case 2: __asm __volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (reg)); break; case 3: __asm __volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (reg)); break; } return (reg); } static uint32_t xscale_pmcn_write(unsigned int pmc, uint32_t reg) { KASSERT(pmc < 4, ("[xscale,%d] illegal PMC number %d", __LINE__, pmc)); switch (pmc) { case 0: __asm __volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (reg)); break; case 1: __asm __volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (reg)); break; case 2: __asm __volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (reg)); break; case 3: __asm __volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (reg)); break; } return (reg); } static int xscale_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { enum pmc_event pe; uint32_t caps, config; int i; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] illegal row index %d", __LINE__, ri)); caps = a->pm_caps; if (a->pm_class != PMC_CLASS_XSCALE) return (EINVAL); pe = a->pm_ev; - for (i = 0; i < xscale_event_codes_size; i++) { + for (i = 0; i < nitems(xscale_event_codes); i++) { if (xscale_event_codes[i].pe_ev == pe) { config = xscale_event_codes[i].pe_code; break; } } - if (i == xscale_event_codes_size) + if (i == nitems(xscale_event_codes)) return EINVAL; /* Generation 1 has fewer events */ if (xscale_gen == 1 && i > PMC_EV_XSCALE_PC_CHANGE) return EINVAL; pm->pm_md.pm_xscale.pm_xscale_evsel = config; PMCDBG2(MDP,ALL,2,"xscale-allocate ri=%d -> config=0x%x", ri, config); return 0; } static int xscale_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] illegal row index %d", __LINE__, ri)); pm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc; tmp = xscale_pmcn_read(ri); PMCDBG2(MDP,REA,2,"xscale-read id=%d -> %jd", ri, tmp); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = XSCALE_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); else *v = tmp; return 0; } static int xscale_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] illegal row-index %d", __LINE__, ri)); pm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = XSCALE_RELOAD_COUNT_TO_PERFCTR_VALUE(v); PMCDBG3(MDP,WRI,1,"xscale-write cpu=%d ri=%d v=%jx", cpu, ri, v); xscale_pmcn_write(ri, v); return 0; } static int xscale_config_pmc(int cpu, int ri, struct pmc *pm) { struct pmc_hw *phw; PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] illegal row-index %d", __LINE__, ri)); phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[xscale,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, pm, phw->phw_pmc)); phw->phw_pmc = pm; return 0; } static int xscale_start_pmc(int cpu, int ri) { uint32_t pmnc, config, evtsel; struct pmc *pm; struct pmc_hw *phw; phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri]; pm = phw->phw_pmc; config = pm->pm_md.pm_xscale.pm_xscale_evsel; /* * Configure the event selection. * * On the XScale 2nd Generation there's no EVTSEL register. */ if (xscale_npmcs == 2) { pmnc = xscale_pmnc_read(); switch (ri) { case 0: pmnc &= ~XSCALE_PMNC_EVT0_MASK; pmnc |= (config << 12) & XSCALE_PMNC_EVT0_MASK; break; case 1: pmnc &= ~XSCALE_PMNC_EVT1_MASK; pmnc |= (config << 20) & XSCALE_PMNC_EVT1_MASK; break; default: /* XXX */ break; } xscale_pmnc_write(pmnc); } else { evtsel = xscale_evtsel_read(); switch (ri) { case 0: evtsel &= ~XSCALE_EVTSEL_EVT0_MASK; evtsel |= config & XSCALE_EVTSEL_EVT0_MASK; break; case 1: evtsel &= ~XSCALE_EVTSEL_EVT1_MASK; evtsel |= (config << 8) & XSCALE_EVTSEL_EVT1_MASK; break; case 2: evtsel &= ~XSCALE_EVTSEL_EVT2_MASK; evtsel |= (config << 16) & XSCALE_EVTSEL_EVT2_MASK; break; case 3: evtsel &= ~XSCALE_EVTSEL_EVT3_MASK; evtsel |= (config << 24) & XSCALE_EVTSEL_EVT3_MASK; break; default: /* XXX */ break; } xscale_evtsel_write(evtsel); } /* * Enable the PMC. * * Note that XScale provides only one bit to enable/disable _all_ * performance monitoring units. */ pmnc = xscale_pmnc_read(); pmnc |= XSCALE_PMNC_ENABLE; xscale_pmnc_write(pmnc); return 0; } static int xscale_stop_pmc(int cpu, int ri) { uint32_t pmnc, evtsel; struct pmc *pm; struct pmc_hw *phw; phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri]; pm = phw->phw_pmc; /* * Disable the PMCs. * * Note that XScale provides only one bit to enable/disable _all_ * performance monitoring units. */ pmnc = xscale_pmnc_read(); pmnc &= ~XSCALE_PMNC_ENABLE; xscale_pmnc_write(pmnc); /* * A value of 0xff makes the corresponding PMU go into * power saving mode. */ if (xscale_npmcs == 2) { pmnc = xscale_pmnc_read(); switch (ri) { case 0: pmnc |= XSCALE_PMNC_EVT0_MASK; break; case 1: pmnc |= XSCALE_PMNC_EVT1_MASK; break; default: /* XXX */ break; } xscale_pmnc_write(pmnc); } else { evtsel = xscale_evtsel_read(); switch (ri) { case 0: evtsel |= XSCALE_EVTSEL_EVT0_MASK; break; case 1: evtsel |= XSCALE_EVTSEL_EVT1_MASK; break; case 2: evtsel |= XSCALE_EVTSEL_EVT2_MASK; break; case 3: evtsel |= XSCALE_EVTSEL_EVT3_MASK; break; default: /* XXX */ break; } xscale_evtsel_write(evtsel); } return 0; } static int xscale_release_pmc(int cpu, int ri, struct pmc *pmc) { struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] illegal row-index %d", __LINE__, ri)); phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[xscale,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); return 0; } static int xscale_intr(int cpu, struct trapframe *tf) { printf("intr\n"); return 0; } static int xscale_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) { int error; struct pmc_hw *phw; char xscale_name[PMC_NAME_MAX]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d], illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < xscale_npmcs, ("[xscale,%d] row-index %d out of range", __LINE__, ri)); phw = &xscale_pcpu[cpu]->pc_xscalepmcs[ri]; snprintf(xscale_name, sizeof(xscale_name), "XSCALE-%d", ri); if ((error = copystr(xscale_name, pi->pm_name, PMC_NAME_MAX, NULL)) != 0) return error; pi->pm_class = PMC_CLASS_XSCALE; if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; *ppmc = phw->phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } return (0); } static int xscale_get_config(int cpu, int ri, struct pmc **ppm) { *ppm = xscale_pcpu[cpu]->pc_xscalepmcs[ri].phw_pmc; return 0; } /* * XXX don't know what we should do here. */ static int xscale_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int xscale_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) { return 0; } static int xscale_pcpu_init(struct pmc_mdep *md, int cpu) { int first_ri, i; struct pmc_cpu *pc; struct xscale_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[xscale,%d] wrong cpu number %d", __LINE__, cpu)); PMCDBG1(MDP,INI,1,"xscale-init cpu=%d", cpu); xscale_pcpu[cpu] = pac = malloc(sizeof(struct xscale_cpu), M_PMC, M_WAITOK|M_ZERO); pac->pc_xscalepmcs = malloc(sizeof(struct pmc_hw) * xscale_npmcs, M_PMC, M_WAITOK|M_ZERO); pc = pmc_pcpu[cpu]; first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE].pcd_ri; KASSERT(pc != NULL, ("[xscale,%d] NULL per-cpu pointer", __LINE__)); for (i = 0, phw = pac->pc_xscalepmcs; i < xscale_npmcs; i++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i); phw->phw_pmc = NULL; pc->pc_hwpmcs[i + first_ri] = phw; } /* * Disable and put the PMUs into power save mode. */ if (xscale_npmcs == 2) { xscale_pmnc_write(XSCALE_PMNC_EVT1_MASK | XSCALE_PMNC_EVT0_MASK); } else { xscale_evtsel_write(XSCALE_EVTSEL_EVT3_MASK | XSCALE_EVTSEL_EVT2_MASK | XSCALE_EVTSEL_EVT1_MASK | XSCALE_EVTSEL_EVT0_MASK); } return 0; } static int xscale_pcpu_fini(struct pmc_mdep *md, int cpu) { return 0; } struct pmc_mdep * pmc_xscale_initialize() { struct pmc_mdep *pmc_mdep; struct pmc_classdep *pcd; uint32_t idreg; /* Get the Core Generation from CP15 */ __asm __volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (idreg)); xscale_gen = (idreg >> 13) & 0x3; switch (xscale_gen) { case 1: xscale_npmcs = 2; break; case 2: case 3: xscale_npmcs = 4; break; default: printf("%s: unknown XScale core generation\n", __func__); return (NULL); } PMCDBG1(MDP,INI,1,"xscale-init npmcs=%d", xscale_npmcs); /* * Allocate space for pointers to PMC HW descriptors and for * the MDEP structure used by MI code. */ xscale_pcpu = malloc(sizeof(struct xscale_cpu *) * pmc_cpu_max(), M_PMC, M_WAITOK|M_ZERO); /* Just one class */ pmc_mdep = pmc_mdep_alloc(1); pmc_mdep->pmd_cputype = PMC_CPU_INTEL_XSCALE; pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE]; pcd->pcd_caps = XSCALE_PMC_CAPS; pcd->pcd_class = PMC_CLASS_XSCALE; pcd->pcd_num = xscale_npmcs; pcd->pcd_ri = pmc_mdep->pmd_npmc; pcd->pcd_width = 32; pcd->pcd_allocate_pmc = xscale_allocate_pmc; pcd->pcd_config_pmc = xscale_config_pmc; pcd->pcd_pcpu_fini = xscale_pcpu_fini; pcd->pcd_pcpu_init = xscale_pcpu_init; pcd->pcd_describe = xscale_describe; pcd->pcd_get_config = xscale_get_config; pcd->pcd_read_pmc = xscale_read_pmc; pcd->pcd_release_pmc = xscale_release_pmc; pcd->pcd_start_pmc = xscale_start_pmc; pcd->pcd_stop_pmc = xscale_stop_pmc; pcd->pcd_write_pmc = xscale_write_pmc; pmc_mdep->pmd_intr = xscale_intr; pmc_mdep->pmd_switch_in = xscale_switch_in; pmc_mdep->pmd_switch_out = xscale_switch_out; pmc_mdep->pmd_npmc += xscale_npmcs; return (pmc_mdep); } void pmc_xscale_finalize(struct pmc_mdep *md) { } Index: head/sys/dev/pccard/pccard_cis_quirks.c =================================================================== --- head/sys/dev/pccard/pccard_cis_quirks.c (revision 298410) +++ head/sys/dev/pccard/pccard_cis_quirks.c (revision 298411) @@ -1,353 +1,350 @@ /* $NetBSD: pcmcia_cis_quirks.c,v 1.6 2000/04/12 21:07:55 scw Exp $ */ #include __FBSDID("$FreeBSD$"); #define PCCARDDEBUG /*- * Copyright (c) 1998 Marc Horowitz. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Marc Horowitz. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "pccarddevs.h" /* There are cards out there whose CIS flat-out lies. This file contains struct pccard_function chains for those devices. */ /* these structures are just static templates which are then copied into "live" allocated structures */ struct pccard_function pccard_3cxem556_func0 = { 0, /* function number */ PCCARD_FUNCTION_NETWORK, 0x07, /* last cfe number */ 0x800, /* ccr_base */ 0x63, /* ccr_mask */ }; struct pccard_config_entry pccard_3cxem556_func0_cfe0 = { 0x07, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IO16 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 4, /* iomask */ { { 0x0010, 0 } }, /* iospace */ 0xffff, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_3cxem556_func1 = { 1, /* function number */ PCCARD_FUNCTION_SERIAL, 0x27, /* last cfe number */ 0x900, /* ccr_base */ 0x63, /* ccr_mask */ }; static struct pccard_config_entry pccard_3cxem556_func1_cfe0 = { 0x27, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 3, /* iomask */ { { 0x0008, 0 } }, /* iospace */ 0xffff, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_3ccfem556bi_func0 = { 0, /* function number */ PCCARD_FUNCTION_NETWORK, 0x07, /* last cfe number */ 0x1000, /* ccr_base */ 0x267, /* ccr_mask */ }; static struct pccard_config_entry pccard_3ccfem556bi_func0_cfe0 = { 0x07, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IO16 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 5, /* iomask */ { { 0x0020, 0 } }, /* iospace */ 0xffff, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_3ccfem556bi_func1 = { 1, /* function number */ PCCARD_FUNCTION_SERIAL, 0x27, /* last cfe number */ 0x1100, /* ccr_base */ 0x277, /* ccr_mask */ }; static struct pccard_config_entry pccard_3ccfem556bi_func1_cfe0 = { 0x27, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 3, /* iomask */ { { 0x0008, 0 } }, /* iospace */ 0xffff, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_3c1_func0 = { 0, /* function number */ PCCARD_FUNCTION_NETWORK, 0x05, /* last cfe number */ 0x400, /* ccr_base */ 0x267, /* ccr_mask */ }; static struct pccard_config_entry pccard_3c1_func0_cfe0 = { 0x05, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IO16 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 5, /* iomask */ { { 0x0010, 0 } }, /* iospace */ 0xffff, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_sveclancard_func0 = { 0, /* function number */ PCCARD_FUNCTION_NETWORK, 0x1, /* last cfe number */ 0x100, /* ccr_base */ 0x1, /* ccr_mask */ }; static struct pccard_config_entry pccard_sveclancard_func0_cfe0 = { 0x1, /* cfe number */ PCCARD_CFE_MWAIT_REQUIRED | PCCARD_CFE_RDYBSY_ACTIVE | PCCARD_CFE_WP_ACTIVE | PCCARD_CFE_BVD_ACTIVE | PCCARD_CFE_IO16, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 5, /* iomask */ { { 0x20, 0x300 } }, /* iospace */ 0xdeb8, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_ndc_nd5100_func0 = { 0, /* function number */ PCCARD_FUNCTION_NETWORK, 0x23, /* last cfe number */ 0x3f8, /* ccr_base */ 0x3, /* ccr_mask */ }; static struct pccard_config_entry pccard_ndc_nd5100_func0_cfe0 = { 0x20, /* cfe number */ PCCARD_CFE_MWAIT_REQUIRED | PCCARD_CFE_IO16 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 5, /* iomask */ { { 0x20, 0x300 } }, /* iospace */ 0xdeb8, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_function pccard_sierra_a555_func1 = { 1, /* function number */ PCCARD_FUNCTION_SERIAL, 0x24, /* last cfe number */ 0x700, /* ccr_base */ 0x73, /* ccr_mask */ }; static struct pccard_config_entry pccard_sierra_a555_func1_cfe0 = { 0x22, /* cfe number */ PCCARD_CFE_IO8 | PCCARD_CFE_IRQLEVEL, PCCARD_IFTYPE_IO, 1, /* num_iospace */ 0, /* iomask */ { { 0x0008, 0x3e8 } }, /* iospace */ 0x3fbc, /* irqmask */ 0, /* num_memspace */ { }, /* memspace */ 0, /* maxtwins */ }; static struct pccard_cis_quirk pccard_cis_quirks[] = { { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CXEM556, PCMCIA_CIS_INVALID, &pccard_3cxem556_func0, &pccard_3cxem556_func0_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CXEM556, PCMCIA_CIS_INVALID, &pccard_3cxem556_func1, &pccard_3cxem556_func1_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CXEM556INT, PCMCIA_CIS_INVALID, &pccard_3cxem556_func0, &pccard_3cxem556_func0_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CXEM556INT, PCMCIA_CIS_INVALID, &pccard_3cxem556_func1, &pccard_3cxem556_func1_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CCFEM556BI, PCMCIA_CIS_INVALID, &pccard_3ccfem556bi_func0, &pccard_3ccfem556bi_func0_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3CCFEM556BI, PCMCIA_CIS_INVALID, &pccard_3ccfem556bi_func1, &pccard_3ccfem556bi_func1_cfe0 }, { PCMCIA_VENDOR_SIERRA, PCMCIA_PRODUCT_SIERRA_A550, PCMCIA_CIS_INVALID, &pccard_sierra_a555_func1, &pccard_sierra_a555_func1_cfe0 }, { PCMCIA_VENDOR_SIERRA, PCMCIA_PRODUCT_SIERRA_A555, PCMCIA_CIS_INVALID, &pccard_sierra_a555_func1, &pccard_sierra_a555_func1_cfe0 }, { PCMCIA_VENDOR_SIERRA, PCMCIA_PRODUCT_SIERRA_A710, PCMCIA_CIS_INVALID, &pccard_sierra_a555_func1, &pccard_sierra_a555_func1_cfe0 }, { PCMCIA_VENDOR_SIERRA, PCMCIA_PRODUCT_SIERRA_AC710, PCMCIA_CIS_INVALID, &pccard_sierra_a555_func1, &pccard_sierra_a555_func1_cfe0 }, { PCMCIA_VENDOR_3COM, PCMCIA_PRODUCT_3COM_3C1, PCMCIA_CIS_INVALID, &pccard_3c1_func0, &pccard_3c1_func0_cfe0 }, { PCMCIA_VENDOR_INVALID, PCMCIA_PRODUCT_INVALID, PCMCIA_CIS_SVEC_LANCARD, &pccard_sveclancard_func0, &pccard_sveclancard_func0_cfe0 }, { PCMCIA_VENDOR_INVALID, PCMCIA_PRODUCT_INVALID, PCMCIA_CIS_NDC_ND5100_E, &pccard_ndc_nd5100_func0, &pccard_ndc_nd5100_func0_cfe0 }, }; -static int n_pccard_cis_quirks = - sizeof(pccard_cis_quirks)/sizeof(pccard_cis_quirks[0]); - static int pccard_cis_quirk_match(struct pccard_softc *sc, struct pccard_cis_quirk *q) { if ((sc->card.manufacturer == q->manufacturer) && (sc->card.product == q->product) && (((sc->card.manufacturer != PCMCIA_VENDOR_INVALID) && (sc->card.product != PCMCIA_PRODUCT_INVALID)) || ((sc->card.manufacturer == PCMCIA_VENDOR_INVALID) && (sc->card.product == PCMCIA_PRODUCT_INVALID) && sc->card.cis1_info[0] && (strcmp(sc->card.cis1_info[0], q->cis1_info[0]) == 0) && sc->card.cis1_info[1] && (strcmp(sc->card.cis1_info[1], q->cis1_info[1]) == 0)))) return (1); return (0); } void pccard_check_cis_quirks(device_t dev) { struct pccard_softc *sc = PCCARD_SOFTC(dev); int wiped = 0; int i, j; struct pccard_function *pf, *pf_next, *pf_last; struct pccard_config_entry *cfe, *cfe_next; struct pccard_cis_quirk *q; pf = NULL; pf_last = NULL; - for (i=0; icard.cis1_info[j] == NULL) break; if (j) printf(", "); printf("%s", sc->card.cis1_info[j]); } printf("\n"); } for (pf = STAILQ_FIRST(&sc->card.pf_head); pf != NULL; pf = pf_next) { for (cfe = STAILQ_FIRST(&pf->cfe_head); cfe != NULL; cfe = cfe_next) { cfe_next = STAILQ_NEXT(cfe, cfe_list); free(cfe, M_DEVBUF); } pf_next = STAILQ_NEXT(pf, pf_list); free(pf, M_DEVBUF); } STAILQ_INIT(&sc->card.pf_head); wiped = 1; } if (pf_last == q->pf) { cfe = malloc(sizeof(*cfe), M_DEVBUF, M_NOWAIT); if (cfe == NULL) { device_printf(dev, "no memory for quirk (1)\n"); continue; } *cfe = *q->cfe; STAILQ_INSERT_TAIL(&pf->cfe_head, cfe, cfe_list); } else { pf = malloc(sizeof(*pf), M_DEVBUF, M_NOWAIT); if (pf == NULL) { device_printf(dev, "no memory for pccard function\n"); continue; } *pf = *q->pf; STAILQ_INIT(&pf->cfe_head); cfe = malloc(sizeof(*cfe), M_DEVBUF, M_NOWAIT); if (cfe == NULL) { free(pf, M_DEVBUF); device_printf(dev, "no memory for quirk (2)\n"); continue; } *cfe = *q->cfe; STAILQ_INSERT_TAIL(&pf->cfe_head, cfe, cfe_list); STAILQ_INSERT_TAIL(&sc->card.pf_head, pf, pf_list); pf_last = q->pf; } } } Index: head/sys/dev/sge/if_sge.c =================================================================== --- head/sys/dev/sge/if_sge.c (revision 298410) +++ head/sys/dev/sge/if_sge.c (revision 298411) @@ -1,1904 +1,1903 @@ /*- * Copyright (c) 2008-2010 Nikolay Denev * Copyright (c) 2007-2008 Alexander Pohoyda * Copyright (c) 1997, 1998, 1999 * Bill Paul . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Bill Paul. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL AUTHORS OR * THE VOICES IN THEIR HEADS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED * OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * SiS 190/191 PCI Ethernet NIC driver. * * Adapted to SiS 190 NIC by Alexander Pohoyda based on the original * SiS 900 driver by Bill Paul, using SiS 190/191 Solaris driver by * Masayuki Murayama and SiS 190/191 GNU/Linux driver by K.M. Liu * . Thanks to Pyun YongHyeon for * review and very useful comments. * * Adapted to SiS 191 NIC by Nikolay Denev with further ideas from the * Linux and Solaris drivers. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MODULE_DEPEND(sge, pci, 1, 1, 1); MODULE_DEPEND(sge, ether, 1, 1, 1); MODULE_DEPEND(sge, miibus, 1, 1, 1); /* "device miibus0" required. See GENERIC if you get errors here. */ #include "miibus_if.h" /* * Various supported device vendors/types and their names. */ static struct sge_type sge_devs[] = { { SIS_VENDORID, SIS_DEVICEID_190, "SiS190 Fast Ethernet" }, { SIS_VENDORID, SIS_DEVICEID_191, "SiS191 Fast/Gigabit Ethernet" }, { 0, 0, NULL } }; static int sge_probe(device_t); static int sge_attach(device_t); static int sge_detach(device_t); static int sge_shutdown(device_t); static int sge_suspend(device_t); static int sge_resume(device_t); static int sge_miibus_readreg(device_t, int, int); static int sge_miibus_writereg(device_t, int, int, int); static void sge_miibus_statchg(device_t); static int sge_newbuf(struct sge_softc *, int); static int sge_encap(struct sge_softc *, struct mbuf **); static __inline void sge_discard_rxbuf(struct sge_softc *, int); static void sge_rxeof(struct sge_softc *); static void sge_txeof(struct sge_softc *); static void sge_intr(void *); static void sge_tick(void *); static void sge_start(struct ifnet *); static void sge_start_locked(struct ifnet *); static int sge_ioctl(struct ifnet *, u_long, caddr_t); static void sge_init(void *); static void sge_init_locked(struct sge_softc *); static void sge_stop(struct sge_softc *); static void sge_watchdog(struct sge_softc *); static int sge_ifmedia_upd(struct ifnet *); static void sge_ifmedia_sts(struct ifnet *, struct ifmediareq *); static int sge_get_mac_addr_apc(struct sge_softc *, uint8_t *); static int sge_get_mac_addr_eeprom(struct sge_softc *, uint8_t *); static uint16_t sge_read_eeprom(struct sge_softc *, int); static void sge_rxfilter(struct sge_softc *); static void sge_setvlan(struct sge_softc *); static void sge_reset(struct sge_softc *); static int sge_list_rx_init(struct sge_softc *); static int sge_list_rx_free(struct sge_softc *); static int sge_list_tx_init(struct sge_softc *); static int sge_list_tx_free(struct sge_softc *); static int sge_dma_alloc(struct sge_softc *); static void sge_dma_free(struct sge_softc *); static void sge_dma_map_addr(void *, bus_dma_segment_t *, int, int); static device_method_t sge_methods[] = { /* Device interface */ DEVMETHOD(device_probe, sge_probe), DEVMETHOD(device_attach, sge_attach), DEVMETHOD(device_detach, sge_detach), DEVMETHOD(device_suspend, sge_suspend), DEVMETHOD(device_resume, sge_resume), DEVMETHOD(device_shutdown, sge_shutdown), /* MII interface */ DEVMETHOD(miibus_readreg, sge_miibus_readreg), DEVMETHOD(miibus_writereg, sge_miibus_writereg), DEVMETHOD(miibus_statchg, sge_miibus_statchg), DEVMETHOD_END }; static driver_t sge_driver = { "sge", sge_methods, sizeof(struct sge_softc) }; static devclass_t sge_devclass; DRIVER_MODULE(sge, pci, sge_driver, sge_devclass, 0, 0); DRIVER_MODULE(miibus, sge, miibus_driver, miibus_devclass, 0, 0); /* * Register space access macros. */ #define CSR_WRITE_4(sc, reg, val) bus_write_4(sc->sge_res, reg, val) #define CSR_WRITE_2(sc, reg, val) bus_write_2(sc->sge_res, reg, val) #define CSR_WRITE_1(cs, reg, val) bus_write_1(sc->sge_res, reg, val) #define CSR_READ_4(sc, reg) bus_read_4(sc->sge_res, reg) #define CSR_READ_2(sc, reg) bus_read_2(sc->sge_res, reg) #define CSR_READ_1(sc, reg) bus_read_1(sc->sge_res, reg) /* Define to show Tx/Rx error status. */ #undef SGE_SHOW_ERRORS #define SGE_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) static void sge_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *p; if (error != 0) return; KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg)); p = arg; *p = segs->ds_addr; } /* * Read a sequence of words from the EEPROM. */ static uint16_t sge_read_eeprom(struct sge_softc *sc, int offset) { uint32_t val; int i; KASSERT(offset <= EI_OFFSET, ("EEPROM offset too big")); CSR_WRITE_4(sc, ROMInterface, EI_REQ | EI_OP_RD | (offset << EI_OFFSET_SHIFT)); DELAY(500); for (i = 0; i < SGE_TIMEOUT; i++) { val = CSR_READ_4(sc, ROMInterface); if ((val & EI_REQ) == 0) break; DELAY(100); } if (i == SGE_TIMEOUT) { device_printf(sc->sge_dev, "EEPROM read timeout : 0x%08x\n", val); return (0xffff); } return ((val & EI_DATA) >> EI_DATA_SHIFT); } static int sge_get_mac_addr_eeprom(struct sge_softc *sc, uint8_t *dest) { uint16_t val; int i; val = sge_read_eeprom(sc, EEPROMSignature); if (val == 0xffff || val == 0) { device_printf(sc->sge_dev, "invalid EEPROM signature : 0x%04x\n", val); return (EINVAL); } for (i = 0; i < ETHER_ADDR_LEN; i += 2) { val = sge_read_eeprom(sc, EEPROMMACAddr + i / 2); dest[i + 0] = (uint8_t)val; dest[i + 1] = (uint8_t)(val >> 8); } if ((sge_read_eeprom(sc, EEPROMInfo) & 0x80) != 0) sc->sge_flags |= SGE_FLAG_RGMII; return (0); } /* * For SiS96x, APC CMOS RAM is used to store ethernet address. * APC CMOS RAM is accessed through ISA bridge. */ static int sge_get_mac_addr_apc(struct sge_softc *sc, uint8_t *dest) { #if defined(__amd64__) || defined(__i386__) devclass_t pci; device_t bus, dev = NULL; device_t *kids; struct apc_tbl { uint16_t vid; uint16_t did; } *tp, apc_tbls[] = { { SIS_VENDORID, 0x0965 }, { SIS_VENDORID, 0x0966 }, { SIS_VENDORID, 0x0968 } }; uint8_t reg; - int busnum, cnt, i, j, numkids; + int busnum, i, j, numkids; - cnt = sizeof(apc_tbls) / sizeof(apc_tbls[0]); pci = devclass_find("pci"); for (busnum = 0; busnum < devclass_get_maxunit(pci); busnum++) { bus = devclass_get_device(pci, busnum); if (!bus) continue; if (device_get_children(bus, &kids, &numkids) != 0) continue; for (i = 0; i < numkids; i++) { dev = kids[i]; if (pci_get_class(dev) == PCIC_BRIDGE && pci_get_subclass(dev) == PCIS_BRIDGE_ISA) { tp = apc_tbls; - for (j = 0; j < cnt; j++) { + for (j = 0; j < nitems(apc_tbls); j++) { if (pci_get_vendor(dev) == tp->vid && pci_get_device(dev) == tp->did) { free(kids, M_TEMP); goto apc_found; } tp++; } } } free(kids, M_TEMP); } device_printf(sc->sge_dev, "couldn't find PCI-ISA bridge\n"); return (EINVAL); apc_found: /* Enable port 0x78 and 0x79 to access APC registers. */ reg = pci_read_config(dev, 0x48, 1); pci_write_config(dev, 0x48, reg & ~0x02, 1); DELAY(50); pci_read_config(dev, 0x48, 1); /* Read stored ethernet address. */ for (i = 0; i < ETHER_ADDR_LEN; i++) { outb(0x78, 0x09 + i); dest[i] = inb(0x79); } outb(0x78, 0x12); if ((inb(0x79) & 0x80) != 0) sc->sge_flags |= SGE_FLAG_RGMII; /* Restore access to APC registers. */ pci_write_config(dev, 0x48, reg, 1); return (0); #else return (EINVAL); #endif } static int sge_miibus_readreg(device_t dev, int phy, int reg) { struct sge_softc *sc; uint32_t val; int i; sc = device_get_softc(dev); CSR_WRITE_4(sc, GMIIControl, (phy << GMI_PHY_SHIFT) | (reg << GMI_REG_SHIFT) | GMI_OP_RD | GMI_REQ); DELAY(10); for (i = 0; i < SGE_TIMEOUT; i++) { val = CSR_READ_4(sc, GMIIControl); if ((val & GMI_REQ) == 0) break; DELAY(10); } if (i == SGE_TIMEOUT) { device_printf(sc->sge_dev, "PHY read timeout : %d\n", reg); return (0); } return ((val & GMI_DATA) >> GMI_DATA_SHIFT); } static int sge_miibus_writereg(device_t dev, int phy, int reg, int data) { struct sge_softc *sc; uint32_t val; int i; sc = device_get_softc(dev); CSR_WRITE_4(sc, GMIIControl, (phy << GMI_PHY_SHIFT) | (reg << GMI_REG_SHIFT) | (data << GMI_DATA_SHIFT) | GMI_OP_WR | GMI_REQ); DELAY(10); for (i = 0; i < SGE_TIMEOUT; i++) { val = CSR_READ_4(sc, GMIIControl); if ((val & GMI_REQ) == 0) break; DELAY(10); } if (i == SGE_TIMEOUT) device_printf(sc->sge_dev, "PHY write timeout : %d\n", reg); return (0); } static void sge_miibus_statchg(device_t dev) { struct sge_softc *sc; struct mii_data *mii; struct ifnet *ifp; uint32_t ctl, speed; sc = device_get_softc(dev); mii = device_get_softc(sc->sge_miibus); ifp = sc->sge_ifp; if (mii == NULL || ifp == NULL || (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return; speed = 0; sc->sge_flags &= ~SGE_FLAG_LINK; if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == (IFM_ACTIVE | IFM_AVALID)) { switch (IFM_SUBTYPE(mii->mii_media_active)) { case IFM_10_T: sc->sge_flags |= SGE_FLAG_LINK; speed = SC_SPEED_10; break; case IFM_100_TX: sc->sge_flags |= SGE_FLAG_LINK; speed = SC_SPEED_100; break; case IFM_1000_T: if ((sc->sge_flags & SGE_FLAG_FASTETHER) == 0) { sc->sge_flags |= SGE_FLAG_LINK; speed = SC_SPEED_1000; } break; default: break; } } if ((sc->sge_flags & SGE_FLAG_LINK) == 0) return; /* Reprogram MAC to resolved speed/duplex/flow-control parameters. */ ctl = CSR_READ_4(sc, StationControl); ctl &= ~(0x0f000000 | SC_FDX | SC_SPEED_MASK); if (speed == SC_SPEED_1000) { ctl |= 0x07000000; sc->sge_flags |= SGE_FLAG_SPEED_1000; } else { ctl |= 0x04000000; sc->sge_flags &= ~SGE_FLAG_SPEED_1000; } #ifdef notyet if ((sc->sge_flags & SGE_FLAG_GMII) != 0) ctl |= 0x03000000; #endif ctl |= speed; if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) { ctl |= SC_FDX; sc->sge_flags |= SGE_FLAG_FDX; } else sc->sge_flags &= ~SGE_FLAG_FDX; CSR_WRITE_4(sc, StationControl, ctl); if ((sc->sge_flags & SGE_FLAG_RGMII) != 0) { CSR_WRITE_4(sc, RGMIIDelay, 0x0441); CSR_WRITE_4(sc, RGMIIDelay, 0x0440); } } static void sge_rxfilter(struct sge_softc *sc) { struct ifnet *ifp; struct ifmultiaddr *ifma; uint32_t crc, hashes[2]; uint16_t rxfilt; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; rxfilt = CSR_READ_2(sc, RxMacControl); rxfilt &= ~(AcceptBroadcast | AcceptAllPhys | AcceptMulticast); rxfilt |= AcceptMyPhys; if ((ifp->if_flags & IFF_BROADCAST) != 0) rxfilt |= AcceptBroadcast; if ((ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) != 0) { if ((ifp->if_flags & IFF_PROMISC) != 0) rxfilt |= AcceptAllPhys; rxfilt |= AcceptMulticast; hashes[0] = 0xFFFFFFFF; hashes[1] = 0xFFFFFFFF; } else { rxfilt |= AcceptMulticast; hashes[0] = hashes[1] = 0; /* Now program new ones. */ if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; crc = ether_crc32_be(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), ETHER_ADDR_LEN); hashes[crc >> 31] |= 1 << ((crc >> 26) & 0x1f); } if_maddr_runlock(ifp); } CSR_WRITE_2(sc, RxMacControl, rxfilt); CSR_WRITE_4(sc, RxHashTable, hashes[0]); CSR_WRITE_4(sc, RxHashTable2, hashes[1]); } static void sge_setvlan(struct sge_softc *sc) { struct ifnet *ifp; uint16_t rxfilt; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; if ((ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) == 0) return; rxfilt = CSR_READ_2(sc, RxMacControl); if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0) rxfilt |= RXMAC_STRIP_VLAN; else rxfilt &= ~RXMAC_STRIP_VLAN; CSR_WRITE_2(sc, RxMacControl, rxfilt); } static void sge_reset(struct sge_softc *sc) { CSR_WRITE_4(sc, IntrMask, 0); CSR_WRITE_4(sc, IntrStatus, 0xffffffff); /* Soft reset. */ CSR_WRITE_4(sc, IntrControl, 0x8000); CSR_READ_4(sc, IntrControl); DELAY(100); CSR_WRITE_4(sc, IntrControl, 0); /* Stop MAC. */ CSR_WRITE_4(sc, TX_CTL, 0x1a00); CSR_WRITE_4(sc, RX_CTL, 0x1a00); CSR_WRITE_4(sc, IntrMask, 0); CSR_WRITE_4(sc, IntrStatus, 0xffffffff); CSR_WRITE_4(sc, GMIIControl, 0); } /* * Probe for an SiS chip. Check the PCI vendor and device * IDs against our list and return a device name if we find a match. */ static int sge_probe(device_t dev) { struct sge_type *t; t = sge_devs; while (t->sge_name != NULL) { if ((pci_get_vendor(dev) == t->sge_vid) && (pci_get_device(dev) == t->sge_did)) { device_set_desc(dev, t->sge_name); return (BUS_PROBE_DEFAULT); } t++; } return (ENXIO); } /* * Attach the interface. Allocate softc structures, do ifmedia * setup and ethernet/BPF attach. */ static int sge_attach(device_t dev) { struct sge_softc *sc; struct ifnet *ifp; uint8_t eaddr[ETHER_ADDR_LEN]; int error = 0, rid; sc = device_get_softc(dev); sc->sge_dev = dev; mtx_init(&sc->sge_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF); callout_init_mtx(&sc->sge_stat_ch, &sc->sge_mtx, 0); /* * Map control/status registers. */ pci_enable_busmaster(dev); /* Allocate resources. */ sc->sge_res_id = PCIR_BAR(0); sc->sge_res_type = SYS_RES_MEMORY; sc->sge_res = bus_alloc_resource_any(dev, sc->sge_res_type, &sc->sge_res_id, RF_ACTIVE); if (sc->sge_res == NULL) { device_printf(dev, "couldn't allocate resource\n"); error = ENXIO; goto fail; } rid = 0; sc->sge_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (sc->sge_irq == NULL) { device_printf(dev, "couldn't allocate IRQ resources\n"); error = ENXIO; goto fail; } sc->sge_rev = pci_get_revid(dev); if (pci_get_device(dev) == SIS_DEVICEID_190) sc->sge_flags |= SGE_FLAG_FASTETHER | SGE_FLAG_SIS190; /* Reset the adapter. */ sge_reset(sc); /* Get MAC address from the EEPROM. */ if ((pci_read_config(dev, 0x73, 1) & 0x01) != 0) sge_get_mac_addr_apc(sc, eaddr); else sge_get_mac_addr_eeprom(sc, eaddr); if ((error = sge_dma_alloc(sc)) != 0) goto fail; ifp = sc->sge_ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { device_printf(dev, "cannot allocate ifnet structure.\n"); error = ENOSPC; goto fail; } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = sge_ioctl; ifp->if_start = sge_start; ifp->if_init = sge_init; ifp->if_snd.ifq_drv_maxlen = SGE_TX_RING_CNT - 1; IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen); IFQ_SET_READY(&ifp->if_snd); ifp->if_capabilities = IFCAP_TXCSUM | IFCAP_RXCSUM | IFCAP_TSO4; ifp->if_hwassist = SGE_CSUM_FEATURES | CSUM_TSO; ifp->if_capenable = ifp->if_capabilities; /* * Do MII setup. */ error = mii_attach(dev, &sc->sge_miibus, ifp, sge_ifmedia_upd, sge_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0); if (error != 0) { device_printf(dev, "attaching PHYs failed\n"); goto fail; } /* * Call MI attach routine. */ ether_ifattach(ifp, eaddr); /* VLAN setup. */ ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; ifp->if_capenable = ifp->if_capabilities; /* Tell the upper layer(s) we support long frames. */ ifp->if_hdrlen = sizeof(struct ether_vlan_header); /* Hook interrupt last to avoid having to lock softc */ error = bus_setup_intr(dev, sc->sge_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, sge_intr, sc, &sc->sge_intrhand); if (error) { device_printf(dev, "couldn't set up irq\n"); ether_ifdetach(ifp); goto fail; } fail: if (error) sge_detach(dev); return (error); } /* * Shutdown hardware and free up resources. This can be called any * time after the mutex has been initialized. It is called in both * the error case in attach and the normal detach case so it needs * to be careful about only freeing resources that have actually been * allocated. */ static int sge_detach(device_t dev) { struct sge_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); ifp = sc->sge_ifp; /* These should only be active if attach succeeded. */ if (device_is_attached(dev)) { ether_ifdetach(ifp); SGE_LOCK(sc); sge_stop(sc); SGE_UNLOCK(sc); callout_drain(&sc->sge_stat_ch); } if (sc->sge_miibus) device_delete_child(dev, sc->sge_miibus); bus_generic_detach(dev); if (sc->sge_intrhand) bus_teardown_intr(dev, sc->sge_irq, sc->sge_intrhand); if (sc->sge_irq) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sge_irq); if (sc->sge_res) bus_release_resource(dev, sc->sge_res_type, sc->sge_res_id, sc->sge_res); if (ifp) if_free(ifp); sge_dma_free(sc); mtx_destroy(&sc->sge_mtx); return (0); } /* * Stop all chip I/O so that the kernel's probe routines don't * get confused by errant DMAs when rebooting. */ static int sge_shutdown(device_t dev) { struct sge_softc *sc; sc = device_get_softc(dev); SGE_LOCK(sc); sge_stop(sc); SGE_UNLOCK(sc); return (0); } static int sge_suspend(device_t dev) { struct sge_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); SGE_LOCK(sc); ifp = sc->sge_ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) sge_stop(sc); SGE_UNLOCK(sc); return (0); } static int sge_resume(device_t dev) { struct sge_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); SGE_LOCK(sc); ifp = sc->sge_ifp; if ((ifp->if_flags & IFF_UP) != 0) sge_init_locked(sc); SGE_UNLOCK(sc); return (0); } static int sge_dma_alloc(struct sge_softc *sc) { struct sge_chain_data *cd; struct sge_list_data *ld; struct sge_rxdesc *rxd; struct sge_txdesc *txd; int error, i; cd = &sc->sge_cdata; ld = &sc->sge_ldata; error = bus_dma_tag_create(bus_get_dma_tag(sc->sge_dev), 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ 1, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &cd->sge_tag); if (error != 0) { device_printf(sc->sge_dev, "could not create parent DMA tag.\n"); goto fail; } /* RX descriptor ring */ error = bus_dma_tag_create(cd->sge_tag, SGE_DESC_ALIGN, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ SGE_RX_RING_SZ, 1, /* maxsize,nsegments */ SGE_RX_RING_SZ, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &cd->sge_rx_tag); if (error != 0) { device_printf(sc->sge_dev, "could not create Rx ring DMA tag.\n"); goto fail; } /* Allocate DMA'able memory and load DMA map for RX ring. */ error = bus_dmamem_alloc(cd->sge_rx_tag, (void **)&ld->sge_rx_ring, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &cd->sge_rx_dmamap); if (error != 0) { device_printf(sc->sge_dev, "could not allocate DMA'able memory for Rx ring.\n"); goto fail; } error = bus_dmamap_load(cd->sge_rx_tag, cd->sge_rx_dmamap, ld->sge_rx_ring, SGE_RX_RING_SZ, sge_dma_map_addr, &ld->sge_rx_paddr, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sge_dev, "could not load DMA'able memory for Rx ring.\n"); } /* TX descriptor ring */ error = bus_dma_tag_create(cd->sge_tag, SGE_DESC_ALIGN, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ SGE_TX_RING_SZ, 1, /* maxsize,nsegments */ SGE_TX_RING_SZ, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockarg */ &cd->sge_tx_tag); if (error != 0) { device_printf(sc->sge_dev, "could not create Rx ring DMA tag.\n"); goto fail; } /* Allocate DMA'able memory and load DMA map for TX ring. */ error = bus_dmamem_alloc(cd->sge_tx_tag, (void **)&ld->sge_tx_ring, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &cd->sge_tx_dmamap); if (error != 0) { device_printf(sc->sge_dev, "could not allocate DMA'able memory for Tx ring.\n"); goto fail; } error = bus_dmamap_load(cd->sge_tx_tag, cd->sge_tx_dmamap, ld->sge_tx_ring, SGE_TX_RING_SZ, sge_dma_map_addr, &ld->sge_tx_paddr, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sge_dev, "could not load DMA'able memory for Rx ring.\n"); goto fail; } /* Create DMA tag for Tx buffers. */ error = bus_dma_tag_create(cd->sge_tag, 1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, SGE_TSO_MAXSIZE, SGE_MAXTXSEGS, SGE_TSO_MAXSEGSIZE, 0, NULL, NULL, &cd->sge_txmbuf_tag); if (error != 0) { device_printf(sc->sge_dev, "could not create Tx mbuf DMA tag.\n"); goto fail; } /* Create DMA tag for Rx buffers. */ error = bus_dma_tag_create(cd->sge_tag, SGE_RX_BUF_ALIGN, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, MCLBYTES, 0, NULL, NULL, &cd->sge_rxmbuf_tag); if (error != 0) { device_printf(sc->sge_dev, "could not create Rx mbuf DMA tag.\n"); goto fail; } /* Create DMA maps for Tx buffers. */ for (i = 0; i < SGE_TX_RING_CNT; i++) { txd = &cd->sge_txdesc[i]; txd->tx_m = NULL; txd->tx_dmamap = NULL; txd->tx_ndesc = 0; error = bus_dmamap_create(cd->sge_txmbuf_tag, 0, &txd->tx_dmamap); if (error != 0) { device_printf(sc->sge_dev, "could not create Tx DMA map.\n"); goto fail; } } /* Create spare DMA map for Rx buffer. */ error = bus_dmamap_create(cd->sge_rxmbuf_tag, 0, &cd->sge_rx_spare_map); if (error != 0) { device_printf(sc->sge_dev, "could not create spare Rx DMA map.\n"); goto fail; } /* Create DMA maps for Rx buffers. */ for (i = 0; i < SGE_RX_RING_CNT; i++) { rxd = &cd->sge_rxdesc[i]; rxd->rx_m = NULL; rxd->rx_dmamap = NULL; error = bus_dmamap_create(cd->sge_rxmbuf_tag, 0, &rxd->rx_dmamap); if (error) { device_printf(sc->sge_dev, "could not create Rx DMA map.\n"); goto fail; } } fail: return (error); } static void sge_dma_free(struct sge_softc *sc) { struct sge_chain_data *cd; struct sge_list_data *ld; struct sge_rxdesc *rxd; struct sge_txdesc *txd; int i; cd = &sc->sge_cdata; ld = &sc->sge_ldata; /* Rx ring. */ if (cd->sge_rx_tag != NULL) { if (ld->sge_rx_paddr != 0) bus_dmamap_unload(cd->sge_rx_tag, cd->sge_rx_dmamap); if (ld->sge_rx_ring != NULL) bus_dmamem_free(cd->sge_rx_tag, ld->sge_rx_ring, cd->sge_rx_dmamap); ld->sge_rx_ring = NULL; ld->sge_rx_paddr = 0; bus_dma_tag_destroy(cd->sge_rx_tag); cd->sge_rx_tag = NULL; } /* Tx ring. */ if (cd->sge_tx_tag != NULL) { if (ld->sge_tx_paddr != 0) bus_dmamap_unload(cd->sge_tx_tag, cd->sge_tx_dmamap); if (ld->sge_tx_ring != NULL) bus_dmamem_free(cd->sge_tx_tag, ld->sge_tx_ring, cd->sge_tx_dmamap); ld->sge_tx_ring = NULL; ld->sge_tx_paddr = 0; bus_dma_tag_destroy(cd->sge_tx_tag); cd->sge_tx_tag = NULL; } /* Rx buffers. */ if (cd->sge_rxmbuf_tag != NULL) { for (i = 0; i < SGE_RX_RING_CNT; i++) { rxd = &cd->sge_rxdesc[i]; if (rxd->rx_dmamap != NULL) { bus_dmamap_destroy(cd->sge_rxmbuf_tag, rxd->rx_dmamap); rxd->rx_dmamap = NULL; } } if (cd->sge_rx_spare_map != NULL) { bus_dmamap_destroy(cd->sge_rxmbuf_tag, cd->sge_rx_spare_map); cd->sge_rx_spare_map = NULL; } bus_dma_tag_destroy(cd->sge_rxmbuf_tag); cd->sge_rxmbuf_tag = NULL; } /* Tx buffers. */ if (cd->sge_txmbuf_tag != NULL) { for (i = 0; i < SGE_TX_RING_CNT; i++) { txd = &cd->sge_txdesc[i]; if (txd->tx_dmamap != NULL) { bus_dmamap_destroy(cd->sge_txmbuf_tag, txd->tx_dmamap); txd->tx_dmamap = NULL; } } bus_dma_tag_destroy(cd->sge_txmbuf_tag); cd->sge_txmbuf_tag = NULL; } if (cd->sge_tag != NULL) bus_dma_tag_destroy(cd->sge_tag); cd->sge_tag = NULL; } /* * Initialize the TX descriptors. */ static int sge_list_tx_init(struct sge_softc *sc) { struct sge_list_data *ld; struct sge_chain_data *cd; SGE_LOCK_ASSERT(sc); ld = &sc->sge_ldata; cd = &sc->sge_cdata; bzero(ld->sge_tx_ring, SGE_TX_RING_SZ); ld->sge_tx_ring[SGE_TX_RING_CNT - 1].sge_flags = htole32(RING_END); bus_dmamap_sync(cd->sge_tx_tag, cd->sge_tx_dmamap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); cd->sge_tx_prod = 0; cd->sge_tx_cons = 0; cd->sge_tx_cnt = 0; return (0); } static int sge_list_tx_free(struct sge_softc *sc) { struct sge_chain_data *cd; struct sge_txdesc *txd; int i; SGE_LOCK_ASSERT(sc); cd = &sc->sge_cdata; for (i = 0; i < SGE_TX_RING_CNT; i++) { txd = &cd->sge_txdesc[i]; if (txd->tx_m != NULL) { bus_dmamap_sync(cd->sge_txmbuf_tag, txd->tx_dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(cd->sge_txmbuf_tag, txd->tx_dmamap); m_freem(txd->tx_m); txd->tx_m = NULL; txd->tx_ndesc = 0; } } return (0); } /* * Initialize the RX descriptors and allocate mbufs for them. Note that * we arrange the descriptors in a closed ring, so that the last descriptor * has RING_END flag set. */ static int sge_list_rx_init(struct sge_softc *sc) { struct sge_chain_data *cd; int i; SGE_LOCK_ASSERT(sc); cd = &sc->sge_cdata; cd->sge_rx_cons = 0; bzero(sc->sge_ldata.sge_rx_ring, SGE_RX_RING_SZ); for (i = 0; i < SGE_RX_RING_CNT; i++) { if (sge_newbuf(sc, i) != 0) return (ENOBUFS); } bus_dmamap_sync(cd->sge_rx_tag, cd->sge_rx_dmamap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return (0); } static int sge_list_rx_free(struct sge_softc *sc) { struct sge_chain_data *cd; struct sge_rxdesc *rxd; int i; SGE_LOCK_ASSERT(sc); cd = &sc->sge_cdata; for (i = 0; i < SGE_RX_RING_CNT; i++) { rxd = &cd->sge_rxdesc[i]; if (rxd->rx_m != NULL) { bus_dmamap_sync(cd->sge_rxmbuf_tag, rxd->rx_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(cd->sge_rxmbuf_tag, rxd->rx_dmamap); m_freem(rxd->rx_m); rxd->rx_m = NULL; } } return (0); } /* * Initialize an RX descriptor and attach an MBUF cluster. */ static int sge_newbuf(struct sge_softc *sc, int prod) { struct mbuf *m; struct sge_desc *desc; struct sge_chain_data *cd; struct sge_rxdesc *rxd; bus_dma_segment_t segs[1]; bus_dmamap_t map; int error, nsegs; SGE_LOCK_ASSERT(sc); cd = &sc->sge_cdata; m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (m == NULL) return (ENOBUFS); m->m_len = m->m_pkthdr.len = MCLBYTES; m_adj(m, SGE_RX_BUF_ALIGN); error = bus_dmamap_load_mbuf_sg(cd->sge_rxmbuf_tag, cd->sge_rx_spare_map, m, segs, &nsegs, 0); if (error != 0) { m_freem(m); return (error); } KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); rxd = &cd->sge_rxdesc[prod]; if (rxd->rx_m != NULL) { bus_dmamap_sync(cd->sge_rxmbuf_tag, rxd->rx_dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(cd->sge_rxmbuf_tag, rxd->rx_dmamap); } map = rxd->rx_dmamap; rxd->rx_dmamap = cd->sge_rx_spare_map; cd->sge_rx_spare_map = map; bus_dmamap_sync(cd->sge_rxmbuf_tag, rxd->rx_dmamap, BUS_DMASYNC_PREREAD); rxd->rx_m = m; desc = &sc->sge_ldata.sge_rx_ring[prod]; desc->sge_sts_size = 0; desc->sge_ptr = htole32(SGE_ADDR_LO(segs[0].ds_addr)); desc->sge_flags = htole32(segs[0].ds_len); if (prod == SGE_RX_RING_CNT - 1) desc->sge_flags |= htole32(RING_END); desc->sge_cmdsts = htole32(RDC_OWN | RDC_INTR); return (0); } static __inline void sge_discard_rxbuf(struct sge_softc *sc, int index) { struct sge_desc *desc; desc = &sc->sge_ldata.sge_rx_ring[index]; desc->sge_sts_size = 0; desc->sge_flags = htole32(MCLBYTES - SGE_RX_BUF_ALIGN); if (index == SGE_RX_RING_CNT - 1) desc->sge_flags |= htole32(RING_END); desc->sge_cmdsts = htole32(RDC_OWN | RDC_INTR); } /* * A frame has been uploaded: pass the resulting mbuf chain up to * the higher level protocols. */ static void sge_rxeof(struct sge_softc *sc) { struct ifnet *ifp; struct mbuf *m; struct sge_chain_data *cd; struct sge_desc *cur_rx; uint32_t rxinfo, rxstat; int cons, prog; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; cd = &sc->sge_cdata; bus_dmamap_sync(cd->sge_rx_tag, cd->sge_rx_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cons = cd->sge_rx_cons; for (prog = 0; prog < SGE_RX_RING_CNT; prog++, SGE_INC(cons, SGE_RX_RING_CNT)) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; cur_rx = &sc->sge_ldata.sge_rx_ring[cons]; rxinfo = le32toh(cur_rx->sge_cmdsts); if ((rxinfo & RDC_OWN) != 0) break; rxstat = le32toh(cur_rx->sge_sts_size); if ((rxstat & RDS_CRCOK) == 0 || SGE_RX_ERROR(rxstat) != 0 || SGE_RX_NSEGS(rxstat) != 1) { /* XXX We don't support multi-segment frames yet. */ #ifdef SGE_SHOW_ERRORS device_printf(sc->sge_dev, "Rx error : 0x%b\n", rxstat, RX_ERR_BITS); #endif sge_discard_rxbuf(sc, cons); if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); continue; } m = cd->sge_rxdesc[cons].rx_m; if (sge_newbuf(sc, cons) != 0) { sge_discard_rxbuf(sc, cons); if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); continue; } if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { if ((rxinfo & RDC_IP_CSUM) != 0 && (rxinfo & RDC_IP_CSUM_OK) != 0) m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID; if (((rxinfo & RDC_TCP_CSUM) != 0 && (rxinfo & RDC_TCP_CSUM_OK) != 0) || ((rxinfo & RDC_UDP_CSUM) != 0 && (rxinfo & RDC_UDP_CSUM_OK) != 0)) { m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR; m->m_pkthdr.csum_data = 0xffff; } } /* Check for VLAN tagged frame. */ if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && (rxstat & RDS_VLAN) != 0) { m->m_pkthdr.ether_vtag = rxinfo & RDC_VLAN_MASK; m->m_flags |= M_VLANTAG; } /* * Account for 10bytes auto padding which is used * to align IP header on 32bit boundary. Also note, * CRC bytes is automatically removed by the * hardware. */ m->m_data += SGE_RX_PAD_BYTES; m->m_pkthdr.len = m->m_len = SGE_RX_BYTES(rxstat) - SGE_RX_PAD_BYTES; m->m_pkthdr.rcvif = ifp; if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); SGE_UNLOCK(sc); (*ifp->if_input)(ifp, m); SGE_LOCK(sc); } if (prog > 0) { bus_dmamap_sync(cd->sge_rx_tag, cd->sge_rx_dmamap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); cd->sge_rx_cons = cons; } } /* * A frame was downloaded to the chip. It's safe for us to clean up * the list buffers. */ static void sge_txeof(struct sge_softc *sc) { struct ifnet *ifp; struct sge_list_data *ld; struct sge_chain_data *cd; struct sge_txdesc *txd; uint32_t txstat; int cons, nsegs, prod; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; ld = &sc->sge_ldata; cd = &sc->sge_cdata; if (cd->sge_tx_cnt == 0) return; bus_dmamap_sync(cd->sge_tx_tag, cd->sge_tx_dmamap, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); cons = cd->sge_tx_cons; prod = cd->sge_tx_prod; for (; cons != prod;) { txstat = le32toh(ld->sge_tx_ring[cons].sge_cmdsts); if ((txstat & TDC_OWN) != 0) break; /* * Only the first descriptor of multi-descriptor transmission * is updated by controller. Driver should skip entire * chained buffers for the transmitted frame. In other words * TDC_OWN bit is valid only at the first descriptor of a * multi-descriptor transmission. */ if (SGE_TX_ERROR(txstat) != 0) { #ifdef SGE_SHOW_ERRORS device_printf(sc->sge_dev, "Tx error : 0x%b\n", txstat, TX_ERR_BITS); #endif if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); } else { #ifdef notyet if_inc_counter(ifp, IFCOUNTER_COLLISIONS, (txstat & 0xFFFF) - 1); #endif if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } txd = &cd->sge_txdesc[cons]; for (nsegs = 0; nsegs < txd->tx_ndesc; nsegs++) { ld->sge_tx_ring[cons].sge_cmdsts = 0; SGE_INC(cons, SGE_TX_RING_CNT); } /* Reclaim transmitted mbuf. */ KASSERT(txd->tx_m != NULL, ("%s: freeing NULL mbuf\n", __func__)); bus_dmamap_sync(cd->sge_txmbuf_tag, txd->tx_dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(cd->sge_txmbuf_tag, txd->tx_dmamap); m_freem(txd->tx_m); txd->tx_m = NULL; cd->sge_tx_cnt -= txd->tx_ndesc; KASSERT(cd->sge_tx_cnt >= 0, ("%s: Active Tx desc counter was garbled\n", __func__)); txd->tx_ndesc = 0; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; } cd->sge_tx_cons = cons; if (cd->sge_tx_cnt == 0) sc->sge_timer = 0; } static void sge_tick(void *arg) { struct sge_softc *sc; struct mii_data *mii; struct ifnet *ifp; sc = arg; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; mii = device_get_softc(sc->sge_miibus); mii_tick(mii); if ((sc->sge_flags & SGE_FLAG_LINK) == 0) { sge_miibus_statchg(sc->sge_dev); if ((sc->sge_flags & SGE_FLAG_LINK) != 0 && !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) sge_start_locked(ifp); } /* * Reclaim transmitted frames here as we do not request * Tx completion interrupt for every queued frames to * reduce excessive interrupts. */ sge_txeof(sc); sge_watchdog(sc); callout_reset(&sc->sge_stat_ch, hz, sge_tick, sc); } static void sge_intr(void *arg) { struct sge_softc *sc; struct ifnet *ifp; uint32_t status; sc = arg; SGE_LOCK(sc); ifp = sc->sge_ifp; status = CSR_READ_4(sc, IntrStatus); if (status == 0xFFFFFFFF || (status & SGE_INTRS) == 0) { /* Not ours. */ SGE_UNLOCK(sc); return; } /* Acknowledge interrupts. */ CSR_WRITE_4(sc, IntrStatus, status); /* Disable further interrupts. */ CSR_WRITE_4(sc, IntrMask, 0); /* * It seems the controller supports some kind of interrupt * moderation mechanism but we still don't know how to * enable that. To reduce number of generated interrupts * under load we check pending interrupts in a loop. This * will increase number of register access and is not correct * way to handle interrupt moderation but there seems to be * no other way at this time. */ for (;;) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) break; if ((status & (INTR_RX_DONE | INTR_RX_IDLE)) != 0) { sge_rxeof(sc); /* Wakeup Rx MAC. */ if ((status & INTR_RX_IDLE) != 0) CSR_WRITE_4(sc, RX_CTL, 0x1a00 | 0x000c | RX_CTL_POLL | RX_CTL_ENB); } if ((status & (INTR_TX_DONE | INTR_TX_IDLE)) != 0) sge_txeof(sc); status = CSR_READ_4(sc, IntrStatus); if ((status & SGE_INTRS) == 0) break; /* Acknowledge interrupts. */ CSR_WRITE_4(sc, IntrStatus, status); } if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { /* Re-enable interrupts */ CSR_WRITE_4(sc, IntrMask, SGE_INTRS); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) sge_start_locked(ifp); } SGE_UNLOCK(sc); } /* * Encapsulate an mbuf chain in a descriptor by coupling the mbuf data * pointers to the fragment pointers. */ static int sge_encap(struct sge_softc *sc, struct mbuf **m_head) { struct mbuf *m; struct sge_desc *desc; struct sge_txdesc *txd; bus_dma_segment_t txsegs[SGE_MAXTXSEGS]; uint32_t cflags, mss; int error, i, nsegs, prod, si; SGE_LOCK_ASSERT(sc); si = prod = sc->sge_cdata.sge_tx_prod; txd = &sc->sge_cdata.sge_txdesc[prod]; if (((*m_head)->m_pkthdr.csum_flags & CSUM_TSO) != 0) { struct ether_header *eh; struct ip *ip; struct tcphdr *tcp; uint32_t ip_off, poff; if (M_WRITABLE(*m_head) == 0) { /* Get a writable copy. */ m = m_dup(*m_head, M_NOWAIT); m_freem(*m_head); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } *m_head = m; } ip_off = sizeof(struct ether_header); m = m_pullup(*m_head, ip_off); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } eh = mtod(m, struct ether_header *); /* Check the existence of VLAN tag. */ if (eh->ether_type == htons(ETHERTYPE_VLAN)) { ip_off = sizeof(struct ether_vlan_header); m = m_pullup(m, ip_off); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } } m = m_pullup(m, ip_off + sizeof(struct ip)); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } ip = (struct ip *)(mtod(m, char *) + ip_off); poff = ip_off + (ip->ip_hl << 2); m = m_pullup(m, poff + sizeof(struct tcphdr)); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } tcp = (struct tcphdr *)(mtod(m, char *) + poff); m = m_pullup(m, poff + (tcp->th_off << 2)); if (m == NULL) { *m_head = NULL; return (ENOBUFS); } /* * Reset IP checksum and recompute TCP pseudo * checksum that NDIS specification requires. */ ip = (struct ip *)(mtod(m, char *) + ip_off); ip->ip_sum = 0; tcp = (struct tcphdr *)(mtod(m, char *) + poff); tcp->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, htons(IPPROTO_TCP)); *m_head = m; } error = bus_dmamap_load_mbuf_sg(sc->sge_cdata.sge_txmbuf_tag, txd->tx_dmamap, *m_head, txsegs, &nsegs, 0); if (error == EFBIG) { m = m_collapse(*m_head, M_NOWAIT, SGE_MAXTXSEGS); if (m == NULL) { m_freem(*m_head); *m_head = NULL; return (ENOBUFS); } *m_head = m; error = bus_dmamap_load_mbuf_sg(sc->sge_cdata.sge_txmbuf_tag, txd->tx_dmamap, *m_head, txsegs, &nsegs, 0); if (error != 0) { m_freem(*m_head); *m_head = NULL; return (error); } } else if (error != 0) return (error); KASSERT(nsegs != 0, ("zero segment returned")); /* Check descriptor overrun. */ if (sc->sge_cdata.sge_tx_cnt + nsegs >= SGE_TX_RING_CNT) { bus_dmamap_unload(sc->sge_cdata.sge_txmbuf_tag, txd->tx_dmamap); return (ENOBUFS); } bus_dmamap_sync(sc->sge_cdata.sge_txmbuf_tag, txd->tx_dmamap, BUS_DMASYNC_PREWRITE); m = *m_head; cflags = 0; mss = 0; if ((m->m_pkthdr.csum_flags & CSUM_TSO) != 0) { cflags |= TDC_LS; mss = (uint32_t)m->m_pkthdr.tso_segsz; mss <<= 16; } else { if (m->m_pkthdr.csum_flags & CSUM_IP) cflags |= TDC_IP_CSUM; if (m->m_pkthdr.csum_flags & CSUM_TCP) cflags |= TDC_TCP_CSUM; if (m->m_pkthdr.csum_flags & CSUM_UDP) cflags |= TDC_UDP_CSUM; } for (i = 0; i < nsegs; i++) { desc = &sc->sge_ldata.sge_tx_ring[prod]; if (i == 0) { desc->sge_sts_size = htole32(m->m_pkthdr.len | mss); desc->sge_cmdsts = 0; } else { desc->sge_sts_size = 0; desc->sge_cmdsts = htole32(TDC_OWN); } desc->sge_ptr = htole32(SGE_ADDR_LO(txsegs[i].ds_addr)); desc->sge_flags = htole32(txsegs[i].ds_len); if (prod == SGE_TX_RING_CNT - 1) desc->sge_flags |= htole32(RING_END); sc->sge_cdata.sge_tx_cnt++; SGE_INC(prod, SGE_TX_RING_CNT); } /* Update producer index. */ sc->sge_cdata.sge_tx_prod = prod; desc = &sc->sge_ldata.sge_tx_ring[si]; /* Configure VLAN. */ if((m->m_flags & M_VLANTAG) != 0) { cflags |= m->m_pkthdr.ether_vtag; desc->sge_sts_size |= htole32(TDS_INS_VLAN); } desc->sge_cmdsts |= htole32(TDC_DEF | TDC_CRC | TDC_PAD | cflags); #if 1 if ((sc->sge_flags & SGE_FLAG_SPEED_1000) != 0) desc->sge_cmdsts |= htole32(TDC_BST); #else if ((sc->sge_flags & SGE_FLAG_FDX) == 0) { desc->sge_cmdsts |= htole32(TDC_COL | TDC_CRS | TDC_BKF); if ((sc->sge_flags & SGE_FLAG_SPEED_1000) != 0) desc->sge_cmdsts |= htole32(TDC_EXT | TDC_BST); } #endif /* Request interrupt and give ownership to controller. */ desc->sge_cmdsts |= htole32(TDC_OWN | TDC_INTR); txd->tx_m = m; txd->tx_ndesc = nsegs; return (0); } static void sge_start(struct ifnet *ifp) { struct sge_softc *sc; sc = ifp->if_softc; SGE_LOCK(sc); sge_start_locked(ifp); SGE_UNLOCK(sc); } static void sge_start_locked(struct ifnet *ifp) { struct sge_softc *sc; struct mbuf *m_head; int queued = 0; sc = ifp->if_softc; SGE_LOCK_ASSERT(sc); if ((sc->sge_flags & SGE_FLAG_LINK) == 0 || (ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; for (queued = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd); ) { if (sc->sge_cdata.sge_tx_cnt > (SGE_TX_RING_CNT - SGE_MAXTXSEGS)) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; if (sge_encap(sc, &m_head)) { if (m_head == NULL) break; IFQ_DRV_PREPEND(&ifp->if_snd, m_head); ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } queued++; /* * If there's a BPF listener, bounce a copy of this frame * to him. */ BPF_MTAP(ifp, m_head); } if (queued > 0) { bus_dmamap_sync(sc->sge_cdata.sge_tx_tag, sc->sge_cdata.sge_tx_dmamap, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); CSR_WRITE_4(sc, TX_CTL, 0x1a00 | TX_CTL_ENB | TX_CTL_POLL); sc->sge_timer = 5; } } static void sge_init(void *arg) { struct sge_softc *sc; sc = arg; SGE_LOCK(sc); sge_init_locked(sc); SGE_UNLOCK(sc); } static void sge_init_locked(struct sge_softc *sc) { struct ifnet *ifp; struct mii_data *mii; uint16_t rxfilt; int i; SGE_LOCK_ASSERT(sc); ifp = sc->sge_ifp; mii = device_get_softc(sc->sge_miibus); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) return; /* * Cancel pending I/O and free all RX/TX buffers. */ sge_stop(sc); sge_reset(sc); /* Init circular RX list. */ if (sge_list_rx_init(sc) == ENOBUFS) { device_printf(sc->sge_dev, "no memory for Rx buffers\n"); sge_stop(sc); return; } /* Init TX descriptors. */ sge_list_tx_init(sc); /* * Load the address of the RX and TX lists. */ CSR_WRITE_4(sc, TX_DESC, SGE_ADDR_LO(sc->sge_ldata.sge_tx_paddr)); CSR_WRITE_4(sc, RX_DESC, SGE_ADDR_LO(sc->sge_ldata.sge_rx_paddr)); CSR_WRITE_4(sc, TxMacControl, 0x60); CSR_WRITE_4(sc, RxWakeOnLan, 0); CSR_WRITE_4(sc, RxWakeOnLanData, 0); /* Allow receiving VLAN frames. */ CSR_WRITE_2(sc, RxMPSControl, ETHER_MAX_LEN + ETHER_VLAN_ENCAP_LEN + SGE_RX_PAD_BYTES); for (i = 0; i < ETHER_ADDR_LEN; i++) CSR_WRITE_1(sc, RxMacAddr + i, IF_LLADDR(ifp)[i]); /* Configure RX MAC. */ rxfilt = RXMAC_STRIP_FCS | RXMAC_PAD_ENB | RXMAC_CSUM_ENB; CSR_WRITE_2(sc, RxMacControl, rxfilt); sge_rxfilter(sc); sge_setvlan(sc); /* Initialize default speed/duplex information. */ if ((sc->sge_flags & SGE_FLAG_FASTETHER) == 0) sc->sge_flags |= SGE_FLAG_SPEED_1000; sc->sge_flags |= SGE_FLAG_FDX; if ((sc->sge_flags & SGE_FLAG_RGMII) != 0) CSR_WRITE_4(sc, StationControl, 0x04008001); else CSR_WRITE_4(sc, StationControl, 0x04000001); /* * XXX Try to mitigate interrupts. */ CSR_WRITE_4(sc, IntrControl, 0x08880000); #ifdef notyet if (sc->sge_intrcontrol != 0) CSR_WRITE_4(sc, IntrControl, sc->sge_intrcontrol); if (sc->sge_intrtimer != 0) CSR_WRITE_4(sc, IntrTimer, sc->sge_intrtimer); #endif /* * Clear and enable interrupts. */ CSR_WRITE_4(sc, IntrStatus, 0xFFFFFFFF); CSR_WRITE_4(sc, IntrMask, SGE_INTRS); /* Enable receiver and transmitter. */ CSR_WRITE_4(sc, TX_CTL, 0x1a00 | TX_CTL_ENB); CSR_WRITE_4(sc, RX_CTL, 0x1a00 | 0x000c | RX_CTL_POLL | RX_CTL_ENB); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sc->sge_flags &= ~SGE_FLAG_LINK; mii_mediachg(mii); callout_reset(&sc->sge_stat_ch, hz, sge_tick, sc); } /* * Set media options. */ static int sge_ifmedia_upd(struct ifnet *ifp) { struct sge_softc *sc; struct mii_data *mii; struct mii_softc *miisc; int error; sc = ifp->if_softc; SGE_LOCK(sc); mii = device_get_softc(sc->sge_miibus); LIST_FOREACH(miisc, &mii->mii_phys, mii_list) PHY_RESET(miisc); error = mii_mediachg(mii); SGE_UNLOCK(sc); return (error); } /* * Report current media status. */ static void sge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct sge_softc *sc; struct mii_data *mii; sc = ifp->if_softc; SGE_LOCK(sc); mii = device_get_softc(sc->sge_miibus); if ((ifp->if_flags & IFF_UP) == 0) { SGE_UNLOCK(sc); return; } mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; SGE_UNLOCK(sc); } static int sge_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { struct sge_softc *sc; struct ifreq *ifr; struct mii_data *mii; int error = 0, mask, reinit; sc = ifp->if_softc; ifr = (struct ifreq *)data; switch(command) { case SIOCSIFFLAGS: SGE_LOCK(sc); if ((ifp->if_flags & IFF_UP) != 0) { if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0 && ((ifp->if_flags ^ sc->sge_if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) != 0) sge_rxfilter(sc); else sge_init_locked(sc); } else if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) sge_stop(sc); sc->sge_if_flags = ifp->if_flags; SGE_UNLOCK(sc); break; case SIOCSIFCAP: SGE_LOCK(sc); reinit = 0; mask = ifr->ifr_reqcap ^ ifp->if_capenable; if ((mask & IFCAP_TXCSUM) != 0 && (ifp->if_capabilities & IFCAP_TXCSUM) != 0) { ifp->if_capenable ^= IFCAP_TXCSUM; if ((ifp->if_capenable & IFCAP_TXCSUM) != 0) ifp->if_hwassist |= SGE_CSUM_FEATURES; else ifp->if_hwassist &= ~SGE_CSUM_FEATURES; } if ((mask & IFCAP_RXCSUM) != 0 && (ifp->if_capabilities & IFCAP_RXCSUM) != 0) ifp->if_capenable ^= IFCAP_RXCSUM; if ((mask & IFCAP_VLAN_HWCSUM) != 0 && (ifp->if_capabilities & IFCAP_VLAN_HWCSUM) != 0) ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; if ((mask & IFCAP_TSO4) != 0 && (ifp->if_capabilities & IFCAP_TSO4) != 0) { ifp->if_capenable ^= IFCAP_TSO4; if ((ifp->if_capenable & IFCAP_TSO4) != 0) ifp->if_hwassist |= CSUM_TSO; else ifp->if_hwassist &= ~CSUM_TSO; } if ((mask & IFCAP_VLAN_HWTSO) != 0 && (ifp->if_capabilities & IFCAP_VLAN_HWTSO) != 0) ifp->if_capenable ^= IFCAP_VLAN_HWTSO; if ((mask & IFCAP_VLAN_HWTAGGING) != 0 && (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING) != 0) { /* * Due to unknown reason, toggling VLAN hardware * tagging require interface reinitialization. */ ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) == 0) ifp->if_capenable &= ~(IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM); reinit = 1; } if (reinit > 0 && (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; sge_init_locked(sc); } SGE_UNLOCK(sc); VLAN_CAPABILITIES(ifp); break; case SIOCADDMULTI: case SIOCDELMULTI: SGE_LOCK(sc); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) sge_rxfilter(sc); SGE_UNLOCK(sc); break; case SIOCGIFMEDIA: case SIOCSIFMEDIA: mii = device_get_softc(sc->sge_miibus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command); break; default: error = ether_ioctl(ifp, command, data); break; } return (error); } static void sge_watchdog(struct sge_softc *sc) { struct ifnet *ifp; SGE_LOCK_ASSERT(sc); if (sc->sge_timer == 0 || --sc->sge_timer > 0) return; ifp = sc->sge_ifp; if ((sc->sge_flags & SGE_FLAG_LINK) == 0) { if (1 || bootverbose) device_printf(sc->sge_dev, "watchdog timeout (lost link)\n"); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; sge_init_locked(sc); return; } device_printf(sc->sge_dev, "watchdog timeout\n"); if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; sge_init_locked(sc); if (!IFQ_DRV_IS_EMPTY(&sc->sge_ifp->if_snd)) sge_start_locked(ifp); } /* * Stop the adapter and free any mbufs allocated to the * RX and TX lists. */ static void sge_stop(struct sge_softc *sc) { struct ifnet *ifp; ifp = sc->sge_ifp; SGE_LOCK_ASSERT(sc); sc->sge_timer = 0; callout_stop(&sc->sge_stat_ch); ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); CSR_WRITE_4(sc, IntrMask, 0); CSR_READ_4(sc, IntrMask); CSR_WRITE_4(sc, IntrStatus, 0xffffffff); /* Stop TX/RX MAC. */ CSR_WRITE_4(sc, TX_CTL, 0x1a00); CSR_WRITE_4(sc, RX_CTL, 0x1a00); /* XXX Can we assume active DMA cycles gone? */ DELAY(2000); CSR_WRITE_4(sc, IntrMask, 0); CSR_WRITE_4(sc, IntrStatus, 0xffffffff); sc->sge_flags &= ~SGE_FLAG_LINK; sge_list_rx_free(sc); sge_list_tx_free(sc); } Index: head/sys/dev/sound/pci/emu10kx.c =================================================================== --- head/sys/dev/sound/pci/emu10kx.c (revision 298410) +++ head/sys/dev/sound/pci/emu10kx.c (revision 298411) @@ -1,3570 +1,3567 @@ /*- * Copyright (c) 1999 Cameron Grant * Copyright (c) 2003-2007 Yuriy Tsibizov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHERIN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for DELAY */ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_snd.h" #endif #include #include #include #include #include /* hw flags */ #define HAS_51 0x0001 #define HAS_71 0x0002 #define HAS_AC97 0x0004 #define IS_EMU10K1 0x0008 #define IS_EMU10K2 0x0010 #define IS_CA0102 0x0020 #define IS_CA0108 0x0040 #define IS_UNKNOWN 0x0080 #define BROKEN_DIGITAL 0x0100 #define DIGITAL_ONLY 0x0200 #define IS_CARDBUS 0x0400 #define MODE_ANALOG 1 #define MODE_DIGITAL 2 #define SPDIF_MODE_PCM 1 #define SPDIF_MODE_AC3 2 #define MACS 0x0 #define MACS1 0x1 #define MACW 0x2 #define MACW1 0x3 #define MACINTS 0x4 #define MACINTW 0x5 #define ACC3 0x6 #define MACMV 0x7 #define ANDXOR 0x8 #define TSTNEG 0x9 #define LIMIT 0xA #define LIMIT1 0xB #define LOG 0xC #define EXP 0xD #define INTERP 0xE #define SKIP 0xF #define GPR(i) (sc->gpr_base+(i)) #define INP(i) (sc->input_base+(i)) #define OUTP(i) (sc->output_base+(i)) #define FX(i) (i) #define FX2(i) (sc->efxc_base+(i)) #define DSP_CONST(i) (sc->dsp_zero+(i)) #define COND_NORMALIZED DSP_CONST(0x1) #define COND_BORROW DSP_CONST(0x2) #define COND_MINUS DSP_CONST(0x3) #define COND_LESS_ZERO DSP_CONST(0x4) #define COND_EQ_ZERO DSP_CONST(0x5) #define COND_SATURATION DSP_CONST(0x6) #define COND_NEQ_ZERO DSP_CONST(0x8) #define DSP_ACCUM DSP_CONST(0x16) #define DSP_CCR DSP_CONST(0x17) /* Live! Inputs */ #define IN_AC97_L 0x00 #define IN_AC97_R 0x01 #define IN_AC97 IN_AC97_L #define IN_SPDIF_CD_L 0x02 #define IN_SPDIF_CD_R 0x03 #define IN_SPDIF_CD IN_SPDIF_CD_L #define IN_ZOOM_L 0x04 #define IN_ZOOM_R 0x05 #define IN_ZOOM IN_ZOOM_L #define IN_TOSLINK_L 0x06 #define IN_TOSLINK_R 0x07 #define IN_TOSLINK IN_TOSLINK_L #define IN_LINE1_L 0x08 #define IN_LINE1_R 0x09 #define IN_LINE1 IN_LINE1_L #define IN_COAX_SPDIF_L 0x0a #define IN_COAX_SPDIF_R 0x0b #define IN_COAX_SPDIF IN_COAX_SPDIF_L #define IN_LINE2_L 0x0c #define IN_LINE2_R 0x0d #define IN_LINE2 IN_LINE2_L #define IN_0E 0x0e #define IN_0F 0x0f /* Outputs */ #define OUT_AC97_L 0x00 #define OUT_AC97_R 0x01 #define OUT_AC97 OUT_AC97_L #define OUT_A_FRONT OUT_AC97 #define OUT_TOSLINK_L 0x02 #define OUT_TOSLINK_R 0x03 #define OUT_TOSLINK OUT_TOSLINK_L #define OUT_D_CENTER 0x04 #define OUT_D_SUB 0x05 #define OUT_HEADPHONE_L 0x06 #define OUT_HEADPHONE_R 0x07 #define OUT_HEADPHONE OUT_HEADPHONE_L #define OUT_REAR_L 0x08 #define OUT_REAR_R 0x09 #define OUT_REAR OUT_REAR_L #define OUT_ADC_REC_L 0x0a #define OUT_ADC_REC_R 0x0b #define OUT_ADC_REC OUT_ADC_REC_L #define OUT_MIC_CAP 0x0c /* Live! 5.1 Digital, non-standard 5.1 (center & sub) outputs */ #define OUT_A_CENTER 0x11 #define OUT_A_SUB 0x12 /* Audigy Inputs */ #define A_IN_AC97_L 0x00 #define A_IN_AC97_R 0x01 #define A_IN_AC97 A_IN_AC97_L #define A_IN_SPDIF_CD_L 0x02 #define A_IN_SPDIF_CD_R 0x03 #define A_IN_SPDIF_CD A_IN_SPDIF_CD_L #define A_IN_O_SPDIF_L 0x04 #define A_IN_O_SPDIF_R 0x05 #define A_IN_O_SPDIF A_IN_O_SPDIF_L #define A_IN_LINE2_L 0x08 #define A_IN_LINE2_R 0x09 #define A_IN_LINE2 A_IN_LINE2_L #define A_IN_R_SPDIF_L 0x0a #define A_IN_R_SPDIF_R 0x0b #define A_IN_R_SPDIF A_IN_R_SPDIF_L #define A_IN_AUX2_L 0x0c #define A_IN_AUX2_R 0x0d #define A_IN_AUX2 A_IN_AUX2_L /* Audigy Outputs */ #define A_OUT_D_FRONT_L 0x00 #define A_OUT_D_FRONT_R 0x01 #define A_OUT_D_FRONT A_OUT_D_FRONT_L #define A_OUT_D_CENTER 0x02 #define A_OUT_D_SUB 0x03 #define A_OUT_D_SIDE_L 0x04 #define A_OUT_D_SIDE_R 0x05 #define A_OUT_D_SIDE A_OUT_D_SIDE_L #define A_OUT_D_REAR_L 0x06 #define A_OUT_D_REAR_R 0x07 #define A_OUT_D_REAR A_OUT_D_REAR_L /* on Audigy Platinum only */ #define A_OUT_HPHONE_L 0x04 #define A_OUT_HPHONE_R 0x05 #define A_OUT_HPHONE A_OUT_HPHONE_L #define A_OUT_A_FRONT_L 0x08 #define A_OUT_A_FRONT_R 0x09 #define A_OUT_A_FRONT A_OUT_A_FRONT_L #define A_OUT_A_CENTER 0x0a #define A_OUT_A_SUB 0x0b #define A_OUT_A_SIDE_L 0x0c #define A_OUT_A_SIDE_R 0x0d #define A_OUT_A_SIDE A_OUT_A_SIDE_L #define A_OUT_A_REAR_L 0x0e #define A_OUT_A_REAR_R 0x0f #define A_OUT_A_REAR A_OUT_A_REAR_L #define A_OUT_AC97_L 0x10 #define A_OUT_AC97_R 0x11 #define A_OUT_AC97 A_OUT_AC97_L #define A_OUT_ADC_REC_L 0x16 #define A_OUT_ADC_REC_R 0x17 #define A_OUT_ADC_REC A_OUT_ADC_REC_L #define EMU_DATA2 0x24 #define EMU_IPR2 0x28 #define EMU_INTE2 0x2c #define EMU_IPR3 0x38 #define EMU_INTE3 0x3c #define EMU_A2_SRCSel 0x60 #define EMU_A2_SRCMULTI_ENABLE 0x6e #define EMU_A_I2S_CAPTURE_96000 0x00000400 #define EMU_A2_MIXER_I2S_ENABLE 0x7B #define EMU_A2_MIXER_SPDIF_ENABLE 0x7A #define C_FRONT_L 0 #define C_FRONT_R 1 #define C_REC_L 2 #define C_REC_R 3 #define C_REAR_L 4 #define C_REAR_R 5 #define C_CENTER 6 #define C_SUB 7 #define C_SIDE_L 8 #define C_SIDE_R 9 #define NUM_CACHES 10 #define CDSPDIFMUTE 0 #define ANALOGMUTE 1 #define NUM_MUTE 2 #define EMU_MAX_GPR 512 #define EMU_MAX_IRQ_CONSUMERS 32 struct emu_voice { int vnum; unsigned int b16:1, stereo:1, busy:1, running:1, ismaster:1; int speed; int start; int end; int vol; uint32_t buf; void *vbuf; struct emu_voice *slave; uint32_t sa; uint32_t ea; uint32_t routing[8]; uint32_t amounts[8]; }; struct emu_memblk { SLIST_ENTRY(emu_memblk) link; void *buf; char owner[16]; bus_addr_t buf_addr; uint32_t pte_start, pte_size; bus_dmamap_t buf_map; }; struct emu_mem { uint8_t bmap[EMU_MAXPAGES / 8]; uint32_t *ptb_pages; void *silent_page; bus_addr_t ptb_pages_addr; bus_addr_t silent_page_addr; bus_dmamap_t ptb_map; bus_dmamap_t silent_map; bus_dma_tag_t dmat; struct emu_sc_info *card; SLIST_HEAD(, emu_memblk) blocks; }; /* rm */ struct emu_rm { struct emu_sc_info *card; struct mtx gpr_lock; signed int allocmap[EMU_MAX_GPR]; int num_gprs; int last_free_gpr; int num_used; }; struct emu_intr_handler { void* softc; uint32_t intr_mask; uint32_t inte_mask; uint32_t(*irq_func) (void *softc, uint32_t irq); }; struct emu_sc_info { struct mtx lock; struct mtx rw; /* Hardware exclusive access lock */ /* Hardware and subdevices */ device_t dev; device_t pcm[RT_COUNT]; device_t midi[2]; uint32_t type; uint32_t rev; bus_space_tag_t st; bus_space_handle_t sh; struct cdev *cdev; /* /dev/emu10k character device */ struct mtx emu10kx_lock; int emu10kx_isopen; struct sbuf emu10kx_sbuf; int emu10kx_bufptr; /* Resources */ struct resource *reg; struct resource *irq; void *ih; /* IRQ handlers */ struct emu_intr_handler ihandler[EMU_MAX_IRQ_CONSUMERS]; /* Card HW configuration */ unsigned int mode; /* analog / digital */ unsigned int mchannel_fx; unsigned int dsp_zero; unsigned int code_base; unsigned int code_size; unsigned int gpr_base; unsigned int num_gprs; unsigned int input_base; unsigned int output_base; unsigned int efxc_base; unsigned int opcode_shift; unsigned int high_operand_shift; unsigned int address_mask; uint32_t is_emu10k1:1, is_emu10k2, is_ca0102, is_ca0108:1, has_ac97:1, has_51:1, has_71:1, enable_ir:1, broken_digital:1, is_cardbus:1; signed int mch_disabled, mch_rec, dbg_level; signed int num_inputs; unsigned int num_outputs; unsigned int num_fxbuses; unsigned int routing_code_start; unsigned int routing_code_end; /* HW resources */ struct emu_voice voice[NUM_G]; /* Hardware voices */ uint32_t irq_mask[EMU_MAX_IRQ_CONSUMERS]; /* IRQ manager data */ int timer[EMU_MAX_IRQ_CONSUMERS]; /* timer */ int timerinterval; struct emu_rm *rm; struct emu_mem mem; /* memory */ /* Mixer */ int mixer_gpr[NUM_MIXERS]; int mixer_volcache[NUM_MIXERS]; int cache_gpr[NUM_CACHES]; int dummy_gpr; int mute_gpr[NUM_MUTE]; struct sysctl_ctx_list *ctx; struct sysctl_oid *root; }; static void emu_setmap(void *arg, bus_dma_segment_t * segs, int nseg, int error); static void* emu_malloc(struct emu_mem *mem, uint32_t sz, bus_addr_t * addr, bus_dmamap_t *map); static void emu_free(struct emu_mem *mem, void *dmabuf, bus_dmamap_t map); static void* emu_memalloc(struct emu_mem *mem, uint32_t sz, bus_addr_t * addr, const char * owner); static int emu_memfree(struct emu_mem *mem, void *membuf); static int emu_memstart(struct emu_mem *mem, void *membuf); /* /dev */ static int emu10kx_dev_init(struct emu_sc_info *sc); static int emu10kx_dev_uninit(struct emu_sc_info *sc); static int emu10kx_prepare(struct emu_sc_info *sc, struct sbuf *s); static void emumix_set_mode(struct emu_sc_info *sc, int mode); static void emumix_set_spdif_mode(struct emu_sc_info *sc, int mode); static void emumix_set_fxvol(struct emu_sc_info *sc, unsigned gpr, int32_t vol); static void emumix_set_gpr(struct emu_sc_info *sc, unsigned gpr, int32_t val); static int sysctl_emu_mixer_control(SYSCTL_HANDLER_ARGS); static int emu_rm_init(struct emu_sc_info *sc); static int emu_rm_uninit(struct emu_sc_info *sc); static int emu_rm_gpr_alloc(struct emu_rm *rm, int count); static unsigned int emu_getcard(device_t dev); static uint32_t emu_rd_nolock(struct emu_sc_info *sc, unsigned int regno, unsigned int size); static void emu_wr_nolock(struct emu_sc_info *sc, unsigned int regno, uint32_t data, unsigned int size); static void emu_wr_cbptr(struct emu_sc_info *sc, uint32_t data); static void emu_vstop(struct emu_sc_info *sc, char channel, int enable); static void emu_intr(void *p); static void emu_wrefx(struct emu_sc_info *sc, unsigned int pc, unsigned int data); static void emu_addefxop(struct emu_sc_info *sc, unsigned int op, unsigned int z, unsigned int w, unsigned int x, unsigned int y, uint32_t * pc); static void emu_initefx(struct emu_sc_info *sc); static int emu_cardbus_init(struct emu_sc_info *sc); static int emu_init(struct emu_sc_info *sc); static int emu_uninit(struct emu_sc_info *sc); static int emu_read_ivar(device_t bus __unused, device_t dev, int ivar_index, uintptr_t * result); static int emu_write_ivar(device_t bus __unused, device_t dev __unused, int ivar_index, uintptr_t value __unused); static int emu_pci_probe(device_t dev); static int emu_pci_attach(device_t dev); static int emu_pci_detach(device_t dev); static int emu_modevent(module_t mod __unused, int cmd, void *data __unused); #ifdef SND_EMU10KX_DEBUG #define EMU_MTX_DEBUG() do { \ if (mtx_owned(&sc->rw)) { \ printf("RW owned in %s line %d for %s\n", __func__, \ __LINE__ , device_get_nameunit(sc->dev)); \ printf("rw lock owned: %d\n", mtx_owned(&sc->rw)); \ printf("rw lock: value %x thread %x\n", \ ((&sc->rw)->mtx_lock & ~MTX_FLAGMASK), \ (uintptr_t)curthread); \ printf("rw lock: recursed %d\n", mtx_recursed(&sc->rw));\ db_show_mtx(&sc->rw); \ } \ } while (0) #else #define EMU_MTX_DEBUG() do { \ } while (0) #endif #define EMU_RWLOCK() do { \ EMU_MTX_DEBUG(); \ mtx_lock(&(sc->rw)); \ } while (0) #define EMU_RWUNLOCK() do { \ mtx_unlock(&(sc->rw)); \ EMU_MTX_DEBUG(); \ } while (0) /* Supported cards */ struct emu_hwinfo { uint16_t vendor; uint16_t device; uint16_t subvendor; uint16_t subdevice; char SBcode[8]; char desc[32]; int flags; }; static struct emu_hwinfo emu_cards[] = { {0xffff, 0xffff, 0xffff, 0xffff, "BADCRD", "Not a compatible card", 0}, /* 0x0020..0x002f 4.0 EMU10K1 cards */ {0x1102, 0x0002, 0x1102, 0x0020, "CT4850", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x0021, "CT4620", "SBLive!", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x002f, "CT????", "SBLive! mainboard implementation", HAS_AC97 | IS_EMU10K1}, /* (range unknown) 5.1 EMU10K1 cards */ {0x1102, 0x0002, 0x1102, 0x100a, "CT????", "SBLive! 5.1", HAS_AC97 | HAS_51 | IS_EMU10K1}, /* 0x80??..0x805? 4.0 EMU10K1 cards */ {0x1102, 0x0002, 0x1102, 0x8022, "CT4780", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8023, "CT4790", "SB PCI512", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8024, "CT4760", "SBLive!", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8025, "CT????", "SBLive! Mainboard Implementation", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8026, "CT4830", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8027, "CT4832", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8028, "CT4760", "SBLive! OEM version", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8031, "CT4831", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8040, "CT4760", "SBLive!", HAS_AC97 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8051, "CT4850", "SBLive! Value", HAS_AC97 | IS_EMU10K1}, /* 0x8061..0x???? 5.1 EMU10K1 cards */ {0x1102, 0x0002, 0x1102, 0x8061, "SB????", "SBLive! Player 5.1", HAS_AC97 | HAS_51 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8062, "CT4830", "SBLive! 1024", HAS_AC97 | HAS_51 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8064, "SB????", "SBLive! 5.1", HAS_AC97 | HAS_51 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8065, "SB0220", "SBLive! 5.1 Digital", HAS_AC97 | HAS_51 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8066, "CT4780", "SBLive! 5.1 Digital", HAS_AC97 | HAS_51 | IS_EMU10K1}, {0x1102, 0x0002, 0x1102, 0x8067, "SB????", "SBLive!", HAS_AC97 | HAS_51 | IS_EMU10K1}, /* Generic SB Live! */ {0x1102, 0x0002, 0x1102, 0x0000, "SB????", "SBLive! (Unknown model)", HAS_AC97 | IS_EMU10K1}, /* 0x0041..0x0043 EMU10K2 (some kind of Audigy) cards */ /* 0x0051..0x0051 5.1 CA0100-IAF cards */ {0x1102, 0x0004, 0x1102, 0x0051, "SB0090", "Audigy", HAS_AC97 | HAS_51 | IS_EMU10K2}, /* ES is CA0100-IDF chip that don't work in digital mode */ {0x1102, 0x0004, 0x1102, 0x0052, "SB0160", "Audigy ES", HAS_AC97 | HAS_71 | IS_EMU10K2 | BROKEN_DIGITAL}, /* 0x0053..0x005C 5.1 CA0101-NAF cards */ {0x1102, 0x0004, 0x1102, 0x0053, "SB0090", "Audigy Player/OEM", HAS_AC97 | HAS_51 | IS_EMU10K2}, {0x1102, 0x0004, 0x1102, 0x0058, "SB0090", "Audigy Player/OEM", HAS_AC97 | HAS_51 | IS_EMU10K2}, /* 0x1002..0x1009 5.1 CA0102-IAT cards */ {0x1102, 0x0004, 0x1102, 0x1002, "SB????", "Audigy 2 Platinum", HAS_51 | IS_CA0102}, {0x1102, 0x0004, 0x1102, 0x1005, "SB????", "Audigy 2 Platinum EX", HAS_51 | IS_CA0102}, {0x1102, 0x0004, 0x1102, 0x1007, "SB0240", "Audigy 2", HAS_AC97 | HAS_51 | IS_CA0102}, /* 0x2001..0x2003 7.1 CA0102-ICT cards */ {0x1102, 0x0004, 0x1102, 0x2001, "SB0350", "Audigy 2 ZS", HAS_AC97 | HAS_71 | IS_CA0102}, {0x1102, 0x0004, 0x1102, 0x2002, "SB0350", "Audigy 2 ZS", HAS_AC97 | HAS_71 | IS_CA0102}, /* XXX No reports about 0x2003 & 0x2004 cards */ {0x1102, 0x0004, 0x1102, 0x2003, "SB0350", "Audigy 2 ZS", HAS_AC97 | HAS_71 | IS_CA0102}, {0x1102, 0x0004, 0x1102, 0x2004, "SB0350", "Audigy 2 ZS", HAS_AC97 | HAS_71 | IS_CA0102}, {0x1102, 0x0004, 0x1102, 0x2005, "SB0350", "Audigy 2 ZS", HAS_AC97 | HAS_71 | IS_CA0102}, /* (range unknown) 7.1 CA0102-xxx Audigy 4 cards */ {0x1102, 0x0004, 0x1102, 0x2007, "SB0380", "Audigy 4 Pro", HAS_AC97 | HAS_71 | IS_CA0102}, /* Generic Audigy or Audigy 2 */ {0x1102, 0x0004, 0x1102, 0x0000, "SB????", "Audigy (Unknown model)", HAS_AC97 | HAS_51 | IS_EMU10K2}, /* We don't support CA0103-DAT (Audigy LS) cards */ /* There is NO CA0104-xxx cards */ /* There is NO CA0105-xxx cards */ /* We don't support CA0106-DAT (SB Live! 24 bit) cards */ /* There is NO CA0107-xxx cards */ /* 0x1000..0x1001 7.1 CA0108-IAT cards */ {0x1102, 0x0008, 0x1102, 0x1000, "SB????", "Audigy 2 LS", HAS_AC97 | HAS_51 | IS_CA0108 | DIGITAL_ONLY}, {0x1102, 0x0008, 0x1102, 0x1001, "SB0400", "Audigy 2 Value", HAS_AC97 | HAS_71 | IS_CA0108 | DIGITAL_ONLY}, {0x1102, 0x0008, 0x1102, 0x1021, "SB0610", "Audigy 4", HAS_AC97 | HAS_71 | IS_CA0108 | DIGITAL_ONLY}, {0x1102, 0x0008, 0x1102, 0x2001, "SB0530", "Audigy 2 ZS CardBus", HAS_AC97 | HAS_71 | IS_CA0108 | IS_CARDBUS}, {0x1102, 0x0008, 0x0000, 0x0000, "SB????", "Audigy 2 Value (Unknown model)", HAS_AC97 | HAS_51 | IS_CA0108}, }; /* Unsupported cards */ static struct emu_hwinfo emu_bad_cards[] = { /* APS cards should be possible to support */ {0x1102, 0x0002, 0x1102, 0x4001, "EMUAPS", "E-mu APS", 0}, {0x1102, 0x0002, 0x1102, 0x4002, "EMUAPS", "E-mu APS", 0}, {0x1102, 0x0004, 0x1102, 0x4001, "EMU???", "E-mu 1212m [4001]", 0}, /* Similar-named ("Live!" or "Audigy") cards on different chipsets */ {0x1102, 0x8064, 0x0000, 0x0000, "SB0100", "SBLive! 5.1 OEM", 0}, {0x1102, 0x0006, 0x0000, 0x0000, "SB0200", "DELL OEM SBLive! Value", 0}, {0x1102, 0x0007, 0x0000, 0x0000, "SB0310", "Audigy LS", 0}, }; /* * Get best known information about device. */ static unsigned int emu_getcard(device_t dev) { uint16_t device; uint16_t subdevice; - int n_cards; unsigned int thiscard; int i; device = pci_read_config(dev, PCIR_DEVICE, /* bytes */ 2); subdevice = pci_read_config(dev, PCIR_SUBDEV_0, /* bytes */ 2); - n_cards = sizeof(emu_cards) / sizeof(struct emu_hwinfo); thiscard = 0; - for (i = 1; i < n_cards; i++) { + for (i = 1; i < nitems(emu_cards); i++) { if (device == emu_cards[i].device) { if (subdevice == emu_cards[i].subdevice) { thiscard = i; break; } if (0x0000 == emu_cards[i].subdevice) { thiscard = i; /* * don't break, we can get more specific card * later in the list. */ } } } - n_cards = sizeof(emu_bad_cards) / sizeof(struct emu_hwinfo); - for (i = 0; i < n_cards; i++) { + for (i = 0; i < nitems(emu_cards); i++) { if (device == emu_bad_cards[i].device) { if (subdevice == emu_bad_cards[i].subdevice) { thiscard = 0; break; } if (0x0000 == emu_bad_cards[i].subdevice) { thiscard = 0; break; /* we avoid all this cards */ } } } return (thiscard); } /* * Base hardware interface are 32 (Audigy) or 64 (Audigy2) registers. * Some of them are used directly, some of them provide pointer / data pairs. */ static uint32_t emu_rd_nolock(struct emu_sc_info *sc, unsigned int regno, unsigned int size) { KASSERT(sc != NULL, ("emu_rd: NULL sc")); switch (size) { case 1: return (bus_space_read_1(sc->st, sc->sh, regno)); case 2: return (bus_space_read_2(sc->st, sc->sh, regno)); case 4: return (bus_space_read_4(sc->st, sc->sh, regno)); } return (0xffffffff); } static void emu_wr_nolock(struct emu_sc_info *sc, unsigned int regno, uint32_t data, unsigned int size) { KASSERT(sc != NULL, ("emu_rd: NULL sc")); switch (size) { case 1: bus_space_write_1(sc->st, sc->sh, regno, data); break; case 2: bus_space_write_2(sc->st, sc->sh, regno, data); break; case 4: bus_space_write_4(sc->st, sc->sh, regno, data); break; } } /* * EMU_PTR / EMU_DATA interface. Access to EMU10Kx is made * via (channel, register) pair. Some registers are channel-specific, * some not. */ uint32_t emu_rdptr(struct emu_sc_info *sc, unsigned int chn, unsigned int reg) { uint32_t ptr, val, mask, size, offset; ptr = ((reg << 16) & sc->address_mask) | (chn & EMU_PTR_CHNO_MASK); EMU_RWLOCK(); emu_wr_nolock(sc, EMU_PTR, ptr, 4); val = emu_rd_nolock(sc, EMU_DATA, 4); EMU_RWUNLOCK(); /* * XXX Some register numbers has data size and offset encoded in * it to get only part of 32bit register. This use is not described * in register name, be careful! */ if (reg & 0xff000000) { size = (reg >> 24) & 0x3f; offset = (reg >> 16) & 0x1f; mask = ((1 << size) - 1) << offset; val &= mask; val >>= offset; } return (val); } void emu_wrptr(struct emu_sc_info *sc, unsigned int chn, unsigned int reg, uint32_t data) { uint32_t ptr, mask, size, offset; ptr = ((reg << 16) & sc->address_mask) | (chn & EMU_PTR_CHNO_MASK); EMU_RWLOCK(); emu_wr_nolock(sc, EMU_PTR, ptr, 4); /* * XXX Another kind of magic encoding in register number. This can * give you side effect - it will read previous data from register * and change only required bits. */ if (reg & 0xff000000) { size = (reg >> 24) & 0x3f; offset = (reg >> 16) & 0x1f; mask = ((1 << size) - 1) << offset; data <<= offset; data &= mask; data |= emu_rd_nolock(sc, EMU_DATA, 4) & ~mask; } emu_wr_nolock(sc, EMU_DATA, data, 4); EMU_RWUNLOCK(); } /* * EMU_A2_PTR / EMU_DATA2 interface. Access to P16v is made * via (channel, register) pair. Some registers are channel-specific, * some not. This interface is supported by CA0102 and CA0108 chips only. */ uint32_t emu_rd_p16vptr(struct emu_sc_info *sc, uint16_t chn, uint16_t reg) { uint32_t val; /* XXX separate lock? */ EMU_RWLOCK(); emu_wr_nolock(sc, EMU_A2_PTR, (reg << 16) | chn, 4); val = emu_rd_nolock(sc, EMU_DATA2, 4); EMU_RWUNLOCK(); return (val); } void emu_wr_p16vptr(struct emu_sc_info *sc, uint16_t chn, uint16_t reg, uint32_t data) { EMU_RWLOCK(); emu_wr_nolock(sc, EMU_A2_PTR, (reg << 16) | chn, 4); emu_wr_nolock(sc, EMU_DATA2, data, 4); EMU_RWUNLOCK(); } /* * XXX CardBus interface. Not tested on any real hardware. */ static void emu_wr_cbptr(struct emu_sc_info *sc, uint32_t data) { uint32_t val; /* * 0x38 is IPE3 (CD S/PDIF interrupt pending register) on CA0102. Seems * to be some reg/value accessible kind of config register on CardBus * CA0108, with value(?) in top 16 bit, address(?) in low 16 */ val = emu_rd_nolock(sc, 0x38, 4); emu_wr_nolock(sc, 0x38, data, 4); val = emu_rd_nolock(sc, 0x38, 4); } /* * Direct hardware register access * Assume that it is never used to access EMU_PTR-based registers and can run unlocked. */ void emu_wr(struct emu_sc_info *sc, unsigned int regno, uint32_t data, unsigned int size) { KASSERT(regno != EMU_PTR, ("emu_wr: attempt to write to EMU_PTR")); KASSERT(regno != EMU_A2_PTR, ("emu_wr: attempt to write to EMU_A2_PTR")); emu_wr_nolock(sc, regno, data, size); } uint32_t emu_rd(struct emu_sc_info *sc, unsigned int regno, unsigned int size) { uint32_t rd; KASSERT(regno != EMU_DATA, ("emu_rd: attempt to read DATA")); KASSERT(regno != EMU_DATA2, ("emu_rd: attempt to read DATA2")); rd = emu_rd_nolock(sc, regno, size); return (rd); } /* * Enabling IR MIDI messages is another kind of black magic. It just * has to be made this way. It really do it. */ void emu_enable_ir(struct emu_sc_info *sc) { uint32_t iocfg; if (sc->is_emu10k2 || sc->is_ca0102) { iocfg = emu_rd_nolock(sc, EMU_A_IOCFG, 2); emu_wr_nolock(sc, EMU_A_IOCFG, iocfg | EMU_A_IOCFG_GPOUT2, 2); DELAY(500); emu_wr_nolock(sc, EMU_A_IOCFG, iocfg | EMU_A_IOCFG_GPOUT1 | EMU_A_IOCFG_GPOUT2, 2); DELAY(500); emu_wr_nolock(sc, EMU_A_IOCFG, iocfg | EMU_A_IOCFG_GPOUT1, 2); DELAY(100); emu_wr_nolock(sc, EMU_A_IOCFG, iocfg, 2); device_printf(sc->dev, "Audigy IR MIDI events enabled.\n"); sc->enable_ir = 1; } if (sc->is_emu10k1) { iocfg = emu_rd_nolock(sc, EMU_HCFG, 4); emu_wr_nolock(sc, EMU_HCFG, iocfg | EMU_HCFG_GPOUT2, 4); DELAY(500); emu_wr_nolock(sc, EMU_HCFG, iocfg | EMU_HCFG_GPOUT1 | EMU_HCFG_GPOUT2, 4); DELAY(100); emu_wr_nolock(sc, EMU_HCFG, iocfg, 4); device_printf(sc->dev, "SB Live! IR MIDI events enabled.\n"); sc->enable_ir = 1; } } /* * emu_timer_ - HW timer management */ int emu_timer_create(struct emu_sc_info *sc) { int i, timer; timer = -1; mtx_lock(&sc->lock); for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) if (sc->timer[i] == 0) { sc->timer[i] = -1; /* disable it */ timer = i; mtx_unlock(&sc->lock); return (timer); } mtx_unlock(&sc->lock); return (-1); } int emu_timer_set(struct emu_sc_info *sc, int timer, int delay) { int i; if (timer < 0) return (-1); RANGE(delay, 16, 1024); RANGE(timer, 0, EMU_MAX_IRQ_CONSUMERS-1); mtx_lock(&sc->lock); sc->timer[timer] = delay; for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) if (sc->timerinterval > sc->timer[i]) sc->timerinterval = sc->timer[i]; /* XXX */ emu_wr(sc, EMU_TIMER, sc->timerinterval & 0x03ff, 2); mtx_unlock(&sc->lock); return (timer); } int emu_timer_enable(struct emu_sc_info *sc, int timer, int go) { uint32_t x; int ena_int; int i; if (timer < 0) return (-1); RANGE(timer, 0, EMU_MAX_IRQ_CONSUMERS-1); mtx_lock(&sc->lock); if ((go == 1) && (sc->timer[timer] < 0)) sc->timer[timer] = -sc->timer[timer]; if ((go == 0) && (sc->timer[timer] > 0)) sc->timer[timer] = -sc->timer[timer]; ena_int = 0; for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) { if (sc->timerinterval > sc->timer[i]) sc->timerinterval = sc->timer[i]; if (sc->timer[i] > 0) ena_int = 1; } emu_wr(sc, EMU_TIMER, sc->timerinterval & 0x03ff, 2); if (ena_int == 1) { x = emu_rd(sc, EMU_INTE, 4); x |= EMU_INTE_INTERTIMERENB; emu_wr(sc, EMU_INTE, x, 4); } else { x = emu_rd(sc, EMU_INTE, 4); x &= ~EMU_INTE_INTERTIMERENB; emu_wr(sc, EMU_INTE, x, 4); } mtx_unlock(&sc->lock); return (0); } int emu_timer_clear(struct emu_sc_info *sc, int timer) { if (timer < 0) return (-1); RANGE(timer, 0, EMU_MAX_IRQ_CONSUMERS-1); emu_timer_enable(sc, timer, 0); mtx_lock(&sc->lock); if (sc->timer[timer] != 0) sc->timer[timer] = 0; mtx_unlock(&sc->lock); return (timer); } /* * emu_intr_ - HW interrupt handler management */ int emu_intr_register(struct emu_sc_info *sc, uint32_t inte_mask, uint32_t intr_mask, uint32_t(*func) (void *softc, uint32_t irq), void *isc) { int i; uint32_t x; mtx_lock(&sc->lock); for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) if (sc->ihandler[i].inte_mask == 0) { sc->ihandler[i].inte_mask = inte_mask; sc->ihandler[i].intr_mask = intr_mask; sc->ihandler[i].softc = isc; sc->ihandler[i].irq_func = func; x = emu_rd(sc, EMU_INTE, 4); x |= inte_mask; emu_wr(sc, EMU_INTE, x, 4); mtx_unlock(&sc->lock); if (sc->dbg_level > 1) device_printf(sc->dev, "ihandle %d registered\n", i); return (i); } mtx_unlock(&sc->lock); if (sc->dbg_level > 1) device_printf(sc->dev, "ihandle not registered\n"); return (-1); } int emu_intr_unregister(struct emu_sc_info *sc, int hnumber) { uint32_t x; int i; mtx_lock(&sc->lock); if (sc->ihandler[hnumber].inte_mask == 0) { mtx_unlock(&sc->lock); return (-1); } x = emu_rd(sc, EMU_INTE, 4); x &= ~sc->ihandler[hnumber].inte_mask; sc->ihandler[hnumber].inte_mask = 0; sc->ihandler[hnumber].intr_mask = 0; sc->ihandler[hnumber].softc = NULL; sc->ihandler[hnumber].irq_func = NULL; /* other interrupt handlers may use this EMU_INTE value */ for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) if (sc->ihandler[i].inte_mask != 0) x |= sc->ihandler[i].inte_mask; emu_wr(sc, EMU_INTE, x, 4); mtx_unlock(&sc->lock); return (hnumber); } static void emu_intr(void *p) { struct emu_sc_info *sc = (struct emu_sc_info *)p; uint32_t stat, ack; int i; for (;;) { stat = emu_rd(sc, EMU_IPR, 4); ack = 0; if (stat == 0) break; emu_wr(sc, EMU_IPR, stat, 4); for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) { if ((((sc->ihandler[i].intr_mask) & stat) != 0) && (((void *)sc->ihandler[i].irq_func) != NULL)) { ack |= sc->ihandler[i].irq_func(sc->ihandler[i].softc, (sc->ihandler[i].intr_mask) & stat); } } if (sc->dbg_level > 1) if (stat & (~ack)) device_printf(sc->dev, "Unhandled interrupt: %08x\n", stat & (~ack)); } if ((sc->is_ca0102) || (sc->is_ca0108)) for (;;) { stat = emu_rd(sc, EMU_IPR2, 4); ack = 0; if (stat == 0) break; emu_wr(sc, EMU_IPR2, stat, 4); if (sc->dbg_level > 1) device_printf(sc->dev, "EMU_IPR2: %08x\n", stat); break; /* to avoid infinite loop. should be removed * after completion of P16V interface. */ } if (sc->is_ca0102) for (;;) { stat = emu_rd(sc, EMU_IPR3, 4); ack = 0; if (stat == 0) break; emu_wr(sc, EMU_IPR3, stat, 4); if (sc->dbg_level > 1) device_printf(sc->dev, "EMU_IPR3: %08x\n", stat); break; /* to avoid infinite loop. should be removed * after completion of S/PDIF interface */ } } /* * Get data from private emu10kx structure for PCM buffer allocation. * Used by PCM code only. */ bus_dma_tag_t emu_gettag(struct emu_sc_info *sc) { return (sc->mem.dmat); } static void emu_setmap(void *arg, bus_dma_segment_t * segs, int nseg, int error) { bus_addr_t *phys = (bus_addr_t *) arg; *phys = error ? 0 : (bus_addr_t) segs->ds_addr; if (bootverbose) { printf("emu10kx: setmap (%lx, %lx), nseg=%d, error=%d\n", (unsigned long)segs->ds_addr, (unsigned long)segs->ds_len, nseg, error); } } static void * emu_malloc(struct emu_mem *mem, uint32_t sz, bus_addr_t * addr, bus_dmamap_t *map) { void *dmabuf; int error; *addr = 0; if ((error = bus_dmamem_alloc(mem->dmat, &dmabuf, BUS_DMA_NOWAIT, map))) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_malloc: failed to alloc DMA map: %d\n", error); return (NULL); } if ((error = bus_dmamap_load(mem->dmat, *map, dmabuf, sz, emu_setmap, addr, 0)) || !*addr) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_malloc: failed to load DMA memory: %d\n", error); bus_dmamem_free(mem->dmat, dmabuf, *map); return (NULL); } return (dmabuf); } static void emu_free(struct emu_mem *mem, void *dmabuf, bus_dmamap_t map) { bus_dmamap_unload(mem->dmat, map); bus_dmamem_free(mem->dmat, dmabuf, map); } static void * emu_memalloc(struct emu_mem *mem, uint32_t sz, bus_addr_t * addr, const char *owner) { uint32_t blksz, start, idx, ofs, tmp, found; struct emu_memblk *blk; void *membuf; blksz = sz / EMUPAGESIZE; if (sz > (blksz * EMUPAGESIZE)) blksz++; if (blksz > EMU_MAX_BUFSZ / EMUPAGESIZE) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_memalloc: memory request tool large\n"); return (NULL); } /* find a free block in the bitmap */ found = 0; start = 1; while (!found && start + blksz < EMU_MAXPAGES) { found = 1; for (idx = start; idx < start + blksz; idx++) if (mem->bmap[idx >> 3] & (1 << (idx & 7))) found = 0; if (!found) start++; } if (!found) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_memalloc: no free space in bitmap\n"); return (NULL); } blk = malloc(sizeof(*blk), M_DEVBUF, M_NOWAIT); if (blk == NULL) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_memalloc: buffer allocation failed\n"); return (NULL); } bzero(blk, sizeof(*blk)); membuf = emu_malloc(mem, sz, &blk->buf_addr, &blk->buf_map); *addr = blk->buf_addr; if (membuf == NULL) { if (mem->card->dbg_level > 2) device_printf(mem->card->dev, "emu_memalloc: can't setup HW memory\n"); free(blk, M_DEVBUF); return (NULL); } blk->buf = membuf; blk->pte_start = start; blk->pte_size = blksz; strncpy(blk->owner, owner, 15); blk->owner[15] = '\0'; ofs = 0; for (idx = start; idx < start + blksz; idx++) { mem->bmap[idx >> 3] |= 1 << (idx & 7); tmp = (uint32_t) (blk->buf_addr + ofs); mem->ptb_pages[idx] = (tmp << 1) | idx; ofs += EMUPAGESIZE; } SLIST_INSERT_HEAD(&mem->blocks, blk, link); return (membuf); } static int emu_memfree(struct emu_mem *mem, void *membuf) { uint32_t idx, tmp; struct emu_memblk *blk, *i; blk = NULL; SLIST_FOREACH(i, &mem->blocks, link) { if (i->buf == membuf) blk = i; } if (blk == NULL) return (EINVAL); SLIST_REMOVE(&mem->blocks, blk, emu_memblk, link); emu_free(mem, membuf, blk->buf_map); tmp = (uint32_t) (mem->silent_page_addr) << 1; for (idx = blk->pte_start; idx < blk->pte_start + blk->pte_size; idx++) { mem->bmap[idx >> 3] &= ~(1 << (idx & 7)); mem->ptb_pages[idx] = tmp | idx; } free(blk, M_DEVBUF); return (0); } static int emu_memstart(struct emu_mem *mem, void *membuf) { struct emu_memblk *blk, *i; blk = NULL; SLIST_FOREACH(i, &mem->blocks, link) { if (i->buf == membuf) blk = i; } if (blk == NULL) return (-1); return (blk->pte_start); } static uint32_t emu_rate_to_pitch(uint32_t rate) { static uint32_t logMagTable[128] = { 0x00000, 0x02dfc, 0x05b9e, 0x088e6, 0x0b5d6, 0x0e26f, 0x10eb3, 0x13aa2, 0x1663f, 0x1918a, 0x1bc84, 0x1e72e, 0x2118b, 0x23b9a, 0x2655d, 0x28ed5, 0x2b803, 0x2e0e8, 0x30985, 0x331db, 0x359eb, 0x381b6, 0x3a93d, 0x3d081, 0x3f782, 0x41e42, 0x444c1, 0x46b01, 0x49101, 0x4b6c4, 0x4dc49, 0x50191, 0x5269e, 0x54b6f, 0x57006, 0x59463, 0x5b888, 0x5dc74, 0x60029, 0x623a7, 0x646ee, 0x66a00, 0x68cdd, 0x6af86, 0x6d1fa, 0x6f43c, 0x7164b, 0x73829, 0x759d4, 0x77b4f, 0x79c9a, 0x7bdb5, 0x7dea1, 0x7ff5e, 0x81fed, 0x8404e, 0x86082, 0x88089, 0x8a064, 0x8c014, 0x8df98, 0x8fef1, 0x91e20, 0x93d26, 0x95c01, 0x97ab4, 0x9993e, 0x9b79f, 0x9d5d9, 0x9f3ec, 0xa11d8, 0xa2f9d, 0xa4d3c, 0xa6ab5, 0xa8808, 0xaa537, 0xac241, 0xadf26, 0xafbe7, 0xb1885, 0xb3500, 0xb5157, 0xb6d8c, 0xb899f, 0xba58f, 0xbc15e, 0xbdd0c, 0xbf899, 0xc1404, 0xc2f50, 0xc4a7b, 0xc6587, 0xc8073, 0xc9b3f, 0xcb5ed, 0xcd07c, 0xceaec, 0xd053f, 0xd1f73, 0xd398a, 0xd5384, 0xd6d60, 0xd8720, 0xda0c3, 0xdba4a, 0xdd3b4, 0xded03, 0xe0636, 0xe1f4e, 0xe384a, 0xe512c, 0xe69f3, 0xe829f, 0xe9b31, 0xeb3a9, 0xecc08, 0xee44c, 0xefc78, 0xf148a, 0xf2c83, 0xf4463, 0xf5c2a, 0xf73da, 0xf8b71, 0xfa2f0, 0xfba57, 0xfd1a7, 0xfe8df }; static char logSlopeTable[128] = { 0x5c, 0x5c, 0x5b, 0x5a, 0x5a, 0x59, 0x58, 0x58, 0x57, 0x56, 0x56, 0x55, 0x55, 0x54, 0x53, 0x53, 0x52, 0x52, 0x51, 0x51, 0x50, 0x50, 0x4f, 0x4f, 0x4e, 0x4d, 0x4d, 0x4d, 0x4c, 0x4c, 0x4b, 0x4b, 0x4a, 0x4a, 0x49, 0x49, 0x48, 0x48, 0x47, 0x47, 0x47, 0x46, 0x46, 0x45, 0x45, 0x45, 0x44, 0x44, 0x43, 0x43, 0x43, 0x42, 0x42, 0x42, 0x41, 0x41, 0x41, 0x40, 0x40, 0x40, 0x3f, 0x3f, 0x3f, 0x3e, 0x3e, 0x3e, 0x3d, 0x3d, 0x3d, 0x3c, 0x3c, 0x3c, 0x3b, 0x3b, 0x3b, 0x3b, 0x3a, 0x3a, 0x3a, 0x39, 0x39, 0x39, 0x39, 0x38, 0x38, 0x38, 0x38, 0x37, 0x37, 0x37, 0x37, 0x36, 0x36, 0x36, 0x36, 0x35, 0x35, 0x35, 0x35, 0x34, 0x34, 0x34, 0x34, 0x34, 0x33, 0x33, 0x33, 0x33, 0x32, 0x32, 0x32, 0x32, 0x32, 0x31, 0x31, 0x31, 0x31, 0x31, 0x30, 0x30, 0x30, 0x30, 0x30, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f }; int i; if (rate == 0) return (0); rate *= 11185; /* Scale 48000 to 0x20002380 */ for (i = 31; i > 0; i--) { if (rate & 0x80000000) { /* Detect leading "1" */ return (((uint32_t) (i - 15) << 20) + logMagTable[0x7f & (rate >> 24)] + (0x7f & (rate >> 17)) * logSlopeTable[0x7f & (rate >> 24)]); } rate <<= 1; } /* NOTREACHED */ return (0); } static uint32_t emu_rate_to_linearpitch(uint32_t rate) { rate = (rate << 8) / 375; return ((rate >> 1) + (rate & 1)); } struct emu_voice * emu_valloc(struct emu_sc_info *sc) { struct emu_voice *v; int i; v = NULL; mtx_lock(&sc->lock); for (i = 0; i < NUM_G && sc->voice[i].busy; i++); if (i < NUM_G) { v = &sc->voice[i]; v->busy = 1; } mtx_unlock(&sc->lock); return (v); } void emu_vfree(struct emu_sc_info *sc, struct emu_voice *v) { int i, r; mtx_lock(&sc->lock); for (i = 0; i < NUM_G; i++) { if (v == &sc->voice[i] && sc->voice[i].busy) { v->busy = 0; /* * XXX What we should do with mono channels? * See -pcm.c emupchan_init for other side of * this problem */ if (v->slave != NULL) r = emu_memfree(&sc->mem, v->vbuf); } } mtx_unlock(&sc->lock); } int emu_vinit(struct emu_sc_info *sc, struct emu_voice *m, struct emu_voice *s, uint32_t sz, struct snd_dbuf *b) { void *vbuf; bus_addr_t tmp_addr; vbuf = emu_memalloc(&sc->mem, sz, &tmp_addr, "vinit"); if (vbuf == NULL) { if(sc->dbg_level > 2) device_printf(sc->dev, "emu_memalloc returns NULL in enu_vinit\n"); return (ENOMEM); } if (b != NULL) sndbuf_setup(b, vbuf, sz); m->start = emu_memstart(&sc->mem, vbuf) * EMUPAGESIZE; if (m->start < 0) { if(sc->dbg_level > 2) device_printf(sc->dev, "emu_memstart returns (-1) in enu_vinit\n"); emu_memfree(&sc->mem, vbuf); return (ENOMEM); } m->end = m->start + sz; m->speed = 0; m->b16 = 0; m->stereo = 0; m->running = 0; m->ismaster = 1; m->vol = 0xff; m->buf = tmp_addr; m->vbuf = vbuf; m->slave = s; if (s != NULL) { s->start = m->start; s->end = m->end; s->speed = 0; s->b16 = 0; s->stereo = 0; s->running = 0; s->ismaster = 0; s->vol = m->vol; s->buf = m->buf; s->vbuf = NULL; s->slave = NULL; } return (0); } void emu_vsetup(struct emu_voice *v, int fmt, int spd) { if (fmt) { v->b16 = (fmt & AFMT_16BIT) ? 1 : 0; v->stereo = (AFMT_CHANNEL(fmt) > 1) ? 1 : 0; if (v->slave != NULL) { v->slave->b16 = v->b16; v->slave->stereo = v->stereo; } } if (spd) { v->speed = spd; if (v->slave != NULL) v->slave->speed = v->speed; } } void emu_vroute(struct emu_sc_info *sc, struct emu_route *rt, struct emu_voice *v) { int i; for (i = 0; i < 8; i++) { v->routing[i] = rt->routing_left[i]; v->amounts[i] = rt->amounts_left[i]; } if ((v->stereo) && (v->ismaster == 0)) for (i = 0; i < 8; i++) { v->routing[i] = rt->routing_right[i]; v->amounts[i] = rt->amounts_right[i]; } if ((v->stereo) && (v->slave != NULL)) emu_vroute(sc, rt, v->slave); } void emu_vwrite(struct emu_sc_info *sc, struct emu_voice *v) { int s; uint32_t start, val, silent_page; s = (v->stereo ? 1 : 0) + (v->b16 ? 1 : 0); v->sa = v->start >> s; v->ea = v->end >> s; if (v->stereo) { emu_wrptr(sc, v->vnum, EMU_CHAN_CPF, EMU_CHAN_CPF_STEREO_MASK); } else { emu_wrptr(sc, v->vnum, EMU_CHAN_CPF, 0); } val = v->stereo ? 28 : 30; val *= v->b16 ? 1 : 2; start = v->sa + val; if (sc->is_emu10k1) { emu_wrptr(sc, v->vnum, EMU_CHAN_FXRT, ((v->routing[3] << 12) | (v->routing[2] << 8) | (v->routing[1] << 4) | (v->routing[0] << 0)) << 16); } else { emu_wrptr(sc, v->vnum, EMU_A_CHAN_FXRT1, (v->routing[3] << 24) | (v->routing[2] << 16) | (v->routing[1] << 8) | (v->routing[0] << 0)); emu_wrptr(sc, v->vnum, EMU_A_CHAN_FXRT2, (v->routing[7] << 24) | (v->routing[6] << 16) | (v->routing[5] << 8) | (v->routing[4] << 0)); emu_wrptr(sc, v->vnum, EMU_A_CHAN_SENDAMOUNTS, (v->amounts[7] << 24) | (v->amounts[6] << 26) | (v->amounts[5] << 8) | (v->amounts[4] << 0)); } emu_wrptr(sc, v->vnum, EMU_CHAN_PTRX, (v->amounts[0] << 8) | (v->amounts[1] << 0)); emu_wrptr(sc, v->vnum, EMU_CHAN_DSL, v->ea | (v->amounts[3] << 24)); emu_wrptr(sc, v->vnum, EMU_CHAN_PSST, v->sa | (v->amounts[2] << 24)); emu_wrptr(sc, v->vnum, EMU_CHAN_CCCA, start | (v->b16 ? 0 : EMU_CHAN_CCCA_8BITSELECT)); emu_wrptr(sc, v->vnum, EMU_CHAN_Z1, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_Z2, 0); silent_page = ((uint32_t) (sc->mem.silent_page_addr) << 1) | EMU_CHAN_MAP_PTI_MASK; emu_wrptr(sc, v->vnum, EMU_CHAN_MAPA, silent_page); emu_wrptr(sc, v->vnum, EMU_CHAN_MAPB, silent_page); emu_wrptr(sc, v->vnum, EMU_CHAN_CVCF, EMU_CHAN_CVCF_CURRFILTER_MASK); emu_wrptr(sc, v->vnum, EMU_CHAN_VTFT, EMU_CHAN_VTFT_FILTERTARGET_MASK); emu_wrptr(sc, v->vnum, EMU_CHAN_ATKHLDM, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_DCYSUSM, EMU_CHAN_DCYSUSM_DECAYTIME_MASK); emu_wrptr(sc, v->vnum, EMU_CHAN_LFOVAL1, 0x8000); emu_wrptr(sc, v->vnum, EMU_CHAN_LFOVAL2, 0x8000); emu_wrptr(sc, v->vnum, EMU_CHAN_FMMOD, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_TREMFRQ, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_FM2FRQ2, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_ENVVAL, 0x8000); emu_wrptr(sc, v->vnum, EMU_CHAN_ATKHLDV, EMU_CHAN_ATKHLDV_HOLDTIME_MASK | EMU_CHAN_ATKHLDV_ATTACKTIME_MASK); emu_wrptr(sc, v->vnum, EMU_CHAN_ENVVOL, 0x8000); emu_wrptr(sc, v->vnum, EMU_CHAN_PEFE_FILTERAMOUNT, 0x7f); emu_wrptr(sc, v->vnum, EMU_CHAN_PEFE_PITCHAMOUNT, 0); if ((v->stereo) && (v->slave != NULL)) emu_vwrite(sc, v->slave); } static void emu_vstop(struct emu_sc_info *sc, char channel, int enable) { int reg; reg = (channel & 0x20) ? EMU_SOLEH : EMU_SOLEL; channel &= 0x1f; reg |= 1 << 24; reg |= channel << 16; emu_wrptr(sc, 0, reg, enable); } void emu_vtrigger(struct emu_sc_info *sc, struct emu_voice *v, int go) { uint32_t pitch_target, initial_pitch; uint32_t cra, cs, ccis; uint32_t sample, i; if (go) { cra = 64; cs = v->stereo ? 4 : 2; ccis = v->stereo ? 28 : 30; ccis *= v->b16 ? 1 : 2; sample = v->b16 ? 0x00000000 : 0x80808080; for (i = 0; i < cs; i++) emu_wrptr(sc, v->vnum, EMU_CHAN_CD0 + i, sample); emu_wrptr(sc, v->vnum, EMU_CHAN_CCR_CACHEINVALIDSIZE, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_CCR_READADDRESS, cra); emu_wrptr(sc, v->vnum, EMU_CHAN_CCR_CACHEINVALIDSIZE, ccis); emu_wrptr(sc, v->vnum, EMU_CHAN_IFATN, 0xff00); emu_wrptr(sc, v->vnum, EMU_CHAN_VTFT, 0xffffffff); emu_wrptr(sc, v->vnum, EMU_CHAN_CVCF, 0xffffffff); emu_wrptr(sc, v->vnum, EMU_CHAN_DCYSUSV, 0x00007f7f); emu_vstop(sc, v->vnum, 0); pitch_target = emu_rate_to_linearpitch(v->speed); initial_pitch = emu_rate_to_pitch(v->speed) >> 8; emu_wrptr(sc, v->vnum, EMU_CHAN_PTRX_PITCHTARGET, pitch_target); emu_wrptr(sc, v->vnum, EMU_CHAN_CPF_PITCH, pitch_target); emu_wrptr(sc, v->vnum, EMU_CHAN_IP, initial_pitch); } else { emu_wrptr(sc, v->vnum, EMU_CHAN_PTRX_PITCHTARGET, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_CPF_PITCH, 0); emu_wrptr(sc, v->vnum, EMU_CHAN_IFATN, 0xffff); emu_wrptr(sc, v->vnum, EMU_CHAN_VTFT, 0x0000ffff); emu_wrptr(sc, v->vnum, EMU_CHAN_CVCF, 0x0000ffff); emu_wrptr(sc, v->vnum, EMU_CHAN_IP, 0); emu_vstop(sc, v->vnum, 1); } if ((v->stereo) && (v->slave != NULL)) emu_vtrigger(sc, v->slave, go); } int emu_vpos(struct emu_sc_info *sc, struct emu_voice *v) { int s, ptr; s = (v->b16 ? 1 : 0) + (v->stereo ? 1 : 0); ptr = (emu_rdptr(sc, v->vnum, EMU_CHAN_CCCA_CURRADDR) - (v->start >> s)) << s; return (ptr & ~0x0000001f); } /* fx */ static void emu_wrefx(struct emu_sc_info *sc, unsigned int pc, unsigned int data) { emu_wrptr(sc, 0, sc->code_base + pc, data); } static void emu_addefxop(struct emu_sc_info *sc, unsigned int op, unsigned int z, unsigned int w, unsigned int x, unsigned int y, uint32_t * pc) { if ((*pc) + 1 > sc->code_size) { device_printf(sc->dev, "DSP CODE OVERRUN: attept to write past code_size (pc=%d)\n", (*pc)); return; } emu_wrefx(sc, (*pc) * 2, (x << sc->high_operand_shift) | y); emu_wrefx(sc, (*pc) * 2 + 1, (op << sc->opcode_shift) | (z << sc->high_operand_shift) | w); (*pc)++; } static int sysctl_emu_mixer_control(SYSCTL_HANDLER_ARGS) { struct emu_sc_info *sc; int mixer_id; int new_vol; int err; sc = arg1; mixer_id = arg2; new_vol = emumix_get_volume(sc, mixer_id); err = sysctl_handle_int(oidp, &new_vol, 0, req); if (err || req->newptr == NULL) return (err); if (new_vol < 0 || new_vol > 100) return (EINVAL); emumix_set_volume(sc, mixer_id, new_vol); return (0); } static int emu_addefxmixer(struct emu_sc_info *sc, const char *mix_name, const int mix_id, uint32_t defvolume) { int volgpr; char sysctl_name[32]; volgpr = emu_rm_gpr_alloc(sc->rm, 1); emumix_set_fxvol(sc, volgpr, defvolume); /* * Mixer controls with NULL mix_name are handled * by AC97 emulation code or PCM mixer. */ if (mix_name != NULL) { /* * Temporary sysctls should start with underscore, * see freebsd-current mailing list, emu10kx driver * discussion around 2006-05-24. */ snprintf(sysctl_name, 32, "_%s", mix_name); SYSCTL_ADD_PROC(sc->ctx, SYSCTL_CHILDREN(sc->root), OID_AUTO, sysctl_name, CTLTYPE_INT | CTLFLAG_RW, sc, mix_id, sysctl_emu_mixer_control, "I", ""); } return (volgpr); } static int sysctl_emu_digitalswitch_control(SYSCTL_HANDLER_ARGS) { struct emu_sc_info *sc; int new_val; int err; sc = arg1; new_val = (sc->mode == MODE_DIGITAL) ? 1 : 0; err = sysctl_handle_int(oidp, &new_val, 0, req); if (err || req->newptr == NULL) return (err); if (new_val < 0 || new_val > 1) return (EINVAL); switch (new_val) { case 0: emumix_set_mode(sc, MODE_ANALOG); break; case 1: emumix_set_mode(sc, MODE_DIGITAL); break; } return (0); } static void emu_digitalswitch(struct emu_sc_info *sc) { /* XXX temporary? */ SYSCTL_ADD_PROC(sc->ctx, SYSCTL_CHILDREN(sc->root), OID_AUTO, "_digital", CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_emu_digitalswitch_control, "I", "Enable digital output"); return; } /* * Allocate cache GPRs that will hold mixed output channels * and clear it on every DSP run. */ #define EFX_CACHE(CACHE_IDX) do { \ sc->cache_gpr[CACHE_IDX] = emu_rm_gpr_alloc(sc->rm, 1); \ emu_addefxop(sc, ACC3, \ GPR(sc->cache_gpr[CACHE_IDX]), \ DSP_CONST(0), \ DSP_CONST(0), \ DSP_CONST(0), \ &pc); \ } while (0) /* Allocate GPR for volume control and route sound: OUT = OUT + IN * VOL */ #define EFX_ROUTE(TITLE, INP_NR, IN_GPR_IDX, OUT_CACHE_IDX, DEF) do { \ sc->mixer_gpr[IN_GPR_IDX] = emu_addefxmixer(sc, TITLE, IN_GPR_IDX, DEF); \ sc->mixer_volcache[IN_GPR_IDX] = DEF; \ emu_addefxop(sc, MACS, \ GPR(sc->cache_gpr[OUT_CACHE_IDX]), \ GPR(sc->cache_gpr[OUT_CACHE_IDX]), \ INP_NR, \ GPR(sc->mixer_gpr[IN_GPR_IDX]), \ &pc); \ } while (0) /* allocate GPR, OUT = IN * VOL */ #define EFX_OUTPUT(TITLE, OUT_CACHE_IDX, OUT_GPR_IDX, OUTP_NR, DEF) do { \ sc->mixer_gpr[OUT_GPR_IDX] = emu_addefxmixer(sc, TITLE, OUT_GPR_IDX, DEF); \ sc->mixer_volcache[OUT_GPR_IDX] = DEF; \ emu_addefxop(sc, MACS, \ OUTP(OUTP_NR), \ DSP_CONST(0), \ GPR(sc->cache_gpr[OUT_CACHE_IDX]), \ GPR(sc->mixer_gpr[OUT_GPR_IDX]), \ &pc); \ } while (0) /* like EFX_OUTPUT, but don't allocate mixer gpr */ #define EFX_OUTPUTD(OUT_CACHE_IDX, OUT_GPR_IDX, OUTP_NR) do { \ emu_addefxop(sc, MACS, \ OUTP(OUTP_NR), \ DSP_CONST(0), \ GPR(sc->cache_gpr[OUT_CACHE_IDX]), \ GPR(sc->mixer_gpr[OUT_GPR_IDX]), \ &pc); \ } while (0) /* skip next OPCOUNT instructions if FLAG != 0 */ #define EFX_SKIP(OPCOUNT, FLAG_GPR) do { \ emu_addefxop(sc, MACS, \ DSP_CONST(0), \ GPR(sc->mute_gpr[FLAG_GPR]), \ DSP_CONST(0), \ DSP_CONST(0), \ &pc); \ emu_addefxop(sc, SKIP, \ DSP_CCR, \ DSP_CCR, \ COND_NEQ_ZERO, \ OPCOUNT, \ &pc); \ } while (0) #define EFX_COPY(TO, FROM) do { \ emu_addefxop(sc, ACC3, \ TO, \ DSP_CONST(0), \ DSP_CONST(0), \ FROM, \ &pc); \ } while (0) static void emu_initefx(struct emu_sc_info *sc) { unsigned int i; uint32_t pc; /* stop DSP */ if (sc->is_emu10k1) { emu_wrptr(sc, 0, EMU_DBG, EMU_DBG_SINGLE_STEP); } else { emu_wrptr(sc, 0, EMU_A_DBG, EMU_A_DBG_SINGLE_STEP); } /* code size is in instructions */ pc = 0; for (i = 0; i < sc->code_size; i++) { if (sc->is_emu10k1) { emu_addefxop(sc, ACC3, DSP_CONST(0x0), DSP_CONST(0x0), DSP_CONST(0x0), DSP_CONST(0x0), &pc); } else { emu_addefxop(sc, SKIP, DSP_CONST(0x0), DSP_CONST(0x0), DSP_CONST(0xf), DSP_CONST(0x0), &pc); } } /* allocate GPRs for mute switches (EFX_SKIP). Mute by default */ for (i = 0; i < NUM_MUTE; i++) { sc->mute_gpr[i] = emu_rm_gpr_alloc(sc->rm, 1); emumix_set_gpr(sc, sc->mute_gpr[i], 1); } emu_digitalswitch(sc); pc = 0; /* * DSP code below is not good, because: * 1. It can be written smaller, if it can use DSP accumulator register * instead of cache_gpr[]. * 2. It can be more careful when volume is 100%, because in DSP * x*0x7fffffff may not be equal to x ! */ /* clean outputs */ for (i = 0; i < 16 ; i++) { emu_addefxop(sc, ACC3, OUTP(i), DSP_CONST(0), DSP_CONST(0), DSP_CONST(0), &pc); } if (sc->is_emu10k1) { EFX_CACHE(C_FRONT_L); EFX_CACHE(C_FRONT_R); EFX_CACHE(C_REC_L); EFX_CACHE(C_REC_R); /* fx0 to front/record, 100%/muted by default */ EFX_ROUTE("pcm_front_l", FX(0), M_FX0_FRONT_L, C_FRONT_L, 100); EFX_ROUTE("pcm_front_r", FX(1), M_FX1_FRONT_R, C_FRONT_R, 100); EFX_ROUTE(NULL, FX(0), M_FX0_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, FX(1), M_FX1_REC_R, C_REC_R, 0); /* in0, from AC97 codec output */ EFX_ROUTE("ac97_front_l", INP(IN_AC97_L), M_IN0_FRONT_L, C_FRONT_L, 0); EFX_ROUTE("ac97_front_r", INP(IN_AC97_R), M_IN0_FRONT_R, C_FRONT_R, 0); EFX_ROUTE("ac97_rec_l", INP(IN_AC97_L), M_IN0_REC_L, C_REC_L, 0); EFX_ROUTE("ac97_rec_r", INP(IN_AC97_R), M_IN0_REC_R, C_REC_R, 0); /* in1, from CD S/PDIF */ /* XXX EFX_SKIP 4 assumes that each EFX_ROUTE is one DSP op */ EFX_SKIP(4, CDSPDIFMUTE); EFX_ROUTE(NULL, INP(IN_SPDIF_CD_L), M_IN1_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(IN_SPDIF_CD_R), M_IN1_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(IN_SPDIF_CD_L), M_IN1_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(IN_SPDIF_CD_R), M_IN1_REC_R, C_REC_R, 0); if (sc->dbg_level > 0) { /* in2, ZoomVide (???) */ EFX_ROUTE("zoom_front_l", INP(IN_ZOOM_L), M_IN2_FRONT_L, C_FRONT_L, 0); EFX_ROUTE("zoom_front_r", INP(IN_ZOOM_R), M_IN2_FRONT_R, C_FRONT_R, 0); EFX_ROUTE("zoom_rec_l", INP(IN_ZOOM_L), M_IN2_REC_L, C_REC_L, 0); EFX_ROUTE("zoom_rec_r", INP(IN_ZOOM_R), M_IN2_REC_R, C_REC_R, 0); } /* in3, TOSLink */ EFX_ROUTE(NULL, INP(IN_TOSLINK_L), M_IN3_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(IN_TOSLINK_R), M_IN3_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(IN_TOSLINK_L), M_IN3_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(IN_TOSLINK_R), M_IN3_REC_R, C_REC_R, 0); /* in4, LineIn */ EFX_ROUTE(NULL, INP(IN_LINE1_L), M_IN4_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(IN_LINE1_R), M_IN4_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(IN_LINE1_L), M_IN4_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(IN_LINE1_R), M_IN4_REC_R, C_REC_R, 0); /* in5, on-card S/PDIF */ EFX_ROUTE(NULL, INP(IN_COAX_SPDIF_L), M_IN5_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(IN_COAX_SPDIF_R), M_IN5_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(IN_COAX_SPDIF_L), M_IN5_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(IN_COAX_SPDIF_R), M_IN5_REC_R, C_REC_R, 0); /* in6, Line2 on Live!Drive */ EFX_ROUTE(NULL, INP(IN_LINE2_L), M_IN6_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(IN_LINE2_R), M_IN6_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(IN_LINE2_L), M_IN6_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(IN_LINE2_R), M_IN6_REC_R, C_REC_R, 0); if (sc->dbg_level > 0) { /* in7, unknown */ EFX_ROUTE("in7_front_l", INP(0xE), M_IN7_FRONT_L, C_FRONT_L, 0); EFX_ROUTE("in7_front_r", INP(0xF), M_IN7_FRONT_R, C_FRONT_R, 0); EFX_ROUTE("in7_rec_l", INP(0xE), M_IN7_REC_L, C_REC_L, 0); EFX_ROUTE("in7_rec_r", INP(0xF), M_IN7_REC_R, C_REC_R, 0); } /* analog and digital */ EFX_OUTPUT("master_front_l", C_FRONT_L, M_MASTER_FRONT_L, OUT_AC97_L, 100); EFX_OUTPUT("master_front_r", C_FRONT_R, M_MASTER_FRONT_R, OUT_AC97_R, 100); /* S/PDIF */ EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, OUT_TOSLINK_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, OUT_TOSLINK_R); /* Headphones */ EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, OUT_HEADPHONE_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, OUT_HEADPHONE_R); /* rec output to "ADC" */ EFX_OUTPUT("master_rec_l", C_REC_L, M_MASTER_REC_L, OUT_ADC_REC_L, 100); EFX_OUTPUT("master_rec_r", C_REC_R, M_MASTER_REC_R, OUT_ADC_REC_R, 100); if (!(sc->mch_disabled)) { /* * Additional channel volume is controlled by mixer in * emu_dspmixer_set() in -pcm.c */ /* fx2/3 (pcm1) to rear */ EFX_CACHE(C_REAR_L); EFX_CACHE(C_REAR_R); EFX_ROUTE(NULL, FX(2), M_FX2_REAR_L, C_REAR_L, 100); EFX_ROUTE(NULL, FX(3), M_FX3_REAR_R, C_REAR_R, 100); EFX_OUTPUT(NULL, C_REAR_L, M_MASTER_REAR_L, OUT_REAR_L, 100); EFX_OUTPUT(NULL, C_REAR_R, M_MASTER_REAR_R, OUT_REAR_R, 100); if (sc->has_51) { /* fx4 (pcm2) to center */ EFX_CACHE(C_CENTER); EFX_ROUTE(NULL, FX(4), M_FX4_CENTER, C_CENTER, 100); EFX_OUTPUT(NULL, C_CENTER, M_MASTER_CENTER, OUT_D_CENTER, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_CENTER, M_MASTER_CENTER, OUT_A_CENTER); /* fx5 (pcm3) to sub */ EFX_CACHE(C_SUB); EFX_ROUTE(NULL, FX(5), M_FX5_SUBWOOFER, C_SUB, 100); EFX_OUTPUT(NULL, C_SUB, M_MASTER_SUBWOOFER, OUT_D_SUB, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_SUB, M_MASTER_SUBWOOFER, OUT_A_SUB); } } else { /* SND_EMU10KX_MULTICHANNEL_DISABLED */ EFX_OUTPUT(NULL, C_FRONT_L, M_MASTER_REAR_L, OUT_REAR_L, 57); /* 75%*75% */ EFX_OUTPUT(NULL, C_FRONT_R, M_MASTER_REAR_R, OUT_REAR_R, 57); /* 75%*75% */ #if 0 /* XXX 5.1 does not work */ if (sc->has_51) { /* (fx0+fx1)/2 to center */ EFX_CACHE(C_CENTER); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_L]), &pc); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_R]), &pc); EFX_OUTPUT(NULL, C_CENTER, M_MASTER_CENTER, OUT_D_CENTER, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_CENTER, M_MASTER_CENTER, OUT_A_CENTER); /* (fx0+fx1)/2 to sub */ EFX_CACHE(C_SUB); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_L]), &pc); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_R]), &pc); /* XXX add lowpass filter here */ EFX_OUTPUT(NULL, C_SUB, M_MASTER_SUBWOOFER, OUT_D_SUB, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_SUB, M_MASTER_SUBWOOFER, OUT_A_SUB); } #endif } /* !mch_disabled */ if (sc->mch_rec) { /* * MCH RECORDING , hight 16 slots. On 5.1 cards first 4 slots * are used as outputs and already filled with data */ /* * XXX On Live! cards stream does not begin at zero offset. * It can be HW, driver or sound buffering problem. * Use sync substream (offset 0x3E) to let userland find * correct data. */ /* * Substream map (in byte offsets, each substream is 2 bytes): * 0x00..0x1E - outputs * 0x20..0x3E - FX, inputs and sync stream */ /* First 2 channels (offset 0x20,0x22) are empty */ for(i = (sc->has_51 ? 2 : 0); i < 2; i++) EFX_COPY(FX2(i), DSP_CONST(0)); /* PCM Playback monitoring, offset 0x24..0x2A */ for(i = 0; i < 4; i++) EFX_COPY(FX2(i+2), FX(i)); /* Copy of some inputs, offset 0x2C..0x3C */ for(i = 0; i < 9; i++) EFX_COPY(FX2(i+8), INP(i)); /* sync data (0xc0de, offset 0x3E) */ sc->dummy_gpr = emu_rm_gpr_alloc(sc->rm, 1); emumix_set_gpr(sc, sc->dummy_gpr, 0xc0de0000); EFX_COPY(FX2(15), GPR(sc->dummy_gpr)); } /* mch_rec */ } else /* emu10k2 and later */ { EFX_CACHE(C_FRONT_L); EFX_CACHE(C_FRONT_R); EFX_CACHE(C_REC_L); EFX_CACHE(C_REC_R); /* fx0 to front/record, 100%/muted by default */ /* * FRONT_[L|R] is controlled by AC97 emulation in * emu_ac97_[read|write]_emulation in -pcm.c */ EFX_ROUTE(NULL, FX(0), M_FX0_FRONT_L, C_FRONT_L, 100); EFX_ROUTE(NULL, FX(1), M_FX1_FRONT_R, C_FRONT_R, 100); EFX_ROUTE(NULL, FX(0), M_FX0_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, FX(1), M_FX1_REC_R, C_REC_R, 0); /* in0, from AC97 codec output */ EFX_ROUTE(NULL, INP(A_IN_AC97_L), M_IN0_FRONT_L, C_FRONT_L, 100); EFX_ROUTE(NULL, INP(A_IN_AC97_R), M_IN0_FRONT_R, C_FRONT_R, 100); EFX_ROUTE(NULL, INP(A_IN_AC97_L), M_IN0_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_AC97_R), M_IN0_REC_R, C_REC_R, 0); /* in1, from CD S/PDIF */ EFX_ROUTE(NULL, INP(A_IN_SPDIF_CD_L), M_IN1_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(A_IN_SPDIF_CD_R), M_IN1_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(A_IN_SPDIF_CD_L), M_IN1_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_SPDIF_CD_R), M_IN1_REC_R, C_REC_R, 0); /* in2, optical & coax S/PDIF on AudigyDrive*/ /* XXX Should be muted when GPRSCS valid stream == 0 */ EFX_ROUTE(NULL, INP(A_IN_O_SPDIF_L), M_IN2_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(A_IN_O_SPDIF_R), M_IN2_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(A_IN_O_SPDIF_L), M_IN2_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_O_SPDIF_R), M_IN2_REC_R, C_REC_R, 0); if (sc->dbg_level > 0) { /* in3, unknown */ EFX_ROUTE("in3_front_l", INP(0x6), M_IN3_FRONT_L, C_FRONT_L, 0); EFX_ROUTE("in3_front_r", INP(0x7), M_IN3_FRONT_R, C_FRONT_R, 0); EFX_ROUTE("in3_rec_l", INP(0x6), M_IN3_REC_L, C_REC_L, 0); EFX_ROUTE("in3_rec_r", INP(0x7), M_IN3_REC_R, C_REC_R, 0); } /* in4, LineIn 2 on AudigyDrive */ EFX_ROUTE(NULL, INP(A_IN_LINE2_L), M_IN4_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(A_IN_LINE2_R), M_IN4_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(A_IN_LINE2_L), M_IN4_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_LINE2_R), M_IN4_REC_R, C_REC_R, 0); /* in5, on-card S/PDIF */ EFX_ROUTE(NULL, INP(A_IN_R_SPDIF_L), M_IN5_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(A_IN_R_SPDIF_R), M_IN5_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(A_IN_R_SPDIF_L), M_IN5_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_R_SPDIF_R), M_IN5_REC_R, C_REC_R, 0); /* in6, AUX2 on AudigyDrive */ EFX_ROUTE(NULL, INP(A_IN_AUX2_L), M_IN6_FRONT_L, C_FRONT_L, 0); EFX_ROUTE(NULL, INP(A_IN_AUX2_R), M_IN6_FRONT_R, C_FRONT_R, 0); EFX_ROUTE(NULL, INP(A_IN_AUX2_L), M_IN6_REC_L, C_REC_L, 0); EFX_ROUTE(NULL, INP(A_IN_AUX2_R), M_IN6_REC_R, C_REC_R, 0); if (sc->dbg_level > 0) { /* in7, unknown */ EFX_ROUTE("in7_front_l", INP(0xE), M_IN7_FRONT_L, C_FRONT_L, 0); EFX_ROUTE("in7_front_r", INP(0xF), M_IN7_FRONT_R, C_FRONT_R, 0); EFX_ROUTE("in7_rec_l", INP(0xE), M_IN7_REC_L, C_REC_L, 0); EFX_ROUTE("in7_rec_r", INP(0xF), M_IN7_REC_R, C_REC_R, 0); } /* front output to headphones and alog and digital *front */ /* volume controlled by AC97 emulation */ EFX_OUTPUT(NULL, C_FRONT_L, M_MASTER_FRONT_L, A_OUT_A_FRONT_L, 100); EFX_OUTPUT(NULL, C_FRONT_R, M_MASTER_FRONT_R, A_OUT_A_FRONT_R, 100); EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, A_OUT_D_FRONT_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, A_OUT_D_FRONT_R); EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, A_OUT_HPHONE_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, A_OUT_HPHONE_R); /* rec output to "ADC" */ /* volume controlled by AC97 emulation */ EFX_OUTPUT(NULL, C_REC_L, M_MASTER_REC_L, A_OUT_ADC_REC_L, 100); EFX_OUTPUT(NULL, C_REC_R, M_MASTER_REC_R, A_OUT_ADC_REC_R, 100); if (!(sc->mch_disabled)) { /* * Additional channel volume is controlled by mixer in * emu_dspmixer_set() in -pcm.c */ /* fx2/3 (pcm1) to rear */ EFX_CACHE(C_REAR_L); EFX_CACHE(C_REAR_R); EFX_ROUTE(NULL, FX(2), M_FX2_REAR_L, C_REAR_L, 100); EFX_ROUTE(NULL, FX(3), M_FX3_REAR_R, C_REAR_R, 100); EFX_OUTPUT(NULL, C_REAR_L, M_MASTER_REAR_L, A_OUT_A_REAR_L, 100); EFX_OUTPUT(NULL, C_REAR_R, M_MASTER_REAR_R, A_OUT_A_REAR_R, 100); EFX_OUTPUTD(C_REAR_L, M_MASTER_REAR_L, A_OUT_D_REAR_L); EFX_OUTPUTD(C_REAR_R, M_MASTER_REAR_R, A_OUT_D_REAR_R); /* fx4 (pcm2) to center */ EFX_CACHE(C_CENTER); EFX_ROUTE(NULL, FX(4), M_FX4_CENTER, C_CENTER, 100); EFX_OUTPUT(NULL, C_CENTER, M_MASTER_CENTER, A_OUT_D_CENTER, 100); #if 0 /* * XXX in digital mode (default) this should be muted * because this output is shared with digital out */ EFX_OUTPUTD(C_CENTER, M_MASTER_CENTER, A_OUT_A_CENTER); #endif /* fx5 (pcm3) to sub */ EFX_CACHE(C_SUB); EFX_ROUTE(NULL, FX(5), M_FX5_SUBWOOFER, C_SUB, 100); EFX_OUTPUT(NULL, C_SUB, M_MASTER_SUBWOOFER, A_OUT_D_SUB, 100); #if 0 /* * XXX in digital mode (default) this should be muted * because this output is shared with digital out */ EFX_OUTPUTD(C_SUB, M_MASTER_SUBWOOFER, A_OUT_A_SUB); #endif if (sc->has_71) { /* XXX this will broke headphones on AudigyDrive */ /* fx6/7 (pcm4) to side */ EFX_CACHE(C_SIDE_L); EFX_CACHE(C_SIDE_R); EFX_ROUTE(NULL, FX(6), M_FX6_SIDE_L, C_SIDE_L, 100); EFX_ROUTE(NULL, FX(7), M_FX7_SIDE_R, C_SIDE_R, 100); EFX_OUTPUT(NULL, C_SIDE_L, M_MASTER_SIDE_L, A_OUT_A_SIDE_L, 100); EFX_OUTPUT(NULL, C_SIDE_R, M_MASTER_SIDE_R, A_OUT_A_SIDE_R, 100); EFX_OUTPUTD(C_SIDE_L, M_MASTER_SIDE_L, A_OUT_D_SIDE_L); EFX_OUTPUTD(C_SIDE_R, M_MASTER_SIDE_R, A_OUT_D_SIDE_R); } } else { /* mch_disabled */ EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, A_OUT_A_REAR_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, A_OUT_A_REAR_R); EFX_OUTPUTD(C_FRONT_L, M_MASTER_FRONT_L, A_OUT_D_REAR_L); EFX_OUTPUTD(C_FRONT_R, M_MASTER_FRONT_R, A_OUT_D_REAR_R); if (sc->has_51) { /* (fx0+fx1)/2 to center */ EFX_CACHE(C_CENTER); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_L]), &pc); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_CENTER]), GPR(sc->cache_gpr[C_CENTER]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_R]), &pc); EFX_OUTPUT(NULL, C_CENTER, M_MASTER_CENTER, A_OUT_D_CENTER, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_CENTER, M_MASTER_CENTER, A_OUT_A_CENTER); /* (fx0+fx1)/2 to sub */ EFX_CACHE(C_SUB); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_SUB]), GPR(sc->cache_gpr[C_SUB]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_L]), &pc); emu_addefxop(sc, MACS, GPR(sc->cache_gpr[C_SUB]), GPR(sc->cache_gpr[C_SUB]), DSP_CONST(0xd), /* = 1/2 */ GPR(sc->cache_gpr[C_FRONT_R]), &pc); /* XXX add lowpass filter here */ EFX_OUTPUT(NULL, C_SUB, M_MASTER_SUBWOOFER, A_OUT_D_SUB, 100); /* XXX in digital mode (default) this should be muted because this output is shared with digital out */ EFX_SKIP(1, ANALOGMUTE); EFX_OUTPUTD(C_SUB, M_MASTER_SUBWOOFER, A_OUT_A_SUB); } } /* mch_disabled */ if (sc->mch_rec) { /* MCH RECORDING, high 32 slots */ /* * Stream map (in byte offsets): * 0x00..0x3E - outputs * 0x40..0x7E - FX, inputs * each substream is 2 bytes. */ /* * XXX Audigy 2 Value cards (and, possibly, * Audigy 4) write some unknown data in place of * some outputs (offsets 0x20..0x3F) and one * input (offset 0x7E). */ /* PCM Playback monitoring, offsets 0x40..0x5E */ for(i = 0; i < 16; i++) EFX_COPY(FX2(i), FX(i)); /* Copy of all inputs, offsets 0x60..0x7E */ for(i = 0; i < 16; i++) EFX_COPY(FX2(i+16), INP(i)); #if 0 /* XXX Audigy seems to work correct and does not need this */ /* sync data (0xc0de), offset 0x7E */ sc->dummy_gpr = emu_rm_gpr_alloc(sc->rm, 1); emumix_set_gpr(sc, sc->dummy_gpr, 0xc0de0000); EFX_COPY(FX2(31), GPR(sc->dummy_gpr)); #endif } /* mch_rec */ } sc->routing_code_end = pc; /* start DSP */ if (sc->is_emu10k1) { emu_wrptr(sc, 0, EMU_DBG, 0); } else { emu_wrptr(sc, 0, EMU_A_DBG, 0); } } /* /dev/em10kx */ static d_open_t emu10kx_open; static d_close_t emu10kx_close; static d_read_t emu10kx_read; static struct cdevsw emu10kx_cdevsw = { .d_open = emu10kx_open, .d_close = emu10kx_close, .d_read = emu10kx_read, .d_name = "emu10kx", .d_version = D_VERSION, }; static int emu10kx_open(struct cdev *i_dev, int flags __unused, int mode __unused, struct thread *td __unused) { int error; struct emu_sc_info *sc; sc = i_dev->si_drv1; mtx_lock(&sc->emu10kx_lock); if (sc->emu10kx_isopen) { mtx_unlock(&sc->emu10kx_lock); return (EBUSY); } sc->emu10kx_isopen = 1; mtx_unlock(&sc->emu10kx_lock); if (sbuf_new(&sc->emu10kx_sbuf, NULL, 4096, 0) == NULL) { error = ENXIO; goto out; } sc->emu10kx_bufptr = 0; error = (emu10kx_prepare(sc, &sc->emu10kx_sbuf) > 0) ? 0 : ENOMEM; out: if (error) { mtx_lock(&sc->emu10kx_lock); sc->emu10kx_isopen = 0; mtx_unlock(&sc->emu10kx_lock); } return (error); } static int emu10kx_close(struct cdev *i_dev, int flags __unused, int mode __unused, struct thread *td __unused) { struct emu_sc_info *sc; sc = i_dev->si_drv1; mtx_lock(&sc->emu10kx_lock); if (!(sc->emu10kx_isopen)) { mtx_unlock(&sc->emu10kx_lock); return (EBADF); } sbuf_delete(&sc->emu10kx_sbuf); sc->emu10kx_isopen = 0; mtx_unlock(&sc->emu10kx_lock); return (0); } static int emu10kx_read(struct cdev *i_dev, struct uio *buf, int flag __unused) { int l, err; struct emu_sc_info *sc; sc = i_dev->si_drv1; mtx_lock(&sc->emu10kx_lock); if (!(sc->emu10kx_isopen)) { mtx_unlock(&sc->emu10kx_lock); return (EBADF); } mtx_unlock(&sc->emu10kx_lock); l = min(buf->uio_resid, sbuf_len(&sc->emu10kx_sbuf) - sc->emu10kx_bufptr); err = (l > 0) ? uiomove(sbuf_data(&sc->emu10kx_sbuf) + sc->emu10kx_bufptr, l, buf) : 0; sc->emu10kx_bufptr += l; return (err); } static int emu10kx_prepare(struct emu_sc_info *sc, struct sbuf *s) { int i; sbuf_printf(s, "FreeBSD EMU10Kx Audio Driver\n"); sbuf_printf(s, "\nHardware resource usage:\n"); sbuf_printf(s, "DSP General Purpose Registers: %d used, %d total\n", sc->rm->num_used, sc->rm->num_gprs); sbuf_printf(s, "DSP Instruction Registers: %d used, %d total\n", sc->routing_code_end, sc->code_size); sbuf_printf(s, "Card supports"); if (sc->has_ac97) { sbuf_printf(s, " AC97 codec"); } else { sbuf_printf(s, " NO AC97 codec"); } if (sc->has_51) { if (sc->has_71) sbuf_printf(s, " and 7.1 output"); else sbuf_printf(s, " and 5.1 output"); } if (sc->is_emu10k1) sbuf_printf(s, ", SBLive! DSP code"); if (sc->is_emu10k2) sbuf_printf(s, ", Audigy DSP code"); if (sc->is_ca0102) sbuf_printf(s, ", Audigy DSP code with Audigy2 hacks"); if (sc->is_ca0108) sbuf_printf(s, ", Audigy DSP code with Audigy2Value hacks"); sbuf_printf(s, "\n"); if (sc->broken_digital) sbuf_printf(s, "Digital mode unsupported\n"); sbuf_printf(s, "\nInstalled devices:\n"); for (i = 0; i < RT_COUNT; i++) if (sc->pcm[i] != NULL) if (device_is_attached(sc->pcm[i])) { sbuf_printf(s, "%s on %s\n", device_get_desc(sc->pcm[i]), device_get_nameunit(sc->pcm[i])); } if (sc->midi[0] != NULL) if (device_is_attached(sc->midi[0])) { sbuf_printf(s, "EMU10Kx MIDI Interface\n"); sbuf_printf(s, "\tOn-card connector on %s\n", device_get_nameunit(sc->midi[0])); } if (sc->midi[1] != NULL) if (device_is_attached(sc->midi[1])) { sbuf_printf(s, "\tOn-Drive connector on %s\n", device_get_nameunit(sc->midi[1])); } if (sc->midi[0] != NULL) if (device_is_attached(sc->midi[0])) { sbuf_printf(s, "\tIR receiver MIDI events %s\n", sc->enable_ir ? "enabled" : "disabled"); } sbuf_printf(s, "Card is in %s mode\n", (sc->mode == MODE_ANALOG) ? "analog" : "digital"); sbuf_finish(s); return (sbuf_len(s)); } /* INIT & UNINIT */ static int emu10kx_dev_init(struct emu_sc_info *sc) { int unit; mtx_init(&sc->emu10kx_lock, device_get_nameunit(sc->dev), "kxdevlock", 0); unit = device_get_unit(sc->dev); sc->cdev = make_dev(&emu10kx_cdevsw, PCMMINOR(unit), UID_ROOT, GID_WHEEL, 0640, "emu10kx%d", unit); if (sc->cdev != NULL) { sc->cdev->si_drv1 = sc; return (0); } return (ENXIO); } static int emu10kx_dev_uninit(struct emu_sc_info *sc) { mtx_lock(&sc->emu10kx_lock); if (sc->emu10kx_isopen) { mtx_unlock(&sc->emu10kx_lock); return (EBUSY); } if (sc->cdev) destroy_dev(sc->cdev); sc->cdev = NULL; mtx_destroy(&sc->emu10kx_lock); return (0); } /* resource manager */ int emu_rm_init(struct emu_sc_info *sc) { int i; int maxcount; struct emu_rm *rm; rm = malloc(sizeof(struct emu_rm), M_DEVBUF, M_NOWAIT | M_ZERO); if (rm == NULL) { return (ENOMEM); } sc->rm = rm; rm->card = sc; maxcount = sc->num_gprs; rm->num_used = 0; mtx_init(&(rm->gpr_lock), device_get_nameunit(sc->dev), "gpr alloc", MTX_DEF); rm->num_gprs = (maxcount < EMU_MAX_GPR ? maxcount : EMU_MAX_GPR); for (i = 0; i < rm->num_gprs; i++) rm->allocmap[i] = 0; /* pre-allocate gpr[0] */ rm->allocmap[0] = 1; rm->last_free_gpr = 1; return (0); } int emu_rm_uninit(struct emu_sc_info *sc) { int i; if (sc->dbg_level > 1) { mtx_lock(&(sc->rm->gpr_lock)); for (i = 1; i < sc->rm->last_free_gpr; i++) if (sc->rm->allocmap[i] > 0) device_printf(sc->dev, "rm: gpr %d not free before uninit\n", i); mtx_unlock(&(sc->rm->gpr_lock)); } mtx_destroy(&(sc->rm->gpr_lock)); free(sc->rm, M_DEVBUF); return (0); } static int emu_rm_gpr_alloc(struct emu_rm *rm, int count) { int i, j; int allocated_gpr; allocated_gpr = rm->num_gprs; /* try fast way first */ mtx_lock(&(rm->gpr_lock)); if (rm->last_free_gpr + count <= rm->num_gprs) { allocated_gpr = rm->last_free_gpr; rm->last_free_gpr += count; rm->allocmap[allocated_gpr] = count; for (i = 1; i < count; i++) rm->allocmap[allocated_gpr + i] = -(count - i); } else { /* longer */ i = 0; allocated_gpr = rm->num_gprs; while (i < rm->last_free_gpr - count) { if (rm->allocmap[i] > 0) { i += rm->allocmap[i]; } else { allocated_gpr = i; for (j = 1; j < count; j++) { if (rm->allocmap[i + j] != 0) allocated_gpr = rm->num_gprs; } if (allocated_gpr == i) break; } } if (allocated_gpr + count < rm->last_free_gpr) { rm->allocmap[allocated_gpr] = count; for (i = 1; i < count; i++) rm->allocmap[allocated_gpr + i] = -(count - i); } } if (allocated_gpr == rm->num_gprs) allocated_gpr = (-1); if (allocated_gpr >= 0) rm->num_used += count; mtx_unlock(&(rm->gpr_lock)); return (allocated_gpr); } /* mixer */ void emumix_set_mode(struct emu_sc_info *sc, int mode) { uint32_t a_iocfg; uint32_t hcfg; uint32_t tmp; switch (mode) { case MODE_DIGITAL: /* FALLTHROUGH */ case MODE_ANALOG: break; default: return; } hcfg = EMU_HCFG_AUDIOENABLE | EMU_HCFG_AUTOMUTE; a_iocfg = 0; if (sc->rev >= 6) hcfg |= EMU_HCFG_JOYENABLE; if (sc->is_emu10k1) hcfg |= EMU_HCFG_LOCKTANKCACHE_MASK; else hcfg |= EMU_HCFG_CODECFMT_I2S | EMU_HCFG_JOYENABLE; if (mode == MODE_DIGITAL) { if (sc->broken_digital) { device_printf(sc->dev, "Digital mode is reported as broken on this card.\n"); } a_iocfg |= EMU_A_IOCFG_GPOUT1; hcfg |= EMU_HCFG_GPOUT0; } if (mode == MODE_ANALOG) emumix_set_spdif_mode(sc, SPDIF_MODE_PCM); if (sc->is_emu10k2) a_iocfg |= 0x80; /* XXX */ if ((sc->is_ca0102) || (sc->is_ca0108)) /* * Setting EMU_A_IOCFG_DISABLE_ANALOG will do opposite things * on diffrerent cards. * "don't disable analog outs" on Audigy 2 (ca0102/ca0108) * "disable analog outs" on Audigy (emu10k2) */ a_iocfg |= EMU_A_IOCFG_DISABLE_ANALOG; if (sc->is_ca0108) a_iocfg |= 0x20; /* XXX */ /* Mute analog center & subwoofer before mode change */ if (mode == MODE_DIGITAL) emumix_set_gpr(sc, sc->mute_gpr[ANALOGMUTE], 1); emu_wr(sc, EMU_HCFG, hcfg, 4); if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) { tmp = emu_rd(sc, EMU_A_IOCFG, 2); tmp = a_iocfg; emu_wr(sc, EMU_A_IOCFG, tmp, 2); } /* Unmute if we have changed mode to analog. */ if (mode == MODE_ANALOG) emumix_set_gpr(sc, sc->mute_gpr[ANALOGMUTE], 0); sc->mode = mode; } void emumix_set_spdif_mode(struct emu_sc_info *sc, int mode) { uint32_t spcs; switch (mode) { case SPDIF_MODE_PCM: break; case SPDIF_MODE_AC3: device_printf(sc->dev, "AC3 mode does not work and disabled\n"); return; default: return; } spcs = EMU_SPCS_CLKACCY_1000PPM | EMU_SPCS_SAMPLERATE_48 | EMU_SPCS_CHANNELNUM_LEFT | EMU_SPCS_SOURCENUM_UNSPEC | EMU_SPCS_GENERATIONSTATUS | 0x00001200 | 0x00000000 | EMU_SPCS_EMPHASIS_NONE | EMU_SPCS_COPYRIGHT; mode = SPDIF_MODE_PCM; emu_wrptr(sc, 0, EMU_SPCS0, spcs); emu_wrptr(sc, 0, EMU_SPCS1, spcs); emu_wrptr(sc, 0, EMU_SPCS2, spcs); } #define L2L_POINTS 10 static int l2l_df[L2L_POINTS] = { 0x572C5CA, /* 100..90 */ 0x3211625, /* 90..80 */ 0x1CC1A76, /* 80..70 */ 0x108428F, /* 70..60 */ 0x097C70A, /* 60..50 */ 0x0572C5C, /* 50..40 */ 0x0321162, /* 40..30 */ 0x01CC1A7, /* 30..20 */ 0x0108428, /* 20..10 */ 0x016493D /* 10..0 */ }; static int l2l_f[L2L_POINTS] = { 0x4984461A, /* 90 */ 0x2A3968A7, /* 80 */ 0x18406003, /* 70 */ 0x0DEDC66D, /* 60 */ 0x07FFFFFF, /* 50 */ 0x04984461, /* 40 */ 0x02A3968A, /* 30 */ 0x01840600, /* 20 */ 0x00DEDC66, /* 10 */ 0x00000000 /* 0 */ }; static int log2lin(int log_t) { int lin_t; int idx, lin; if (log_t <= 0) { lin_t = 0x00000000; return (lin_t); } if (log_t >= 100) { lin_t = 0x7fffffff; return (lin_t); } idx = (L2L_POINTS - 1) - log_t / (L2L_POINTS); lin = log_t % (L2L_POINTS); lin_t = l2l_df[idx] * lin + l2l_f[idx]; return (lin_t); } void emumix_set_fxvol(struct emu_sc_info *sc, unsigned gpr, int32_t vol) { vol = log2lin(vol); emumix_set_gpr(sc, gpr, vol); } void emumix_set_gpr(struct emu_sc_info *sc, unsigned gpr, int32_t val) { if (sc->dbg_level > 1) if (gpr == 0) { device_printf(sc->dev, "Zero gpr write access\n"); #ifdef KDB kdb_backtrace(); #endif return; } emu_wrptr(sc, 0, GPR(gpr), val); } void emumix_set_volume(struct emu_sc_info *sc, int mixer_idx, int volume) { RANGE(volume, 0, 100); if (mixer_idx < NUM_MIXERS) { sc->mixer_volcache[mixer_idx] = volume; emumix_set_fxvol(sc, sc->mixer_gpr[mixer_idx], volume); } } int emumix_get_volume(struct emu_sc_info *sc, int mixer_idx) { if ((mixer_idx < NUM_MIXERS) && (mixer_idx >= 0)) return (sc->mixer_volcache[mixer_idx]); return (-1); } /* Init CardBus part */ static int emu_cardbus_init(struct emu_sc_info *sc) { /* * XXX May not need this if we have EMU_IPR3 handler. * Is it a real init calls, or EMU_IPR3 interrupt acknowledgments? * Looks much like "(data << 16) | register". */ emu_wr_cbptr(sc, (0x00d0 << 16) | 0x0000); emu_wr_cbptr(sc, (0x00d0 << 16) | 0x0001); emu_wr_cbptr(sc, (0x00d0 << 16) | 0x005f); emu_wr_cbptr(sc, (0x00d0 << 16) | 0x007f); emu_wr_cbptr(sc, (0x0090 << 16) | 0x007f); return (0); } /* Probe and attach the card */ static int emu_init(struct emu_sc_info *sc) { uint32_t ch, tmp; uint32_t spdif_sr; uint32_t ac97slot; int def_mode; int i; /* disable audio and lock cache */ emu_wr(sc, EMU_HCFG, EMU_HCFG_LOCKSOUNDCACHE | EMU_HCFG_LOCKTANKCACHE_MASK | EMU_HCFG_MUTEBUTTONENABLE, 4); /* reset recording buffers */ emu_wrptr(sc, 0, EMU_MICBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_MICBA, 0); emu_wrptr(sc, 0, EMU_FXBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_FXBA, 0); emu_wrptr(sc, 0, EMU_ADCBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_ADCBA, 0); /* disable channel interrupt */ emu_wr(sc, EMU_INTE, EMU_INTE_INTERTIMERENB | EMU_INTE_SAMPLERATER | EMU_INTE_PCIERRENABLE, 4); emu_wrptr(sc, 0, EMU_CLIEL, 0); emu_wrptr(sc, 0, EMU_CLIEH, 0); emu_wrptr(sc, 0, EMU_SOLEL, 0); emu_wrptr(sc, 0, EMU_SOLEH, 0); /* disable P16V and S/PDIF interrupts */ if ((sc->is_ca0102) || (sc->is_ca0108)) emu_wr(sc, EMU_INTE2, 0, 4); if (sc->is_ca0102) emu_wr(sc, EMU_INTE3, 0, 4); /* init phys inputs and outputs */ ac97slot = 0; if (sc->has_51) ac97slot = EMU_AC97SLOT_CENTER | EMU_AC97SLOT_LFE; if (sc->has_71) ac97slot = EMU_AC97SLOT_CENTER | EMU_AC97SLOT_LFE | EMU_AC97SLOT_REAR_LEFT | EMU_AC97SLOT_REAR_RIGHT; if (sc->is_emu10k2) ac97slot |= 0x40; emu_wrptr(sc, 0, EMU_AC97SLOT, ac97slot); if (sc->is_emu10k2) /* XXX for later cards? */ emu_wrptr(sc, 0, EMU_SPBYPASS, 0xf00); /* What will happen if * we write 1 here? */ if (bus_dma_tag_create( /* parent */ bus_get_dma_tag(sc->dev), /* alignment */ 2, /* boundary */ 0, /* lowaddr */ (1U << 31) - 1, /* can only access 0-2gb */ /* highaddr */ BUS_SPACE_MAXADDR, /* filter */ NULL, /* filterarg */ NULL, /* maxsize */ EMU_MAX_BUFSZ, /* nsegments */ 1, /* maxsegz */ 0x3ffff, /* flags */ 0, /* lockfunc */ busdma_lock_mutex, /* lockarg */ &Giant, &(sc->mem.dmat)) != 0) { device_printf(sc->dev, "unable to create dma tag\n"); bus_dma_tag_destroy(sc->mem.dmat); return (ENOMEM); } sc->mem.card = sc; SLIST_INIT(&sc->mem.blocks); sc->mem.ptb_pages = emu_malloc(&sc->mem, EMU_MAXPAGES * sizeof(uint32_t), &sc->mem.ptb_pages_addr, &sc->mem.ptb_map); if (sc->mem.ptb_pages == NULL) return (ENOMEM); sc->mem.silent_page = emu_malloc(&sc->mem, EMUPAGESIZE, &sc->mem.silent_page_addr, &sc->mem.silent_map); if (sc->mem.silent_page == NULL) { emu_free(&sc->mem, sc->mem.ptb_pages, sc->mem.ptb_map); return (ENOMEM); } /* Clear page with silence & setup all pointers to this page */ bzero(sc->mem.silent_page, EMUPAGESIZE); tmp = (uint32_t) (sc->mem.silent_page_addr) << 1; for (i = 0; i < EMU_MAXPAGES; i++) sc->mem.ptb_pages[i] = tmp | i; for (ch = 0; ch < NUM_G; ch++) { emu_wrptr(sc, ch, EMU_CHAN_MAPA, tmp | EMU_CHAN_MAP_PTI_MASK); emu_wrptr(sc, ch, EMU_CHAN_MAPB, tmp | EMU_CHAN_MAP_PTI_MASK); } emu_wrptr(sc, 0, EMU_PTB, (sc->mem.ptb_pages_addr)); emu_wrptr(sc, 0, EMU_TCB, 0); /* taken from original driver */ emu_wrptr(sc, 0, EMU_TCBS, 0); /* taken from original driver */ /* init envelope engine */ for (ch = 0; ch < NUM_G; ch++) { emu_wrptr(sc, ch, EMU_CHAN_DCYSUSV, 0); emu_wrptr(sc, ch, EMU_CHAN_IP, 0); emu_wrptr(sc, ch, EMU_CHAN_VTFT, 0xffff); emu_wrptr(sc, ch, EMU_CHAN_CVCF, 0xffff); emu_wrptr(sc, ch, EMU_CHAN_PTRX, 0); emu_wrptr(sc, ch, EMU_CHAN_CPF, 0); emu_wrptr(sc, ch, EMU_CHAN_CCR, 0); emu_wrptr(sc, ch, EMU_CHAN_PSST, 0); emu_wrptr(sc, ch, EMU_CHAN_DSL, 0x10); emu_wrptr(sc, ch, EMU_CHAN_CCCA, 0); emu_wrptr(sc, ch, EMU_CHAN_Z1, 0); emu_wrptr(sc, ch, EMU_CHAN_Z2, 0); emu_wrptr(sc, ch, EMU_CHAN_FXRT, 0xd01c0000); emu_wrptr(sc, ch, EMU_CHAN_ATKHLDM, 0); emu_wrptr(sc, ch, EMU_CHAN_DCYSUSM, 0); emu_wrptr(sc, ch, EMU_CHAN_IFATN, 0xffff); emu_wrptr(sc, ch, EMU_CHAN_PEFE, 0); emu_wrptr(sc, ch, EMU_CHAN_FMMOD, 0); emu_wrptr(sc, ch, EMU_CHAN_TREMFRQ, 24); /* 1 Hz */ emu_wrptr(sc, ch, EMU_CHAN_FM2FRQ2, 24); /* 1 Hz */ emu_wrptr(sc, ch, EMU_CHAN_TEMPENV, 0); /*** these are last so OFF prevents writing ***/ emu_wrptr(sc, ch, EMU_CHAN_LFOVAL2, 0); emu_wrptr(sc, ch, EMU_CHAN_LFOVAL1, 0); emu_wrptr(sc, ch, EMU_CHAN_ATKHLDV, 0); emu_wrptr(sc, ch, EMU_CHAN_ENVVOL, 0); emu_wrptr(sc, ch, EMU_CHAN_ENVVAL, 0); if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) { emu_wrptr(sc, ch, 0x4c, 0x0); emu_wrptr(sc, ch, 0x4d, 0x0); emu_wrptr(sc, ch, 0x4e, 0x0); emu_wrptr(sc, ch, 0x4f, 0x0); emu_wrptr(sc, ch, EMU_A_CHAN_FXRT1, 0x3f3f3f3f); emu_wrptr(sc, ch, EMU_A_CHAN_FXRT2, 0x3f3f3f3f); emu_wrptr(sc, ch, EMU_A_CHAN_SENDAMOUNTS, 0x0); } } emumix_set_spdif_mode(sc, SPDIF_MODE_PCM); if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) emu_wrptr(sc, 0, EMU_A_SPDIF_SAMPLERATE, EMU_A_SPDIF_48000); /* * CAxxxx cards needs additional setup: * 1. Set I2S capture sample rate to 96000 * 2. Disable P16v / P17v proceesing * 3. Allow EMU10K DSP inputs */ if ((sc->is_ca0102) || (sc->is_ca0108)) { spdif_sr = emu_rdptr(sc, 0, EMU_A_SPDIF_SAMPLERATE); spdif_sr &= 0xfffff1ff; spdif_sr |= EMU_A_I2S_CAPTURE_96000; emu_wrptr(sc, 0, EMU_A_SPDIF_SAMPLERATE, spdif_sr); /* Disable P16v processing */ emu_wr_p16vptr(sc, 0, EMU_A2_SRCSel, 0x14); /* Setup P16v/P17v sound routing */ if (sc->is_ca0102) emu_wr_p16vptr(sc, 0, EMU_A2_SRCMULTI_ENABLE, 0xFF00FF00); else { emu_wr_p16vptr(sc, 0, EMU_A2_MIXER_I2S_ENABLE, 0xFF000000); emu_wr_p16vptr(sc, 0, EMU_A2_MIXER_SPDIF_ENABLE, 0xFF000000); tmp = emu_rd(sc, EMU_A_IOCFG, 2); emu_wr(sc, EMU_A_IOCFG, tmp & ~0x8, 2); } } emu_initefx(sc); def_mode = MODE_ANALOG; if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) def_mode = MODE_DIGITAL; if (((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) && (sc->broken_digital)) { device_printf(sc->dev, "Audigy card initialized in analog mode.\n"); def_mode = MODE_ANALOG; } emumix_set_mode(sc, def_mode); if (bootverbose) { tmp = emu_rd(sc, EMU_HCFG, 4); device_printf(sc->dev, "Card Configuration ( 0x%08x )\n", tmp); device_printf(sc->dev, "Card Configuration ( & 0xff000000 ) : %s%s%s%s%s%s%s%s\n", (tmp & 0x80000000 ? "[Legacy MPIC] " : ""), (tmp & 0x40000000 ? "[0x40] " : ""), (tmp & 0x20000000 ? "[0x20] " : ""), (tmp & 0x10000000 ? "[0x10] " : ""), (tmp & 0x08000000 ? "[0x08] " : ""), (tmp & 0x04000000 ? "[0x04] " : ""), (tmp & 0x02000000 ? "[0x02] " : ""), (tmp & 0x01000000 ? "[0x01]" : " ")); device_printf(sc->dev, "Card Configuration ( & 0x00ff0000 ) : %s%s%s%s%s%s%s%s\n", (tmp & 0x00800000 ? "[0x80] " : ""), (tmp & 0x00400000 ? "[0x40] " : ""), (tmp & 0x00200000 ? "[Legacy INT] " : ""), (tmp & 0x00100000 ? "[0x10] " : ""), (tmp & 0x00080000 ? "[0x08] " : ""), (tmp & 0x00040000 ? "[Codec4] " : ""), (tmp & 0x00020000 ? "[Codec2] " : ""), (tmp & 0x00010000 ? "[I2S Codec]" : " ")); device_printf(sc->dev, "Card Configuration ( & 0x0000ff00 ) : %s%s%s%s%s%s%s%s\n", (tmp & 0x00008000 ? "[0x80] " : ""), (tmp & 0x00004000 ? "[GPINPUT0] " : ""), (tmp & 0x00002000 ? "[GPINPUT1] " : ""), (tmp & 0x00001000 ? "[GPOUT0] " : ""), (tmp & 0x00000800 ? "[GPOUT1] " : ""), (tmp & 0x00000400 ? "[GPOUT2] " : ""), (tmp & 0x00000200 ? "[Joystick] " : ""), (tmp & 0x00000100 ? "[0x01]" : " ")); device_printf(sc->dev, "Card Configuration ( & 0x000000ff ) : %s%s%s%s%s%s%s%s\n", (tmp & 0x00000080 ? "[0x80] " : ""), (tmp & 0x00000040 ? "[0x40] " : ""), (tmp & 0x00000020 ? "[0x20] " : ""), (tmp & 0x00000010 ? "[AUTOMUTE] " : ""), (tmp & 0x00000008 ? "[LOCKSOUNDCACHE] " : ""), (tmp & 0x00000004 ? "[LOCKTANKCACHE] " : ""), (tmp & 0x00000002 ? "[MUTEBUTTONENABLE] " : ""), (tmp & 0x00000001 ? "[AUDIOENABLE]" : " ")); if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) { tmp = emu_rd(sc, EMU_A_IOCFG, 2); device_printf(sc->dev, "Audigy Card Configuration ( 0x%04x )\n", tmp); device_printf(sc->dev, "Audigy Card Configuration ( & 0xff00 )"); printf(" : %s%s%s%s%s%s%s%s\n", (tmp & 0x8000 ? "[Rear Speakers] " : ""), (tmp & 0x4000 ? "[Front Speakers] " : ""), (tmp & 0x2000 ? "[0x20] " : ""), (tmp & 0x1000 ? "[0x10] " : ""), (tmp & 0x0800 ? "[0x08] " : ""), (tmp & 0x0400 ? "[0x04] " : ""), (tmp & 0x0200 ? "[0x02] " : ""), (tmp & 0x0100 ? "[AudigyDrive Phones]" : " ")); device_printf(sc->dev, "Audigy Card Configuration ( & 0x00ff )"); printf(" : %s%s%s%s%s%s%s%s\n", (tmp & 0x0080 ? "[0x80] " : ""), (tmp & 0x0040 ? "[Mute AnalogOut] " : ""), (tmp & 0x0020 ? "[0x20] " : ""), (tmp & 0x0010 ? "[0x10] " : ""), (tmp & 0x0008 ? "[0x08] " : ""), (tmp & 0x0004 ? "[GPOUT0] " : ""), (tmp & 0x0002 ? "[GPOUT1] " : ""), (tmp & 0x0001 ? "[GPOUT2]" : " ")); } /* is_emu10k2 or ca* */ } /* bootverbose */ return (0); } static int emu_uninit(struct emu_sc_info *sc) { uint32_t ch; struct emu_memblk *blk; emu_wr(sc, EMU_INTE, 0, 4); for (ch = 0; ch < NUM_G; ch++) emu_wrptr(sc, ch, EMU_CHAN_DCYSUSV, 0); for (ch = 0; ch < NUM_G; ch++) { emu_wrptr(sc, ch, EMU_CHAN_VTFT, 0); emu_wrptr(sc, ch, EMU_CHAN_CVCF, 0); emu_wrptr(sc, ch, EMU_CHAN_PTRX, 0); emu_wrptr(sc, ch, EMU_CHAN_CPF, 0); } /* disable audio and lock cache */ emu_wr(sc, EMU_HCFG, EMU_HCFG_LOCKSOUNDCACHE | EMU_HCFG_LOCKTANKCACHE_MASK | EMU_HCFG_MUTEBUTTONENABLE, 4); emu_wrptr(sc, 0, EMU_PTB, 0); /* reset recording buffers */ emu_wrptr(sc, 0, EMU_MICBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_MICBA, 0); emu_wrptr(sc, 0, EMU_FXBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_FXBA, 0); emu_wrptr(sc, 0, EMU_FXWC, 0); emu_wrptr(sc, 0, EMU_ADCBS, EMU_RECBS_BUFSIZE_NONE); emu_wrptr(sc, 0, EMU_ADCBA, 0); emu_wrptr(sc, 0, EMU_TCB, 0); emu_wrptr(sc, 0, EMU_TCBS, 0); /* disable channel interrupt */ emu_wrptr(sc, 0, EMU_CLIEL, 0); emu_wrptr(sc, 0, EMU_CLIEH, 0); emu_wrptr(sc, 0, EMU_SOLEL, 0); emu_wrptr(sc, 0, EMU_SOLEH, 0); if (!SLIST_EMPTY(&sc->mem.blocks)) device_printf(sc->dev, "warning: memblock list not empty\n"); SLIST_FOREACH(blk, &sc->mem.blocks, link) if (blk != NULL) device_printf(sc->dev, "lost %d for %s\n", blk->pte_size, blk->owner); emu_free(&sc->mem, sc->mem.ptb_pages, sc->mem.ptb_map); emu_free(&sc->mem, sc->mem.silent_page, sc->mem.silent_map); return (0); } static int emu_read_ivar(device_t bus, device_t dev, int ivar_index, uintptr_t * result) { struct sndcard_func *func = device_get_ivars(dev); struct emu_sc_info *sc = device_get_softc(bus); if (func==NULL) return (ENOMEM); if (sc == NULL) return (ENOMEM); switch (ivar_index) { case EMU_VAR_FUNC: *result = func->func; break; case EMU_VAR_ROUTE: if (func->varinfo == NULL) return (ENOMEM); *result = ((struct emu_pcminfo *)func->varinfo)->route; break; case EMU_VAR_ISEMU10K1: *result = sc->is_emu10k1; break; case EMU_VAR_MCH_DISABLED: *result = sc->mch_disabled; break; case EMU_VAR_MCH_REC: *result = sc->mch_rec; break; default: return (ENOENT); } return (0); } static int emu_write_ivar(device_t bus __unused, device_t dev __unused, int ivar_index, uintptr_t value __unused) { switch (ivar_index) { case 0: return (EINVAL); default: return (ENOENT); } } static int emu_pci_probe(device_t dev) { struct sbuf *s; unsigned int thiscard = 0; uint16_t vendor; vendor = pci_read_config(dev, PCIR_DEVVENDOR, /* bytes */ 2); if (vendor != 0x1102) return (ENXIO); /* Not Creative */ thiscard = emu_getcard(dev); if (thiscard == 0) return (ENXIO); s = sbuf_new(NULL, NULL, 4096, 0); if (s == NULL) return (ENOMEM); sbuf_printf(s, "Creative %s [%s]", emu_cards[thiscard].desc, emu_cards[thiscard].SBcode); sbuf_finish(s); device_set_desc_copy(dev, sbuf_data(s)); sbuf_delete(s); return (BUS_PROBE_DEFAULT); } static int emu_pci_attach(device_t dev) { struct sndcard_func *func; struct emu_sc_info *sc; struct emu_pcminfo *pcminfo; #if 0 struct emu_midiinfo *midiinfo; #endif int i; int device_flags; char status[255]; int error = ENXIO; int unit; sc = device_get_softc(dev); unit = device_get_unit(dev); /* Get configuration */ sc->ctx = device_get_sysctl_ctx(dev); if (sc->ctx == NULL) goto bad; sc->root = device_get_sysctl_tree(dev); if (sc->root == NULL) goto bad; if (resource_int_value("emu10kx", unit, "multichannel_disabled", &(sc->mch_disabled))) RANGE(sc->mch_disabled, 0, 1); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "multichannel_disabled", CTLFLAG_RD, &(sc->mch_disabled), 0, "Multichannel playback setting"); if (resource_int_value("emu10kx", unit, "multichannel_recording", &(sc->mch_rec))) RANGE(sc->mch_rec, 0, 1); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "multichannel_recording", CTLFLAG_RD, &(sc->mch_rec), 0, "Multichannel recording setting"); if (resource_int_value("emu10kx", unit, "debug", &(sc->dbg_level))) RANGE(sc->mch_rec, 0, 2); SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug", CTLFLAG_RW, &(sc->dbg_level), 0, "Debug level"); /* Fill in the softc. */ mtx_init(&sc->lock, device_get_nameunit(dev), "bridge conf", MTX_DEF); mtx_init(&sc->rw, device_get_nameunit(dev), "exclusive io", MTX_DEF); sc->dev = dev; sc->type = pci_get_devid(dev); sc->rev = pci_get_revid(dev); sc->enable_ir = 0; sc->has_ac97 = 0; sc->has_51 = 0; sc->has_71 = 0; sc->broken_digital = 0; sc->is_emu10k1 = 0; sc->is_emu10k2 = 0; sc->is_ca0102 = 0; sc->is_ca0108 = 0; sc->is_cardbus = 0; device_flags = emu_cards[emu_getcard(dev)].flags; if (device_flags & HAS_51) sc->has_51 = 1; if (device_flags & HAS_71) { sc->has_51 = 1; sc->has_71 = 1; } if (device_flags & IS_EMU10K1) sc->is_emu10k1 = 1; if (device_flags & IS_EMU10K2) sc->is_emu10k2 = 1; if (device_flags & IS_CA0102) sc->is_ca0102 = 1; if (device_flags & IS_CA0108) sc->is_ca0108 = 1; if ((sc->is_emu10k2) && (sc->rev == 4)) { sc->is_emu10k2 = 0; sc->is_ca0102 = 1; /* for unknown Audigy 2 cards */ } if ((sc->is_ca0102 == 1) || (sc->is_ca0108 == 1)) if (device_flags & IS_CARDBUS) sc->is_cardbus = 1; if ((sc->is_emu10k1 + sc->is_emu10k2 + sc->is_ca0102 + sc->is_ca0108) != 1) { device_printf(sc->dev, "Unable to detect HW chipset\n"); goto bad; } if (device_flags & BROKEN_DIGITAL) sc->broken_digital = 1; if (device_flags & HAS_AC97) sc->has_ac97 = 1; sc->opcode_shift = 0; if ((sc->is_emu10k2) || (sc->is_ca0102) || (sc->is_ca0108)) { sc->opcode_shift = 24; sc->high_operand_shift = 12; /* DSP map */ /* sc->fx_base = 0x0 */ sc->input_base = 0x40; /* sc->p16vinput_base = 0x50; */ sc->output_base = 0x60; sc->efxc_base = 0x80; /* sc->output32h_base = 0xa0; */ /* sc->output32l_base = 0xb0; */ sc->dsp_zero = 0xc0; /* 0xe0...0x100 are unknown */ /* sc->tram_base = 0x200 */ /* sc->tram_addr_base = 0x300 */ sc->gpr_base = EMU_A_FXGPREGBASE; sc->num_gprs = 0x200; sc->code_base = EMU_A_MICROCODEBASE; sc->code_size = 0x800 / 2; /* 0x600-0xdff, 2048 words, * 1024 instructions */ sc->mchannel_fx = 8; sc->num_fxbuses = 16; sc->num_inputs = 8; sc->num_outputs = 16; sc->address_mask = EMU_A_PTR_ADDR_MASK; } if (sc->is_emu10k1) { sc->has_51 = 0; /* We don't support 5.1 sound on SB Live! 5.1 */ sc->opcode_shift = 20; sc->high_operand_shift = 10; sc->code_base = EMU_MICROCODEBASE; sc->code_size = 0x400 / 2; /* 0x400-0x7ff, 1024 words, * 512 instructions */ sc->gpr_base = EMU_FXGPREGBASE; sc->num_gprs = 0x100; sc->input_base = 0x10; sc->output_base = 0x20; /* * XXX 5.1 Analog outputs are inside efxc address space! * They use output+0x11/+0x12 (=efxc+1/+2). * Don't use this efx registers for recording on SB Live! 5.1! */ sc->efxc_base = 0x30; sc->dsp_zero = 0x40; sc->mchannel_fx = 0; sc->num_fxbuses = 8; sc->num_inputs = 8; sc->num_outputs = 16; sc->address_mask = EMU_PTR_ADDR_MASK; } if (sc->opcode_shift == 0) goto bad; pci_enable_busmaster(dev); i = PCIR_BAR(0); sc->reg = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &i, RF_ACTIVE); if (sc->reg == NULL) { device_printf(dev, "unable to map register space\n"); goto bad; } sc->st = rman_get_bustag(sc->reg); sc->sh = rman_get_bushandle(sc->reg); for (i = 0; i < EMU_MAX_IRQ_CONSUMERS; i++) sc->timer[i] = 0; /* disable it */ i = 0; sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i, RF_ACTIVE | RF_SHAREABLE); if ((sc->irq == NULL) || bus_setup_intr(dev, sc->irq, INTR_MPSAFE | INTR_TYPE_AV, NULL, emu_intr, sc, &sc->ih)) { device_printf(dev, "unable to map interrupt\n"); goto bad; } if (emu_rm_init(sc) != 0) { device_printf(dev, "unable to create resource manager\n"); goto bad; } if (sc->is_cardbus) if (emu_cardbus_init(sc) != 0) { device_printf(dev, "unable to initialize CardBus interface\n"); goto bad; } if (emu_init(sc) != 0) { device_printf(dev, "unable to initialize the card\n"); goto bad; } if (emu10kx_dev_init(sc) != 0) { device_printf(dev, "unable to create control device\n"); goto bad; } snprintf(status, 255, "rev %d at io 0x%jx irq %jd", sc->rev, rman_get_start(sc->reg), rman_get_start(sc->irq)); /* Voices */ for (i = 0; i < NUM_G; i++) { sc->voice[i].vnum = i; sc->voice[i].slave = NULL; sc->voice[i].busy = 0; sc->voice[i].ismaster = 0; sc->voice[i].running = 0; sc->voice[i].b16 = 0; sc->voice[i].stereo = 0; sc->voice[i].speed = 0; sc->voice[i].start = 0; sc->voice[i].end = 0; } /* PCM Audio */ for (i = 0; i < RT_COUNT; i++) sc->pcm[i] = NULL; /* FRONT */ func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_FRONT; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_FRONT] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_FRONT], func); if (!(sc->mch_disabled)) { /* REAR */ func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_REAR; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_REAR] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_REAR], func); if (sc->has_51) { /* CENTER */ func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_CENTER; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_CENTER] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_CENTER], func); /* SUB */ func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_SUB; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_SUB] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_SUB], func); } if (sc->has_71) { /* SIDE */ func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_SIDE; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_SIDE] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_SIDE], func); } } /* mch_disabled */ if (sc->mch_rec) { func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } pcminfo = malloc(sizeof(struct emu_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (pcminfo == NULL) { error = ENOMEM; goto bad; } pcminfo->card = sc; pcminfo->route = RT_MCHRECORD; func->func = SCF_PCM; func->varinfo = pcminfo; sc->pcm[RT_MCHRECORD] = device_add_child(dev, "pcm", -1); device_set_ivars(sc->pcm[RT_MCHRECORD], func); } /*mch_rec */ for (i = 0; i < 2; i++) sc->midi[i] = NULL; /* MIDI has some memory mangament and (possible) locking problems */ #if 0 /* Midi Interface 1: Live!, Audigy, Audigy 2 */ if ((sc->is_emu10k1) || (sc->is_emu10k2) || (sc->is_ca0102)) { func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } midiinfo = malloc(sizeof(struct emu_midiinfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (midiinfo == NULL) { error = ENOMEM; goto bad; } midiinfo->card = sc; if (sc->is_emu10k2 || (sc->is_ca0102)) { midiinfo->port = EMU_A_MUDATA1; midiinfo->portnr = 1; } if (sc->is_emu10k1) { midiinfo->port = MUDATA; midiinfo->portnr = 1; } func->func = SCF_MIDI; func->varinfo = midiinfo; sc->midi[0] = device_add_child(dev, "midi", -1); device_set_ivars(sc->midi[0], func); } /* Midi Interface 2: Audigy, Audigy 2 (on AudigyDrive) */ if (sc->is_emu10k2 || (sc->is_ca0102)) { func = malloc(sizeof(struct sndcard_func), M_DEVBUF, M_NOWAIT | M_ZERO); if (func == NULL) { error = ENOMEM; goto bad; } midiinfo = malloc(sizeof(struct emu_midiinfo), M_DEVBUF, M_NOWAIT | M_ZERO); if (midiinfo == NULL) { error = ENOMEM; goto bad; } midiinfo->card = sc; midiinfo->port = EMU_A_MUDATA2; midiinfo->portnr = 2; func->func = SCF_MIDI; func->varinfo = midiinfo; sc->midi[1] = device_add_child(dev, "midi", -1); device_set_ivars(sc->midi[1], func); } #endif return (bus_generic_attach(dev)); bad: /* XXX can we just call emu_pci_detach here? */ if (sc->cdev) emu10kx_dev_uninit(sc); if (sc->rm != NULL) emu_rm_uninit(sc); if (sc->reg) bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), sc->reg); if (sc->ih) bus_teardown_intr(dev, sc->irq, sc->ih); if (sc->irq) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->irq); mtx_destroy(&sc->rw); mtx_destroy(&sc->lock); return (error); } static int emu_pci_detach(device_t dev) { struct emu_sc_info *sc; struct sndcard_func *func; int devcount, i; device_t *childlist; int r = 0; sc = device_get_softc(dev); for (i = 0; i < RT_COUNT; i++) { if (sc->pcm[i] != NULL) { func = device_get_ivars(sc->pcm[i]); if (func != NULL && func->func == SCF_PCM) { device_set_ivars(sc->pcm[i], NULL); free(func->varinfo, M_DEVBUF); free(func, M_DEVBUF); } r = device_delete_child(dev, sc->pcm[i]); if (r) return (r); } } if (sc->midi[0] != NULL) { func = device_get_ivars(sc->midi[0]); if (func != NULL && func->func == SCF_MIDI) { device_set_ivars(sc->midi[0], NULL); free(func->varinfo, M_DEVBUF); free(func, M_DEVBUF); } r = device_delete_child(dev, sc->midi[0]); if (r) return (r); } if (sc->midi[1] != NULL) { func = device_get_ivars(sc->midi[1]); if (func != NULL && func->func == SCF_MIDI) { device_set_ivars(sc->midi[1], NULL); free(func->varinfo, M_DEVBUF); free(func, M_DEVBUF); } r = device_delete_child(dev, sc->midi[1]); if (r) return (r); } if (device_get_children(dev, &childlist, &devcount) == 0) for (i = 0; i < devcount - 1; i++) { device_printf(dev, "removing stale child %d (unit %d)\n", i, device_get_unit(childlist[i])); func = device_get_ivars(childlist[i]); if (func != NULL && (func->func == SCF_MIDI || func->func == SCF_PCM)) { device_set_ivars(childlist[i], NULL); free(func->varinfo, M_DEVBUF); free(func, M_DEVBUF); } device_delete_child(dev, childlist[i]); } if (childlist != NULL) free(childlist, M_TEMP); r = emu10kx_dev_uninit(sc); if (r) return (r); /* shutdown chip */ emu_uninit(sc); emu_rm_uninit(sc); if (sc->mem.dmat) bus_dma_tag_destroy(sc->mem.dmat); if (sc->reg) bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), sc->reg); bus_teardown_intr(dev, sc->irq, sc->ih); bus_release_resource(dev, SYS_RES_IRQ, 0, sc->irq); mtx_destroy(&sc->rw); mtx_destroy(&sc->lock); return (bus_generic_detach(dev)); } /* add suspend, resume */ static device_method_t emu_methods[] = { /* Device interface */ DEVMETHOD(device_probe, emu_pci_probe), DEVMETHOD(device_attach, emu_pci_attach), DEVMETHOD(device_detach, emu_pci_detach), /* Bus methods */ DEVMETHOD(bus_read_ivar, emu_read_ivar), DEVMETHOD(bus_write_ivar, emu_write_ivar), DEVMETHOD_END }; static driver_t emu_driver = { "emu10kx", emu_methods, sizeof(struct emu_sc_info), NULL, 0, NULL }; static int emu_modevent(module_t mod __unused, int cmd, void *data __unused) { int err = 0; switch (cmd) { case MOD_LOAD: break; /* Success */ case MOD_UNLOAD: case MOD_SHUTDOWN: /* XXX Should we check state of pcm & midi subdevices here? */ break; /* Success */ default: err = EINVAL; break; } return (err); } static devclass_t emu_devclass; DRIVER_MODULE(snd_emu10kx, pci, emu_driver, emu_devclass, emu_modevent, NULL); MODULE_VERSION(snd_emu10kx, SND_EMU10KX_PREFVER); Index: head/sys/dev/uart/uart_subr.c =================================================================== --- head/sys/dev/uart/uart_subr.c (revision 298410) +++ head/sys/dev/uart/uart_subr.c (revision 298411) @@ -1,321 +1,320 @@ /*- * Copyright (c) 2004 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #define UART_TAG_BR 0 #define UART_TAG_CH 1 #define UART_TAG_DB 2 #define UART_TAG_DT 3 #define UART_TAG_IO 4 #define UART_TAG_MM 5 #define UART_TAG_PA 6 #define UART_TAG_RS 7 #define UART_TAG_SB 8 #define UART_TAG_XO 9 static struct uart_class *uart_classes[] = { &uart_ns8250_class, &uart_sab82532_class, &uart_z8530_class, #if defined(__arm__) &uart_s3c2410_class, #endif }; -static size_t uart_nclasses = sizeof(uart_classes) / sizeof(uart_classes[0]); static bus_addr_t uart_parse_addr(const char **p) { return (strtoul(*p, (char**)(uintptr_t)p, 0)); } static struct uart_class * uart_parse_class(struct uart_class *class, const char **p) { struct uart_class *uc; const char *nm; size_t len; u_int i; - for (i = 0; i < uart_nclasses; i++) { + for (i = 0; i < nitems(uart_classes); i++) { uc = uart_classes[i]; nm = uart_getname(uc); if (nm == NULL || *nm == '\0') continue; len = strlen(nm); if (strncmp(nm, *p, len) == 0) { *p += len; return (uc); } } return (class); } static long uart_parse_long(const char **p) { return (strtol(*p, (char**)(uintptr_t)p, 0)); } static int uart_parse_parity(const char **p) { if (!strncmp(*p, "even", 4)) { *p += 4; return UART_PARITY_EVEN; } if (!strncmp(*p, "mark", 4)) { *p += 4; return UART_PARITY_MARK; } if (!strncmp(*p, "none", 4)) { *p += 4; return UART_PARITY_NONE; } if (!strncmp(*p, "odd", 3)) { *p += 3; return UART_PARITY_ODD; } if (!strncmp(*p, "space", 5)) { *p += 5; return UART_PARITY_SPACE; } return (-1); } static int uart_parse_tag(const char **p) { int tag; if ((*p)[0] == 'b' && (*p)[1] == 'r') { tag = UART_TAG_BR; goto out; } if ((*p)[0] == 'c' && (*p)[1] == 'h') { tag = UART_TAG_CH; goto out; } if ((*p)[0] == 'd' && (*p)[1] == 'b') { tag = UART_TAG_DB; goto out; } if ((*p)[0] == 'd' && (*p)[1] == 't') { tag = UART_TAG_DT; goto out; } if ((*p)[0] == 'i' && (*p)[1] == 'o') { tag = UART_TAG_IO; goto out; } if ((*p)[0] == 'm' && (*p)[1] == 'm') { tag = UART_TAG_MM; goto out; } if ((*p)[0] == 'p' && (*p)[1] == 'a') { tag = UART_TAG_PA; goto out; } if ((*p)[0] == 'r' && (*p)[1] == 's') { tag = UART_TAG_RS; goto out; } if ((*p)[0] == 's' && (*p)[1] == 'b') { tag = UART_TAG_SB; goto out; } if ((*p)[0] == 'x' && (*p)[1] == 'o') { tag = UART_TAG_XO; goto out; } return (-1); out: *p += 2; if ((*p)[0] != ':') return (-1); (*p)++; return (tag); } /* * Parse a device specification. The specification is a list of attributes * separated by commas. Each attribute is a tag-value pair with the tag and * value separated by a colon. Supported tags are: * * br = Baudrate * ch = Channel * db = Data bits * dt = Device type * io = I/O port address * mm = Memory mapped I/O address * pa = Parity * rs = Register shift * sb = Stopbits * xo = Device clock (xtal oscillator) * * The io and mm tags are mutually exclusive. */ int uart_getenv(int devtype, struct uart_devinfo *di, struct uart_class *class) { const char *spec; char *cp; bus_addr_t addr = ~0U; int error; /* * All uart_class references are weak. Make sure the default * device class has been compiled-in. */ if (class == NULL) return (ENXIO); /* * Check the environment variables "hw.uart.console" and * "hw.uart.dbgport". These variables, when present, specify * which UART port is to be used as serial console or debug * port (resp). */ switch (devtype) { case UART_DEV_CONSOLE: cp = kern_getenv("hw.uart.console"); break; case UART_DEV_DBGPORT: cp = kern_getenv("hw.uart.dbgport"); break; default: cp = NULL; break; } if (cp == NULL) return (ENXIO); /* Set defaults. */ di->bas.chan = 0; di->bas.regshft = 0; di->bas.rclk = 0; di->baudrate = 0; di->databits = 8; di->stopbits = 1; di->parity = UART_PARITY_NONE; /* Parse the attributes. */ spec = cp; for (;;) { switch (uart_parse_tag(&spec)) { case UART_TAG_BR: di->baudrate = uart_parse_long(&spec); break; case UART_TAG_CH: di->bas.chan = uart_parse_long(&spec); break; case UART_TAG_DB: di->databits = uart_parse_long(&spec); break; case UART_TAG_DT: class = uart_parse_class(class, &spec); break; case UART_TAG_IO: di->bas.bst = uart_bus_space_io; addr = uart_parse_addr(&spec); break; case UART_TAG_MM: di->bas.bst = uart_bus_space_mem; addr = uart_parse_addr(&spec); break; case UART_TAG_PA: di->parity = uart_parse_parity(&spec); break; case UART_TAG_RS: di->bas.regshft = uart_parse_long(&spec); break; case UART_TAG_SB: di->stopbits = uart_parse_long(&spec); break; case UART_TAG_XO: di->bas.rclk = uart_parse_long(&spec); break; default: freeenv(cp); return (EINVAL); } if (*spec == '\0') break; if (*spec != ',') { freeenv(cp); return (EINVAL); } spec++; } freeenv(cp); /* * If we still have an invalid address, the specification must be * missing an I/O port or memory address. We don't like that. */ if (addr == ~0U) return (EINVAL); /* * Accept only the well-known baudrates. Any invalid baudrate * is silently replaced with a 0-valued baudrate. The 0 baudrate * has special meaning. It means that we're not supposed to * program the baudrate and simply communicate with whatever * speed the hardware is currently programmed for. */ if (di->baudrate >= 19200) { if (di->baudrate % 19200) di->baudrate = 0; } else if (di->baudrate >= 1200) { if (di->baudrate % 1200) di->baudrate = 0; } else if (di->baudrate > 0) { if (di->baudrate % 75) di->baudrate = 0; } else di->baudrate = 0; /* Set the ops and create a bus space handle. */ di->ops = uart_getops(class); error = bus_space_map(di->bas.bst, addr, uart_getrange(class), 0, &di->bas.bsh); return (error); } Index: head/sys/kern/subr_witness.c =================================================================== --- head/sys/kern/subr_witness.c (revision 298410) +++ head/sys/kern/subr_witness.c (revision 298411) @@ -1,3019 +1,3018 @@ /*- * Copyright (c) 2008 Isilon Systems, Inc. * Copyright (c) 2008 Ilya Maykov * Copyright (c) 1998 Berkeley Software Design, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI $Id: mutex_witness.c,v 1.1.2.20 2000/04/27 03:10:27 cp Exp $ * and BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ */ /* * Implementation of the `witness' lock verifier. Originally implemented for * mutexes in BSD/OS. Extended to handle generic lock objects and lock * classes in FreeBSD. */ /* * Main Entry: witness * Pronunciation: 'wit-n&s * Function: noun * Etymology: Middle English witnesse, from Old English witnes knowledge, * testimony, witness, from 2wit * Date: before 12th century * 1 : attestation of a fact or event : TESTIMONY * 2 : one that gives evidence; specifically : one who testifies in * a cause or before a judicial tribunal * 3 : one asked to be present at a transaction so as to be able to * testify to its having taken place * 4 : one who has personal knowledge of something * 5 a : something serving as evidence or proof : SIGN * b : public affirmation by word or example of usually * religious faith or conviction * 6 capitalized : a member of the Jehovah's Witnesses */ /* * Special rules concerning Giant and lock orders: * * 1) Giant must be acquired before any other mutexes. Stated another way, * no other mutex may be held when Giant is acquired. * * 2) Giant must be released when blocking on a sleepable lock. * * This rule is less obvious, but is a result of Giant providing the same * semantics as spl(). Basically, when a thread sleeps, it must release * Giant. When a thread blocks on a sleepable lock, it sleeps. Hence rule * 2). * * 3) Giant may be acquired before or after sleepable locks. * * This rule is also not quite as obvious. Giant may be acquired after * a sleepable lock because it is a non-sleepable lock and non-sleepable * locks may always be acquired while holding a sleepable lock. The second * case, Giant before a sleepable lock, follows from rule 2) above. Suppose * you have two threads T1 and T2 and a sleepable lock X. Suppose that T1 * acquires X and blocks on Giant. Then suppose that T2 acquires Giant and * blocks on X. When T2 blocks on X, T2 will release Giant allowing T1 to * execute. Thus, acquiring Giant both before and after a sleepable lock * will not result in a lock order reversal. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_hwpmc_hooks.h" #include "opt_stack.h" #include "opt_witness.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #include #if !defined(DDB) && !defined(STACK) #error "DDB or STACK options are required for WITNESS" #endif /* Note that these traces do not work with KTR_ALQ. */ #if 0 #define KTR_WITNESS KTR_SUBSYS #else #define KTR_WITNESS 0 #endif #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */ #define LI_NORELEASE 0x00020000 /* Lock not allowed to be released. */ /* Define this to check for blessed mutexes */ #undef BLESSING #ifndef WITNESS_COUNT #define WITNESS_COUNT 1536 #endif #define WITNESS_HASH_SIZE 251 /* Prime, gives load factor < 2 */ #define WITNESS_PENDLIST (1024 + MAXCPU) /* Allocate 256 KB of stack data space */ #define WITNESS_LO_DATA_COUNT 2048 /* Prime, gives load factor of ~2 at full load */ #define WITNESS_LO_HASH_SIZE 1021 /* * XXX: This is somewhat bogus, as we assume here that at most 2048 threads * will hold LOCK_NCHILDREN locks. We handle failure ok, and we should * probably be safe for the most part, but it's still a SWAG. */ #define LOCK_NCHILDREN 5 #define LOCK_CHILDCOUNT 2048 #define MAX_W_NAME 64 #define FULLGRAPH_SBUF_SIZE 512 /* * These flags go in the witness relationship matrix and describe the * relationship between any two struct witness objects. */ #define WITNESS_UNRELATED 0x00 /* No lock order relation. */ #define WITNESS_PARENT 0x01 /* Parent, aka direct ancestor. */ #define WITNESS_ANCESTOR 0x02 /* Direct or indirect ancestor. */ #define WITNESS_CHILD 0x04 /* Child, aka direct descendant. */ #define WITNESS_DESCENDANT 0x08 /* Direct or indirect descendant. */ #define WITNESS_ANCESTOR_MASK (WITNESS_PARENT | WITNESS_ANCESTOR) #define WITNESS_DESCENDANT_MASK (WITNESS_CHILD | WITNESS_DESCENDANT) #define WITNESS_RELATED_MASK \ (WITNESS_ANCESTOR_MASK | WITNESS_DESCENDANT_MASK) #define WITNESS_REVERSAL 0x10 /* A lock order reversal has been * observed. */ #define WITNESS_RESERVED1 0x20 /* Unused flag, reserved. */ #define WITNESS_RESERVED2 0x40 /* Unused flag, reserved. */ #define WITNESS_LOCK_ORDER_KNOWN 0x80 /* This lock order is known. */ /* Descendant to ancestor flags */ #define WITNESS_DTOA(x) (((x) & WITNESS_RELATED_MASK) >> 2) /* Ancestor to descendant flags */ #define WITNESS_ATOD(x) (((x) & WITNESS_RELATED_MASK) << 2) #define WITNESS_INDEX_ASSERT(i) \ MPASS((i) > 0 && (i) <= w_max_used_index && (i) < witness_count) static MALLOC_DEFINE(M_WITNESS, "Witness", "Witness"); /* * Lock instances. A lock instance is the data associated with a lock while * it is held by witness. For example, a lock instance will hold the * recursion count of a lock. Lock instances are held in lists. Spin locks * are held in a per-cpu list while sleep locks are held in per-thread list. */ struct lock_instance { struct lock_object *li_lock; const char *li_file; int li_line; u_int li_flags; }; /* * A simple list type used to build the list of locks held by a thread * or CPU. We can't simply embed the list in struct lock_object since a * lock may be held by more than one thread if it is a shared lock. Locks * are added to the head of the list, so we fill up each list entry from * "the back" logically. To ease some of the arithmetic, we actually fill * in each list entry the normal way (children[0] then children[1], etc.) but * when we traverse the list we read children[count-1] as the first entry * down to children[0] as the final entry. */ struct lock_list_entry { struct lock_list_entry *ll_next; struct lock_instance ll_children[LOCK_NCHILDREN]; u_int ll_count; }; /* * The main witness structure. One of these per named lock type in the system * (for example, "vnode interlock"). */ struct witness { char w_name[MAX_W_NAME]; uint32_t w_index; /* Index in the relationship matrix */ struct lock_class *w_class; STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */ STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */ struct witness *w_hash_next; /* Linked list in hash buckets. */ const char *w_file; /* File where last acquired */ uint32_t w_line; /* Line where last acquired */ uint32_t w_refcount; uint16_t w_num_ancestors; /* direct/indirect * ancestor count */ uint16_t w_num_descendants; /* direct/indirect * descendant count */ int16_t w_ddb_level; unsigned w_displayed:1; unsigned w_reversed:1; }; STAILQ_HEAD(witness_list, witness); /* * The witness hash table. Keys are witness names (const char *), elements are * witness objects (struct witness *). */ struct witness_hash { struct witness *wh_array[WITNESS_HASH_SIZE]; uint32_t wh_size; uint32_t wh_count; }; /* * Key type for the lock order data hash table. */ struct witness_lock_order_key { uint16_t from; uint16_t to; }; struct witness_lock_order_data { struct stack wlod_stack; struct witness_lock_order_key wlod_key; struct witness_lock_order_data *wlod_next; }; /* * The witness lock order data hash table. Keys are witness index tuples * (struct witness_lock_order_key), elements are lock order data objects * (struct witness_lock_order_data). */ struct witness_lock_order_hash { struct witness_lock_order_data *wloh_array[WITNESS_LO_HASH_SIZE]; u_int wloh_size; u_int wloh_count; }; #ifdef BLESSING struct witness_blessed { const char *b_lock1; const char *b_lock2; }; #endif struct witness_pendhelp { const char *wh_type; struct lock_object *wh_lock; }; struct witness_order_list_entry { const char *w_name; struct lock_class *w_class; }; /* * Returns 0 if one of the locks is a spin lock and the other is not. * Returns 1 otherwise. */ static __inline int witness_lock_type_equal(struct witness *w1, struct witness *w2) { return ((w1->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) == (w2->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK))); } static __inline int witness_lock_order_key_equal(const struct witness_lock_order_key *a, const struct witness_lock_order_key *b) { return (a->from == b->from && a->to == b->to); } static int _isitmyx(struct witness *w1, struct witness *w2, int rmask, const char *fname); static void adopt(struct witness *parent, struct witness *child); #ifdef BLESSING static int blessed(struct witness *, struct witness *); #endif static void depart(struct witness *w); static struct witness *enroll(const char *description, struct lock_class *lock_class); static struct lock_instance *find_instance(struct lock_list_entry *list, const struct lock_object *lock); static int isitmychild(struct witness *parent, struct witness *child); static int isitmydescendant(struct witness *parent, struct witness *child); static void itismychild(struct witness *parent, struct witness *child); static int sysctl_debug_witness_badstacks(SYSCTL_HANDLER_ARGS); static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS); static int sysctl_debug_witness_fullgraph(SYSCTL_HANDLER_ARGS); static int sysctl_debug_witness_channel(SYSCTL_HANDLER_ARGS); static void witness_add_fullgraph(struct sbuf *sb, struct witness *parent); #ifdef DDB static void witness_ddb_compute_levels(void); static void witness_ddb_display(int(*)(const char *fmt, ...)); static void witness_ddb_display_descendants(int(*)(const char *fmt, ...), struct witness *, int indent); static void witness_ddb_display_list(int(*prnt)(const char *fmt, ...), struct witness_list *list); static void witness_ddb_level_descendants(struct witness *parent, int l); static void witness_ddb_list(struct thread *td); #endif static void witness_debugger(int cond, const char *msg); static void witness_free(struct witness *m); static struct witness *witness_get(void); static uint32_t witness_hash_djb2(const uint8_t *key, uint32_t size); static struct witness *witness_hash_get(const char *key); static void witness_hash_put(struct witness *w); static void witness_init_hash_tables(void); static void witness_increment_graph_generation(void); static void witness_lock_list_free(struct lock_list_entry *lle); static struct lock_list_entry *witness_lock_list_get(void); static int witness_lock_order_add(struct witness *parent, struct witness *child); static int witness_lock_order_check(struct witness *parent, struct witness *child); static struct witness_lock_order_data *witness_lock_order_get( struct witness *parent, struct witness *child); static void witness_list_lock(struct lock_instance *instance, int (*prnt)(const char *fmt, ...)); static int witness_output(const char *fmt, ...) __printflike(1, 2); static int witness_voutput(const char *fmt, va_list ap) __printflike(1, 0); static void witness_setflag(struct lock_object *lock, int flag, int set); static SYSCTL_NODE(_debug, OID_AUTO, witness, CTLFLAG_RW, NULL, "Witness Locking"); /* * If set to 0, lock order checking is disabled. If set to -1, * witness is completely disabled. Otherwise witness performs full * lock order checking for all locks. At runtime, lock order checking * may be toggled. However, witness cannot be reenabled once it is * completely disabled. */ static int witness_watch = 1; SYSCTL_PROC(_debug_witness, OID_AUTO, watch, CTLFLAG_RWTUN | CTLTYPE_INT, NULL, 0, sysctl_debug_witness_watch, "I", "witness is watching lock operations"); #ifdef KDB /* * When KDB is enabled and witness_kdb is 1, it will cause the system * to drop into kdebug() when: * - a lock hierarchy violation occurs * - locks are held when going to sleep. */ #ifdef WITNESS_KDB int witness_kdb = 1; #else int witness_kdb = 0; #endif SYSCTL_INT(_debug_witness, OID_AUTO, kdb, CTLFLAG_RWTUN, &witness_kdb, 0, ""); #endif /* KDB */ #if defined(DDB) || defined(KDB) /* * When DDB or KDB is enabled and witness_trace is 1, it will cause the system * to print a stack trace: * - a lock hierarchy violation occurs * - locks are held when going to sleep. */ int witness_trace = 1; SYSCTL_INT(_debug_witness, OID_AUTO, trace, CTLFLAG_RWTUN, &witness_trace, 0, ""); #endif /* DDB || KDB */ #ifdef WITNESS_SKIPSPIN int witness_skipspin = 1; #else int witness_skipspin = 0; #endif SYSCTL_INT(_debug_witness, OID_AUTO, skipspin, CTLFLAG_RDTUN, &witness_skipspin, 0, ""); int badstack_sbuf_size; int witness_count = WITNESS_COUNT; SYSCTL_INT(_debug_witness, OID_AUTO, witness_count, CTLFLAG_RDTUN, &witness_count, 0, ""); /* * Output channel for witness messages. By default we print to the console. */ enum witness_channel { WITNESS_CONSOLE, WITNESS_LOG, WITNESS_NONE, }; static enum witness_channel witness_channel = WITNESS_CONSOLE; SYSCTL_PROC(_debug_witness, OID_AUTO, output_channel, CTLTYPE_STRING | CTLFLAG_RWTUN, NULL, 0, sysctl_debug_witness_channel, "A", "Output channel for warnings"); /* * Call this to print out the relations between locks. */ SYSCTL_PROC(_debug_witness, OID_AUTO, fullgraph, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, sysctl_debug_witness_fullgraph, "A", "Show locks relation graphs"); /* * Call this to print out the witness faulty stacks. */ SYSCTL_PROC(_debug_witness, OID_AUTO, badstacks, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, sysctl_debug_witness_badstacks, "A", "Show bad witness stacks"); static struct mtx w_mtx; /* w_list */ static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free); static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all); /* w_typelist */ static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin); static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep); /* lock list */ static struct lock_list_entry *w_lock_list_free = NULL; static struct witness_pendhelp pending_locks[WITNESS_PENDLIST]; static u_int pending_cnt; static int w_free_cnt, w_spin_cnt, w_sleep_cnt; SYSCTL_INT(_debug_witness, OID_AUTO, free_cnt, CTLFLAG_RD, &w_free_cnt, 0, ""); SYSCTL_INT(_debug_witness, OID_AUTO, spin_cnt, CTLFLAG_RD, &w_spin_cnt, 0, ""); SYSCTL_INT(_debug_witness, OID_AUTO, sleep_cnt, CTLFLAG_RD, &w_sleep_cnt, 0, ""); static struct witness *w_data; static uint8_t **w_rmatrix; static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT]; static struct witness_hash w_hash; /* The witness hash table. */ /* The lock order data hash */ static struct witness_lock_order_data w_lodata[WITNESS_LO_DATA_COUNT]; static struct witness_lock_order_data *w_lofree = NULL; static struct witness_lock_order_hash w_lohash; static int w_max_used_index = 0; static unsigned int w_generation = 0; static const char w_notrunning[] = "Witness not running\n"; static const char w_stillcold[] = "Witness is still cold\n"; static struct witness_order_list_entry order_lists[] = { /* * sx locks */ { "proctree", &lock_class_sx }, { "allproc", &lock_class_sx }, { "allprison", &lock_class_sx }, { NULL, NULL }, /* * Various mutexes */ { "Giant", &lock_class_mtx_sleep }, { "pipe mutex", &lock_class_mtx_sleep }, { "sigio lock", &lock_class_mtx_sleep }, { "process group", &lock_class_mtx_sleep }, { "process lock", &lock_class_mtx_sleep }, { "session", &lock_class_mtx_sleep }, { "uidinfo hash", &lock_class_rw }, #ifdef HWPMC_HOOKS { "pmc-sleep", &lock_class_mtx_sleep }, #endif { "time lock", &lock_class_mtx_sleep }, { NULL, NULL }, /* * umtx */ { "umtx lock", &lock_class_mtx_sleep }, { NULL, NULL }, /* * Sockets */ { "accept", &lock_class_mtx_sleep }, { "so_snd", &lock_class_mtx_sleep }, { "so_rcv", &lock_class_mtx_sleep }, { "sellck", &lock_class_mtx_sleep }, { NULL, NULL }, /* * Routing */ { "so_rcv", &lock_class_mtx_sleep }, { "radix node head", &lock_class_rw }, { "rtentry", &lock_class_mtx_sleep }, { "ifaddr", &lock_class_mtx_sleep }, { NULL, NULL }, /* * IPv4 multicast: * protocol locks before interface locks, after UDP locks. */ { "udpinp", &lock_class_rw }, { "in_multi_mtx", &lock_class_mtx_sleep }, { "igmp_mtx", &lock_class_mtx_sleep }, { "if_addr_lock", &lock_class_rw }, { NULL, NULL }, /* * IPv6 multicast: * protocol locks before interface locks, after UDP locks. */ { "udpinp", &lock_class_rw }, { "in6_multi_mtx", &lock_class_mtx_sleep }, { "mld_mtx", &lock_class_mtx_sleep }, { "if_addr_lock", &lock_class_rw }, { NULL, NULL }, /* * UNIX Domain Sockets */ { "unp_link_rwlock", &lock_class_rw }, { "unp_list_lock", &lock_class_mtx_sleep }, { "unp", &lock_class_mtx_sleep }, { "so_snd", &lock_class_mtx_sleep }, { NULL, NULL }, /* * UDP/IP */ { "udp", &lock_class_rw }, { "udpinp", &lock_class_rw }, { "so_snd", &lock_class_mtx_sleep }, { NULL, NULL }, /* * TCP/IP */ { "tcp", &lock_class_rw }, { "tcpinp", &lock_class_rw }, { "so_snd", &lock_class_mtx_sleep }, { NULL, NULL }, /* * BPF */ { "bpf global lock", &lock_class_mtx_sleep }, { "bpf interface lock", &lock_class_rw }, { "bpf cdev lock", &lock_class_mtx_sleep }, { NULL, NULL }, /* * NFS server */ { "nfsd_mtx", &lock_class_mtx_sleep }, { "so_snd", &lock_class_mtx_sleep }, { NULL, NULL }, /* * IEEE 802.11 */ { "802.11 com lock", &lock_class_mtx_sleep}, { NULL, NULL }, /* * Network drivers */ { "network driver", &lock_class_mtx_sleep}, { NULL, NULL }, /* * Netgraph */ { "ng_node", &lock_class_mtx_sleep }, { "ng_worklist", &lock_class_mtx_sleep }, { NULL, NULL }, /* * CDEV */ { "vm map (system)", &lock_class_mtx_sleep }, { "vm page queue", &lock_class_mtx_sleep }, { "vnode interlock", &lock_class_mtx_sleep }, { "cdev", &lock_class_mtx_sleep }, { NULL, NULL }, /* * VM */ { "vm map (user)", &lock_class_sx }, { "vm object", &lock_class_rw }, { "vm page", &lock_class_mtx_sleep }, { "vm page queue", &lock_class_mtx_sleep }, { "pmap pv global", &lock_class_rw }, { "pmap", &lock_class_mtx_sleep }, { "pmap pv list", &lock_class_rw }, { "vm page free queue", &lock_class_mtx_sleep }, { NULL, NULL }, /* * kqueue/VFS interaction */ { "kqueue", &lock_class_mtx_sleep }, { "struct mount mtx", &lock_class_mtx_sleep }, { "vnode interlock", &lock_class_mtx_sleep }, { NULL, NULL }, /* * ZFS locking */ { "dn->dn_mtx", &lock_class_sx }, { "dr->dt.di.dr_mtx", &lock_class_sx }, { "db->db_mtx", &lock_class_sx }, { NULL, NULL }, /* * spin locks */ #ifdef SMP { "ap boot", &lock_class_mtx_spin }, #endif { "rm.mutex_mtx", &lock_class_mtx_spin }, { "sio", &lock_class_mtx_spin }, { "scrlock", &lock_class_mtx_spin }, #ifdef __i386__ { "cy", &lock_class_mtx_spin }, #endif #ifdef __sparc64__ { "pcib_mtx", &lock_class_mtx_spin }, { "rtc_mtx", &lock_class_mtx_spin }, #endif { "scc_hwmtx", &lock_class_mtx_spin }, { "uart_hwmtx", &lock_class_mtx_spin }, { "fast_taskqueue", &lock_class_mtx_spin }, { "intr table", &lock_class_mtx_spin }, #ifdef HWPMC_HOOKS { "pmc-per-proc", &lock_class_mtx_spin }, #endif { "process slock", &lock_class_mtx_spin }, { "sleepq chain", &lock_class_mtx_spin }, { "rm_spinlock", &lock_class_mtx_spin }, { "turnstile chain", &lock_class_mtx_spin }, { "turnstile lock", &lock_class_mtx_spin }, { "sched lock", &lock_class_mtx_spin }, { "td_contested", &lock_class_mtx_spin }, { "callout", &lock_class_mtx_spin }, { "entropy harvest mutex", &lock_class_mtx_spin }, { "syscons video lock", &lock_class_mtx_spin }, #ifdef SMP { "smp rendezvous", &lock_class_mtx_spin }, #endif #ifdef __powerpc__ { "tlb0", &lock_class_mtx_spin }, #endif /* * leaf locks */ { "intrcnt", &lock_class_mtx_spin }, { "icu", &lock_class_mtx_spin }, #if defined(SMP) && defined(__sparc64__) { "ipi", &lock_class_mtx_spin }, #endif #ifdef __i386__ { "allpmaps", &lock_class_mtx_spin }, { "descriptor tables", &lock_class_mtx_spin }, #endif { "clk", &lock_class_mtx_spin }, { "cpuset", &lock_class_mtx_spin }, { "mprof lock", &lock_class_mtx_spin }, { "zombie lock", &lock_class_mtx_spin }, { "ALD Queue", &lock_class_mtx_spin }, #if defined(__i386__) || defined(__amd64__) { "pcicfg", &lock_class_mtx_spin }, { "NDIS thread lock", &lock_class_mtx_spin }, #endif { "tw_osl_io_lock", &lock_class_mtx_spin }, { "tw_osl_q_lock", &lock_class_mtx_spin }, { "tw_cl_io_lock", &lock_class_mtx_spin }, { "tw_cl_intr_lock", &lock_class_mtx_spin }, { "tw_cl_gen_lock", &lock_class_mtx_spin }, #ifdef HWPMC_HOOKS { "pmc-leaf", &lock_class_mtx_spin }, #endif { "blocked lock", &lock_class_mtx_spin }, { NULL, NULL }, { NULL, NULL } }; #ifdef BLESSING /* * Pairs of locks which have been blessed * Don't complain about order problems with blessed locks */ static struct witness_blessed blessed_list[] = { }; -static int blessed_count = nitems(blessed_list); #endif /* * This global is set to 0 once it becomes safe to use the witness code. */ static int witness_cold = 1; /* * This global is set to 1 once the static lock orders have been enrolled * so that a warning can be issued for any spin locks enrolled later. */ static int witness_spin_warn = 0; /* Trim useless garbage from filenames. */ static const char * fixup_filename(const char *file) { if (file == NULL) return (NULL); while (strncmp(file, "../", 3) == 0) file += 3; return (file); } /* * The WITNESS-enabled diagnostic code. Note that the witness code does * assume that the early boot is single-threaded at least until after this * routine is completed. */ static void witness_initialize(void *dummy __unused) { struct lock_object *lock; struct witness_order_list_entry *order; struct witness *w, *w1; int i; w_data = malloc(sizeof (struct witness) * witness_count, M_WITNESS, M_WAITOK | M_ZERO); w_rmatrix = malloc(sizeof(*w_rmatrix) * (witness_count + 1), M_WITNESS, M_WAITOK | M_ZERO); for (i = 0; i < witness_count + 1; i++) { w_rmatrix[i] = malloc(sizeof(*w_rmatrix[i]) * (witness_count + 1), M_WITNESS, M_WAITOK | M_ZERO); } badstack_sbuf_size = witness_count * 256; /* * We have to release Giant before initializing its witness * structure so that WITNESS doesn't get confused. */ mtx_unlock(&Giant); mtx_assert(&Giant, MA_NOTOWNED); CTR1(KTR_WITNESS, "%s: initializing witness", __func__); mtx_init(&w_mtx, "witness lock", NULL, MTX_SPIN | MTX_QUIET | MTX_NOWITNESS | MTX_NOPROFILE); for (i = witness_count - 1; i >= 0; i--) { w = &w_data[i]; memset(w, 0, sizeof(*w)); w_data[i].w_index = i; /* Witness index never changes. */ witness_free(w); } KASSERT(STAILQ_FIRST(&w_free)->w_index == 0, ("%s: Invalid list of free witness objects", __func__)); /* Witness with index 0 is not used to aid in debugging. */ STAILQ_REMOVE_HEAD(&w_free, w_list); w_free_cnt--; for (i = 0; i < witness_count; i++) { memset(w_rmatrix[i], 0, sizeof(*w_rmatrix[i]) * (witness_count + 1)); } for (i = 0; i < LOCK_CHILDCOUNT; i++) witness_lock_list_free(&w_locklistdata[i]); witness_init_hash_tables(); /* First add in all the specified order lists. */ for (order = order_lists; order->w_name != NULL; order++) { w = enroll(order->w_name, order->w_class); if (w == NULL) continue; w->w_file = "order list"; for (order++; order->w_name != NULL; order++) { w1 = enroll(order->w_name, order->w_class); if (w1 == NULL) continue; w1->w_file = "order list"; itismychild(w, w1); w = w1; } } witness_spin_warn = 1; /* Iterate through all locks and add them to witness. */ for (i = 0; pending_locks[i].wh_lock != NULL; i++) { lock = pending_locks[i].wh_lock; KASSERT(lock->lo_flags & LO_WITNESS, ("%s: lock %s is on pending list but not LO_WITNESS", __func__, lock->lo_name)); lock->lo_witness = enroll(pending_locks[i].wh_type, LOCK_CLASS(lock)); } /* Mark the witness code as being ready for use. */ witness_cold = 0; mtx_lock(&Giant); } SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL); void witness_init(struct lock_object *lock, const char *type) { struct lock_class *class; /* Various sanity checks. */ class = LOCK_CLASS(lock); if ((lock->lo_flags & LO_RECURSABLE) != 0 && (class->lc_flags & LC_RECURSABLE) == 0) kassert_panic("%s: lock (%s) %s can not be recursable", __func__, class->lc_name, lock->lo_name); if ((lock->lo_flags & LO_SLEEPABLE) != 0 && (class->lc_flags & LC_SLEEPABLE) == 0) kassert_panic("%s: lock (%s) %s can not be sleepable", __func__, class->lc_name, lock->lo_name); if ((lock->lo_flags & LO_UPGRADABLE) != 0 && (class->lc_flags & LC_UPGRADABLE) == 0) kassert_panic("%s: lock (%s) %s can not be upgradable", __func__, class->lc_name, lock->lo_name); /* * If we shouldn't watch this lock, then just clear lo_witness. * Otherwise, if witness_cold is set, then it is too early to * enroll this lock, so defer it to witness_initialize() by adding * it to the pending_locks list. If it is not too early, then enroll * the lock now. */ if (witness_watch < 1 || panicstr != NULL || (lock->lo_flags & LO_WITNESS) == 0) lock->lo_witness = NULL; else if (witness_cold) { pending_locks[pending_cnt].wh_lock = lock; pending_locks[pending_cnt++].wh_type = type; if (pending_cnt > WITNESS_PENDLIST) panic("%s: pending locks list is too small, " "increase WITNESS_PENDLIST\n", __func__); } else lock->lo_witness = enroll(type, class); } void witness_destroy(struct lock_object *lock) { struct lock_class *class; struct witness *w; class = LOCK_CLASS(lock); if (witness_cold) panic("lock (%s) %s destroyed while witness_cold", class->lc_name, lock->lo_name); /* XXX: need to verify that no one holds the lock */ if ((lock->lo_flags & LO_WITNESS) == 0 || lock->lo_witness == NULL) return; w = lock->lo_witness; mtx_lock_spin(&w_mtx); MPASS(w->w_refcount > 0); w->w_refcount--; if (w->w_refcount == 0) depart(w); mtx_unlock_spin(&w_mtx); } #ifdef DDB static void witness_ddb_compute_levels(void) { struct witness *w; /* * First clear all levels. */ STAILQ_FOREACH(w, &w_all, w_list) w->w_ddb_level = -1; /* * Look for locks with no parents and level all their descendants. */ STAILQ_FOREACH(w, &w_all, w_list) { /* If the witness has ancestors (is not a root), skip it. */ if (w->w_num_ancestors > 0) continue; witness_ddb_level_descendants(w, 0); } } static void witness_ddb_level_descendants(struct witness *w, int l) { int i; if (w->w_ddb_level >= l) return; w->w_ddb_level = l; l++; for (i = 1; i <= w_max_used_index; i++) { if (w_rmatrix[w->w_index][i] & WITNESS_PARENT) witness_ddb_level_descendants(&w_data[i], l); } } static void witness_ddb_display_descendants(int(*prnt)(const char *fmt, ...), struct witness *w, int indent) { int i; for (i = 0; i < indent; i++) prnt(" "); prnt("%s (type: %s, depth: %d, active refs: %d)", w->w_name, w->w_class->lc_name, w->w_ddb_level, w->w_refcount); if (w->w_displayed) { prnt(" -- (already displayed)\n"); return; } w->w_displayed = 1; if (w->w_file != NULL && w->w_line != 0) prnt(" -- last acquired @ %s:%d\n", fixup_filename(w->w_file), w->w_line); else prnt(" -- never acquired\n"); indent++; WITNESS_INDEX_ASSERT(w->w_index); for (i = 1; i <= w_max_used_index; i++) { if (db_pager_quit) return; if (w_rmatrix[w->w_index][i] & WITNESS_PARENT) witness_ddb_display_descendants(prnt, &w_data[i], indent); } } static void witness_ddb_display_list(int(*prnt)(const char *fmt, ...), struct witness_list *list) { struct witness *w; STAILQ_FOREACH(w, list, w_typelist) { if (w->w_file == NULL || w->w_ddb_level > 0) continue; /* This lock has no anscestors - display its descendants. */ witness_ddb_display_descendants(prnt, w, 0); if (db_pager_quit) return; } } static void witness_ddb_display(int(*prnt)(const char *fmt, ...)) { struct witness *w; KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); witness_ddb_compute_levels(); /* Clear all the displayed flags. */ STAILQ_FOREACH(w, &w_all, w_list) w->w_displayed = 0; /* * First, handle sleep locks which have been acquired at least * once. */ prnt("Sleep locks:\n"); witness_ddb_display_list(prnt, &w_sleep); if (db_pager_quit) return; /* * Now do spin locks which have been acquired at least once. */ prnt("\nSpin locks:\n"); witness_ddb_display_list(prnt, &w_spin); if (db_pager_quit) return; /* * Finally, any locks which have not been acquired yet. */ prnt("\nLocks which were never acquired:\n"); STAILQ_FOREACH(w, &w_all, w_list) { if (w->w_file != NULL || w->w_refcount == 0) continue; prnt("%s (type: %s, depth: %d)\n", w->w_name, w->w_class->lc_name, w->w_ddb_level); if (db_pager_quit) return; } } #endif /* DDB */ int witness_defineorder(struct lock_object *lock1, struct lock_object *lock2) { if (witness_watch == -1 || panicstr != NULL) return (0); /* Require locks that witness knows about. */ if (lock1 == NULL || lock1->lo_witness == NULL || lock2 == NULL || lock2->lo_witness == NULL) return (EINVAL); mtx_assert(&w_mtx, MA_NOTOWNED); mtx_lock_spin(&w_mtx); /* * If we already have either an explicit or implied lock order that * is the other way around, then return an error. */ if (witness_watch && isitmydescendant(lock2->lo_witness, lock1->lo_witness)) { mtx_unlock_spin(&w_mtx); return (EDOOFUS); } /* Try to add the new order. */ CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, lock2->lo_witness->w_name, lock1->lo_witness->w_name); itismychild(lock1->lo_witness, lock2->lo_witness); mtx_unlock_spin(&w_mtx); return (0); } void witness_checkorder(struct lock_object *lock, int flags, const char *file, int line, struct lock_object *interlock) { struct lock_list_entry *lock_list, *lle; struct lock_instance *lock1, *lock2, *plock; struct lock_class *class, *iclass; struct witness *w, *w1; struct thread *td; int i, j; if (witness_cold || witness_watch < 1 || lock->lo_witness == NULL || panicstr != NULL) return; w = lock->lo_witness; class = LOCK_CLASS(lock); td = curthread; if (class->lc_flags & LC_SLEEPLOCK) { /* * Since spin locks include a critical section, this check * implicitly enforces a lock order of all sleep locks before * all spin locks. */ if (td->td_critnest != 0 && !kdb_active) kassert_panic("acquiring blockable sleep lock with " "spinlock or critical section held (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); /* * If this is the first lock acquired then just return as * no order checking is needed. */ lock_list = td->td_sleeplocks; if (lock_list == NULL || lock_list->ll_count == 0) return; } else { /* * If this is the first lock, just return as no order * checking is needed. Avoid problems with thread * migration pinning the thread while checking if * spinlocks are held. If at least one spinlock is held * the thread is in a safe path and it is allowed to * unpin it. */ sched_pin(); lock_list = PCPU_GET(spinlocks); if (lock_list == NULL || lock_list->ll_count == 0) { sched_unpin(); return; } sched_unpin(); } /* * Check to see if we are recursing on a lock we already own. If * so, make sure that we don't mismatch exclusive and shared lock * acquires. */ lock1 = find_instance(lock_list, lock); if (lock1 != NULL) { if ((lock1->li_flags & LI_EXCLUSIVE) != 0 && (flags & LOP_EXCLUSIVE) == 0) { witness_output("shared lock of (%s) %s @ %s:%d\n", class->lc_name, lock->lo_name, fixup_filename(file), line); witness_output("while exclusively locked from %s:%d\n", fixup_filename(lock1->li_file), lock1->li_line); kassert_panic("excl->share"); } if ((lock1->li_flags & LI_EXCLUSIVE) == 0 && (flags & LOP_EXCLUSIVE) != 0) { witness_output("exclusive lock of (%s) %s @ %s:%d\n", class->lc_name, lock->lo_name, fixup_filename(file), line); witness_output("while share locked from %s:%d\n", fixup_filename(lock1->li_file), lock1->li_line); kassert_panic("share->excl"); } return; } /* Warn if the interlock is not locked exactly once. */ if (interlock != NULL) { iclass = LOCK_CLASS(interlock); lock1 = find_instance(lock_list, interlock); if (lock1 == NULL) kassert_panic("interlock (%s) %s not locked @ %s:%d", iclass->lc_name, interlock->lo_name, fixup_filename(file), line); else if ((lock1->li_flags & LI_RECURSEMASK) != 0) kassert_panic("interlock (%s) %s recursed @ %s:%d", iclass->lc_name, interlock->lo_name, fixup_filename(file), line); } /* * Find the previously acquired lock, but ignore interlocks. */ plock = &lock_list->ll_children[lock_list->ll_count - 1]; if (interlock != NULL && plock->li_lock == interlock) { if (lock_list->ll_count > 1) plock = &lock_list->ll_children[lock_list->ll_count - 2]; else { lle = lock_list->ll_next; /* * The interlock is the only lock we hold, so * simply return. */ if (lle == NULL) return; plock = &lle->ll_children[lle->ll_count - 1]; } } /* * Try to perform most checks without a lock. If this succeeds we * can skip acquiring the lock and return success. Otherwise we redo * the check with the lock held to handle races with concurrent updates. */ w1 = plock->li_lock->lo_witness; if (witness_lock_order_check(w1, w)) return; mtx_lock_spin(&w_mtx); if (witness_lock_order_check(w1, w)) { mtx_unlock_spin(&w_mtx); return; } witness_lock_order_add(w1, w); /* * Check for duplicate locks of the same type. Note that we only * have to check for this on the last lock we just acquired. Any * other cases will be caught as lock order violations. */ if (w1 == w) { i = w->w_index; if (!(lock->lo_flags & LO_DUPOK) && !(flags & LOP_DUPOK) && !(w_rmatrix[i][i] & WITNESS_REVERSAL)) { w_rmatrix[i][i] |= WITNESS_REVERSAL; w->w_reversed = 1; mtx_unlock_spin(&w_mtx); witness_output( "acquiring duplicate lock of same type: \"%s\"\n", w->w_name); witness_output(" 1st %s @ %s:%d\n", plock->li_lock->lo_name, fixup_filename(plock->li_file), plock->li_line); witness_output(" 2nd %s @ %s:%d\n", lock->lo_name, fixup_filename(file), line); witness_debugger(1, __func__); } else mtx_unlock_spin(&w_mtx); return; } mtx_assert(&w_mtx, MA_OWNED); /* * If we know that the lock we are acquiring comes after * the lock we most recently acquired in the lock order tree, * then there is no need for any further checks. */ if (isitmychild(w1, w)) goto out; for (j = 0, lle = lock_list; lle != NULL; lle = lle->ll_next) { for (i = lle->ll_count - 1; i >= 0; i--, j++) { MPASS(j < LOCK_CHILDCOUNT * LOCK_NCHILDREN); lock1 = &lle->ll_children[i]; /* * Ignore the interlock. */ if (interlock == lock1->li_lock) continue; /* * If this lock doesn't undergo witness checking, * then skip it. */ w1 = lock1->li_lock->lo_witness; if (w1 == NULL) { KASSERT((lock1->li_lock->lo_flags & LO_WITNESS) == 0, ("lock missing witness structure")); continue; } /* * If we are locking Giant and this is a sleepable * lock, then skip it. */ if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0 && lock == &Giant.lock_object) continue; /* * If we are locking a sleepable lock and this lock * is Giant, then skip it. */ if ((lock->lo_flags & LO_SLEEPABLE) != 0 && lock1->li_lock == &Giant.lock_object) continue; /* * If we are locking a sleepable lock and this lock * isn't sleepable, we want to treat it as a lock * order violation to enfore a general lock order of * sleepable locks before non-sleepable locks. */ if (((lock->lo_flags & LO_SLEEPABLE) != 0 && (lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0)) goto reversal; /* * If we are locking Giant and this is a non-sleepable * lock, then treat it as a reversal. */ if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0 && lock == &Giant.lock_object) goto reversal; /* * Check the lock order hierarchy for a reveresal. */ if (!isitmydescendant(w, w1)) continue; reversal: /* * We have a lock order violation, check to see if it * is allowed or has already been yelled about. */ #ifdef BLESSING /* * If the lock order is blessed, just bail. We don't * look for other lock order violations though, which * may be a bug. */ if (blessed(w, w1)) goto out; #endif /* Bail if this violation is known */ if (w_rmatrix[w1->w_index][w->w_index] & WITNESS_REVERSAL) goto out; /* Record this as a violation */ w_rmatrix[w1->w_index][w->w_index] |= WITNESS_REVERSAL; w_rmatrix[w->w_index][w1->w_index] |= WITNESS_REVERSAL; w->w_reversed = w1->w_reversed = 1; witness_increment_graph_generation(); mtx_unlock_spin(&w_mtx); #ifdef WITNESS_NO_VNODE /* * There are known LORs between VNODE locks. They are * not an indication of a bug. VNODE locks are flagged * as such (LO_IS_VNODE) and we don't yell if the LOR * is between 2 VNODE locks. */ if ((lock->lo_flags & LO_IS_VNODE) != 0 && (lock1->li_lock->lo_flags & LO_IS_VNODE) != 0) return; #endif /* * Ok, yell about it. */ if (((lock->lo_flags & LO_SLEEPABLE) != 0 && (lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0)) witness_output( "lock order reversal: (sleepable after non-sleepable)\n"); else if ((lock1->li_lock->lo_flags & LO_SLEEPABLE) == 0 && lock == &Giant.lock_object) witness_output( "lock order reversal: (Giant after non-sleepable)\n"); else witness_output("lock order reversal:\n"); /* * Try to locate an earlier lock with * witness w in our list. */ do { lock2 = &lle->ll_children[i]; MPASS(lock2->li_lock != NULL); if (lock2->li_lock->lo_witness == w) break; if (i == 0 && lle->ll_next != NULL) { lle = lle->ll_next; i = lle->ll_count - 1; MPASS(i >= 0 && i < LOCK_NCHILDREN); } else i--; } while (i >= 0); if (i < 0) { witness_output(" 1st %p %s (%s) @ %s:%d\n", lock1->li_lock, lock1->li_lock->lo_name, w1->w_name, fixup_filename(lock1->li_file), lock1->li_line); witness_output(" 2nd %p %s (%s) @ %s:%d\n", lock, lock->lo_name, w->w_name, fixup_filename(file), line); } else { witness_output(" 1st %p %s (%s) @ %s:%d\n", lock2->li_lock, lock2->li_lock->lo_name, lock2->li_lock->lo_witness->w_name, fixup_filename(lock2->li_file), lock2->li_line); witness_output(" 2nd %p %s (%s) @ %s:%d\n", lock1->li_lock, lock1->li_lock->lo_name, w1->w_name, fixup_filename(lock1->li_file), lock1->li_line); witness_output(" 3rd %p %s (%s) @ %s:%d\n", lock, lock->lo_name, w->w_name, fixup_filename(file), line); } witness_debugger(1, __func__); return; } } /* * If requested, build a new lock order. However, don't build a new * relationship between a sleepable lock and Giant if it is in the * wrong direction. The correct lock order is that sleepable locks * always come before Giant. */ if (flags & LOP_NEWORDER && !(plock->li_lock == &Giant.lock_object && (lock->lo_flags & LO_SLEEPABLE) != 0)) { CTR3(KTR_WITNESS, "%s: adding %s as a child of %s", __func__, w->w_name, plock->li_lock->lo_witness->w_name); itismychild(plock->li_lock->lo_witness, w); } out: mtx_unlock_spin(&w_mtx); } void witness_lock(struct lock_object *lock, int flags, const char *file, int line) { struct lock_list_entry **lock_list, *lle; struct lock_instance *instance; struct witness *w; struct thread *td; if (witness_cold || witness_watch == -1 || lock->lo_witness == NULL || panicstr != NULL) return; w = lock->lo_witness; td = curthread; /* Determine lock list for this lock. */ if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK) lock_list = &td->td_sleeplocks; else lock_list = PCPU_PTR(spinlocks); /* Check to see if we are recursing on a lock we already own. */ instance = find_instance(*lock_list, lock); if (instance != NULL) { instance->li_flags++; CTR4(KTR_WITNESS, "%s: pid %d recursed on %s r=%d", __func__, td->td_proc->p_pid, lock->lo_name, instance->li_flags & LI_RECURSEMASK); instance->li_file = file; instance->li_line = line; return; } /* Update per-witness last file and line acquire. */ w->w_file = file; w->w_line = line; /* Find the next open lock instance in the list and fill it. */ lle = *lock_list; if (lle == NULL || lle->ll_count == LOCK_NCHILDREN) { lle = witness_lock_list_get(); if (lle == NULL) return; lle->ll_next = *lock_list; CTR3(KTR_WITNESS, "%s: pid %d added lle %p", __func__, td->td_proc->p_pid, lle); *lock_list = lle; } instance = &lle->ll_children[lle->ll_count++]; instance->li_lock = lock; instance->li_line = line; instance->li_file = file; if ((flags & LOP_EXCLUSIVE) != 0) instance->li_flags = LI_EXCLUSIVE; else instance->li_flags = 0; CTR4(KTR_WITNESS, "%s: pid %d added %s as lle[%d]", __func__, td->td_proc->p_pid, lock->lo_name, lle->ll_count - 1); } void witness_upgrade(struct lock_object *lock, int flags, const char *file, int line) { struct lock_instance *instance; struct lock_class *class; KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if (witness_watch) { if ((lock->lo_flags & LO_UPGRADABLE) == 0) kassert_panic( "upgrade of non-upgradable lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((class->lc_flags & LC_SLEEPLOCK) == 0) kassert_panic( "upgrade of non-sleep lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); } instance = find_instance(curthread->td_sleeplocks, lock); if (instance == NULL) { kassert_panic("upgrade of unlocked lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); return; } if (witness_watch) { if ((instance->li_flags & LI_EXCLUSIVE) != 0) kassert_panic( "upgrade of exclusive lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((instance->li_flags & LI_RECURSEMASK) != 0) kassert_panic( "upgrade of recursed lock (%s) %s r=%d @ %s:%d", class->lc_name, lock->lo_name, instance->li_flags & LI_RECURSEMASK, fixup_filename(file), line); } instance->li_flags |= LI_EXCLUSIVE; } void witness_downgrade(struct lock_object *lock, int flags, const char *file, int line) { struct lock_instance *instance; struct lock_class *class; KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if (witness_watch) { if ((lock->lo_flags & LO_UPGRADABLE) == 0) kassert_panic( "downgrade of non-upgradable lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((class->lc_flags & LC_SLEEPLOCK) == 0) kassert_panic( "downgrade of non-sleep lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); } instance = find_instance(curthread->td_sleeplocks, lock); if (instance == NULL) { kassert_panic("downgrade of unlocked lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); return; } if (witness_watch) { if ((instance->li_flags & LI_EXCLUSIVE) == 0) kassert_panic( "downgrade of shared lock (%s) %s @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((instance->li_flags & LI_RECURSEMASK) != 0) kassert_panic( "downgrade of recursed lock (%s) %s r=%d @ %s:%d", class->lc_name, lock->lo_name, instance->li_flags & LI_RECURSEMASK, fixup_filename(file), line); } instance->li_flags &= ~LI_EXCLUSIVE; } void witness_unlock(struct lock_object *lock, int flags, const char *file, int line) { struct lock_list_entry **lock_list, *lle; struct lock_instance *instance; struct lock_class *class; struct thread *td; register_t s; int i, j; if (witness_cold || lock->lo_witness == NULL || panicstr != NULL) return; td = curthread; class = LOCK_CLASS(lock); /* Find lock instance associated with this lock. */ if (class->lc_flags & LC_SLEEPLOCK) lock_list = &td->td_sleeplocks; else lock_list = PCPU_PTR(spinlocks); lle = *lock_list; for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next) for (i = 0; i < (*lock_list)->ll_count; i++) { instance = &(*lock_list)->ll_children[i]; if (instance->li_lock == lock) goto found; } /* * When disabling WITNESS through witness_watch we could end up in * having registered locks in the td_sleeplocks queue. * We have to make sure we flush these queues, so just search for * eventual register locks and remove them. */ if (witness_watch > 0) { kassert_panic("lock (%s) %s not locked @ %s:%d", class->lc_name, lock->lo_name, fixup_filename(file), line); return; } else { return; } found: /* First, check for shared/exclusive mismatches. */ if ((instance->li_flags & LI_EXCLUSIVE) != 0 && witness_watch > 0 && (flags & LOP_EXCLUSIVE) == 0) { witness_output("shared unlock of (%s) %s @ %s:%d\n", class->lc_name, lock->lo_name, fixup_filename(file), line); witness_output("while exclusively locked from %s:%d\n", fixup_filename(instance->li_file), instance->li_line); kassert_panic("excl->ushare"); } if ((instance->li_flags & LI_EXCLUSIVE) == 0 && witness_watch > 0 && (flags & LOP_EXCLUSIVE) != 0) { witness_output("exclusive unlock of (%s) %s @ %s:%d\n", class->lc_name, lock->lo_name, fixup_filename(file), line); witness_output("while share locked from %s:%d\n", fixup_filename(instance->li_file), instance->li_line); kassert_panic("share->uexcl"); } /* If we are recursed, unrecurse. */ if ((instance->li_flags & LI_RECURSEMASK) > 0) { CTR4(KTR_WITNESS, "%s: pid %d unrecursed on %s r=%d", __func__, td->td_proc->p_pid, instance->li_lock->lo_name, instance->li_flags); instance->li_flags--; return; } /* The lock is now being dropped, check for NORELEASE flag */ if ((instance->li_flags & LI_NORELEASE) != 0 && witness_watch > 0) { witness_output("forbidden unlock of (%s) %s @ %s:%d\n", class->lc_name, lock->lo_name, fixup_filename(file), line); kassert_panic("lock marked norelease"); } /* Otherwise, remove this item from the list. */ s = intr_disable(); CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__, td->td_proc->p_pid, instance->li_lock->lo_name, (*lock_list)->ll_count - 1); for (j = i; j < (*lock_list)->ll_count - 1; j++) (*lock_list)->ll_children[j] = (*lock_list)->ll_children[j + 1]; (*lock_list)->ll_count--; intr_restore(s); /* * In order to reduce contention on w_mtx, we want to keep always an * head object into lists so that frequent allocation from the * free witness pool (and subsequent locking) is avoided. * In order to maintain the current code simple, when the head * object is totally unloaded it means also that we do not have * further objects in the list, so the list ownership needs to be * hand over to another object if the current head needs to be freed. */ if ((*lock_list)->ll_count == 0) { if (*lock_list == lle) { if (lle->ll_next == NULL) return; } else lle = *lock_list; *lock_list = lle->ll_next; CTR3(KTR_WITNESS, "%s: pid %d removed lle %p", __func__, td->td_proc->p_pid, lle); witness_lock_list_free(lle); } } void witness_thread_exit(struct thread *td) { struct lock_list_entry *lle; int i, n; lle = td->td_sleeplocks; if (lle == NULL || panicstr != NULL) return; if (lle->ll_count != 0) { for (n = 0; lle != NULL; lle = lle->ll_next) for (i = lle->ll_count - 1; i >= 0; i--) { if (n == 0) witness_output( "Thread %p exiting with the following locks held:\n", td); n++; witness_list_lock(&lle->ll_children[i], witness_output); } kassert_panic( "Thread %p cannot exit while holding sleeplocks\n", td); } witness_lock_list_free(lle); } /* * Warn if any locks other than 'lock' are held. Flags can be passed in to * exempt Giant and sleepable locks from the checks as well. If any * non-exempt locks are held, then a supplied message is printed to the * output channel along with a list of the offending locks. If indicated in the * flags then a failure results in a panic as well. */ int witness_warn(int flags, struct lock_object *lock, const char *fmt, ...) { struct lock_list_entry *lock_list, *lle; struct lock_instance *lock1; struct thread *td; va_list ap; int i, n; if (witness_cold || witness_watch < 1 || panicstr != NULL) return (0); n = 0; td = curthread; for (lle = td->td_sleeplocks; lle != NULL; lle = lle->ll_next) for (i = lle->ll_count - 1; i >= 0; i--) { lock1 = &lle->ll_children[i]; if (lock1->li_lock == lock) continue; if (flags & WARN_GIANTOK && lock1->li_lock == &Giant.lock_object) continue; if (flags & WARN_SLEEPOK && (lock1->li_lock->lo_flags & LO_SLEEPABLE) != 0) continue; if (n == 0) { va_start(ap, fmt); witness_voutput(fmt, ap); va_end(ap); witness_output( " with the following %slocks held:\n", (flags & WARN_SLEEPOK) != 0 ? "non-sleepable " : ""); } n++; witness_list_lock(lock1, witness_output); } /* * Pin the thread in order to avoid problems with thread migration. * Once that all verifies are passed about spinlocks ownership, * the thread is in a safe path and it can be unpinned. */ sched_pin(); lock_list = PCPU_GET(spinlocks); if (lock_list != NULL && lock_list->ll_count != 0) { sched_unpin(); /* * We should only have one spinlock and as long as * the flags cannot match for this locks class, * check if the first spinlock is the one curthread * should hold. */ lock1 = &lock_list->ll_children[lock_list->ll_count - 1]; if (lock_list->ll_count == 1 && lock_list->ll_next == NULL && lock1->li_lock == lock && n == 0) return (0); va_start(ap, fmt); witness_voutput(fmt, ap); va_end(ap); witness_output(" with the following %slocks held:\n", (flags & WARN_SLEEPOK) != 0 ? "non-sleepable " : ""); n += witness_list_locks(&lock_list, witness_output); } else sched_unpin(); if (flags & WARN_PANIC && n) kassert_panic("%s", __func__); else witness_debugger(n, __func__); return (n); } const char * witness_file(struct lock_object *lock) { struct witness *w; if (witness_cold || witness_watch < 1 || lock->lo_witness == NULL) return ("?"); w = lock->lo_witness; return (w->w_file); } int witness_line(struct lock_object *lock) { struct witness *w; if (witness_cold || witness_watch < 1 || lock->lo_witness == NULL) return (0); w = lock->lo_witness; return (w->w_line); } static struct witness * enroll(const char *description, struct lock_class *lock_class) { struct witness *w; struct witness_list *typelist; MPASS(description != NULL); if (witness_watch == -1 || panicstr != NULL) return (NULL); if ((lock_class->lc_flags & LC_SPINLOCK)) { if (witness_skipspin) return (NULL); else typelist = &w_spin; } else if ((lock_class->lc_flags & LC_SLEEPLOCK)) { typelist = &w_sleep; } else { kassert_panic("lock class %s is not sleep or spin", lock_class->lc_name); return (NULL); } mtx_lock_spin(&w_mtx); w = witness_hash_get(description); if (w) goto found; if ((w = witness_get()) == NULL) return (NULL); MPASS(strlen(description) < MAX_W_NAME); strcpy(w->w_name, description); w->w_class = lock_class; w->w_refcount = 1; STAILQ_INSERT_HEAD(&w_all, w, w_list); if (lock_class->lc_flags & LC_SPINLOCK) { STAILQ_INSERT_HEAD(&w_spin, w, w_typelist); w_spin_cnt++; } else if (lock_class->lc_flags & LC_SLEEPLOCK) { STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist); w_sleep_cnt++; } /* Insert new witness into the hash */ witness_hash_put(w); witness_increment_graph_generation(); mtx_unlock_spin(&w_mtx); return (w); found: w->w_refcount++; mtx_unlock_spin(&w_mtx); if (lock_class != w->w_class) kassert_panic( "lock (%s) %s does not match earlier (%s) lock", description, lock_class->lc_name, w->w_class->lc_name); return (w); } static void depart(struct witness *w) { struct witness_list *list; MPASS(w->w_refcount == 0); if (w->w_class->lc_flags & LC_SLEEPLOCK) { list = &w_sleep; w_sleep_cnt--; } else { list = &w_spin; w_spin_cnt--; } /* * Set file to NULL as it may point into a loadable module. */ w->w_file = NULL; w->w_line = 0; witness_increment_graph_generation(); } static void adopt(struct witness *parent, struct witness *child) { int pi, ci, i, j; if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); /* If the relationship is already known, there's no work to be done. */ if (isitmychild(parent, child)) return; /* When the structure of the graph changes, bump up the generation. */ witness_increment_graph_generation(); /* * The hard part ... create the direct relationship, then propagate all * indirect relationships. */ pi = parent->w_index; ci = child->w_index; WITNESS_INDEX_ASSERT(pi); WITNESS_INDEX_ASSERT(ci); MPASS(pi != ci); w_rmatrix[pi][ci] |= WITNESS_PARENT; w_rmatrix[ci][pi] |= WITNESS_CHILD; /* * If parent was not already an ancestor of child, * then we increment the descendant and ancestor counters. */ if ((w_rmatrix[pi][ci] & WITNESS_ANCESTOR) == 0) { parent->w_num_descendants++; child->w_num_ancestors++; } /* * Find each ancestor of 'pi'. Note that 'pi' itself is counted as * an ancestor of 'pi' during this loop. */ for (i = 1; i <= w_max_used_index; i++) { if ((w_rmatrix[i][pi] & WITNESS_ANCESTOR_MASK) == 0 && (i != pi)) continue; /* Find each descendant of 'i' and mark it as a descendant. */ for (j = 1; j <= w_max_used_index; j++) { /* * Skip children that are already marked as * descendants of 'i'. */ if (w_rmatrix[i][j] & WITNESS_ANCESTOR_MASK) continue; /* * We are only interested in descendants of 'ci'. Note * that 'ci' itself is counted as a descendant of 'ci'. */ if ((w_rmatrix[ci][j] & WITNESS_ANCESTOR_MASK) == 0 && (j != ci)) continue; w_rmatrix[i][j] |= WITNESS_ANCESTOR; w_rmatrix[j][i] |= WITNESS_DESCENDANT; w_data[i].w_num_descendants++; w_data[j].w_num_ancestors++; /* * Make sure we aren't marking a node as both an * ancestor and descendant. We should have caught * this as a lock order reversal earlier. */ if ((w_rmatrix[i][j] & WITNESS_ANCESTOR_MASK) && (w_rmatrix[i][j] & WITNESS_DESCENDANT_MASK)) { printf("witness rmatrix paradox! [%d][%d]=%d " "both ancestor and descendant\n", i, j, w_rmatrix[i][j]); kdb_backtrace(); printf("Witness disabled.\n"); witness_watch = -1; } if ((w_rmatrix[j][i] & WITNESS_ANCESTOR_MASK) && (w_rmatrix[j][i] & WITNESS_DESCENDANT_MASK)) { printf("witness rmatrix paradox! [%d][%d]=%d " "both ancestor and descendant\n", j, i, w_rmatrix[j][i]); kdb_backtrace(); printf("Witness disabled.\n"); witness_watch = -1; } } } } static void itismychild(struct witness *parent, struct witness *child) { int unlocked; MPASS(child != NULL && parent != NULL); if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); if (!witness_lock_type_equal(parent, child)) { if (witness_cold == 0) { unlocked = 1; mtx_unlock_spin(&w_mtx); } else { unlocked = 0; } kassert_panic( "%s: parent \"%s\" (%s) and child \"%s\" (%s) are not " "the same lock type", __func__, parent->w_name, parent->w_class->lc_name, child->w_name, child->w_class->lc_name); if (unlocked) mtx_lock_spin(&w_mtx); } adopt(parent, child); } /* * Generic code for the isitmy*() functions. The rmask parameter is the * expected relationship of w1 to w2. */ static int _isitmyx(struct witness *w1, struct witness *w2, int rmask, const char *fname) { unsigned char r1, r2; int i1, i2; i1 = w1->w_index; i2 = w2->w_index; WITNESS_INDEX_ASSERT(i1); WITNESS_INDEX_ASSERT(i2); r1 = w_rmatrix[i1][i2] & WITNESS_RELATED_MASK; r2 = w_rmatrix[i2][i1] & WITNESS_RELATED_MASK; /* The flags on one better be the inverse of the flags on the other */ if (!((WITNESS_ATOD(r1) == r2 && WITNESS_DTOA(r2) == r1) || (WITNESS_DTOA(r1) == r2 && WITNESS_ATOD(r2) == r1))) { /* Don't squawk if we're potentially racing with an update. */ if (!mtx_owned(&w_mtx)) return (0); printf("%s: rmatrix mismatch between %s (index %d) and %s " "(index %d): w_rmatrix[%d][%d] == %hhx but " "w_rmatrix[%d][%d] == %hhx\n", fname, w1->w_name, i1, w2->w_name, i2, i1, i2, r1, i2, i1, r2); kdb_backtrace(); printf("Witness disabled.\n"); witness_watch = -1; } return (r1 & rmask); } /* * Checks if @child is a direct child of @parent. */ static int isitmychild(struct witness *parent, struct witness *child) { return (_isitmyx(parent, child, WITNESS_PARENT, __func__)); } /* * Checks if @descendant is a direct or inderect descendant of @ancestor. */ static int isitmydescendant(struct witness *ancestor, struct witness *descendant) { return (_isitmyx(ancestor, descendant, WITNESS_ANCESTOR_MASK, __func__)); } #ifdef BLESSING static int blessed(struct witness *w1, struct witness *w2) { int i; struct witness_blessed *b; - for (i = 0; i < blessed_count; i++) { + for (i = 0; i < nitems(blessed_list); i++) { b = &blessed_list[i]; if (strcmp(w1->w_name, b->b_lock1) == 0) { if (strcmp(w2->w_name, b->b_lock2) == 0) return (1); continue; } if (strcmp(w1->w_name, b->b_lock2) == 0) if (strcmp(w2->w_name, b->b_lock1) == 0) return (1); } return (0); } #endif static struct witness * witness_get(void) { struct witness *w; int index; if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); if (witness_watch == -1) { mtx_unlock_spin(&w_mtx); return (NULL); } if (STAILQ_EMPTY(&w_free)) { witness_watch = -1; mtx_unlock_spin(&w_mtx); printf("WITNESS: unable to allocate a new witness object\n"); return (NULL); } w = STAILQ_FIRST(&w_free); STAILQ_REMOVE_HEAD(&w_free, w_list); w_free_cnt--; index = w->w_index; MPASS(index > 0 && index == w_max_used_index+1 && index < witness_count); bzero(w, sizeof(*w)); w->w_index = index; if (index > w_max_used_index) w_max_used_index = index; return (w); } static void witness_free(struct witness *w) { STAILQ_INSERT_HEAD(&w_free, w, w_list); w_free_cnt++; } static struct lock_list_entry * witness_lock_list_get(void) { struct lock_list_entry *lle; if (witness_watch == -1) return (NULL); mtx_lock_spin(&w_mtx); lle = w_lock_list_free; if (lle == NULL) { witness_watch = -1; mtx_unlock_spin(&w_mtx); printf("%s: witness exhausted\n", __func__); return (NULL); } w_lock_list_free = lle->ll_next; mtx_unlock_spin(&w_mtx); bzero(lle, sizeof(*lle)); return (lle); } static void witness_lock_list_free(struct lock_list_entry *lle) { mtx_lock_spin(&w_mtx); lle->ll_next = w_lock_list_free; w_lock_list_free = lle; mtx_unlock_spin(&w_mtx); } static struct lock_instance * find_instance(struct lock_list_entry *list, const struct lock_object *lock) { struct lock_list_entry *lle; struct lock_instance *instance; int i; for (lle = list; lle != NULL; lle = lle->ll_next) for (i = lle->ll_count - 1; i >= 0; i--) { instance = &lle->ll_children[i]; if (instance->li_lock == lock) return (instance); } return (NULL); } static void witness_list_lock(struct lock_instance *instance, int (*prnt)(const char *fmt, ...)) { struct lock_object *lock; lock = instance->li_lock; prnt("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ? "exclusive" : "shared", LOCK_CLASS(lock)->lc_name, lock->lo_name); if (lock->lo_witness->w_name != lock->lo_name) prnt(" (%s)", lock->lo_witness->w_name); prnt(" r = %d (%p) locked @ %s:%d\n", instance->li_flags & LI_RECURSEMASK, lock, fixup_filename(instance->li_file), instance->li_line); } static int witness_output(const char *fmt, ...) { va_list ap; int ret; va_start(ap, fmt); ret = witness_voutput(fmt, ap); va_end(ap); return (ret); } static int witness_voutput(const char *fmt, va_list ap) { int ret; ret = 0; switch (witness_channel) { case WITNESS_CONSOLE: ret = vprintf(fmt, ap); break; case WITNESS_LOG: vlog(LOG_NOTICE, fmt, ap); break; case WITNESS_NONE: break; } return (ret); } #ifdef DDB static int witness_thread_has_locks(struct thread *td) { if (td->td_sleeplocks == NULL) return (0); return (td->td_sleeplocks->ll_count != 0); } static int witness_proc_has_locks(struct proc *p) { struct thread *td; FOREACH_THREAD_IN_PROC(p, td) { if (witness_thread_has_locks(td)) return (1); } return (0); } #endif int witness_list_locks(struct lock_list_entry **lock_list, int (*prnt)(const char *fmt, ...)) { struct lock_list_entry *lle; int i, nheld; nheld = 0; for (lle = *lock_list; lle != NULL; lle = lle->ll_next) for (i = lle->ll_count - 1; i >= 0; i--) { witness_list_lock(&lle->ll_children[i], prnt); nheld++; } return (nheld); } /* * This is a bit risky at best. We call this function when we have timed * out acquiring a spin lock, and we assume that the other CPU is stuck * with this lock held. So, we go groveling around in the other CPU's * per-cpu data to try to find the lock instance for this spin lock to * see when it was last acquired. */ void witness_display_spinlock(struct lock_object *lock, struct thread *owner, int (*prnt)(const char *fmt, ...)) { struct lock_instance *instance; struct pcpu *pc; if (owner->td_critnest == 0 || owner->td_oncpu == NOCPU) return; pc = pcpu_find(owner->td_oncpu); instance = find_instance(pc->pc_spinlocks, lock); if (instance != NULL) witness_list_lock(instance, prnt); } void witness_save(struct lock_object *lock, const char **filep, int *linep) { struct lock_list_entry *lock_list; struct lock_instance *instance; struct lock_class *class; /* * This function is used independently in locking code to deal with * Giant, SCHEDULER_STOPPED() check can be removed here after Giant * is gone. */ if (SCHEDULER_STOPPED()) return; KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if (class->lc_flags & LC_SLEEPLOCK) lock_list = curthread->td_sleeplocks; else { if (witness_skipspin) return; lock_list = PCPU_GET(spinlocks); } instance = find_instance(lock_list, lock); if (instance == NULL) { kassert_panic("%s: lock (%s) %s not locked", __func__, class->lc_name, lock->lo_name); return; } *filep = instance->li_file; *linep = instance->li_line; } void witness_restore(struct lock_object *lock, const char *file, int line) { struct lock_list_entry *lock_list; struct lock_instance *instance; struct lock_class *class; /* * This function is used independently in locking code to deal with * Giant, SCHEDULER_STOPPED() check can be removed here after Giant * is gone. */ if (SCHEDULER_STOPPED()) return; KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if (class->lc_flags & LC_SLEEPLOCK) lock_list = curthread->td_sleeplocks; else { if (witness_skipspin) return; lock_list = PCPU_GET(spinlocks); } instance = find_instance(lock_list, lock); if (instance == NULL) kassert_panic("%s: lock (%s) %s not locked", __func__, class->lc_name, lock->lo_name); lock->lo_witness->w_file = file; lock->lo_witness->w_line = line; if (instance == NULL) return; instance->li_file = file; instance->li_line = line; } void witness_assert(const struct lock_object *lock, int flags, const char *file, int line) { #ifdef INVARIANT_SUPPORT struct lock_instance *instance; struct lock_class *class; if (lock->lo_witness == NULL || witness_watch < 1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if ((class->lc_flags & LC_SLEEPLOCK) != 0) instance = find_instance(curthread->td_sleeplocks, lock); else if ((class->lc_flags & LC_SPINLOCK) != 0) instance = find_instance(PCPU_GET(spinlocks), lock); else { kassert_panic("Lock (%s) %s is not sleep or spin!", class->lc_name, lock->lo_name); return; } switch (flags) { case LA_UNLOCKED: if (instance != NULL) kassert_panic("Lock (%s) %s locked @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); break; case LA_LOCKED: case LA_LOCKED | LA_RECURSED: case LA_LOCKED | LA_NOTRECURSED: case LA_SLOCKED: case LA_SLOCKED | LA_RECURSED: case LA_SLOCKED | LA_NOTRECURSED: case LA_XLOCKED: case LA_XLOCKED | LA_RECURSED: case LA_XLOCKED | LA_NOTRECURSED: if (instance == NULL) { kassert_panic("Lock (%s) %s not locked @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); break; } if ((flags & LA_XLOCKED) != 0 && (instance->li_flags & LI_EXCLUSIVE) == 0) kassert_panic( "Lock (%s) %s not exclusively locked @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((flags & LA_SLOCKED) != 0 && (instance->li_flags & LI_EXCLUSIVE) != 0) kassert_panic( "Lock (%s) %s exclusively locked @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((flags & LA_RECURSED) != 0 && (instance->li_flags & LI_RECURSEMASK) == 0) kassert_panic("Lock (%s) %s not recursed @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); if ((flags & LA_NOTRECURSED) != 0 && (instance->li_flags & LI_RECURSEMASK) != 0) kassert_panic("Lock (%s) %s recursed @ %s:%d.", class->lc_name, lock->lo_name, fixup_filename(file), line); break; default: kassert_panic("Invalid lock assertion at %s:%d.", fixup_filename(file), line); } #endif /* INVARIANT_SUPPORT */ } static void witness_setflag(struct lock_object *lock, int flag, int set) { struct lock_list_entry *lock_list; struct lock_instance *instance; struct lock_class *class; if (lock->lo_witness == NULL || witness_watch == -1 || panicstr != NULL) return; class = LOCK_CLASS(lock); if (class->lc_flags & LC_SLEEPLOCK) lock_list = curthread->td_sleeplocks; else { if (witness_skipspin) return; lock_list = PCPU_GET(spinlocks); } instance = find_instance(lock_list, lock); if (instance == NULL) { kassert_panic("%s: lock (%s) %s not locked", __func__, class->lc_name, lock->lo_name); return; } if (set) instance->li_flags |= flag; else instance->li_flags &= ~flag; } void witness_norelease(struct lock_object *lock) { witness_setflag(lock, LI_NORELEASE, 1); } void witness_releaseok(struct lock_object *lock) { witness_setflag(lock, LI_NORELEASE, 0); } #ifdef DDB static void witness_ddb_list(struct thread *td) { KASSERT(witness_cold == 0, ("%s: witness_cold", __func__)); KASSERT(kdb_active, ("%s: not in the debugger", __func__)); if (witness_watch < 1) return; witness_list_locks(&td->td_sleeplocks, db_printf); /* * We only handle spinlocks if td == curthread. This is somewhat broken * if td is currently executing on some other CPU and holds spin locks * as we won't display those locks. If we had a MI way of getting * the per-cpu data for a given cpu then we could use * td->td_oncpu to get the list of spinlocks for this thread * and "fix" this. * * That still wouldn't really fix this unless we locked the scheduler * lock or stopped the other CPU to make sure it wasn't changing the * list out from under us. It is probably best to just not try to * handle threads on other CPU's for now. */ if (td == curthread && PCPU_GET(spinlocks) != NULL) witness_list_locks(PCPU_PTR(spinlocks), db_printf); } DB_SHOW_COMMAND(locks, db_witness_list) { struct thread *td; if (have_addr) td = db_lookup_thread(addr, true); else td = kdb_thread; witness_ddb_list(td); } DB_SHOW_ALL_COMMAND(locks, db_witness_list_all) { struct thread *td; struct proc *p; /* * It would be nice to list only threads and processes that actually * held sleep locks, but that information is currently not exported * by WITNESS. */ FOREACH_PROC_IN_SYSTEM(p) { if (!witness_proc_has_locks(p)) continue; FOREACH_THREAD_IN_PROC(p, td) { if (!witness_thread_has_locks(td)) continue; db_printf("Process %d (%s) thread %p (%d)\n", p->p_pid, p->p_comm, td, td->td_tid); witness_ddb_list(td); if (db_pager_quit) return; } } } DB_SHOW_ALIAS(alllocks, db_witness_list_all) DB_SHOW_COMMAND(witness, db_witness_display) { witness_ddb_display(db_printf); } #endif static int sysctl_debug_witness_badstacks(SYSCTL_HANDLER_ARGS) { struct witness_lock_order_data *data1, *data2, *tmp_data1, *tmp_data2; struct witness *tmp_w1, *tmp_w2, *w1, *w2; struct sbuf *sb; u_int w_rmatrix1, w_rmatrix2; int error, generation, i, j; tmp_data1 = NULL; tmp_data2 = NULL; tmp_w1 = NULL; tmp_w2 = NULL; if (witness_watch < 1) { error = SYSCTL_OUT(req, w_notrunning, sizeof(w_notrunning)); return (error); } if (witness_cold) { error = SYSCTL_OUT(req, w_stillcold, sizeof(w_stillcold)); return (error); } error = 0; sb = sbuf_new(NULL, NULL, badstack_sbuf_size, SBUF_AUTOEXTEND); if (sb == NULL) return (ENOMEM); /* Allocate and init temporary storage space. */ tmp_w1 = malloc(sizeof(struct witness), M_TEMP, M_WAITOK | M_ZERO); tmp_w2 = malloc(sizeof(struct witness), M_TEMP, M_WAITOK | M_ZERO); tmp_data1 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, M_WAITOK | M_ZERO); tmp_data2 = malloc(sizeof(struct witness_lock_order_data), M_TEMP, M_WAITOK | M_ZERO); stack_zero(&tmp_data1->wlod_stack); stack_zero(&tmp_data2->wlod_stack); restart: mtx_lock_spin(&w_mtx); generation = w_generation; mtx_unlock_spin(&w_mtx); sbuf_printf(sb, "Number of known direct relationships is %d\n", w_lohash.wloh_count); for (i = 1; i < w_max_used_index; i++) { mtx_lock_spin(&w_mtx); if (generation != w_generation) { mtx_unlock_spin(&w_mtx); /* The graph has changed, try again. */ req->oldidx = 0; sbuf_clear(sb); goto restart; } w1 = &w_data[i]; if (w1->w_reversed == 0) { mtx_unlock_spin(&w_mtx); continue; } /* Copy w1 locally so we can release the spin lock. */ *tmp_w1 = *w1; mtx_unlock_spin(&w_mtx); if (tmp_w1->w_reversed == 0) continue; for (j = 1; j < w_max_used_index; j++) { if ((w_rmatrix[i][j] & WITNESS_REVERSAL) == 0 || i > j) continue; mtx_lock_spin(&w_mtx); if (generation != w_generation) { mtx_unlock_spin(&w_mtx); /* The graph has changed, try again. */ req->oldidx = 0; sbuf_clear(sb); goto restart; } w2 = &w_data[j]; data1 = witness_lock_order_get(w1, w2); data2 = witness_lock_order_get(w2, w1); /* * Copy information locally so we can release the * spin lock. */ *tmp_w2 = *w2; w_rmatrix1 = (unsigned int)w_rmatrix[i][j]; w_rmatrix2 = (unsigned int)w_rmatrix[j][i]; if (data1) { stack_zero(&tmp_data1->wlod_stack); stack_copy(&data1->wlod_stack, &tmp_data1->wlod_stack); } if (data2 && data2 != data1) { stack_zero(&tmp_data2->wlod_stack); stack_copy(&data2->wlod_stack, &tmp_data2->wlod_stack); } mtx_unlock_spin(&w_mtx); sbuf_printf(sb, "\nLock order reversal between \"%s\"(%s) and \"%s\"(%s)!\n", tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); if (data1) { sbuf_printf(sb, "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", tmp_w1->w_name, tmp_w1->w_class->lc_name, tmp_w2->w_name, tmp_w2->w_class->lc_name); stack_sbuf_print(sb, &tmp_data1->wlod_stack); sbuf_printf(sb, "\n"); } if (data2 && data2 != data1) { sbuf_printf(sb, "Lock order \"%s\"(%s) -> \"%s\"(%s) first seen at:\n", tmp_w2->w_name, tmp_w2->w_class->lc_name, tmp_w1->w_name, tmp_w1->w_class->lc_name); stack_sbuf_print(sb, &tmp_data2->wlod_stack); sbuf_printf(sb, "\n"); } } } mtx_lock_spin(&w_mtx); if (generation != w_generation) { mtx_unlock_spin(&w_mtx); /* * The graph changed while we were printing stack data, * try again. */ req->oldidx = 0; sbuf_clear(sb); goto restart; } mtx_unlock_spin(&w_mtx); /* Free temporary storage space. */ free(tmp_data1, M_TEMP); free(tmp_data2, M_TEMP); free(tmp_w1, M_TEMP); free(tmp_w2, M_TEMP); sbuf_finish(sb); error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); sbuf_delete(sb); return (error); } static int sysctl_debug_witness_channel(SYSCTL_HANDLER_ARGS) { static const struct { enum witness_channel channel; const char *name; } channels[] = { { WITNESS_CONSOLE, "console" }, { WITNESS_LOG, "log" }, { WITNESS_NONE, "none" }, }; char buf[16]; u_int i; int error; buf[0] = '\0'; for (i = 0; i < nitems(channels); i++) if (witness_channel == channels[i].channel) { snprintf(buf, sizeof(buf), "%s", channels[i].name); break; } error = sysctl_handle_string(oidp, buf, sizeof(buf), req); if (error != 0 || req->newptr == NULL) return (error); error = EINVAL; for (i = 0; i < nitems(channels); i++) if (strcmp(channels[i].name, buf) == 0) { witness_channel = channels[i].channel; error = 0; break; } return (error); } static int sysctl_debug_witness_fullgraph(SYSCTL_HANDLER_ARGS) { struct witness *w; struct sbuf *sb; int error; if (witness_watch < 1) { error = SYSCTL_OUT(req, w_notrunning, sizeof(w_notrunning)); return (error); } if (witness_cold) { error = SYSCTL_OUT(req, w_stillcold, sizeof(w_stillcold)); return (error); } error = 0; error = sysctl_wire_old_buffer(req, 0); if (error != 0) return (error); sb = sbuf_new_for_sysctl(NULL, NULL, FULLGRAPH_SBUF_SIZE, req); if (sb == NULL) return (ENOMEM); sbuf_printf(sb, "\n"); mtx_lock_spin(&w_mtx); STAILQ_FOREACH(w, &w_all, w_list) w->w_displayed = 0; STAILQ_FOREACH(w, &w_all, w_list) witness_add_fullgraph(sb, w); mtx_unlock_spin(&w_mtx); /* * Close the sbuf and return to userland. */ error = sbuf_finish(sb); sbuf_delete(sb); return (error); } static int sysctl_debug_witness_watch(SYSCTL_HANDLER_ARGS) { int error, value; value = witness_watch; error = sysctl_handle_int(oidp, &value, 0, req); if (error != 0 || req->newptr == NULL) return (error); if (value > 1 || value < -1 || (witness_watch == -1 && value != witness_watch)) return (EINVAL); witness_watch = value; return (0); } static void witness_add_fullgraph(struct sbuf *sb, struct witness *w) { int i; if (w->w_displayed != 0 || (w->w_file == NULL && w->w_line == 0)) return; w->w_displayed = 1; WITNESS_INDEX_ASSERT(w->w_index); for (i = 1; i <= w_max_used_index; i++) { if (w_rmatrix[w->w_index][i] & WITNESS_PARENT) { sbuf_printf(sb, "\"%s\",\"%s\"\n", w->w_name, w_data[i].w_name); witness_add_fullgraph(sb, &w_data[i]); } } } /* * A simple hash function. Takes a key pointer and a key size. If size == 0, * interprets the key as a string and reads until the null * terminator. Otherwise, reads the first size bytes. Returns an unsigned 32-bit * hash value computed from the key. */ static uint32_t witness_hash_djb2(const uint8_t *key, uint32_t size) { unsigned int hash = 5381; int i; /* hash = hash * 33 + key[i] */ if (size) for (i = 0; i < size; i++) hash = ((hash << 5) + hash) + (unsigned int)key[i]; else for (i = 0; key[i] != 0; i++) hash = ((hash << 5) + hash) + (unsigned int)key[i]; return (hash); } /* * Initializes the two witness hash tables. Called exactly once from * witness_initialize(). */ static void witness_init_hash_tables(void) { int i; MPASS(witness_cold); /* Initialize the hash tables. */ for (i = 0; i < WITNESS_HASH_SIZE; i++) w_hash.wh_array[i] = NULL; w_hash.wh_size = WITNESS_HASH_SIZE; w_hash.wh_count = 0; /* Initialize the lock order data hash. */ w_lofree = NULL; for (i = 0; i < WITNESS_LO_DATA_COUNT; i++) { memset(&w_lodata[i], 0, sizeof(w_lodata[i])); w_lodata[i].wlod_next = w_lofree; w_lofree = &w_lodata[i]; } w_lohash.wloh_size = WITNESS_LO_HASH_SIZE; w_lohash.wloh_count = 0; for (i = 0; i < WITNESS_LO_HASH_SIZE; i++) w_lohash.wloh_array[i] = NULL; } static struct witness * witness_hash_get(const char *key) { struct witness *w; uint32_t hash; MPASS(key != NULL); if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); hash = witness_hash_djb2(key, 0) % w_hash.wh_size; w = w_hash.wh_array[hash]; while (w != NULL) { if (strcmp(w->w_name, key) == 0) goto out; w = w->w_hash_next; } out: return (w); } static void witness_hash_put(struct witness *w) { uint32_t hash; MPASS(w != NULL); MPASS(w->w_name != NULL); if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); KASSERT(witness_hash_get(w->w_name) == NULL, ("%s: trying to add a hash entry that already exists!", __func__)); KASSERT(w->w_hash_next == NULL, ("%s: w->w_hash_next != NULL", __func__)); hash = witness_hash_djb2(w->w_name, 0) % w_hash.wh_size; w->w_hash_next = w_hash.wh_array[hash]; w_hash.wh_array[hash] = w; w_hash.wh_count++; } static struct witness_lock_order_data * witness_lock_order_get(struct witness *parent, struct witness *child) { struct witness_lock_order_data *data = NULL; struct witness_lock_order_key key; unsigned int hash; MPASS(parent != NULL && child != NULL); key.from = parent->w_index; key.to = child->w_index; WITNESS_INDEX_ASSERT(key.from); WITNESS_INDEX_ASSERT(key.to); if ((w_rmatrix[parent->w_index][child->w_index] & WITNESS_LOCK_ORDER_KNOWN) == 0) goto out; hash = witness_hash_djb2((const char*)&key, sizeof(key)) % w_lohash.wloh_size; data = w_lohash.wloh_array[hash]; while (data != NULL) { if (witness_lock_order_key_equal(&data->wlod_key, &key)) break; data = data->wlod_next; } out: return (data); } /* * Verify that parent and child have a known relationship, are not the same, * and child is actually a child of parent. This is done without w_mtx * to avoid contention in the common case. */ static int witness_lock_order_check(struct witness *parent, struct witness *child) { if (parent != child && w_rmatrix[parent->w_index][child->w_index] & WITNESS_LOCK_ORDER_KNOWN && isitmychild(parent, child)) return (1); return (0); } static int witness_lock_order_add(struct witness *parent, struct witness *child) { struct witness_lock_order_data *data = NULL; struct witness_lock_order_key key; unsigned int hash; MPASS(parent != NULL && child != NULL); key.from = parent->w_index; key.to = child->w_index; WITNESS_INDEX_ASSERT(key.from); WITNESS_INDEX_ASSERT(key.to); if (w_rmatrix[parent->w_index][child->w_index] & WITNESS_LOCK_ORDER_KNOWN) return (1); hash = witness_hash_djb2((const char*)&key, sizeof(key)) % w_lohash.wloh_size; w_rmatrix[parent->w_index][child->w_index] |= WITNESS_LOCK_ORDER_KNOWN; data = w_lofree; if (data == NULL) return (0); w_lofree = data->wlod_next; data->wlod_next = w_lohash.wloh_array[hash]; data->wlod_key = key; w_lohash.wloh_array[hash] = data; w_lohash.wloh_count++; stack_zero(&data->wlod_stack); stack_save(&data->wlod_stack); return (1); } /* Call this whenver the structure of the witness graph changes. */ static void witness_increment_graph_generation(void) { if (witness_cold == 0) mtx_assert(&w_mtx, MA_OWNED); w_generation++; } static int witness_output_drain(void *arg __unused, const char *data, int len) { witness_output("%.*s", len, data); return (len); } static void witness_debugger(int cond, const char *msg) { char buf[32]; struct sbuf sb; struct stack st; if (!cond) return; if (witness_trace) { sbuf_new(&sb, buf, sizeof(buf), SBUF_FIXEDLEN); sbuf_set_drain(&sb, witness_output_drain, NULL); stack_zero(&st); stack_save(&st); witness_output("stack backtrace:\n"); stack_sbuf_print_ddb(&sb, &st); sbuf_finish(&sb); } #ifdef KDB if (witness_kdb) kdb_enter(KDB_WHY_WITNESS, msg); #endif } Index: head/sys/mips/nlm/xlp_machdep.c =================================================================== --- head/sys/mips/nlm/xlp_machdep.c (revision 298410) +++ head/sys/mips/nlm/xlp_machdep.c (revision 298411) @@ -1,735 +1,734 @@ /*- * Copyright 2003-2011 Netlogic Microsystems (Netlogic). All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY Netlogic Microsystems ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NETLOGIC OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. * * NETLOGIC_BSD */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include /* cinit() */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #endif /* 4KB static data aread to keep a copy of the bootload env until the dynamic kenv is setup */ char boot1_env[4096]; uint64_t xlp_cpu_frequency; uint64_t xlp_io_base = MIPS_PHYS_TO_DIRECT_UNCACHED(XLP_DEFAULT_IO_BASE); int xlp_ncores; int xlp_threads_per_core; uint32_t xlp_hw_thread_mask; int xlp_cpuid_to_hwtid[MAXCPU]; int xlp_hwtid_to_cpuid[MAXCPU]; uint64_t xlp_pic_base; static int xlp_mmuval; extern uint32_t _end; extern char XLPResetEntry[], XLPResetEntryEnd[]; static void xlp_setup_core(void) { uint64_t reg; reg = nlm_mfcr(LSU_DEFEATURE); /* Enable Unaligned and L2HPE */ reg |= (1 << 30) | (1 << 23); /* * Experimental : Enable SUE * Speculative Unmap Enable. Enable speculative L2 cache request for * unmapped access. */ reg |= (1ull << 31); /* Clear S1RCM - A0 errata */ reg &= ~0xeull; nlm_mtcr(LSU_DEFEATURE, reg); reg = nlm_mfcr(SCHED_DEFEATURE); /* Experimental: Disable BRU accepting ALU ops - A0 errata */ reg |= (1 << 24); nlm_mtcr(SCHED_DEFEATURE, reg); } static void xlp_setup_mmu(void) { uint32_t pagegrain; if (nlm_threadid() == 0) { nlm_setup_extended_pagemask(0); nlm_large_variable_tlb_en(1); nlm_extended_tlb_en(1); nlm_mmu_setup(0, 0, 0); } /* Enable no-read, no-exec, large-physical-address */ pagegrain = mips_rd_pagegrain(); pagegrain |= (1U << 31) | /* RIE */ (1 << 30) | /* XIE */ (1 << 29); /* ELPA */ mips_wr_pagegrain(pagegrain); } static void xlp_enable_blocks(void) { uint64_t sysbase; int i; for (i = 0; i < XLP_MAX_NODES; i++) { if (!nlm_dev_exists(XLP_IO_SYS_OFFSET(i))) continue; sysbase = nlm_get_sys_regbase(i); nlm_sys_enable_block(sysbase, DFS_DEVICE_RSA); } } static void xlp_parse_mmu_options(void) { uint64_t sysbase; uint32_t cpu_map = xlp_hw_thread_mask; uint32_t core0_thr_mask, core_thr_mask, cpu_rst_mask; int i, j, k; #ifdef SMP if (cpu_map == 0) cpu_map = 0xffffffff; #else /* Uniprocessor! */ if (cpu_map == 0) cpu_map = 0x1; else if (cpu_map != 0x1) { printf("WARNING: Starting uniprocessor kernel on cpumask [0x%lx]!\n" "WARNING: Other CPUs will be unused.\n", (u_long)cpu_map); cpu_map = 0x1; } #endif xlp_ncores = 1; core0_thr_mask = cpu_map & 0xf; switch (core0_thr_mask) { case 1: xlp_threads_per_core = 1; xlp_mmuval = 0; break; case 3: xlp_threads_per_core = 2; xlp_mmuval = 2; break; case 0xf: xlp_threads_per_core = 4; xlp_mmuval = 3; break; default: goto unsupp; } /* Try to find the enabled cores from SYS block */ sysbase = nlm_get_sys_regbase(0); cpu_rst_mask = nlm_read_sys_reg(sysbase, SYS_CPU_RESET) & 0xff; /* XLP 416 does not report this correctly, fix */ if (nlm_processor_id() == CHIP_PROCESSOR_ID_XLP_416) cpu_rst_mask = 0xe; /* Take out cores which do not exist on chip */ for (i = 1; i < XLP_MAX_CORES; i++) { if ((cpu_rst_mask & (1 << i)) == 0) cpu_map &= ~(0xfu << (4 * i)); } /* Verify other cores' CPU masks */ for (i = 1; i < XLP_MAX_CORES; i++) { core_thr_mask = (cpu_map >> (4 * i)) & 0xf; if (core_thr_mask == 0) continue; if (core_thr_mask != core0_thr_mask) goto unsupp; xlp_ncores++; } xlp_hw_thread_mask = cpu_map; /* setup hardware processor id to cpu id mapping */ for (i = 0; i< MAXCPU; i++) xlp_cpuid_to_hwtid[i] = xlp_hwtid_to_cpuid[i] = -1; for (i = 0, k = 0; i < XLP_MAX_CORES; i++) { if (((cpu_map >> (i * 4)) & 0xf) == 0) continue; for (j = 0; j < xlp_threads_per_core; j++) { xlp_cpuid_to_hwtid[k] = i * 4 + j; xlp_hwtid_to_cpuid[i * 4 + j] = k; k++; } } return; unsupp: printf("ERROR : Unsupported CPU mask [use 1,2 or 4 threads per core].\n" "\tcore0 thread mask [%lx], boot cpu mask [%lx].\n", (u_long)core0_thr_mask, (u_long)cpu_map); panic("Invalid CPU mask - halting.\n"); return; } /* Parse cmd line args as env - copied from ar71xx */ static void xlp_parse_bootargs(char *cmdline) { char *n, *v; while ((v = strsep(&cmdline, " \n")) != NULL) { if (*v == '\0') continue; if (*v == '-') { while (*v != '\0') { v++; switch (*v) { case 'a': boothowto |= RB_ASKNAME; break; case 'd': boothowto |= RB_KDB; break; case 'g': boothowto |= RB_GDB; break; case 's': boothowto |= RB_SINGLE; break; case 'v': boothowto |= RB_VERBOSE; break; } } } else { n = strsep(&v, "="); if (v == NULL) kern_setenv(n, "1"); else kern_setenv(n, v); } } } #ifdef FDT static void xlp_bootargs_init(__register_t arg) { char buf[2048]; /* early stack is big enough */ void *dtbp; phandle_t chosen; ihandle_t mask; dtbp = (void *)(intptr_t)arg; #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not passed as argument try * to use the statically embedded one. */ if (dtbp == NULL) dtbp = &fdt_static_dtb; #endif if (OF_install(OFW_FDT, 0) == FALSE) while (1); if (OF_init((void *)dtbp) != 0) while (1); OF_interpret("perform-fixup", 0); chosen = OF_finddevice("/chosen"); if (OF_getprop(chosen, "cpumask", &mask, sizeof(mask)) != -1) { xlp_hw_thread_mask = mask; } if (OF_getprop(chosen, "bootargs", buf, sizeof(buf)) != -1) xlp_parse_bootargs(buf); } #else /* * arg is a pointer to the environment block, the format of the block is * a=xyz\0b=pqr\0\0 */ static void xlp_bootargs_init(__register_t arg) { char buf[2048]; /* early stack is big enough */ char *p, *v, *n; uint32_t mask; /* * provide backward compat for passing cpu mask as arg */ if (arg & 1) { xlp_hw_thread_mask = arg; return; } p = (void *)(intptr_t)arg; while (*p != '\0') { strlcpy(buf, p, sizeof(buf)); v = buf; n = strsep(&v, "="); if (v == NULL) kern_setenv(n, "1"); else kern_setenv(n, v); p += strlen(p) + 1; } /* CPU mask can be passed thru env */ if (getenv_uint("cpumask", &mask) != 0) xlp_hw_thread_mask = mask; /* command line argument */ v = kern_getenv("bootargs"); if (v != NULL) { strlcpy(buf, v, sizeof(buf)); xlp_parse_bootargs(buf); freeenv(v); } } #endif static void mips_init(void) { init_param1(); init_param2(physmem); mips_cpu_init(); cpuinfo.cache_coherent_dma = TRUE; pmap_bootstrap(); mips_proc0_init(); mutex_init(); #ifdef DDB kdb_init(); if (boothowto & RB_KDB) { kdb_enter("Boot flags requested debugger", NULL); } #endif } unsigned int platform_get_timecount(struct timecounter *tc __unused) { uint64_t count = nlm_pic_read_timer(xlp_pic_base, PIC_CLOCK_TIMER); return (unsigned int)~count; } static void xlp_pic_init(void) { struct timecounter pic_timecounter = { platform_get_timecount, /* get_timecount */ 0, /* no poll_pps */ ~0U, /* counter_mask */ XLP_IO_CLK, /* frequency */ "XLRPIC", /* name */ 2000, /* quality (adjusted in code) */ }; int i; int maxirt; xlp_pic_base = nlm_get_pic_regbase(0); /* TOOD: Add other nodes */ maxirt = nlm_read_reg(nlm_get_pic_pcibase(nlm_nodeid()), XLP_PCI_DEVINFO_REG0); printf("Initializing PIC...@%jx %d IRTs\n", (uintmax_t)xlp_pic_base, maxirt); /* Bind all PIC irqs to cpu 0 */ for (i = 0; i < maxirt; i++) nlm_pic_write_irt(xlp_pic_base, i, 0, 0, 1, 0, 1, 0, 0x1); nlm_pic_set_timer(xlp_pic_base, PIC_CLOCK_TIMER, ~0ULL, 0, 0); platform_timecounter = &pic_timecounter; } #if defined(__mips_n32) || defined(__mips_n64) /* PHYSADDR_64_BIT */ #ifdef XLP_SIM #define XLP_MEM_LIM 0x200000000ULL #else #define XLP_MEM_LIM 0x10000000000ULL #endif #else #define XLP_MEM_LIM 0xfffff000UL #endif static vm_paddr_t xlp_mem_excl[] = { 0, 0, /* for kernel image region, see xlp_mem_init */ 0x0c000000, 0x14000000, /* uboot area, cms queue and other stuff */ 0x1fc00000, 0x1fd00000, /* reset vec */ 0x1e000000, 0x1e200000, /* poe buffers */ }; static int mem_exclude_add(vm_paddr_t *avail, vm_paddr_t mstart, vm_paddr_t mend) { - int nreg = sizeof(xlp_mem_excl)/sizeof(xlp_mem_excl[0]); int i, pos; pos = 0; - for (i = 0; i < nreg; i += 2) { + for (i = 0; i < nitems(xlp_mem_excl); i += 2) { if (mstart > xlp_mem_excl[i + 1]) continue; if (mstart < xlp_mem_excl[i]) { avail[pos++] = mstart; if (mend < xlp_mem_excl[i]) avail[pos++] = mend; else avail[pos++] = xlp_mem_excl[i]; } mstart = xlp_mem_excl[i + 1]; if (mend <= mstart) break; } if (mstart < mend) { avail[pos++] = mstart; avail[pos++] = mend; } return (pos); } static void xlp_mem_init(void) { vm_paddr_t physsz, tmp; uint64_t bridgebase, base, lim, val; int i, j, k, n; /* update kernel image area in exclude regions */ tmp = (vm_paddr_t)MIPS_KSEG0_TO_PHYS(&_end); tmp = round_page(tmp) + 0x20000; /* round up */ xlp_mem_excl[1] = tmp; printf("Memory (from DRAM BARs):\n"); bridgebase = nlm_get_bridge_regbase(0); /* TODO: Add other nodes */ physsz = 0; for (i = 0, j = 0; i < 8; i++) { val = nlm_read_bridge_reg(bridgebase, BRIDGE_DRAM_BAR(i)); val = (val >> 12) & 0xfffff; base = val << 20; val = nlm_read_bridge_reg(bridgebase, BRIDGE_DRAM_LIMIT(i)); val = (val >> 12) & 0xfffff; if (val == 0) /* BAR not enabled */ continue; lim = (val + 1) << 20; printf(" BAR %d: %#jx - %#jx : ", i, (intmax_t)base, (intmax_t)lim); if (lim <= base) { printf("\tskipped - malformed %#jx -> %#jx\n", (intmax_t)base, (intmax_t)lim); continue; } else if (base >= XLP_MEM_LIM) { printf(" skipped - outside usable limit %#jx.\n", (intmax_t)XLP_MEM_LIM); continue; } else if (lim >= XLP_MEM_LIM) { lim = XLP_MEM_LIM; printf(" truncated to %#jx.\n", (intmax_t)XLP_MEM_LIM); } else printf(" usable\n"); /* exclude unusable regions from BAR and add rest */ n = mem_exclude_add(&phys_avail[j], base, lim); for (k = j; k < j + n; k += 2) { physsz += phys_avail[k + 1] - phys_avail[k]; printf("\tMem[%d]: %#jx - %#jx\n", k/2, (intmax_t)phys_avail[k], (intmax_t)phys_avail[k+1]); } j = k; } /* setup final entry with 0 */ phys_avail[j] = phys_avail[j + 1] = 0; /* copy phys_avail to dump_avail */ for (i = 0; i <= j + 1; i++) dump_avail[i] = phys_avail[i]; realmem = physmem = btoc(physsz); } void platform_start(__register_t a0 __unused, __register_t a1 __unused, __register_t a2 __unused, __register_t a3 __unused) { /* Initialize pcpu stuff */ mips_pcpu0_init(); /* initialize console so that we have printf */ boothowto |= (RB_SERIAL | RB_MULTIPLE); /* Use multiple consoles */ init_static_kenv(boot1_env, sizeof(boot1_env)); xlp_bootargs_init(a0); /* clockrate used by delay, so initialize it here */ xlp_cpu_frequency = xlp_get_cpu_frequency(0, 0); cpu_clock = xlp_cpu_frequency / 1000000; mips_timer_early_init(xlp_cpu_frequency); /* Init console please */ cninit(); /* Early core init and fixes for errata */ xlp_setup_core(); xlp_parse_mmu_options(); xlp_mem_init(); bcopy(XLPResetEntry, (void *)MIPS_RESET_EXC_VEC, XLPResetEntryEnd - XLPResetEntry); #ifdef SMP /* * We will enable the other threads in core 0 here * so that the TLB and cache info is correct when * mips_init runs */ xlp_enable_threads(xlp_mmuval); #endif /* setup for the startup core */ xlp_setup_mmu(); xlp_enable_blocks(); /* Read/Guess/setup board information */ nlm_board_info_setup(); /* MIPS generic init */ mips_init(); /* * XLP specific post initialization * initialize other on chip stuff */ xlp_pic_init(); mips_timer_init_params(xlp_cpu_frequency, 0); } void platform_cpu_init() { } void platform_reset(void) { uint64_t sysbase = nlm_get_sys_regbase(0); nlm_write_sys_reg(sysbase, SYS_CHIP_RESET, 1); for( ; ; ) __asm __volatile("wait"); } #ifdef SMP /* * XLP threads are started simultaneously when we enable threads, this will * ensure that the threads are blocked in platform_init_ap, until they are * ready to proceed to smp_init_secondary() */ static volatile int thr_unblock[4]; int platform_start_ap(int cpuid) { uint32_t coremask, val; uint64_t sysbase = nlm_get_sys_regbase(0); int hwtid = xlp_cpuid_to_hwtid[cpuid]; int core, thr; core = hwtid / 4; thr = hwtid % 4; if (thr == 0) { /* First thread in core, do core wake up */ coremask = 1u << core; /* Enable core clock */ val = nlm_read_sys_reg(sysbase, SYS_CORE_DFS_DIS_CTRL); val &= ~coremask; nlm_write_sys_reg(sysbase, SYS_CORE_DFS_DIS_CTRL, val); /* Remove CPU Reset */ val = nlm_read_sys_reg(sysbase, SYS_CPU_RESET); val &= ~coremask & 0xff; nlm_write_sys_reg(sysbase, SYS_CPU_RESET, val); if (bootverbose) printf("Waking up core %d ...", core); /* Poll for CPU to mark itself coherent */ do { val = nlm_read_sys_reg(sysbase, SYS_CPU_NONCOHERENT_MODE); } while ((val & coremask) != 0); if (bootverbose) printf("Done\n"); } else { /* otherwise release the threads stuck in platform_init_ap */ thr_unblock[thr] = 1; } return (0); } void platform_init_ap(int cpuid) { uint32_t stat; int thr; /* The first thread has to setup the MMU and enable other threads */ thr = nlm_threadid(); if (thr == 0) { xlp_setup_core(); xlp_enable_threads(xlp_mmuval); } else { /* * FIXME busy wait here eats too many cycles, especially * in the core 0 while bootup */ while (thr_unblock[thr] == 0) __asm__ __volatile__ ("nop;nop;nop;nop"); thr_unblock[thr] = 0; } xlp_setup_mmu(); stat = mips_rd_status(); KASSERT((stat & MIPS_SR_INT_IE) == 0, ("Interrupts enabled in %s!", __func__)); stat |= MIPS_SR_COP_2_BIT | MIPS_SR_COP_0_BIT; mips_wr_status(stat); nlm_write_c0_eimr(0ull); xlp_enable_irq(IRQ_IPI); xlp_enable_irq(IRQ_TIMER); xlp_enable_irq(IRQ_MSGRING); return; } int platform_ipi_intrnum(void) { return (IRQ_IPI); } void platform_ipi_send(int cpuid) { nlm_pic_send_ipi(xlp_pic_base, xlp_cpuid_to_hwtid[cpuid], platform_ipi_intrnum(), 0); } void platform_ipi_clear(void) { } int platform_processor_id(void) { return (xlp_hwtid_to_cpuid[nlm_cpuid()]); } void platform_cpu_mask(cpuset_t *mask) { int i, s; CPU_ZERO(mask); s = xlp_ncores * xlp_threads_per_core; for (i = 0; i < s; i++) CPU_SET(i, mask); } struct cpu_group * platform_smp_topo() { return (smp_topo_2level(CG_SHARE_L2, xlp_ncores, CG_SHARE_L1, xlp_threads_per_core, CG_FLAG_THREAD)); } #endif Index: head/sys/net/netisr.c =================================================================== --- head/sys/net/netisr.c (revision 298410) +++ head/sys/net/netisr.c (revision 298411) @@ -1,1372 +1,1370 @@ /*- * Copyright (c) 2007-2009 Robert N. M. Watson * Copyright (c) 2010-2011 Juniper Networks, Inc. * All rights reserved. * * This software was developed by Robert N. M. Watson under contract * to Juniper Networks, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * netisr is a packet dispatch service, allowing synchronous (directly * dispatched) and asynchronous (deferred dispatch) processing of packets by * registered protocol handlers. Callers pass a protocol identifier and * packet to netisr, along with a direct dispatch hint, and work will either * be immediately processed by the registered handler, or passed to a * software interrupt (SWI) thread for deferred dispatch. Callers will * generally select one or the other based on: * * - Whether directly dispatching a netisr handler lead to code reentrance or * lock recursion, such as entering the socket code from the socket code. * - Whether directly dispatching a netisr handler lead to recursive * processing, such as when decapsulating several wrapped layers of tunnel * information (IPSEC within IPSEC within ...). * * Maintaining ordering for protocol streams is a critical design concern. * Enforcing ordering limits the opportunity for concurrency, but maintains * the strong ordering requirements found in some protocols, such as TCP. Of * related concern is CPU affinity--it is desirable to process all data * associated with a particular stream on the same CPU over time in order to * avoid acquiring locks associated with the connection on different CPUs, * keep connection data in one cache, and to generally encourage associated * user threads to live on the same CPU as the stream. It's also desirable * to avoid lock migration and contention where locks are associated with * more than one flow. * * netisr supports several policy variations, represented by the * NETISR_POLICY_* constants, allowing protocols to play various roles in * identifying flows, assigning work to CPUs, etc. These are described in * netisr.h. */ #include "opt_ddb.h" #include "opt_device_polling.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #define _WANT_NETISR_INTERNAL /* Enable definitions from netisr_internal.h */ #include #include #include #include #include /*- * Synchronize use and modification of the registered netisr data structures; * acquire a read lock while modifying the set of registered protocols to * prevent partially registered or unregistered protocols from being run. * * The following data structures and fields are protected by this lock: * * - The netisr_proto array, including all fields of struct netisr_proto. * - The nws array, including all fields of struct netisr_worker. * - The nws_array array. * * Note: the NETISR_LOCKING define controls whether read locks are acquired * in packet processing paths requiring netisr registration stability. This * is disabled by default as it can lead to measurable performance * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and * because netisr registration and unregistration is extremely rare at * runtime. If it becomes more common, this decision should be revisited. * * XXXRW: rmlocks don't support assertions. */ static struct rmlock netisr_rmlock; #define NETISR_LOCK_INIT() rm_init_flags(&netisr_rmlock, "netisr", \ RM_NOWITNESS) #define NETISR_LOCK_ASSERT() #define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) #define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) #define NETISR_WLOCK() rm_wlock(&netisr_rmlock) #define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) /* #define NETISR_LOCKING */ static SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr"); /*- * Three global direct dispatch policies are supported: * * NETISR_DISPATCH_DEFERRED: All work is deferred for a netisr, regardless of * context (may be overriden by protocols). * * NETISR_DISPATCH_HYBRID: If the executing context allows direct dispatch, * and we're running on the CPU the work would be performed on, then direct * dispatch it if it wouldn't violate ordering constraints on the workstream. * * NETISR_DISPATCH_DIRECT: If the executing context allows direct dispatch, * always direct dispatch. (The default.) * * Notice that changing the global policy could lead to short periods of * misordered processing, but this is considered acceptable as compared to * the complexity of enforcing ordering during policy changes. Protocols can * override the global policy (when they're not doing that, they select * NETISR_DISPATCH_DEFAULT). */ #define NETISR_DISPATCH_POLICY_DEFAULT NETISR_DISPATCH_DIRECT #define NETISR_DISPATCH_POLICY_MAXSTR 20 /* Used for temporary buffers. */ static u_int netisr_dispatch_policy = NETISR_DISPATCH_POLICY_DEFAULT; static int sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_net_isr, OID_AUTO, dispatch, CTLTYPE_STRING | CTLFLAG_RWTUN, 0, 0, sysctl_netisr_dispatch_policy, "A", "netisr dispatch policy"); /* * Allow the administrator to limit the number of threads (CPUs) to use for * netisr. We don't check netisr_maxthreads before creating the thread for * CPU 0. This must be set at boot. We will create at most one thread per CPU. * By default we initialize this to 1 which would assign just 1 cpu (cpu0) and * therefore only 1 workstream. If set to -1, netisr would use all cpus * (mp_ncpus) and therefore would have those many workstreams. One workstream * per thread (CPU). */ static int netisr_maxthreads = 1; /* Max number of threads. */ SYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RDTUN, &netisr_maxthreads, 0, "Use at most this many CPUs for netisr processing"); static int netisr_bindthreads = 0; /* Bind threads to CPUs. */ SYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RDTUN, &netisr_bindthreads, 0, "Bind netisr threads to CPUs."); /* * Limit per-workstream mbuf queue limits s to at most net.isr.maxqlimit, * both for initial configuration and later modification using * netisr_setqlimit(). */ #define NETISR_DEFAULT_MAXQLIMIT 10240 static u_int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; SYSCTL_UINT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RDTUN, &netisr_maxqlimit, 0, "Maximum netisr per-protocol, per-CPU queue depth."); /* * The default per-workstream mbuf queue limit for protocols that don't * initialize the nh_qlimit field of their struct netisr_handler. If this is * set above netisr_maxqlimit, we truncate it to the maximum during boot. */ #define NETISR_DEFAULT_DEFAULTQLIMIT 256 static u_int netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT; SYSCTL_UINT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RDTUN, &netisr_defaultqlimit, 0, "Default netisr per-protocol, per-CPU queue limit if not set by protocol"); /* * Store and export the compile-time constant NETISR_MAXPROT limit on the * number of protocols that can register with netisr at a time. This is * required for crashdump analysis, as it sizes netisr_proto[]. */ static u_int netisr_maxprot = NETISR_MAXPROT; SYSCTL_UINT(_net_isr, OID_AUTO, maxprot, CTLFLAG_RD, &netisr_maxprot, 0, "Compile-time limit on the number of protocols supported by netisr."); /* * The netisr_proto array describes all registered protocols, indexed by * protocol number. See netisr_internal.h for more details. */ static struct netisr_proto netisr_proto[NETISR_MAXPROT]; /* * Per-CPU workstream data. See netisr_internal.h for more details. */ DPCPU_DEFINE(struct netisr_workstream, nws); /* * Map contiguous values between 0 and nws_count into CPU IDs appropriate for * accessing workstreams. This allows constructions of the form * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws). */ static u_int nws_array[MAXCPU]; /* * Number of registered workstreams. Will be at most the number of running * CPUs once fully started. */ static u_int nws_count; SYSCTL_UINT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD, &nws_count, 0, "Number of extant netisr threads."); /* * Synchronization for each workstream: a mutex protects all mutable fields * in each stream, including per-protocol state (mbuf queues). The SWI is * woken up if asynchronous dispatch is required. */ #define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) #define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) #define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) #define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) /* * Utility routines for protocols that implement their own mapping of flows * to CPUs. */ u_int netisr_get_cpucount(void) { return (nws_count); } u_int netisr_get_cpuid(u_int cpunumber) { KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber, nws_count)); return (nws_array[cpunumber]); } /* * The default implementation of flow -> CPU ID mapping. * * Non-static so that protocols can use it to map their own work to specific * CPUs in a manner consistent to netisr for affinity purposes. */ u_int netisr_default_flow2cpu(u_int flowid) { return (nws_array[flowid % nws_count]); } /* * Dispatch tunable and sysctl configuration. */ struct netisr_dispatch_table_entry { u_int ndte_policy; const char *ndte_policy_str; }; static const struct netisr_dispatch_table_entry netisr_dispatch_table[] = { { NETISR_DISPATCH_DEFAULT, "default" }, { NETISR_DISPATCH_DEFERRED, "deferred" }, { NETISR_DISPATCH_HYBRID, "hybrid" }, { NETISR_DISPATCH_DIRECT, "direct" }, }; -static const u_int netisr_dispatch_table_len = - (sizeof(netisr_dispatch_table) / sizeof(netisr_dispatch_table[0])); static void netisr_dispatch_policy_to_str(u_int dispatch_policy, char *buffer, u_int buflen) { const struct netisr_dispatch_table_entry *ndtep; const char *str; u_int i; str = "unknown"; - for (i = 0; i < netisr_dispatch_table_len; i++) { + for (i = 0; i < nitems(netisr_dispatch_table); i++) { ndtep = &netisr_dispatch_table[i]; if (ndtep->ndte_policy == dispatch_policy) { str = ndtep->ndte_policy_str; break; } } snprintf(buffer, buflen, "%s", str); } static int netisr_dispatch_policy_from_str(const char *str, u_int *dispatch_policyp) { const struct netisr_dispatch_table_entry *ndtep; u_int i; - for (i = 0; i < netisr_dispatch_table_len; i++) { + for (i = 0; i < nitems(netisr_dispatch_table); i++) { ndtep = &netisr_dispatch_table[i]; if (strcmp(ndtep->ndte_policy_str, str) == 0) { *dispatch_policyp = ndtep->ndte_policy; return (0); } } return (EINVAL); } static int sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS) { char tmp[NETISR_DISPATCH_POLICY_MAXSTR]; u_int dispatch_policy; int error; netisr_dispatch_policy_to_str(netisr_dispatch_policy, tmp, sizeof(tmp)); error = sysctl_handle_string(oidp, tmp, sizeof(tmp), req); if (error == 0 && req->newptr != NULL) { error = netisr_dispatch_policy_from_str(tmp, &dispatch_policy); if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT) error = EINVAL; if (error == 0) netisr_dispatch_policy = dispatch_policy; } return (error); } /* * Register a new netisr handler, which requires initializing per-protocol * fields for each workstream. All netisr work is briefly suspended while * the protocol is installed. */ void netisr_register(const struct netisr_handler *nhp) { struct netisr_work *npwp; const char *name; u_int i, proto; proto = nhp->nh_proto; name = nhp->nh_name; /* * Test that the requested registration is valid. */ KASSERT(nhp->nh_name != NULL, ("%s: nh_name NULL for %u", __func__, proto)); KASSERT(nhp->nh_handler != NULL, ("%s: nh_handler NULL for %s", __func__, name)); KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || nhp->nh_policy == NETISR_POLICY_FLOW || nhp->nh_policy == NETISR_POLICY_CPU, ("%s: unsupported nh_policy %u for %s", __func__, nhp->nh_policy, name)); KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || nhp->nh_m2flow == NULL, ("%s: nh_policy != FLOW but m2flow defined for %s", __func__, name)); KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL, ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__, name)); KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL, ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__, name)); KASSERT(nhp->nh_dispatch == NETISR_DISPATCH_DEFAULT || nhp->nh_dispatch == NETISR_DISPATCH_DEFERRED || nhp->nh_dispatch == NETISR_DISPATCH_HYBRID || nhp->nh_dispatch == NETISR_DISPATCH_DIRECT, ("%s: invalid nh_dispatch (%u)", __func__, nhp->nh_dispatch)); KASSERT(proto < NETISR_MAXPROT, ("%s(%u, %s): protocol too big", __func__, proto, name)); /* * Test that no existing registration exists for this protocol. */ NETISR_WLOCK(); KASSERT(netisr_proto[proto].np_name == NULL, ("%s(%u, %s): name present", __func__, proto, name)); KASSERT(netisr_proto[proto].np_handler == NULL, ("%s(%u, %s): handler present", __func__, proto, name)); netisr_proto[proto].np_name = name; netisr_proto[proto].np_handler = nhp->nh_handler; netisr_proto[proto].np_m2flow = nhp->nh_m2flow; netisr_proto[proto].np_m2cpuid = nhp->nh_m2cpuid; netisr_proto[proto].np_drainedcpu = nhp->nh_drainedcpu; if (nhp->nh_qlimit == 0) netisr_proto[proto].np_qlimit = netisr_defaultqlimit; else if (nhp->nh_qlimit > netisr_maxqlimit) { printf("%s: %s requested queue limit %u capped to " "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit, netisr_maxqlimit); netisr_proto[proto].np_qlimit = netisr_maxqlimit; } else netisr_proto[proto].np_qlimit = nhp->nh_qlimit; netisr_proto[proto].np_policy = nhp->nh_policy; netisr_proto[proto].np_dispatch = nhp->nh_dispatch; CPU_FOREACH(i) { npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; bzero(npwp, sizeof(*npwp)); npwp->nw_qlimit = netisr_proto[proto].np_qlimit; } NETISR_WUNLOCK(); } /* * Clear drop counters across all workstreams for a protocol. */ void netisr_clearqdrops(const struct netisr_handler *nhp) { struct netisr_work *npwp; #ifdef INVARIANTS const char *name; #endif u_int i, proto; proto = nhp->nh_proto; #ifdef INVARIANTS name = nhp->nh_name; #endif KASSERT(proto < NETISR_MAXPROT, ("%s(%u): protocol too big for %s", __func__, proto, name)); NETISR_WLOCK(); KASSERT(netisr_proto[proto].np_handler != NULL, ("%s(%u): protocol not registered for %s", __func__, proto, name)); CPU_FOREACH(i) { npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; npwp->nw_qdrops = 0; } NETISR_WUNLOCK(); } /* * Query current drop counters across all workstreams for a protocol. */ void netisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) { struct netisr_work *npwp; struct rm_priotracker tracker; #ifdef INVARIANTS const char *name; #endif u_int i, proto; *qdropp = 0; proto = nhp->nh_proto; #ifdef INVARIANTS name = nhp->nh_name; #endif KASSERT(proto < NETISR_MAXPROT, ("%s(%u): protocol too big for %s", __func__, proto, name)); NETISR_RLOCK(&tracker); KASSERT(netisr_proto[proto].np_handler != NULL, ("%s(%u): protocol not registered for %s", __func__, proto, name)); CPU_FOREACH(i) { npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; *qdropp += npwp->nw_qdrops; } NETISR_RUNLOCK(&tracker); } /* * Query current per-workstream queue limit for a protocol. */ void netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) { struct rm_priotracker tracker; #ifdef INVARIANTS const char *name; #endif u_int proto; proto = nhp->nh_proto; #ifdef INVARIANTS name = nhp->nh_name; #endif KASSERT(proto < NETISR_MAXPROT, ("%s(%u): protocol too big for %s", __func__, proto, name)); NETISR_RLOCK(&tracker); KASSERT(netisr_proto[proto].np_handler != NULL, ("%s(%u): protocol not registered for %s", __func__, proto, name)); *qlimitp = netisr_proto[proto].np_qlimit; NETISR_RUNLOCK(&tracker); } /* * Update the queue limit across per-workstream queues for a protocol. We * simply change the limits, and don't drain overflowed packets as they will * (hopefully) take care of themselves shortly. */ int netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit) { struct netisr_work *npwp; #ifdef INVARIANTS const char *name; #endif u_int i, proto; if (qlimit > netisr_maxqlimit) return (EINVAL); proto = nhp->nh_proto; #ifdef INVARIANTS name = nhp->nh_name; #endif KASSERT(proto < NETISR_MAXPROT, ("%s(%u): protocol too big for %s", __func__, proto, name)); NETISR_WLOCK(); KASSERT(netisr_proto[proto].np_handler != NULL, ("%s(%u): protocol not registered for %s", __func__, proto, name)); netisr_proto[proto].np_qlimit = qlimit; CPU_FOREACH(i) { npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; npwp->nw_qlimit = qlimit; } NETISR_WUNLOCK(); return (0); } /* * Drain all packets currently held in a particular protocol work queue. */ static void netisr_drain_proto(struct netisr_work *npwp) { struct mbuf *m; /* * We would assert the lock on the workstream but it's not passed in. */ while ((m = npwp->nw_head) != NULL) { npwp->nw_head = m->m_nextpkt; m->m_nextpkt = NULL; if (npwp->nw_head == NULL) npwp->nw_tail = NULL; npwp->nw_len--; m_freem(m); } KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__)); KASSERT(npwp->nw_len == 0, ("%s: len", __func__)); } /* * Remove the registration of a network protocol, which requires clearing * per-protocol fields across all workstreams, including freeing all mbufs in * the queues at time of unregister. All work in netisr is briefly suspended * while this takes place. */ void netisr_unregister(const struct netisr_handler *nhp) { struct netisr_work *npwp; #ifdef INVARIANTS const char *name; #endif u_int i, proto; proto = nhp->nh_proto; #ifdef INVARIANTS name = nhp->nh_name; #endif KASSERT(proto < NETISR_MAXPROT, ("%s(%u): protocol too big for %s", __func__, proto, name)); NETISR_WLOCK(); KASSERT(netisr_proto[proto].np_handler != NULL, ("%s(%u): protocol not registered for %s", __func__, proto, name)); netisr_proto[proto].np_name = NULL; netisr_proto[proto].np_handler = NULL; netisr_proto[proto].np_m2flow = NULL; netisr_proto[proto].np_m2cpuid = NULL; netisr_proto[proto].np_qlimit = 0; netisr_proto[proto].np_policy = 0; CPU_FOREACH(i) { npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; netisr_drain_proto(npwp); bzero(npwp, sizeof(*npwp)); } NETISR_WUNLOCK(); } /* * Compose the global and per-protocol policies on dispatch, and return the * dispatch policy to use. */ static u_int netisr_get_dispatch(struct netisr_proto *npp) { /* * Protocol-specific configuration overrides the global default. */ if (npp->np_dispatch != NETISR_DISPATCH_DEFAULT) return (npp->np_dispatch); return (netisr_dispatch_policy); } /* * Look up the workstream given a packet and source identifier. Do this by * checking the protocol's policy, and optionally call out to the protocol * for assistance if required. */ static struct mbuf * netisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy, uintptr_t source, struct mbuf *m, u_int *cpuidp) { struct ifnet *ifp; u_int policy; NETISR_LOCK_ASSERT(); /* * In the event we have only one worker, shortcut and deliver to it * without further ado. */ if (nws_count == 1) { *cpuidp = nws_array[0]; return (m); } /* * What happens next depends on the policy selected by the protocol. * If we want to support per-interface policies, we should do that * here first. */ policy = npp->np_policy; if (policy == NETISR_POLICY_CPU) { m = npp->np_m2cpuid(m, source, cpuidp); if (m == NULL) return (NULL); /* * It's possible for a protocol not to have a good idea about * where to process a packet, in which case we fall back on * the netisr code to decide. In the hybrid case, return the * current CPU ID, which will force an immediate direct * dispatch. In the queued case, fall back on the SOURCE * policy. */ if (*cpuidp != NETISR_CPUID_NONE) return (m); if (dispatch_policy == NETISR_DISPATCH_HYBRID) { *cpuidp = curcpu; return (m); } policy = NETISR_POLICY_SOURCE; } if (policy == NETISR_POLICY_FLOW) { if (M_HASHTYPE_GET(m) == M_HASHTYPE_NONE && npp->np_m2flow != NULL) { m = npp->np_m2flow(m, source); if (m == NULL) return (NULL); } if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { *cpuidp = netisr_default_flow2cpu(m->m_pkthdr.flowid); return (m); } policy = NETISR_POLICY_SOURCE; } KASSERT(policy == NETISR_POLICY_SOURCE, ("%s: invalid policy %u for %s", __func__, npp->np_policy, npp->np_name)); ifp = m->m_pkthdr.rcvif; if (ifp != NULL) *cpuidp = nws_array[(ifp->if_index + source) % nws_count]; else *cpuidp = nws_array[source % nws_count]; return (m); } /* * Process packets associated with a workstream and protocol. For reasons of * fairness, we process up to one complete netisr queue at a time, moving the * queue to a stack-local queue for processing, but do not loop refreshing * from the global queue. The caller is responsible for deciding whether to * loop, and for setting the NWS_RUNNING flag. The passed workstream will be * locked on entry and relocked before return, but will be released while * processing. The number of packets processed is returned. */ static u_int netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) { struct netisr_work local_npw, *npwp; u_int handled; struct mbuf *m; NETISR_LOCK_ASSERT(); NWS_LOCK_ASSERT(nwsp); KASSERT(nwsp->nws_flags & NWS_RUNNING, ("%s(%u): not running", __func__, proto)); KASSERT(proto >= 0 && proto < NETISR_MAXPROT, ("%s(%u): invalid proto\n", __func__, proto)); npwp = &nwsp->nws_work[proto]; if (npwp->nw_len == 0) return (0); /* * Move the global work queue to a thread-local work queue. * * Notice that this means the effective maximum length of the queue * is actually twice that of the maximum queue length specified in * the protocol registration call. */ handled = npwp->nw_len; local_npw = *npwp; npwp->nw_head = NULL; npwp->nw_tail = NULL; npwp->nw_len = 0; nwsp->nws_pendingbits &= ~(1 << proto); NWS_UNLOCK(nwsp); while ((m = local_npw.nw_head) != NULL) { local_npw.nw_head = m->m_nextpkt; m->m_nextpkt = NULL; if (local_npw.nw_head == NULL) local_npw.nw_tail = NULL; local_npw.nw_len--; VNET_ASSERT(m->m_pkthdr.rcvif != NULL, ("%s:%d rcvif == NULL: m=%p", __func__, __LINE__, m)); CURVNET_SET(m->m_pkthdr.rcvif->if_vnet); netisr_proto[proto].np_handler(m); CURVNET_RESTORE(); } KASSERT(local_npw.nw_len == 0, ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); if (netisr_proto[proto].np_drainedcpu) netisr_proto[proto].np_drainedcpu(nwsp->nws_cpu); NWS_LOCK(nwsp); npwp->nw_handled += handled; return (handled); } /* * SWI handler for netisr -- processes packets in a set of workstreams that * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already * being direct dispatched, go back to sleep and wait for the dispatching * thread to wake us up again. */ static void swi_net(void *arg) { #ifdef NETISR_LOCKING struct rm_priotracker tracker; #endif struct netisr_workstream *nwsp; u_int bits, prot; nwsp = arg; #ifdef DEVICE_POLLING KASSERT(nws_count == 1, ("%s: device_polling but nws_count != 1", __func__)); netisr_poll(); #endif #ifdef NETISR_LOCKING NETISR_RLOCK(&tracker); #endif NWS_LOCK(nwsp); KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); if (nwsp->nws_flags & NWS_DISPATCHING) goto out; nwsp->nws_flags |= NWS_RUNNING; nwsp->nws_flags &= ~NWS_SCHEDULED; while ((bits = nwsp->nws_pendingbits) != 0) { while ((prot = ffs(bits)) != 0) { prot--; bits &= ~(1 << prot); (void)netisr_process_workstream_proto(nwsp, prot); } } nwsp->nws_flags &= ~NWS_RUNNING; out: NWS_UNLOCK(nwsp); #ifdef NETISR_LOCKING NETISR_RUNLOCK(&tracker); #endif #ifdef DEVICE_POLLING netisr_pollmore(); #endif } static int netisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto, struct netisr_work *npwp, struct mbuf *m, int *dosignalp) { NWS_LOCK_ASSERT(nwsp); *dosignalp = 0; if (npwp->nw_len < npwp->nw_qlimit) { m->m_nextpkt = NULL; if (npwp->nw_head == NULL) { npwp->nw_head = m; npwp->nw_tail = m; } else { npwp->nw_tail->m_nextpkt = m; npwp->nw_tail = m; } npwp->nw_len++; if (npwp->nw_len > npwp->nw_watermark) npwp->nw_watermark = npwp->nw_len; /* * We must set the bit regardless of NWS_RUNNING, so that * swi_net() keeps calling netisr_process_workstream_proto(). */ nwsp->nws_pendingbits |= (1 << proto); if (!(nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) { nwsp->nws_flags |= NWS_SCHEDULED; *dosignalp = 1; /* Defer until unlocked. */ } npwp->nw_queued++; return (0); } else { m_freem(m); npwp->nw_qdrops++; return (ENOBUFS); } } static int netisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) { struct netisr_workstream *nwsp; struct netisr_work *npwp; int dosignal, error; #ifdef NETISR_LOCKING NETISR_LOCK_ASSERT(); #endif KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__, cpuid, mp_maxid)); KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); dosignal = 0; error = 0; nwsp = DPCPU_ID_PTR(cpuid, nws); npwp = &nwsp->nws_work[proto]; NWS_LOCK(nwsp); error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); NWS_UNLOCK(nwsp); if (dosignal) NWS_SIGNAL(nwsp); return (error); } int netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m) { #ifdef NETISR_LOCKING struct rm_priotracker tracker; #endif u_int cpuid; int error; KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING NETISR_RLOCK(&tracker); #endif KASSERT(netisr_proto[proto].np_handler != NULL, ("%s: invalid proto %u", __func__, proto)); m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_DEFERRED, source, m, &cpuid); if (m != NULL) { KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); error = netisr_queue_internal(proto, m, cpuid); } else error = ENOBUFS; #ifdef NETISR_LOCKING NETISR_RUNLOCK(&tracker); #endif return (error); } int netisr_queue(u_int proto, struct mbuf *m) { return (netisr_queue_src(proto, 0, m)); } /* * Dispatch a packet for netisr processing; direct dispatch is permitted by * calling context. */ int netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) { #ifdef NETISR_LOCKING struct rm_priotracker tracker; #endif struct netisr_workstream *nwsp; struct netisr_proto *npp; struct netisr_work *npwp; int dosignal, error; u_int cpuid, dispatch_policy; KASSERT(proto < NETISR_MAXPROT, ("%s: invalid proto %u", __func__, proto)); #ifdef NETISR_LOCKING NETISR_RLOCK(&tracker); #endif npp = &netisr_proto[proto]; KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__, proto)); dispatch_policy = netisr_get_dispatch(npp); if (dispatch_policy == NETISR_DISPATCH_DEFERRED) return (netisr_queue_src(proto, source, m)); /* * If direct dispatch is forced, then unconditionally dispatch * without a formal CPU selection. Borrow the current CPU's stats, * even if there's no worker on it. In this case we don't update * nws_flags because all netisr processing will be source ordered due * to always being forced to directly dispatch. */ if (dispatch_policy == NETISR_DISPATCH_DIRECT) { nwsp = DPCPU_PTR(nws); npwp = &nwsp->nws_work[proto]; npwp->nw_dispatched++; npwp->nw_handled++; netisr_proto[proto].np_handler(m); error = 0; goto out_unlock; } KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID, ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy)); /* * Otherwise, we execute in a hybrid mode where we will try to direct * dispatch if we're on the right CPU and the netisr worker isn't * already running. */ sched_pin(); m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID, source, m, &cpuid); if (m == NULL) { error = ENOBUFS; goto out_unpin; } KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); if (cpuid != curcpu) goto queue_fallback; nwsp = DPCPU_PTR(nws); npwp = &nwsp->nws_work[proto]; /*- * We are willing to direct dispatch only if three conditions hold: * * (1) The netisr worker isn't already running, * (2) Another thread isn't already directly dispatching, and * (3) The netisr hasn't already been woken up. */ NWS_LOCK(nwsp); if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); NWS_UNLOCK(nwsp); if (dosignal) NWS_SIGNAL(nwsp); goto out_unpin; } /* * The current thread is now effectively the netisr worker, so set * the dispatching flag to prevent concurrent processing of the * stream from another thread (even the netisr worker), which could * otherwise lead to effective misordering of the stream. */ nwsp->nws_flags |= NWS_DISPATCHING; NWS_UNLOCK(nwsp); netisr_proto[proto].np_handler(m); NWS_LOCK(nwsp); nwsp->nws_flags &= ~NWS_DISPATCHING; npwp->nw_handled++; npwp->nw_hybrid_dispatched++; /* * If other work was enqueued by another thread while we were direct * dispatching, we need to signal the netisr worker to do that work. * In the future, we might want to do some of that work in the * current thread, rather than trigger further context switches. If * so, we'll want to establish a reasonable bound on the work done in * the "borrowed" context. */ if (nwsp->nws_pendingbits != 0) { nwsp->nws_flags |= NWS_SCHEDULED; dosignal = 1; } else dosignal = 0; NWS_UNLOCK(nwsp); if (dosignal) NWS_SIGNAL(nwsp); error = 0; goto out_unpin; queue_fallback: error = netisr_queue_internal(proto, m, cpuid); out_unpin: sched_unpin(); out_unlock: #ifdef NETISR_LOCKING NETISR_RUNLOCK(&tracker); #endif return (error); } int netisr_dispatch(u_int proto, struct mbuf *m) { return (netisr_dispatch_src(proto, 0, m)); } #ifdef DEVICE_POLLING /* * Kernel polling borrows a netisr thread to run interface polling in; this * function allows kernel polling to request that the netisr thread be * scheduled even if no packets are pending for protocols. */ void netisr_sched_poll(void) { struct netisr_workstream *nwsp; nwsp = DPCPU_ID_PTR(nws_array[0], nws); NWS_SIGNAL(nwsp); } #endif static void netisr_start_swi(u_int cpuid, struct pcpu *pc) { char swiname[12]; struct netisr_workstream *nwsp; int error; KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); nwsp = DPCPU_ID_PTR(cpuid, nws); mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF); nwsp->nws_cpu = cpuid; snprintf(swiname, sizeof(swiname), "netisr %u", cpuid); error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); if (error) panic("%s: swi_add %d", __func__, error); pc->pc_netisr = nwsp->nws_intr_event; if (netisr_bindthreads) { error = intr_event_bind(nwsp->nws_intr_event, cpuid); if (error != 0) printf("%s: cpu %u: intr_event_bind: %d", __func__, cpuid, error); } NETISR_WLOCK(); nws_array[nws_count] = nwsp->nws_cpu; nws_count++; NETISR_WUNLOCK(); } /* * Initialize the netisr subsystem. We rely on BSS and static initialization * of most fields in global data structures. * * Start a worker thread for the boot CPU so that we can support network * traffic immediately in case the network stack is used before additional * CPUs are started (for example, diskless boot). */ static void netisr_init(void *arg) { KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__)); NETISR_LOCK_INIT(); if (netisr_maxthreads == 0 || netisr_maxthreads < -1 ) netisr_maxthreads = 1; /* default behavior */ else if (netisr_maxthreads == -1) netisr_maxthreads = mp_ncpus; /* use max cpus */ if (netisr_maxthreads > mp_ncpus) { printf("netisr_init: forcing maxthreads from %d to %d\n", netisr_maxthreads, mp_ncpus); netisr_maxthreads = mp_ncpus; } if (netisr_defaultqlimit > netisr_maxqlimit) { printf("netisr_init: forcing defaultqlimit from %d to %d\n", netisr_defaultqlimit, netisr_maxqlimit); netisr_defaultqlimit = netisr_maxqlimit; } #ifdef DEVICE_POLLING /* * The device polling code is not yet aware of how to deal with * multiple netisr threads, so for the time being compiling in device * polling disables parallel netisr workers. */ if (netisr_maxthreads != 1 || netisr_bindthreads != 0) { printf("netisr_init: forcing maxthreads to 1 and " "bindthreads to 0 for device polling\n"); netisr_maxthreads = 1; netisr_bindthreads = 0; } #endif netisr_start_swi(curcpu, pcpu_find(curcpu)); } SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); /* * Start worker threads for additional CPUs. No attempt to gracefully handle * work reassignment, we don't yet support dynamic reconfiguration. */ static void netisr_start(void *arg) { struct pcpu *pc; STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { if (nws_count >= netisr_maxthreads) break; /* Worker will already be present for boot CPU. */ if (pc->pc_netisr != NULL) continue; netisr_start_swi(pc->pc_cpuid, pc); } } SYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL); /* * Sysctl monitoring for netisr: query a list of registered protocols. */ static int sysctl_netisr_proto(SYSCTL_HANDLER_ARGS) { struct rm_priotracker tracker; struct sysctl_netisr_proto *snpp, *snp_array; struct netisr_proto *npp; u_int counter, proto; int error; if (req->newptr != NULL) return (EINVAL); snp_array = malloc(sizeof(*snp_array) * NETISR_MAXPROT, M_TEMP, M_ZERO | M_WAITOK); counter = 0; NETISR_RLOCK(&tracker); for (proto = 0; proto < NETISR_MAXPROT; proto++) { npp = &netisr_proto[proto]; if (npp->np_name == NULL) continue; snpp = &snp_array[counter]; snpp->snp_version = sizeof(*snpp); strlcpy(snpp->snp_name, npp->np_name, NETISR_NAMEMAXLEN); snpp->snp_proto = proto; snpp->snp_qlimit = npp->np_qlimit; snpp->snp_policy = npp->np_policy; snpp->snp_dispatch = npp->np_dispatch; if (npp->np_m2flow != NULL) snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW; if (npp->np_m2cpuid != NULL) snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID; if (npp->np_drainedcpu != NULL) snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU; counter++; } NETISR_RUNLOCK(&tracker); KASSERT(counter <= NETISR_MAXPROT, ("sysctl_netisr_proto: counter too big (%d)", counter)); error = SYSCTL_OUT(req, snp_array, sizeof(*snp_array) * counter); free(snp_array, M_TEMP); return (error); } SYSCTL_PROC(_net_isr, OID_AUTO, proto, CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_proto, "S,sysctl_netisr_proto", "Return list of protocols registered with netisr"); /* * Sysctl monitoring for netisr: query a list of workstreams. */ static int sysctl_netisr_workstream(SYSCTL_HANDLER_ARGS) { struct rm_priotracker tracker; struct sysctl_netisr_workstream *snwsp, *snws_array; struct netisr_workstream *nwsp; u_int counter, cpuid; int error; if (req->newptr != NULL) return (EINVAL); snws_array = malloc(sizeof(*snws_array) * MAXCPU, M_TEMP, M_ZERO | M_WAITOK); counter = 0; NETISR_RLOCK(&tracker); CPU_FOREACH(cpuid) { nwsp = DPCPU_ID_PTR(cpuid, nws); if (nwsp->nws_intr_event == NULL) continue; NWS_LOCK(nwsp); snwsp = &snws_array[counter]; snwsp->snws_version = sizeof(*snwsp); /* * For now, we equate workstream IDs and CPU IDs in the * kernel, but expose them independently to userspace in case * that assumption changes in the future. */ snwsp->snws_wsid = cpuid; snwsp->snws_cpu = cpuid; if (nwsp->nws_intr_event != NULL) snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR; NWS_UNLOCK(nwsp); counter++; } NETISR_RUNLOCK(&tracker); KASSERT(counter <= MAXCPU, ("sysctl_netisr_workstream: counter too big (%d)", counter)); error = SYSCTL_OUT(req, snws_array, sizeof(*snws_array) * counter); free(snws_array, M_TEMP); return (error); } SYSCTL_PROC(_net_isr, OID_AUTO, workstream, CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_workstream, "S,sysctl_netisr_workstream", "Return list of workstreams implemented by netisr"); /* * Sysctl monitoring for netisr: query per-protocol data across all * workstreams. */ static int sysctl_netisr_work(SYSCTL_HANDLER_ARGS) { struct rm_priotracker tracker; struct sysctl_netisr_work *snwp, *snw_array; struct netisr_workstream *nwsp; struct netisr_proto *npp; struct netisr_work *nwp; u_int counter, cpuid, proto; int error; if (req->newptr != NULL) return (EINVAL); snw_array = malloc(sizeof(*snw_array) * MAXCPU * NETISR_MAXPROT, M_TEMP, M_ZERO | M_WAITOK); counter = 0; NETISR_RLOCK(&tracker); CPU_FOREACH(cpuid) { nwsp = DPCPU_ID_PTR(cpuid, nws); if (nwsp->nws_intr_event == NULL) continue; NWS_LOCK(nwsp); for (proto = 0; proto < NETISR_MAXPROT; proto++) { npp = &netisr_proto[proto]; if (npp->np_name == NULL) continue; nwp = &nwsp->nws_work[proto]; snwp = &snw_array[counter]; snwp->snw_version = sizeof(*snwp); snwp->snw_wsid = cpuid; /* See comment above. */ snwp->snw_proto = proto; snwp->snw_len = nwp->nw_len; snwp->snw_watermark = nwp->nw_watermark; snwp->snw_dispatched = nwp->nw_dispatched; snwp->snw_hybrid_dispatched = nwp->nw_hybrid_dispatched; snwp->snw_qdrops = nwp->nw_qdrops; snwp->snw_queued = nwp->nw_queued; snwp->snw_handled = nwp->nw_handled; counter++; } NWS_UNLOCK(nwsp); } KASSERT(counter <= MAXCPU * NETISR_MAXPROT, ("sysctl_netisr_work: counter too big (%d)", counter)); NETISR_RUNLOCK(&tracker); error = SYSCTL_OUT(req, snw_array, sizeof(*snw_array) * counter); free(snw_array, M_TEMP); return (error); } SYSCTL_PROC(_net_isr, OID_AUTO, work, CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_work, "S,sysctl_netisr_work", "Return list of per-workstream, per-protocol work in netisr"); #ifdef DDB DB_SHOW_COMMAND(netisr, db_show_netisr) { struct netisr_workstream *nwsp; struct netisr_work *nwp; int first, proto; u_int cpuid; db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto", "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue"); CPU_FOREACH(cpuid) { nwsp = DPCPU_ID_PTR(cpuid, nws); if (nwsp->nws_intr_event == NULL) continue; first = 1; for (proto = 0; proto < NETISR_MAXPROT; proto++) { if (netisr_proto[proto].np_handler == NULL) continue; nwp = &nwsp->nws_work[proto]; if (first) { db_printf("%3d ", cpuid); first = 0; } else db_printf("%3s ", ""); db_printf( "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n", netisr_proto[proto].np_name, nwp->nw_len, nwp->nw_watermark, nwp->nw_qlimit, nwp->nw_dispatched, nwp->nw_hybrid_dispatched, nwp->nw_qdrops, nwp->nw_queued); } } } #endif Index: head/sys/netgraph/bluetooth/socket/ng_btsocket.c =================================================================== --- head/sys/netgraph/bluetooth/socket/ng_btsocket.c (revision 298410) +++ head/sys/netgraph/bluetooth/socket/ng_btsocket.c (revision 298411) @@ -1,290 +1,289 @@ /* * ng_btsocket.c */ /*- * Copyright (c) 2001-2002 Maksim Yevmenkin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: ng_btsocket.c,v 1.4 2003/09/14 23:29:06 max Exp $ * $FreeBSD$ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int ng_btsocket_modevent (module_t, int, void *); static struct domain ng_btsocket_domain; /* * Bluetooth raw HCI sockets */ static struct pr_usrreqs ng_btsocket_hci_raw_usrreqs = { .pru_abort = ng_btsocket_hci_raw_abort, .pru_attach = ng_btsocket_hci_raw_attach, .pru_bind = ng_btsocket_hci_raw_bind, .pru_connect = ng_btsocket_hci_raw_connect, .pru_control = ng_btsocket_hci_raw_control, .pru_detach = ng_btsocket_hci_raw_detach, .pru_disconnect = ng_btsocket_hci_raw_disconnect, .pru_peeraddr = ng_btsocket_hci_raw_peeraddr, .pru_send = ng_btsocket_hci_raw_send, .pru_shutdown = NULL, .pru_sockaddr = ng_btsocket_hci_raw_sockaddr, .pru_close = ng_btsocket_hci_raw_close, }; /* * Bluetooth raw L2CAP sockets */ static struct pr_usrreqs ng_btsocket_l2cap_raw_usrreqs = { .pru_abort = ng_btsocket_l2cap_raw_abort, .pru_attach = ng_btsocket_l2cap_raw_attach, .pru_bind = ng_btsocket_l2cap_raw_bind, .pru_connect = ng_btsocket_l2cap_raw_connect, .pru_control = ng_btsocket_l2cap_raw_control, .pru_detach = ng_btsocket_l2cap_raw_detach, .pru_disconnect = ng_btsocket_l2cap_raw_disconnect, .pru_peeraddr = ng_btsocket_l2cap_raw_peeraddr, .pru_send = ng_btsocket_l2cap_raw_send, .pru_shutdown = NULL, .pru_sockaddr = ng_btsocket_l2cap_raw_sockaddr, .pru_close = ng_btsocket_l2cap_raw_close, }; /* * Bluetooth SEQPACKET L2CAP sockets */ static struct pr_usrreqs ng_btsocket_l2cap_usrreqs = { .pru_abort = ng_btsocket_l2cap_abort, .pru_accept = ng_btsocket_l2cap_accept, .pru_attach = ng_btsocket_l2cap_attach, .pru_bind = ng_btsocket_l2cap_bind, .pru_connect = ng_btsocket_l2cap_connect, .pru_control = ng_btsocket_l2cap_control, .pru_detach = ng_btsocket_l2cap_detach, .pru_disconnect = ng_btsocket_l2cap_disconnect, .pru_listen = ng_btsocket_l2cap_listen, .pru_peeraddr = ng_btsocket_l2cap_peeraddr, .pru_send = ng_btsocket_l2cap_send, .pru_shutdown = NULL, .pru_sockaddr = ng_btsocket_l2cap_sockaddr, .pru_close = ng_btsocket_l2cap_close, }; /* * Bluetooth STREAM RFCOMM sockets */ static struct pr_usrreqs ng_btsocket_rfcomm_usrreqs = { .pru_abort = ng_btsocket_rfcomm_abort, .pru_accept = ng_btsocket_rfcomm_accept, .pru_attach = ng_btsocket_rfcomm_attach, .pru_bind = ng_btsocket_rfcomm_bind, .pru_connect = ng_btsocket_rfcomm_connect, .pru_control = ng_btsocket_rfcomm_control, .pru_detach = ng_btsocket_rfcomm_detach, .pru_disconnect = ng_btsocket_rfcomm_disconnect, .pru_listen = ng_btsocket_rfcomm_listen, .pru_peeraddr = ng_btsocket_rfcomm_peeraddr, .pru_send = ng_btsocket_rfcomm_send, .pru_shutdown = NULL, .pru_sockaddr = ng_btsocket_rfcomm_sockaddr, .pru_close = ng_btsocket_rfcomm_close, }; /* * Bluetooth SEQPACKET SCO sockets */ static struct pr_usrreqs ng_btsocket_sco_usrreqs = { .pru_abort = ng_btsocket_sco_abort, .pru_accept = ng_btsocket_sco_accept, .pru_attach = ng_btsocket_sco_attach, .pru_bind = ng_btsocket_sco_bind, .pru_connect = ng_btsocket_sco_connect, .pru_control = ng_btsocket_sco_control, .pru_detach = ng_btsocket_sco_detach, .pru_disconnect = ng_btsocket_sco_disconnect, .pru_listen = ng_btsocket_sco_listen, .pru_peeraddr = ng_btsocket_sco_peeraddr, .pru_send = ng_btsocket_sco_send, .pru_shutdown = NULL, .pru_sockaddr = ng_btsocket_sco_sockaddr, .pru_close = ng_btsocket_sco_close, }; /* * Definitions of protocols supported in the BLUETOOTH domain */ static struct protosw ng_btsocket_protosw[] = { { .pr_type = SOCK_RAW, .pr_domain = &ng_btsocket_domain, .pr_protocol = BLUETOOTH_PROTO_HCI, .pr_flags = PR_ATOMIC|PR_ADDR, .pr_ctloutput = ng_btsocket_hci_raw_ctloutput, .pr_init = ng_btsocket_hci_raw_init, .pr_usrreqs = &ng_btsocket_hci_raw_usrreqs, }, { .pr_type = SOCK_RAW, .pr_domain = &ng_btsocket_domain, .pr_protocol = BLUETOOTH_PROTO_L2CAP, .pr_flags = PR_ATOMIC|PR_ADDR, .pr_init = ng_btsocket_l2cap_raw_init, .pr_usrreqs = &ng_btsocket_l2cap_raw_usrreqs, }, { .pr_type = SOCK_SEQPACKET, .pr_domain = &ng_btsocket_domain, .pr_protocol = BLUETOOTH_PROTO_L2CAP, .pr_flags = PR_ATOMIC|PR_CONNREQUIRED, .pr_ctloutput = ng_btsocket_l2cap_ctloutput, .pr_init = ng_btsocket_l2cap_init, .pr_usrreqs = &ng_btsocket_l2cap_usrreqs, }, { .pr_type = SOCK_STREAM, .pr_domain = &ng_btsocket_domain, .pr_protocol = BLUETOOTH_PROTO_RFCOMM, .pr_flags = PR_CONNREQUIRED, .pr_ctloutput = ng_btsocket_rfcomm_ctloutput, .pr_init = ng_btsocket_rfcomm_init, .pr_usrreqs = &ng_btsocket_rfcomm_usrreqs, }, { .pr_type = SOCK_SEQPACKET, .pr_domain = &ng_btsocket_domain, .pr_protocol = BLUETOOTH_PROTO_SCO, .pr_flags = PR_ATOMIC|PR_CONNREQUIRED, .pr_ctloutput = ng_btsocket_sco_ctloutput, .pr_init = ng_btsocket_sco_init, .pr_usrreqs = &ng_btsocket_sco_usrreqs, }, }; -#define ng_btsocket_protosw_size \ - (sizeof(ng_btsocket_protosw)/sizeof(ng_btsocket_protosw[0])) + #define ng_btsocket_protosw_end \ - &ng_btsocket_protosw[ng_btsocket_protosw_size] + &ng_btsocket_protosw[nitems(ng_btsocket_protosw)] /* * BLUETOOTH domain */ static struct domain ng_btsocket_domain = { .dom_family = AF_BLUETOOTH, .dom_name = "bluetooth", .dom_protosw = ng_btsocket_protosw, .dom_protoswNPROTOSW = ng_btsocket_protosw_end }; /* * Socket sysctl tree */ SYSCTL_NODE(_net_bluetooth_hci, OID_AUTO, sockets, CTLFLAG_RW, 0, "Bluetooth HCI sockets family"); SYSCTL_NODE(_net_bluetooth_l2cap, OID_AUTO, sockets, CTLFLAG_RW, 0, "Bluetooth L2CAP sockets family"); SYSCTL_NODE(_net_bluetooth_rfcomm, OID_AUTO, sockets, CTLFLAG_RW, 0, "Bluetooth RFCOMM sockets family"); SYSCTL_NODE(_net_bluetooth_sco, OID_AUTO, sockets, CTLFLAG_RW, 0, "Bluetooth SCO sockets family"); /* * Module */ static moduledata_t ng_btsocket_mod = { "ng_btsocket", ng_btsocket_modevent, NULL }; DECLARE_MODULE(ng_btsocket, ng_btsocket_mod, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY); MODULE_VERSION(ng_btsocket, NG_BLUETOOTH_VERSION); MODULE_DEPEND(ng_btsocket, ng_bluetooth, NG_BLUETOOTH_VERSION, NG_BLUETOOTH_VERSION, NG_BLUETOOTH_VERSION); MODULE_DEPEND(ng_btsocket, netgraph, NG_ABI_VERSION, NG_ABI_VERSION, NG_ABI_VERSION); /* * Handle loading and unloading for this node type. * This is to handle auxiliary linkages (e.g protocol domain addition). */ static int ng_btsocket_modevent(module_t mod, int event, void *data) { int error = 0; switch (event) { case MOD_LOAD: break; case MOD_UNLOAD: /* XXX can't unload protocol domain yet */ error = EBUSY; break; default: error = EOPNOTSUPP; break; } return (error); } /* ng_btsocket_modevent */ VNET_DOMAIN_SET(ng_btsocket_); Index: head/sys/nlm/nlm_prot_impl.c =================================================================== --- head/sys/nlm/nlm_prot_impl.c (revision 298410) +++ head/sys/nlm/nlm_prot_impl.c (revision 298411) @@ -1,2433 +1,2432 @@ /*- * Copyright (c) 2008 Isilon Inc http://www.isilon.com/ * Authors: Doug Rabson * Developed with Red Inc: Alfred Perlstein * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_inet6.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #if __FreeBSD_version >= 700000 #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_NLM, "NLM", "Network Lock Manager"); /* * If a host is inactive (and holds no locks) for this amount of * seconds, we consider it idle and stop tracking it. */ #define NLM_IDLE_TIMEOUT 30 /* * We check the host list for idle every few seconds. */ #define NLM_IDLE_PERIOD 5 /* * We only look for GRANTED_RES messages for a little while. */ #define NLM_EXPIRE_TIMEOUT 10 /* * Support for sysctl vfs.nlm.sysid */ static SYSCTL_NODE(_vfs, OID_AUTO, nlm, CTLFLAG_RW, NULL, "Network Lock Manager"); static SYSCTL_NODE(_vfs_nlm, OID_AUTO, sysid, CTLFLAG_RW, NULL, ""); /* * Syscall hooks */ static int nlm_syscall_offset = SYS_nlm_syscall; static struct sysent nlm_syscall_prev_sysent; #if __FreeBSD_version < 700000 static struct sysent nlm_syscall_sysent = { (sizeof(struct nlm_syscall_args) / sizeof(register_t)) | SYF_MPSAFE, (sy_call_t *) nlm_syscall }; #else MAKE_SYSENT(nlm_syscall); #endif static bool_t nlm_syscall_registered = FALSE; /* * Debug level passed in from userland. We also support a sysctl hook * so that it can be changed on a live system. */ static int nlm_debug_level; SYSCTL_INT(_debug, OID_AUTO, nlm_debug, CTLFLAG_RW, &nlm_debug_level, 0, ""); #define NLM_DEBUG(_level, args...) \ do { \ if (nlm_debug_level >= (_level)) \ log(LOG_DEBUG, args); \ } while(0) #define NLM_ERR(args...) \ do { \ log(LOG_ERR, args); \ } while(0) /* * Grace period handling. The value of nlm_grace_threshold is the * value of time_uptime after which we are serving requests normally. */ static time_t nlm_grace_threshold; /* * We check for idle hosts if time_uptime is greater than * nlm_next_idle_check, */ static time_t nlm_next_idle_check; /* * A flag to indicate the server is already running. */ static int nlm_is_running; /* * A socket to use for RPC - shared by all IPv4 RPC clients. */ static struct socket *nlm_socket; #ifdef INET6 /* * A socket to use for RPC - shared by all IPv6 RPC clients. */ static struct socket *nlm_socket6; #endif /* * An RPC client handle that can be used to communicate with the local * NSM. */ static CLIENT *nlm_nsm; /* * An AUTH handle for the server's creds. */ static AUTH *nlm_auth; /* * A zero timeval for sending async RPC messages. */ struct timeval nlm_zero_tv = { 0, 0 }; /* * The local NSM state number */ int nlm_nsm_state; /* * A lock to protect the host list and waiting lock list. */ static struct mtx nlm_global_lock; /* * Locks: * (l) locked by nh_lock * (s) only accessed via server RPC which is single threaded * (g) locked by nlm_global_lock * (c) const until freeing * (a) modified using atomic ops */ /* * A pending client-side lock request, stored on the nlm_waiting_locks * list. */ struct nlm_waiting_lock { TAILQ_ENTRY(nlm_waiting_lock) nw_link; /* (g) */ bool_t nw_waiting; /* (g) */ nlm4_lock nw_lock; /* (c) */ union nfsfh nw_fh; /* (c) */ struct vnode *nw_vp; /* (c) */ }; TAILQ_HEAD(nlm_waiting_lock_list, nlm_waiting_lock); struct nlm_waiting_lock_list nlm_waiting_locks; /* (g) */ /* * A pending server-side asynchronous lock request, stored on the * nh_pending list of the NLM host. */ struct nlm_async_lock { TAILQ_ENTRY(nlm_async_lock) af_link; /* (l) host's list of locks */ struct task af_task; /* (c) async callback details */ void *af_cookie; /* (l) lock manager cancel token */ struct vnode *af_vp; /* (l) vnode to lock */ struct flock af_fl; /* (c) lock details */ struct nlm_host *af_host; /* (c) host which is locking */ CLIENT *af_rpc; /* (c) rpc client to send message */ nlm4_testargs af_granted; /* (c) notification details */ time_t af_expiretime; /* (c) notification time */ }; TAILQ_HEAD(nlm_async_lock_list, nlm_async_lock); /* * NLM host. */ enum nlm_host_state { NLM_UNMONITORED, NLM_MONITORED, NLM_MONITOR_FAILED, NLM_RECOVERING }; struct nlm_rpc { CLIENT *nr_client; /* (l) RPC client handle */ time_t nr_create_time; /* (l) when client was created */ }; struct nlm_host { struct mtx nh_lock; volatile u_int nh_refs; /* (a) reference count */ TAILQ_ENTRY(nlm_host) nh_link; /* (g) global list of hosts */ char nh_caller_name[MAXNAMELEN]; /* (c) printable name of host */ uint32_t nh_sysid; /* (c) our allocaed system ID */ char nh_sysid_string[10]; /* (c) string rep. of sysid */ struct sockaddr_storage nh_addr; /* (s) remote address of host */ struct nlm_rpc nh_srvrpc; /* (l) RPC for server replies */ struct nlm_rpc nh_clntrpc; /* (l) RPC for client requests */ rpcvers_t nh_vers; /* (s) NLM version of host */ int nh_state; /* (s) last seen NSM state of host */ enum nlm_host_state nh_monstate; /* (l) local NSM monitoring state */ time_t nh_idle_timeout; /* (s) Time at which host is idle */ struct sysctl_ctx_list nh_sysctl; /* (c) vfs.nlm.sysid nodes */ uint32_t nh_grantcookie; /* (l) grant cookie counter */ struct nlm_async_lock_list nh_pending; /* (l) pending async locks */ struct nlm_async_lock_list nh_granted; /* (l) granted locks */ struct nlm_async_lock_list nh_finished; /* (l) finished async locks */ }; TAILQ_HEAD(nlm_host_list, nlm_host); static struct nlm_host_list nlm_hosts; /* (g) */ static uint32_t nlm_next_sysid = 1; /* (g) */ static void nlm_host_unmonitor(struct nlm_host *); struct nlm_grantcookie { uint32_t ng_sysid; uint32_t ng_cookie; }; static inline uint32_t ng_sysid(struct netobj *src) { return ((struct nlm_grantcookie *)src->n_bytes)->ng_sysid; } static inline uint32_t ng_cookie(struct netobj *src) { return ((struct nlm_grantcookie *)src->n_bytes)->ng_cookie; } /**********************************************************************/ /* * Initialise NLM globals. */ static void nlm_init(void *dummy) { int error; mtx_init(&nlm_global_lock, "nlm_global_lock", NULL, MTX_DEF); TAILQ_INIT(&nlm_waiting_locks); TAILQ_INIT(&nlm_hosts); error = syscall_register(&nlm_syscall_offset, &nlm_syscall_sysent, &nlm_syscall_prev_sysent, SY_THR_STATIC_KLD); if (error) NLM_ERR("Can't register NLM syscall\n"); else nlm_syscall_registered = TRUE; } SYSINIT(nlm_init, SI_SUB_LOCK, SI_ORDER_FIRST, nlm_init, NULL); static void nlm_uninit(void *dummy) { if (nlm_syscall_registered) syscall_deregister(&nlm_syscall_offset, &nlm_syscall_prev_sysent); } SYSUNINIT(nlm_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, nlm_uninit, NULL); /* * Create a netobj from an arbitrary source. */ void nlm_make_netobj(struct netobj *dst, caddr_t src, size_t srcsize, struct malloc_type *type) { dst->n_len = srcsize; dst->n_bytes = malloc(srcsize, type, M_WAITOK); memcpy(dst->n_bytes, src, srcsize); } /* * Copy a struct netobj. */ void nlm_copy_netobj(struct netobj *dst, struct netobj *src, struct malloc_type *type) { nlm_make_netobj(dst, src->n_bytes, src->n_len, type); } /* * Create an RPC client handle for the given (address,prog,vers) * triple using UDP. */ static CLIENT * nlm_get_rpc(struct sockaddr *sa, rpcprog_t prog, rpcvers_t vers) { char *wchan = "nlmrcv"; const char* protofmly; struct sockaddr_storage ss; struct socket *so; CLIENT *rpcb; struct timeval timo; RPCB parms; char *uaddr; enum clnt_stat stat = RPC_SUCCESS; int rpcvers = RPCBVERS4; bool_t do_tcp = FALSE; bool_t tryagain = FALSE; struct portmap mapping; u_short port = 0; /* * First we need to contact the remote RPCBIND service to find * the right port. */ memcpy(&ss, sa, sa->sa_len); switch (ss.ss_family) { case AF_INET: ((struct sockaddr_in *)&ss)->sin_port = htons(111); protofmly = "inet"; so = nlm_socket; break; #ifdef INET6 case AF_INET6: ((struct sockaddr_in6 *)&ss)->sin6_port = htons(111); protofmly = "inet6"; so = nlm_socket6; break; #endif default: /* * Unsupported address family - fail. */ return (NULL); } rpcb = clnt_dg_create(so, (struct sockaddr *)&ss, RPCBPROG, rpcvers, 0, 0); if (!rpcb) return (NULL); try_tcp: parms.r_prog = prog; parms.r_vers = vers; if (do_tcp) parms.r_netid = "tcp"; else parms.r_netid = "udp"; parms.r_addr = ""; parms.r_owner = ""; /* * Use the default timeout. */ timo.tv_sec = 25; timo.tv_usec = 0; again: switch (rpcvers) { case RPCBVERS4: case RPCBVERS: /* * Try RPCBIND 4 then 3. */ uaddr = NULL; stat = CLNT_CALL(rpcb, (rpcprog_t) RPCBPROC_GETADDR, (xdrproc_t) xdr_rpcb, &parms, (xdrproc_t) xdr_wrapstring, &uaddr, timo); if (stat == RPC_SUCCESS) { /* * We have a reply from the remote RPCBIND - turn it * into an appropriate address and make a new client * that can talk to the remote NLM. * * XXX fixup IPv6 scope ID. */ struct netbuf *a; a = __rpc_uaddr2taddr_af(ss.ss_family, uaddr); if (!a) { tryagain = TRUE; } else { tryagain = FALSE; memcpy(&ss, a->buf, a->len); free(a->buf, M_RPC); free(a, M_RPC); xdr_free((xdrproc_t) xdr_wrapstring, &uaddr); } } if (tryagain || stat == RPC_PROGVERSMISMATCH) { if (rpcvers == RPCBVERS4) rpcvers = RPCBVERS; else if (rpcvers == RPCBVERS) rpcvers = PMAPVERS; CLNT_CONTROL(rpcb, CLSET_VERS, &rpcvers); goto again; } break; case PMAPVERS: /* * Try portmap. */ mapping.pm_prog = parms.r_prog; mapping.pm_vers = parms.r_vers; mapping.pm_prot = do_tcp ? IPPROTO_TCP : IPPROTO_UDP; mapping.pm_port = 0; stat = CLNT_CALL(rpcb, (rpcprog_t) PMAPPROC_GETPORT, (xdrproc_t) xdr_portmap, &mapping, (xdrproc_t) xdr_u_short, &port, timo); if (stat == RPC_SUCCESS) { switch (ss.ss_family) { case AF_INET: ((struct sockaddr_in *)&ss)->sin_port = htons(port); break; #ifdef INET6 case AF_INET6: ((struct sockaddr_in6 *)&ss)->sin6_port = htons(port); break; #endif } } break; default: panic("invalid rpcvers %d", rpcvers); } /* * We may have a positive response from the portmapper, but the NLM * service was not found. Make sure we received a valid port. */ switch (ss.ss_family) { case AF_INET: port = ((struct sockaddr_in *)&ss)->sin_port; break; #ifdef INET6 case AF_INET6: port = ((struct sockaddr_in6 *)&ss)->sin6_port; break; #endif } if (stat != RPC_SUCCESS || !port) { /* * If we were able to talk to rpcbind or portmap, but the udp * variant wasn't available, ask about tcp. * * XXX - We could also check for a TCP portmapper, but * if the host is running a portmapper at all, we should be able * to hail it over UDP. */ if (stat == RPC_SUCCESS && !do_tcp) { do_tcp = TRUE; goto try_tcp; } /* Otherwise, bad news. */ NLM_ERR("NLM: failed to contact remote rpcbind, " "stat = %d, port = %d\n", (int) stat, port); CLNT_DESTROY(rpcb); return (NULL); } if (do_tcp) { /* * Destroy the UDP client we used to speak to rpcbind and * recreate as a TCP client. */ struct netconfig *nconf = NULL; CLNT_DESTROY(rpcb); switch (ss.ss_family) { case AF_INET: nconf = getnetconfigent("tcp"); break; #ifdef INET6 case AF_INET6: nconf = getnetconfigent("tcp6"); break; #endif } rpcb = clnt_reconnect_create(nconf, (struct sockaddr *)&ss, prog, vers, 0, 0); CLNT_CONTROL(rpcb, CLSET_WAITCHAN, wchan); rpcb->cl_auth = nlm_auth; } else { /* * Re-use the client we used to speak to rpcbind. */ CLNT_CONTROL(rpcb, CLSET_SVC_ADDR, &ss); CLNT_CONTROL(rpcb, CLSET_PROG, &prog); CLNT_CONTROL(rpcb, CLSET_VERS, &vers); CLNT_CONTROL(rpcb, CLSET_WAITCHAN, wchan); rpcb->cl_auth = nlm_auth; } return (rpcb); } /* * This async callback after when an async lock request has been * granted. We notify the host which initiated the request. */ static void nlm_lock_callback(void *arg, int pending) { struct nlm_async_lock *af = (struct nlm_async_lock *) arg; struct rpc_callextra ext; NLM_DEBUG(2, "NLM: async lock %p for %s (sysid %d) granted," " cookie %d:%d\n", af, af->af_host->nh_caller_name, af->af_host->nh_sysid, ng_sysid(&af->af_granted.cookie), ng_cookie(&af->af_granted.cookie)); /* * Send the results back to the host. * * Note: there is a possible race here with nlm_host_notify * destroying the RPC client. To avoid problems, the first * thing nlm_host_notify does is to cancel pending async lock * requests. */ memset(&ext, 0, sizeof(ext)); ext.rc_auth = nlm_auth; if (af->af_host->nh_vers == NLM_VERS4) { nlm4_granted_msg_4(&af->af_granted, NULL, af->af_rpc, &ext, nlm_zero_tv); } else { /* * Back-convert to legacy protocol */ nlm_testargs granted; granted.cookie = af->af_granted.cookie; granted.exclusive = af->af_granted.exclusive; granted.alock.caller_name = af->af_granted.alock.caller_name; granted.alock.fh = af->af_granted.alock.fh; granted.alock.oh = af->af_granted.alock.oh; granted.alock.svid = af->af_granted.alock.svid; granted.alock.l_offset = af->af_granted.alock.l_offset; granted.alock.l_len = af->af_granted.alock.l_len; nlm_granted_msg_1(&granted, NULL, af->af_rpc, &ext, nlm_zero_tv); } /* * Move this entry to the nh_granted list. */ af->af_expiretime = time_uptime + NLM_EXPIRE_TIMEOUT; mtx_lock(&af->af_host->nh_lock); TAILQ_REMOVE(&af->af_host->nh_pending, af, af_link); TAILQ_INSERT_TAIL(&af->af_host->nh_granted, af, af_link); mtx_unlock(&af->af_host->nh_lock); } /* * Free an async lock request. The request must have been removed from * any list. */ static void nlm_free_async_lock(struct nlm_async_lock *af) { /* * Free an async lock. */ if (af->af_rpc) CLNT_RELEASE(af->af_rpc); xdr_free((xdrproc_t) xdr_nlm4_testargs, &af->af_granted); if (af->af_vp) vrele(af->af_vp); free(af, M_NLM); } /* * Cancel our async request - this must be called with * af->nh_host->nh_lock held. This is slightly complicated by a * potential race with our own callback. If we fail to cancel the * lock, it must already have been granted - we make sure our async * task has completed by calling taskqueue_drain in this case. */ static int nlm_cancel_async_lock(struct nlm_async_lock *af) { struct nlm_host *host = af->af_host; int error; mtx_assert(&host->nh_lock, MA_OWNED); mtx_unlock(&host->nh_lock); error = VOP_ADVLOCKASYNC(af->af_vp, NULL, F_CANCEL, &af->af_fl, F_REMOTE, NULL, &af->af_cookie); if (error) { /* * We failed to cancel - make sure our callback has * completed before we continue. */ taskqueue_drain(taskqueue_thread, &af->af_task); } mtx_lock(&host->nh_lock); if (!error) { NLM_DEBUG(2, "NLM: async lock %p for %s (sysid %d) " "cancelled\n", af, host->nh_caller_name, host->nh_sysid); /* * Remove from the nh_pending list and free now that * we are safe from the callback. */ TAILQ_REMOVE(&host->nh_pending, af, af_link); mtx_unlock(&host->nh_lock); nlm_free_async_lock(af); mtx_lock(&host->nh_lock); } return (error); } static void nlm_check_expired_locks(struct nlm_host *host) { struct nlm_async_lock *af; time_t uptime = time_uptime; mtx_lock(&host->nh_lock); while ((af = TAILQ_FIRST(&host->nh_granted)) != NULL && uptime >= af->af_expiretime) { NLM_DEBUG(2, "NLM: async lock %p for %s (sysid %d) expired," " cookie %d:%d\n", af, af->af_host->nh_caller_name, af->af_host->nh_sysid, ng_sysid(&af->af_granted.cookie), ng_cookie(&af->af_granted.cookie)); TAILQ_REMOVE(&host->nh_granted, af, af_link); mtx_unlock(&host->nh_lock); nlm_free_async_lock(af); mtx_lock(&host->nh_lock); } while ((af = TAILQ_FIRST(&host->nh_finished)) != NULL) { TAILQ_REMOVE(&host->nh_finished, af, af_link); mtx_unlock(&host->nh_lock); nlm_free_async_lock(af); mtx_lock(&host->nh_lock); } mtx_unlock(&host->nh_lock); } /* * Free resources used by a host. This is called after the reference * count has reached zero so it doesn't need to worry about locks. */ static void nlm_host_destroy(struct nlm_host *host) { mtx_lock(&nlm_global_lock); TAILQ_REMOVE(&nlm_hosts, host, nh_link); mtx_unlock(&nlm_global_lock); if (host->nh_srvrpc.nr_client) CLNT_RELEASE(host->nh_srvrpc.nr_client); if (host->nh_clntrpc.nr_client) CLNT_RELEASE(host->nh_clntrpc.nr_client); mtx_destroy(&host->nh_lock); sysctl_ctx_free(&host->nh_sysctl); free(host, M_NLM); } /* * Thread start callback for client lock recovery */ static void nlm_client_recovery_start(void *arg) { struct nlm_host *host = (struct nlm_host *) arg; NLM_DEBUG(1, "NLM: client lock recovery for %s started\n", host->nh_caller_name); nlm_client_recovery(host); NLM_DEBUG(1, "NLM: client lock recovery for %s completed\n", host->nh_caller_name); host->nh_monstate = NLM_MONITORED; nlm_host_release(host); kthread_exit(); } /* * This is called when we receive a host state change notification. We * unlock any active locks owned by the host. When rpc.lockd is * shutting down, this function is called with newstate set to zero * which allows us to cancel any pending async locks and clear the * locking state. */ static void nlm_host_notify(struct nlm_host *host, int newstate) { struct nlm_async_lock *af; if (newstate) { NLM_DEBUG(1, "NLM: host %s (sysid %d) rebooted, new " "state is %d\n", host->nh_caller_name, host->nh_sysid, newstate); } /* * Cancel any pending async locks for this host. */ mtx_lock(&host->nh_lock); while ((af = TAILQ_FIRST(&host->nh_pending)) != NULL) { /* * nlm_cancel_async_lock will remove the entry from * nh_pending and free it. */ nlm_cancel_async_lock(af); } mtx_unlock(&host->nh_lock); nlm_check_expired_locks(host); /* * The host just rebooted - trash its locks. */ lf_clearremotesys(host->nh_sysid); host->nh_state = newstate; /* * If we have any remote locks for this host (i.e. it * represents a remote NFS server that our local NFS client * has locks for), start a recovery thread. */ if (newstate != 0 && host->nh_monstate != NLM_RECOVERING && lf_countlocks(NLM_SYSID_CLIENT | host->nh_sysid) > 0) { struct thread *td; host->nh_monstate = NLM_RECOVERING; refcount_acquire(&host->nh_refs); kthread_add(nlm_client_recovery_start, host, curproc, &td, 0, 0, "NFS lock recovery for %s", host->nh_caller_name); } } /* * Sysctl handler to count the number of locks for a sysid. */ static int nlm_host_lock_count_sysctl(SYSCTL_HANDLER_ARGS) { struct nlm_host *host; int count; host = oidp->oid_arg1; count = lf_countlocks(host->nh_sysid); return sysctl_handle_int(oidp, &count, 0, req); } /* * Sysctl handler to count the number of client locks for a sysid. */ static int nlm_host_client_lock_count_sysctl(SYSCTL_HANDLER_ARGS) { struct nlm_host *host; int count; host = oidp->oid_arg1; count = lf_countlocks(NLM_SYSID_CLIENT | host->nh_sysid); return sysctl_handle_int(oidp, &count, 0, req); } /* * Create a new NLM host. */ static struct nlm_host * nlm_create_host(const char* caller_name) { struct nlm_host *host; struct sysctl_oid *oid; mtx_assert(&nlm_global_lock, MA_OWNED); NLM_DEBUG(1, "NLM: new host %s (sysid %d)\n", caller_name, nlm_next_sysid); host = malloc(sizeof(struct nlm_host), M_NLM, M_NOWAIT|M_ZERO); if (!host) return (NULL); mtx_init(&host->nh_lock, "nh_lock", NULL, MTX_DEF); host->nh_refs = 1; strlcpy(host->nh_caller_name, caller_name, MAXNAMELEN); host->nh_sysid = nlm_next_sysid++; snprintf(host->nh_sysid_string, sizeof(host->nh_sysid_string), "%d", host->nh_sysid); host->nh_vers = 0; host->nh_state = 0; host->nh_monstate = NLM_UNMONITORED; host->nh_grantcookie = 1; TAILQ_INIT(&host->nh_pending); TAILQ_INIT(&host->nh_granted); TAILQ_INIT(&host->nh_finished); TAILQ_INSERT_TAIL(&nlm_hosts, host, nh_link); mtx_unlock(&nlm_global_lock); sysctl_ctx_init(&host->nh_sysctl); oid = SYSCTL_ADD_NODE(&host->nh_sysctl, SYSCTL_STATIC_CHILDREN(_vfs_nlm_sysid), OID_AUTO, host->nh_sysid_string, CTLFLAG_RD, NULL, ""); SYSCTL_ADD_STRING(&host->nh_sysctl, SYSCTL_CHILDREN(oid), OID_AUTO, "hostname", CTLFLAG_RD, host->nh_caller_name, 0, ""); SYSCTL_ADD_UINT(&host->nh_sysctl, SYSCTL_CHILDREN(oid), OID_AUTO, "version", CTLFLAG_RD, &host->nh_vers, 0, ""); SYSCTL_ADD_UINT(&host->nh_sysctl, SYSCTL_CHILDREN(oid), OID_AUTO, "monitored", CTLFLAG_RD, &host->nh_monstate, 0, ""); SYSCTL_ADD_PROC(&host->nh_sysctl, SYSCTL_CHILDREN(oid), OID_AUTO, "lock_count", CTLTYPE_INT | CTLFLAG_RD, host, 0, nlm_host_lock_count_sysctl, "I", ""); SYSCTL_ADD_PROC(&host->nh_sysctl, SYSCTL_CHILDREN(oid), OID_AUTO, "client_lock_count", CTLTYPE_INT | CTLFLAG_RD, host, 0, nlm_host_client_lock_count_sysctl, "I", ""); mtx_lock(&nlm_global_lock); return (host); } /* * Acquire the next sysid for remote locks not handled by the NLM. */ uint32_t nlm_acquire_next_sysid(void) { uint32_t next_sysid; mtx_lock(&nlm_global_lock); next_sysid = nlm_next_sysid++; mtx_unlock(&nlm_global_lock); return (next_sysid); } /* * Return non-zero if the address parts of the two sockaddrs are the * same. */ static int nlm_compare_addr(const struct sockaddr *a, const struct sockaddr *b) { const struct sockaddr_in *a4, *b4; #ifdef INET6 const struct sockaddr_in6 *a6, *b6; #endif if (a->sa_family != b->sa_family) return (FALSE); switch (a->sa_family) { case AF_INET: a4 = (const struct sockaddr_in *) a; b4 = (const struct sockaddr_in *) b; return !memcmp(&a4->sin_addr, &b4->sin_addr, sizeof(a4->sin_addr)); #ifdef INET6 case AF_INET6: a6 = (const struct sockaddr_in6 *) a; b6 = (const struct sockaddr_in6 *) b; return !memcmp(&a6->sin6_addr, &b6->sin6_addr, sizeof(a6->sin6_addr)); #endif } return (0); } /* * Check for idle hosts and stop monitoring them. We could also free * the host structure here, possibly after a larger timeout but that * would require some care to avoid races with * e.g. nlm_host_lock_count_sysctl. */ static void nlm_check_idle(void) { struct nlm_host *host; mtx_assert(&nlm_global_lock, MA_OWNED); if (time_uptime <= nlm_next_idle_check) return; nlm_next_idle_check = time_uptime + NLM_IDLE_PERIOD; TAILQ_FOREACH(host, &nlm_hosts, nh_link) { if (host->nh_monstate == NLM_MONITORED && time_uptime > host->nh_idle_timeout) { mtx_unlock(&nlm_global_lock); if (lf_countlocks(host->nh_sysid) > 0 || lf_countlocks(NLM_SYSID_CLIENT + host->nh_sysid)) { host->nh_idle_timeout = time_uptime + NLM_IDLE_TIMEOUT; mtx_lock(&nlm_global_lock); continue; } nlm_host_unmonitor(host); mtx_lock(&nlm_global_lock); } } } /* * Search for an existing NLM host that matches the given name * (typically the caller_name element of an nlm4_lock). If none is * found, create a new host. If 'addr' is non-NULL, record the remote * address of the host so that we can call it back for async * responses. If 'vers' is greater than zero then record the NLM * program version to use to communicate with this client. */ struct nlm_host * nlm_find_host_by_name(const char *name, const struct sockaddr *addr, rpcvers_t vers) { struct nlm_host *host; mtx_lock(&nlm_global_lock); /* * The remote host is determined by caller_name. */ TAILQ_FOREACH(host, &nlm_hosts, nh_link) { if (!strcmp(host->nh_caller_name, name)) break; } if (!host) { host = nlm_create_host(name); if (!host) { mtx_unlock(&nlm_global_lock); return (NULL); } } refcount_acquire(&host->nh_refs); host->nh_idle_timeout = time_uptime + NLM_IDLE_TIMEOUT; /* * If we have an address for the host, record it so that we * can send async replies etc. */ if (addr) { KASSERT(addr->sa_len < sizeof(struct sockaddr_storage), ("Strange remote transport address length")); /* * If we have seen an address before and we currently * have an RPC client handle, make sure the address is * the same, otherwise discard the client handle. */ if (host->nh_addr.ss_len && host->nh_srvrpc.nr_client) { if (!nlm_compare_addr( (struct sockaddr *) &host->nh_addr, addr) || host->nh_vers != vers) { CLIENT *client; mtx_lock(&host->nh_lock); client = host->nh_srvrpc.nr_client; host->nh_srvrpc.nr_client = NULL; mtx_unlock(&host->nh_lock); if (client) { CLNT_RELEASE(client); } } } memcpy(&host->nh_addr, addr, addr->sa_len); host->nh_vers = vers; } nlm_check_idle(); mtx_unlock(&nlm_global_lock); return (host); } /* * Search for an existing NLM host that matches the given remote * address. If none is found, create a new host with the requested * address and remember 'vers' as the NLM protocol version to use for * that host. */ struct nlm_host * nlm_find_host_by_addr(const struct sockaddr *addr, int vers) { /* * Fake up a name using inet_ntop. This buffer is * large enough for an IPv6 address. */ char tmp[sizeof "ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255"]; struct nlm_host *host; switch (addr->sa_family) { case AF_INET: inet_ntop(AF_INET, &((const struct sockaddr_in *) addr)->sin_addr, tmp, sizeof tmp); break; #ifdef INET6 case AF_INET6: inet_ntop(AF_INET6, &((const struct sockaddr_in6 *) addr)->sin6_addr, tmp, sizeof tmp); break; #endif default: strlcpy(tmp, "", sizeof(tmp)); } mtx_lock(&nlm_global_lock); /* * The remote host is determined by caller_name. */ TAILQ_FOREACH(host, &nlm_hosts, nh_link) { if (nlm_compare_addr(addr, (const struct sockaddr *) &host->nh_addr)) break; } if (!host) { host = nlm_create_host(tmp); if (!host) { mtx_unlock(&nlm_global_lock); return (NULL); } memcpy(&host->nh_addr, addr, addr->sa_len); host->nh_vers = vers; } refcount_acquire(&host->nh_refs); host->nh_idle_timeout = time_uptime + NLM_IDLE_TIMEOUT; nlm_check_idle(); mtx_unlock(&nlm_global_lock); return (host); } /* * Find the NLM host that matches the value of 'sysid'. If none * exists, return NULL. */ static struct nlm_host * nlm_find_host_by_sysid(int sysid) { struct nlm_host *host; TAILQ_FOREACH(host, &nlm_hosts, nh_link) { if (host->nh_sysid == sysid) { refcount_acquire(&host->nh_refs); return (host); } } return (NULL); } void nlm_host_release(struct nlm_host *host) { if (refcount_release(&host->nh_refs)) { /* * Free the host */ nlm_host_destroy(host); } } /* * Unregister this NLM host with the local NSM due to idleness. */ static void nlm_host_unmonitor(struct nlm_host *host) { mon_id smmonid; sm_stat_res smstat; struct timeval timo; enum clnt_stat stat; NLM_DEBUG(1, "NLM: unmonitoring %s (sysid %d)\n", host->nh_caller_name, host->nh_sysid); /* * We put our assigned system ID value in the priv field to * make it simpler to find the host if we are notified of a * host restart. */ smmonid.mon_name = host->nh_caller_name; smmonid.my_id.my_name = "localhost"; smmonid.my_id.my_prog = NLM_PROG; smmonid.my_id.my_vers = NLM_SM; smmonid.my_id.my_proc = NLM_SM_NOTIFY; timo.tv_sec = 25; timo.tv_usec = 0; stat = CLNT_CALL(nlm_nsm, SM_UNMON, (xdrproc_t) xdr_mon, &smmonid, (xdrproc_t) xdr_sm_stat, &smstat, timo); if (stat != RPC_SUCCESS) { NLM_ERR("Failed to contact local NSM - rpc error %d\n", stat); return; } if (smstat.res_stat == stat_fail) { NLM_ERR("Local NSM refuses to unmonitor %s\n", host->nh_caller_name); return; } host->nh_monstate = NLM_UNMONITORED; } /* * Register this NLM host with the local NSM so that we can be * notified if it reboots. */ void nlm_host_monitor(struct nlm_host *host, int state) { mon smmon; sm_stat_res smstat; struct timeval timo; enum clnt_stat stat; if (state && !host->nh_state) { /* * This is the first time we have seen an NSM state * value for this host. We record it here to help * detect host reboots. */ host->nh_state = state; NLM_DEBUG(1, "NLM: host %s (sysid %d) has NSM state %d\n", host->nh_caller_name, host->nh_sysid, state); } mtx_lock(&host->nh_lock); if (host->nh_monstate != NLM_UNMONITORED) { mtx_unlock(&host->nh_lock); return; } host->nh_monstate = NLM_MONITORED; mtx_unlock(&host->nh_lock); NLM_DEBUG(1, "NLM: monitoring %s (sysid %d)\n", host->nh_caller_name, host->nh_sysid); /* * We put our assigned system ID value in the priv field to * make it simpler to find the host if we are notified of a * host restart. */ smmon.mon_id.mon_name = host->nh_caller_name; smmon.mon_id.my_id.my_name = "localhost"; smmon.mon_id.my_id.my_prog = NLM_PROG; smmon.mon_id.my_id.my_vers = NLM_SM; smmon.mon_id.my_id.my_proc = NLM_SM_NOTIFY; memcpy(smmon.priv, &host->nh_sysid, sizeof(host->nh_sysid)); timo.tv_sec = 25; timo.tv_usec = 0; stat = CLNT_CALL(nlm_nsm, SM_MON, (xdrproc_t) xdr_mon, &smmon, (xdrproc_t) xdr_sm_stat, &smstat, timo); if (stat != RPC_SUCCESS) { NLM_ERR("Failed to contact local NSM - rpc error %d\n", stat); return; } if (smstat.res_stat == stat_fail) { NLM_ERR("Local NSM refuses to monitor %s\n", host->nh_caller_name); mtx_lock(&host->nh_lock); host->nh_monstate = NLM_MONITOR_FAILED; mtx_unlock(&host->nh_lock); return; } host->nh_monstate = NLM_MONITORED; } /* * Return an RPC client handle that can be used to talk to the NLM * running on the given host. */ CLIENT * nlm_host_get_rpc(struct nlm_host *host, bool_t isserver) { struct nlm_rpc *rpc; CLIENT *client; mtx_lock(&host->nh_lock); if (isserver) rpc = &host->nh_srvrpc; else rpc = &host->nh_clntrpc; /* * We can't hold onto RPC handles for too long - the async * call/reply protocol used by some NLM clients makes it hard * to tell when they change port numbers (e.g. after a * reboot). Note that if a client reboots while it isn't * holding any locks, it won't bother to notify us. We * expire the RPC handles after two minutes. */ if (rpc->nr_client && time_uptime > rpc->nr_create_time + 2*60) { client = rpc->nr_client; rpc->nr_client = NULL; mtx_unlock(&host->nh_lock); CLNT_RELEASE(client); mtx_lock(&host->nh_lock); } if (!rpc->nr_client) { mtx_unlock(&host->nh_lock); client = nlm_get_rpc((struct sockaddr *)&host->nh_addr, NLM_PROG, host->nh_vers); mtx_lock(&host->nh_lock); if (client) { if (rpc->nr_client) { mtx_unlock(&host->nh_lock); CLNT_DESTROY(client); mtx_lock(&host->nh_lock); } else { rpc->nr_client = client; rpc->nr_create_time = time_uptime; } } } client = rpc->nr_client; if (client) CLNT_ACQUIRE(client); mtx_unlock(&host->nh_lock); return (client); } int nlm_host_get_sysid(struct nlm_host *host) { return (host->nh_sysid); } int nlm_host_get_state(struct nlm_host *host) { return (host->nh_state); } void * nlm_register_wait_lock(struct nlm4_lock *lock, struct vnode *vp) { struct nlm_waiting_lock *nw; nw = malloc(sizeof(struct nlm_waiting_lock), M_NLM, M_WAITOK); nw->nw_lock = *lock; memcpy(&nw->nw_fh.fh_bytes, nw->nw_lock.fh.n_bytes, nw->nw_lock.fh.n_len); nw->nw_lock.fh.n_bytes = nw->nw_fh.fh_bytes; nw->nw_waiting = TRUE; nw->nw_vp = vp; mtx_lock(&nlm_global_lock); TAILQ_INSERT_TAIL(&nlm_waiting_locks, nw, nw_link); mtx_unlock(&nlm_global_lock); return nw; } void nlm_deregister_wait_lock(void *handle) { struct nlm_waiting_lock *nw = handle; mtx_lock(&nlm_global_lock); TAILQ_REMOVE(&nlm_waiting_locks, nw, nw_link); mtx_unlock(&nlm_global_lock); free(nw, M_NLM); } int nlm_wait_lock(void *handle, int timo) { struct nlm_waiting_lock *nw = handle; int error; /* * If the granted message arrived before we got here, * nw->nw_waiting will be FALSE - in that case, don't sleep. */ mtx_lock(&nlm_global_lock); error = 0; if (nw->nw_waiting) error = msleep(nw, &nlm_global_lock, PCATCH, "nlmlock", timo); TAILQ_REMOVE(&nlm_waiting_locks, nw, nw_link); if (error) { /* * The granted message may arrive after the * interrupt/timeout but before we manage to lock the * mutex. Detect this by examining nw_lock. */ if (!nw->nw_waiting) error = 0; } else { /* * If nlm_cancel_wait is called, then error will be * zero but nw_waiting will still be TRUE. We * translate this into EINTR. */ if (nw->nw_waiting) error = EINTR; } mtx_unlock(&nlm_global_lock); free(nw, M_NLM); return (error); } void nlm_cancel_wait(struct vnode *vp) { struct nlm_waiting_lock *nw; mtx_lock(&nlm_global_lock); TAILQ_FOREACH(nw, &nlm_waiting_locks, nw_link) { if (nw->nw_vp == vp) { wakeup(nw); } } mtx_unlock(&nlm_global_lock); } /**********************************************************************/ /* * Syscall interface with userland. */ extern void nlm_prog_0(struct svc_req *rqstp, SVCXPRT *transp); extern void nlm_prog_1(struct svc_req *rqstp, SVCXPRT *transp); extern void nlm_prog_3(struct svc_req *rqstp, SVCXPRT *transp); extern void nlm_prog_4(struct svc_req *rqstp, SVCXPRT *transp); static int nlm_register_services(SVCPOOL *pool, int addr_count, char **addrs) { static rpcvers_t versions[] = { NLM_SM, NLM_VERS, NLM_VERSX, NLM_VERS4 }; static void (*dispatchers[])(struct svc_req *, SVCXPRT *) = { nlm_prog_0, nlm_prog_1, nlm_prog_3, nlm_prog_4 }; - static const int version_count = sizeof(versions) / sizeof(versions[0]); SVCXPRT **xprts; char netid[16]; char uaddr[128]; struct netconfig *nconf; int i, j, error; if (!addr_count) { NLM_ERR("NLM: no service addresses given - can't start server"); return (EINVAL); } if (addr_count < 0 || addr_count > 256 ) { NLM_ERR("NLM: too many service addresses (%d) given, " "max 256 - can't start server\n", addr_count); return (EINVAL); } xprts = malloc(addr_count * sizeof(SVCXPRT *), M_NLM, M_WAITOK|M_ZERO); - for (i = 0; i < version_count; i++) { + for (i = 0; i < nitems(versions); i++) { for (j = 0; j < addr_count; j++) { /* * Create transports for the first version and * then just register everything else to the * same transports. */ if (i == 0) { char *up; error = copyin(&addrs[2*j], &up, sizeof(char*)); if (error) goto out; error = copyinstr(up, netid, sizeof(netid), NULL); if (error) goto out; error = copyin(&addrs[2*j+1], &up, sizeof(char*)); if (error) goto out; error = copyinstr(up, uaddr, sizeof(uaddr), NULL); if (error) goto out; nconf = getnetconfigent(netid); if (!nconf) { NLM_ERR("Can't lookup netid %s\n", netid); error = EINVAL; goto out; } xprts[j] = svc_tp_create(pool, dispatchers[i], NLM_PROG, versions[i], uaddr, nconf); if (!xprts[j]) { NLM_ERR("NLM: unable to create " "(NLM_PROG, %d).\n", versions[i]); error = EINVAL; goto out; } freenetconfigent(nconf); } else { nconf = getnetconfigent(xprts[j]->xp_netid); rpcb_unset(NLM_PROG, versions[i], nconf); if (!svc_reg(xprts[j], NLM_PROG, versions[i], dispatchers[i], nconf)) { NLM_ERR("NLM: can't register " "(NLM_PROG, %d)\n", versions[i]); error = EINVAL; goto out; } } } } error = 0; out: for (j = 0; j < addr_count; j++) { if (xprts[j]) SVC_RELEASE(xprts[j]); } free(xprts, M_NLM); return (error); } /* * Main server entry point. Contacts the local NSM to get its current * state and send SM_UNMON_ALL. Registers the NLM services and then * services requests. Does not return until the server is interrupted * by a signal. */ static int nlm_server_main(int addr_count, char **addrs) { struct thread *td = curthread; int error; SVCPOOL *pool = NULL; struct sockopt opt; int portlow; #ifdef INET6 struct sockaddr_in6 sin6; #endif struct sockaddr_in sin; my_id id; sm_stat smstat; struct timeval timo; enum clnt_stat stat; struct nlm_host *host, *nhost; struct nlm_waiting_lock *nw; vop_advlock_t *old_nfs_advlock; vop_reclaim_t *old_nfs_reclaim; if (nlm_is_running != 0) { NLM_ERR("NLM: can't start server - " "it appears to be running already\n"); return (EPERM); } if (nlm_socket == NULL) { memset(&opt, 0, sizeof(opt)); error = socreate(AF_INET, &nlm_socket, SOCK_DGRAM, 0, td->td_ucred, td); if (error) { NLM_ERR("NLM: can't create IPv4 socket - error %d\n", error); return (error); } opt.sopt_dir = SOPT_SET; opt.sopt_level = IPPROTO_IP; opt.sopt_name = IP_PORTRANGE; portlow = IP_PORTRANGE_LOW; opt.sopt_val = &portlow; opt.sopt_valsize = sizeof(portlow); sosetopt(nlm_socket, &opt); #ifdef INET6 nlm_socket6 = NULL; error = socreate(AF_INET6, &nlm_socket6, SOCK_DGRAM, 0, td->td_ucred, td); if (error) { NLM_ERR("NLM: can't create IPv6 socket - error %d\n", error); soclose(nlm_socket); nlm_socket = NULL; return (error); } opt.sopt_dir = SOPT_SET; opt.sopt_level = IPPROTO_IPV6; opt.sopt_name = IPV6_PORTRANGE; portlow = IPV6_PORTRANGE_LOW; opt.sopt_val = &portlow; opt.sopt_valsize = sizeof(portlow); sosetopt(nlm_socket6, &opt); #endif } nlm_auth = authunix_create(curthread->td_ucred); #ifdef INET6 memset(&sin6, 0, sizeof(sin6)); sin6.sin6_len = sizeof(sin6); sin6.sin6_family = AF_INET6; sin6.sin6_addr = in6addr_loopback; nlm_nsm = nlm_get_rpc((struct sockaddr *) &sin6, SM_PROG, SM_VERS); if (!nlm_nsm) { #endif memset(&sin, 0, sizeof(sin)); sin.sin_len = sizeof(sin); sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); nlm_nsm = nlm_get_rpc((struct sockaddr *) &sin, SM_PROG, SM_VERS); #ifdef INET6 } #endif if (!nlm_nsm) { NLM_ERR("Can't start NLM - unable to contact NSM\n"); error = EINVAL; goto out; } pool = svcpool_create("NLM", NULL); error = nlm_register_services(pool, addr_count, addrs); if (error) goto out; memset(&id, 0, sizeof(id)); id.my_name = "NFS NLM"; timo.tv_sec = 25; timo.tv_usec = 0; stat = CLNT_CALL(nlm_nsm, SM_UNMON_ALL, (xdrproc_t) xdr_my_id, &id, (xdrproc_t) xdr_sm_stat, &smstat, timo); if (stat != RPC_SUCCESS) { struct rpc_err err; CLNT_GETERR(nlm_nsm, &err); NLM_ERR("NLM: unexpected error contacting NSM, " "stat=%d, errno=%d\n", stat, err.re_errno); error = EINVAL; goto out; } nlm_is_running = 1; NLM_DEBUG(1, "NLM: local NSM state is %d\n", smstat.state); nlm_nsm_state = smstat.state; old_nfs_advlock = nfs_advlock_p; nfs_advlock_p = nlm_advlock; old_nfs_reclaim = nfs_reclaim_p; nfs_reclaim_p = nlm_reclaim; svc_run(pool); error = 0; nfs_advlock_p = old_nfs_advlock; nfs_reclaim_p = old_nfs_reclaim; out: nlm_is_running = 0; if (pool) svcpool_destroy(pool); /* * We are finished communicating with the NSM. */ if (nlm_nsm) { CLNT_RELEASE(nlm_nsm); nlm_nsm = NULL; } /* * Trash all the existing state so that if the server * restarts, it gets a clean slate. This is complicated by the * possibility that there may be other threads trying to make * client locking requests. * * First we fake a client reboot notification which will * cancel any pending async locks and purge remote lock state * from the local lock manager. We release the reference from * nlm_hosts to the host (which may remove it from the list * and free it). After this phase, the only entries in the * nlm_host list should be from other threads performing * client lock requests. */ mtx_lock(&nlm_global_lock); TAILQ_FOREACH(nw, &nlm_waiting_locks, nw_link) { wakeup(nw); } TAILQ_FOREACH_SAFE(host, &nlm_hosts, nh_link, nhost) { mtx_unlock(&nlm_global_lock); nlm_host_notify(host, 0); nlm_host_release(host); mtx_lock(&nlm_global_lock); } mtx_unlock(&nlm_global_lock); AUTH_DESTROY(nlm_auth); return (error); } int sys_nlm_syscall(struct thread *td, struct nlm_syscall_args *uap) { int error; #if __FreeBSD_version >= 700000 error = priv_check(td, PRIV_NFS_LOCKD); #else error = suser(td); #endif if (error) return (error); nlm_debug_level = uap->debug_level; nlm_grace_threshold = time_uptime + uap->grace_period; nlm_next_idle_check = time_uptime + NLM_IDLE_PERIOD; return nlm_server_main(uap->addr_count, uap->addrs); } /**********************************************************************/ /* * NLM implementation details, called from the RPC stubs. */ void nlm_sm_notify(struct nlm_sm_status *argp) { uint32_t sysid; struct nlm_host *host; NLM_DEBUG(3, "nlm_sm_notify(): mon_name = %s\n", argp->mon_name); memcpy(&sysid, &argp->priv, sizeof(sysid)); host = nlm_find_host_by_sysid(sysid); if (host) { nlm_host_notify(host, argp->state); nlm_host_release(host); } } static void nlm_convert_to_fhandle_t(fhandle_t *fhp, struct netobj *p) { memcpy(fhp, p->n_bytes, sizeof(fhandle_t)); } struct vfs_state { struct mount *vs_mp; struct vnode *vs_vp; int vs_vnlocked; }; static int nlm_get_vfs_state(struct nlm_host *host, struct svc_req *rqstp, fhandle_t *fhp, struct vfs_state *vs, accmode_t accmode) { int error, exflags; struct ucred *cred = NULL, *credanon = NULL; memset(vs, 0, sizeof(*vs)); vs->vs_mp = vfs_getvfs(&fhp->fh_fsid); if (!vs->vs_mp) { return (ESTALE); } /* accmode == 0 means don't check, since it is an unlock. */ if (accmode != 0) { error = VFS_CHECKEXP(vs->vs_mp, (struct sockaddr *)&host->nh_addr, &exflags, &credanon, NULL, NULL); if (error) goto out; if (exflags & MNT_EXRDONLY || (vs->vs_mp->mnt_flag & MNT_RDONLY)) { error = EROFS; goto out; } } error = VFS_FHTOVP(vs->vs_mp, &fhp->fh_fid, LK_EXCLUSIVE, &vs->vs_vp); if (error) goto out; vs->vs_vnlocked = TRUE; if (accmode != 0) { if (!svc_getcred(rqstp, &cred, NULL)) { error = EINVAL; goto out; } if (cred->cr_uid == 0 || (exflags & MNT_EXPORTANON)) { crfree(cred); cred = credanon; credanon = NULL; } /* * Check cred. */ error = VOP_ACCESS(vs->vs_vp, accmode, cred, curthread); /* * If this failed and accmode != VWRITE, try again with * VWRITE to maintain backwards compatibility with the * old code that always used VWRITE. */ if (error != 0 && accmode != VWRITE) error = VOP_ACCESS(vs->vs_vp, VWRITE, cred, curthread); if (error) goto out; } #if __FreeBSD_version < 800011 VOP_UNLOCK(vs->vs_vp, 0, curthread); #else VOP_UNLOCK(vs->vs_vp, 0); #endif vs->vs_vnlocked = FALSE; out: if (cred) crfree(cred); if (credanon) crfree(credanon); return (error); } static void nlm_release_vfs_state(struct vfs_state *vs) { if (vs->vs_vp) { if (vs->vs_vnlocked) vput(vs->vs_vp); else vrele(vs->vs_vp); } if (vs->vs_mp) vfs_rel(vs->vs_mp); } static nlm4_stats nlm_convert_error(int error) { if (error == ESTALE) return nlm4_stale_fh; else if (error == EROFS) return nlm4_rofs; else return nlm4_failed; } int nlm_do_test(nlm4_testargs *argp, nlm4_testres *result, struct svc_req *rqstp, CLIENT **rpcp) { fhandle_t fh; struct vfs_state vs; struct nlm_host *host, *bhost; int error, sysid; struct flock fl; accmode_t accmode; memset(result, 0, sizeof(*result)); memset(&vs, 0, sizeof(vs)); host = nlm_find_host_by_name(argp->alock.caller_name, svc_getrpccaller(rqstp), rqstp->rq_vers); if (!host) { result->stat.stat = nlm4_denied_nolocks; return (ENOMEM); } NLM_DEBUG(3, "nlm_do_test(): caller_name = %s (sysid = %d)\n", host->nh_caller_name, host->nh_sysid); nlm_check_expired_locks(host); sysid = host->nh_sysid; nlm_convert_to_fhandle_t(&fh, &argp->alock.fh); nlm_copy_netobj(&result->cookie, &argp->cookie, M_RPC); if (time_uptime < nlm_grace_threshold) { result->stat.stat = nlm4_denied_grace_period; goto out; } accmode = argp->exclusive ? VWRITE : VREAD; error = nlm_get_vfs_state(host, rqstp, &fh, &vs, accmode); if (error) { result->stat.stat = nlm_convert_error(error); goto out; } fl.l_start = argp->alock.l_offset; fl.l_len = argp->alock.l_len; fl.l_pid = argp->alock.svid; fl.l_sysid = sysid; fl.l_whence = SEEK_SET; if (argp->exclusive) fl.l_type = F_WRLCK; else fl.l_type = F_RDLCK; error = VOP_ADVLOCK(vs.vs_vp, NULL, F_GETLK, &fl, F_REMOTE); if (error) { result->stat.stat = nlm4_failed; goto out; } if (fl.l_type == F_UNLCK) { result->stat.stat = nlm4_granted; } else { result->stat.stat = nlm4_denied; result->stat.nlm4_testrply_u.holder.exclusive = (fl.l_type == F_WRLCK); result->stat.nlm4_testrply_u.holder.svid = fl.l_pid; bhost = nlm_find_host_by_sysid(fl.l_sysid); if (bhost) { /* * We don't have any useful way of recording * the value of oh used in the original lock * request. Ideally, the test reply would have * a space for the owning host's name allowing * our caller's NLM to keep track. * * As far as I can see, Solaris uses an eight * byte structure for oh which contains a four * byte pid encoded in local byte order and * the first four bytes of the host * name. Linux uses a variable length string * 'pid@hostname' in ascii but doesn't even * return that in test replies. * * For the moment, return nothing in oh * (already zero'ed above). */ nlm_host_release(bhost); } result->stat.nlm4_testrply_u.holder.l_offset = fl.l_start; result->stat.nlm4_testrply_u.holder.l_len = fl.l_len; } out: nlm_release_vfs_state(&vs); if (rpcp) *rpcp = nlm_host_get_rpc(host, TRUE); nlm_host_release(host); return (0); } int nlm_do_lock(nlm4_lockargs *argp, nlm4_res *result, struct svc_req *rqstp, bool_t monitor, CLIENT **rpcp) { fhandle_t fh; struct vfs_state vs; struct nlm_host *host; int error, sysid; struct flock fl; accmode_t accmode; memset(result, 0, sizeof(*result)); memset(&vs, 0, sizeof(vs)); host = nlm_find_host_by_name(argp->alock.caller_name, svc_getrpccaller(rqstp), rqstp->rq_vers); if (!host) { result->stat.stat = nlm4_denied_nolocks; return (ENOMEM); } NLM_DEBUG(3, "nlm_do_lock(): caller_name = %s (sysid = %d)\n", host->nh_caller_name, host->nh_sysid); if (monitor && host->nh_state && argp->state && host->nh_state != argp->state) { /* * The host rebooted without telling us. Trash its * locks. */ nlm_host_notify(host, argp->state); } nlm_check_expired_locks(host); sysid = host->nh_sysid; nlm_convert_to_fhandle_t(&fh, &argp->alock.fh); nlm_copy_netobj(&result->cookie, &argp->cookie, M_RPC); if (time_uptime < nlm_grace_threshold && !argp->reclaim) { result->stat.stat = nlm4_denied_grace_period; goto out; } accmode = argp->exclusive ? VWRITE : VREAD; error = nlm_get_vfs_state(host, rqstp, &fh, &vs, accmode); if (error) { result->stat.stat = nlm_convert_error(error); goto out; } fl.l_start = argp->alock.l_offset; fl.l_len = argp->alock.l_len; fl.l_pid = argp->alock.svid; fl.l_sysid = sysid; fl.l_whence = SEEK_SET; if (argp->exclusive) fl.l_type = F_WRLCK; else fl.l_type = F_RDLCK; if (argp->block) { struct nlm_async_lock *af; CLIENT *client; struct nlm_grantcookie cookie; /* * First, make sure we can contact the host's NLM. */ client = nlm_host_get_rpc(host, TRUE); if (!client) { result->stat.stat = nlm4_failed; goto out; } /* * First we need to check and see if there is an * existing blocked lock that matches. This could be a * badly behaved client or an RPC re-send. If we find * one, just return nlm4_blocked. */ mtx_lock(&host->nh_lock); TAILQ_FOREACH(af, &host->nh_pending, af_link) { if (af->af_fl.l_start == fl.l_start && af->af_fl.l_len == fl.l_len && af->af_fl.l_pid == fl.l_pid && af->af_fl.l_type == fl.l_type) { break; } } if (!af) { cookie.ng_sysid = host->nh_sysid; cookie.ng_cookie = host->nh_grantcookie++; } mtx_unlock(&host->nh_lock); if (af) { CLNT_RELEASE(client); result->stat.stat = nlm4_blocked; goto out; } af = malloc(sizeof(struct nlm_async_lock), M_NLM, M_WAITOK|M_ZERO); TASK_INIT(&af->af_task, 0, nlm_lock_callback, af); af->af_vp = vs.vs_vp; af->af_fl = fl; af->af_host = host; af->af_rpc = client; /* * We use M_RPC here so that we can xdr_free the thing * later. */ nlm_make_netobj(&af->af_granted.cookie, (caddr_t)&cookie, sizeof(cookie), M_RPC); af->af_granted.exclusive = argp->exclusive; af->af_granted.alock.caller_name = strdup(argp->alock.caller_name, M_RPC); nlm_copy_netobj(&af->af_granted.alock.fh, &argp->alock.fh, M_RPC); nlm_copy_netobj(&af->af_granted.alock.oh, &argp->alock.oh, M_RPC); af->af_granted.alock.svid = argp->alock.svid; af->af_granted.alock.l_offset = argp->alock.l_offset; af->af_granted.alock.l_len = argp->alock.l_len; /* * Put the entry on the pending list before calling * VOP_ADVLOCKASYNC. We do this in case the lock * request was blocked (returning EINPROGRESS) but * then granted before we manage to run again. The * client may receive the granted message before we * send our blocked reply but thats their problem. */ mtx_lock(&host->nh_lock); TAILQ_INSERT_TAIL(&host->nh_pending, af, af_link); mtx_unlock(&host->nh_lock); error = VOP_ADVLOCKASYNC(vs.vs_vp, NULL, F_SETLK, &fl, F_REMOTE, &af->af_task, &af->af_cookie); /* * If the lock completed synchronously, just free the * tracking structure now. */ if (error != EINPROGRESS) { CLNT_RELEASE(af->af_rpc); mtx_lock(&host->nh_lock); TAILQ_REMOVE(&host->nh_pending, af, af_link); mtx_unlock(&host->nh_lock); xdr_free((xdrproc_t) xdr_nlm4_testargs, &af->af_granted); free(af, M_NLM); } else { NLM_DEBUG(2, "NLM: pending async lock %p for %s " "(sysid %d)\n", af, host->nh_caller_name, sysid); /* * Don't vrele the vnode just yet - this must * wait until either the async callback * happens or the lock is cancelled. */ vs.vs_vp = NULL; } } else { error = VOP_ADVLOCK(vs.vs_vp, NULL, F_SETLK, &fl, F_REMOTE); } if (error) { if (error == EINPROGRESS) { result->stat.stat = nlm4_blocked; } else if (error == EDEADLK) { result->stat.stat = nlm4_deadlck; } else if (error == EAGAIN) { result->stat.stat = nlm4_denied; } else { result->stat.stat = nlm4_failed; } } else { if (monitor) nlm_host_monitor(host, argp->state); result->stat.stat = nlm4_granted; } out: nlm_release_vfs_state(&vs); if (rpcp) *rpcp = nlm_host_get_rpc(host, TRUE); nlm_host_release(host); return (0); } int nlm_do_cancel(nlm4_cancargs *argp, nlm4_res *result, struct svc_req *rqstp, CLIENT **rpcp) { fhandle_t fh; struct vfs_state vs; struct nlm_host *host; int error, sysid; struct flock fl; struct nlm_async_lock *af; memset(result, 0, sizeof(*result)); memset(&vs, 0, sizeof(vs)); host = nlm_find_host_by_name(argp->alock.caller_name, svc_getrpccaller(rqstp), rqstp->rq_vers); if (!host) { result->stat.stat = nlm4_denied_nolocks; return (ENOMEM); } NLM_DEBUG(3, "nlm_do_cancel(): caller_name = %s (sysid = %d)\n", host->nh_caller_name, host->nh_sysid); nlm_check_expired_locks(host); sysid = host->nh_sysid; nlm_convert_to_fhandle_t(&fh, &argp->alock.fh); nlm_copy_netobj(&result->cookie, &argp->cookie, M_RPC); if (time_uptime < nlm_grace_threshold) { result->stat.stat = nlm4_denied_grace_period; goto out; } error = nlm_get_vfs_state(host, rqstp, &fh, &vs, (accmode_t)0); if (error) { result->stat.stat = nlm_convert_error(error); goto out; } fl.l_start = argp->alock.l_offset; fl.l_len = argp->alock.l_len; fl.l_pid = argp->alock.svid; fl.l_sysid = sysid; fl.l_whence = SEEK_SET; if (argp->exclusive) fl.l_type = F_WRLCK; else fl.l_type = F_RDLCK; /* * First we need to try and find the async lock request - if * there isn't one, we give up and return nlm4_denied. */ mtx_lock(&host->nh_lock); TAILQ_FOREACH(af, &host->nh_pending, af_link) { if (af->af_fl.l_start == fl.l_start && af->af_fl.l_len == fl.l_len && af->af_fl.l_pid == fl.l_pid && af->af_fl.l_type == fl.l_type) { break; } } if (!af) { mtx_unlock(&host->nh_lock); result->stat.stat = nlm4_denied; goto out; } error = nlm_cancel_async_lock(af); if (error) { result->stat.stat = nlm4_denied; } else { result->stat.stat = nlm4_granted; } mtx_unlock(&host->nh_lock); out: nlm_release_vfs_state(&vs); if (rpcp) *rpcp = nlm_host_get_rpc(host, TRUE); nlm_host_release(host); return (0); } int nlm_do_unlock(nlm4_unlockargs *argp, nlm4_res *result, struct svc_req *rqstp, CLIENT **rpcp) { fhandle_t fh; struct vfs_state vs; struct nlm_host *host; int error, sysid; struct flock fl; memset(result, 0, sizeof(*result)); memset(&vs, 0, sizeof(vs)); host = nlm_find_host_by_name(argp->alock.caller_name, svc_getrpccaller(rqstp), rqstp->rq_vers); if (!host) { result->stat.stat = nlm4_denied_nolocks; return (ENOMEM); } NLM_DEBUG(3, "nlm_do_unlock(): caller_name = %s (sysid = %d)\n", host->nh_caller_name, host->nh_sysid); nlm_check_expired_locks(host); sysid = host->nh_sysid; nlm_convert_to_fhandle_t(&fh, &argp->alock.fh); nlm_copy_netobj(&result->cookie, &argp->cookie, M_RPC); if (time_uptime < nlm_grace_threshold) { result->stat.stat = nlm4_denied_grace_period; goto out; } error = nlm_get_vfs_state(host, rqstp, &fh, &vs, (accmode_t)0); if (error) { result->stat.stat = nlm_convert_error(error); goto out; } fl.l_start = argp->alock.l_offset; fl.l_len = argp->alock.l_len; fl.l_pid = argp->alock.svid; fl.l_sysid = sysid; fl.l_whence = SEEK_SET; fl.l_type = F_UNLCK; error = VOP_ADVLOCK(vs.vs_vp, NULL, F_UNLCK, &fl, F_REMOTE); /* * Ignore the error - there is no result code for failure, * only for grace period. */ result->stat.stat = nlm4_granted; out: nlm_release_vfs_state(&vs); if (rpcp) *rpcp = nlm_host_get_rpc(host, TRUE); nlm_host_release(host); return (0); } int nlm_do_granted(nlm4_testargs *argp, nlm4_res *result, struct svc_req *rqstp, CLIENT **rpcp) { struct nlm_host *host; struct nlm_waiting_lock *nw; memset(result, 0, sizeof(*result)); host = nlm_find_host_by_addr(svc_getrpccaller(rqstp), rqstp->rq_vers); if (!host) { result->stat.stat = nlm4_denied_nolocks; return (ENOMEM); } nlm_copy_netobj(&result->cookie, &argp->cookie, M_RPC); result->stat.stat = nlm4_denied; KFAIL_POINT_CODE(DEBUG_FP, nlm_deny_grant, goto out); mtx_lock(&nlm_global_lock); TAILQ_FOREACH(nw, &nlm_waiting_locks, nw_link) { if (!nw->nw_waiting) continue; if (argp->alock.svid == nw->nw_lock.svid && argp->alock.l_offset == nw->nw_lock.l_offset && argp->alock.l_len == nw->nw_lock.l_len && argp->alock.fh.n_len == nw->nw_lock.fh.n_len && !memcmp(argp->alock.fh.n_bytes, nw->nw_lock.fh.n_bytes, nw->nw_lock.fh.n_len)) { nw->nw_waiting = FALSE; wakeup(nw); result->stat.stat = nlm4_granted; break; } } mtx_unlock(&nlm_global_lock); out: if (rpcp) *rpcp = nlm_host_get_rpc(host, TRUE); nlm_host_release(host); return (0); } void nlm_do_granted_res(nlm4_res *argp, struct svc_req *rqstp) { struct nlm_host *host = NULL; struct nlm_async_lock *af = NULL; int error; if (argp->cookie.n_len != sizeof(struct nlm_grantcookie)) { NLM_DEBUG(1, "NLM: bogus grant cookie"); goto out; } host = nlm_find_host_by_sysid(ng_sysid(&argp->cookie)); if (!host) { NLM_DEBUG(1, "NLM: Unknown host rejected our grant"); goto out; } mtx_lock(&host->nh_lock); TAILQ_FOREACH(af, &host->nh_granted, af_link) if (ng_cookie(&argp->cookie) == ng_cookie(&af->af_granted.cookie)) break; if (af) TAILQ_REMOVE(&host->nh_granted, af, af_link); mtx_unlock(&host->nh_lock); if (!af) { NLM_DEBUG(1, "NLM: host %s (sysid %d) replied to our grant " "with unrecognized cookie %d:%d", host->nh_caller_name, host->nh_sysid, ng_sysid(&argp->cookie), ng_cookie(&argp->cookie)); goto out; } if (argp->stat.stat != nlm4_granted) { af->af_fl.l_type = F_UNLCK; error = VOP_ADVLOCK(af->af_vp, NULL, F_UNLCK, &af->af_fl, F_REMOTE); if (error) { NLM_DEBUG(1, "NLM: host %s (sysid %d) rejected our grant " "and we failed to unlock (%d)", host->nh_caller_name, host->nh_sysid, error); goto out; } NLM_DEBUG(5, "NLM: async lock %p rejected by host %s (sysid %d)", af, host->nh_caller_name, host->nh_sysid); } else { NLM_DEBUG(5, "NLM: async lock %p accepted by host %s (sysid %d)", af, host->nh_caller_name, host->nh_sysid); } out: if (af) nlm_free_async_lock(af); if (host) nlm_host_release(host); } void nlm_do_free_all(nlm4_notify *argp) { struct nlm_host *host, *thost; TAILQ_FOREACH_SAFE(host, &nlm_hosts, nh_link, thost) { if (!strcmp(host->nh_caller_name, argp->name)) nlm_host_notify(host, argp->state); } } /* * Kernel module glue */ static int nfslockd_modevent(module_t mod, int type, void *data) { switch (type) { case MOD_LOAD: return (0); case MOD_UNLOAD: /* The NLM module cannot be safely unloaded. */ /* FALLTHROUGH */ default: return (EOPNOTSUPP); } } static moduledata_t nfslockd_mod = { "nfslockd", nfslockd_modevent, NULL, }; DECLARE_MODULE(nfslockd, nfslockd_mod, SI_SUB_VFS, SI_ORDER_ANY); /* So that loader and kldload(2) can find us, wherever we are.. */ MODULE_DEPEND(nfslockd, krpc, 1, 1, 1); MODULE_DEPEND(nfslockd, nfslock, 1, 1, 1); MODULE_VERSION(nfslockd, 1); Index: head/sys/security/audit/audit_bsm_klib.c =================================================================== --- head/sys/security/audit/audit_bsm_klib.c (revision 298410) +++ head/sys/security/audit/audit_bsm_klib.c (revision 298411) @@ -1,573 +1,571 @@ /* * Copyright (c) 1999-2009 Apple Inc. * Copyright (c) 2005 Robert N. M. Watson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of Apple Inc. ("Apple") nor the names of * its contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Hash table functions for the audit event number to event class mask * mapping. */ #define EVCLASSMAP_HASH_TABLE_SIZE 251 struct evclass_elem { au_event_t event; au_class_t class; LIST_ENTRY(evclass_elem) entry; }; struct evclass_list { LIST_HEAD(, evclass_elem) head; }; static MALLOC_DEFINE(M_AUDITEVCLASS, "audit_evclass", "Audit event class"); static struct rwlock evclass_lock; static struct evclass_list evclass_hash[EVCLASSMAP_HASH_TABLE_SIZE]; #define EVCLASS_LOCK_INIT() rw_init(&evclass_lock, "evclass_lock") #define EVCLASS_RLOCK() rw_rlock(&evclass_lock) #define EVCLASS_RUNLOCK() rw_runlock(&evclass_lock) #define EVCLASS_WLOCK() rw_wlock(&evclass_lock) #define EVCLASS_WUNLOCK() rw_wunlock(&evclass_lock) struct aue_open_event { int aoe_flags; au_event_t aoe_event; }; static const struct aue_open_event aue_open[] = { { O_RDONLY, AUE_OPEN_R }, { (O_RDONLY | O_CREAT), AUE_OPEN_RC }, { (O_RDONLY | O_CREAT | O_TRUNC), AUE_OPEN_RTC }, { (O_RDONLY | O_TRUNC), AUE_OPEN_RT }, { O_RDWR, AUE_OPEN_RW }, { (O_RDWR | O_CREAT), AUE_OPEN_RWC }, { (O_RDWR | O_CREAT | O_TRUNC), AUE_OPEN_RWTC }, { (O_RDWR | O_TRUNC), AUE_OPEN_RWT }, { O_WRONLY, AUE_OPEN_W }, { (O_WRONLY | O_CREAT), AUE_OPEN_WC }, { (O_WRONLY | O_CREAT | O_TRUNC), AUE_OPEN_WTC }, { (O_WRONLY | O_TRUNC), AUE_OPEN_WT }, }; -static const int aue_open_count = sizeof(aue_open) / sizeof(aue_open[0]); static const struct aue_open_event aue_openat[] = { { O_RDONLY, AUE_OPENAT_R }, { (O_RDONLY | O_CREAT), AUE_OPENAT_RC }, { (O_RDONLY | O_CREAT | O_TRUNC), AUE_OPENAT_RTC }, { (O_RDONLY | O_TRUNC), AUE_OPENAT_RT }, { O_RDWR, AUE_OPENAT_RW }, { (O_RDWR | O_CREAT), AUE_OPENAT_RWC }, { (O_RDWR | O_CREAT | O_TRUNC), AUE_OPENAT_RWTC }, { (O_RDWR | O_TRUNC), AUE_OPENAT_RWT }, { O_WRONLY, AUE_OPENAT_W }, { (O_WRONLY | O_CREAT), AUE_OPENAT_WC }, { (O_WRONLY | O_CREAT | O_TRUNC), AUE_OPENAT_WTC }, { (O_WRONLY | O_TRUNC), AUE_OPENAT_WT }, }; -static const int aue_openat_count = sizeof(aue_openat) / sizeof(aue_openat[0]); /* * Look up the class for an audit event in the class mapping table. */ au_class_t au_event_class(au_event_t event) { struct evclass_list *evcl; struct evclass_elem *evc; au_class_t class; EVCLASS_RLOCK(); evcl = &evclass_hash[event % EVCLASSMAP_HASH_TABLE_SIZE]; class = 0; LIST_FOREACH(evc, &evcl->head, entry) { if (evc->event == event) { class = evc->class; goto out; } } out: EVCLASS_RUNLOCK(); return (class); } /* * Insert a event to class mapping. If the event already exists in the * mapping, then replace the mapping with the new one. * * XXX There is currently no constraints placed on the number of mappings. * May want to either limit to a number, or in terms of memory usage. */ void au_evclassmap_insert(au_event_t event, au_class_t class) { struct evclass_list *evcl; struct evclass_elem *evc, *evc_new; /* * Pessimistically, always allocate storage before acquiring mutex. * Free if there is already a mapping for this event. */ evc_new = malloc(sizeof(*evc), M_AUDITEVCLASS, M_WAITOK); EVCLASS_WLOCK(); evcl = &evclass_hash[event % EVCLASSMAP_HASH_TABLE_SIZE]; LIST_FOREACH(evc, &evcl->head, entry) { if (evc->event == event) { evc->class = class; EVCLASS_WUNLOCK(); free(evc_new, M_AUDITEVCLASS); return; } } evc = evc_new; evc->event = event; evc->class = class; LIST_INSERT_HEAD(&evcl->head, evc, entry); EVCLASS_WUNLOCK(); } void au_evclassmap_init(void) { int i; EVCLASS_LOCK_INIT(); for (i = 0; i < EVCLASSMAP_HASH_TABLE_SIZE; i++) LIST_INIT(&evclass_hash[i].head); /* * Set up the initial event to class mapping for system calls. * * XXXRW: Really, this should walk all possible audit events, not all * native ABI system calls, as there may be audit events reachable * only through non-native system calls. It also seems a shame to * frob the mutex this early. */ for (i = 0; i < SYS_MAXSYSCALL; i++) { if (sysent[i].sy_auevent != AUE_NULL) au_evclassmap_insert(sysent[i].sy_auevent, 0); } } /* * Check whether an event is aditable by comparing the mask of classes this * event is part of against the given mask. */ int au_preselect(au_event_t event, au_class_t class, au_mask_t *mask_p, int sorf) { au_class_t effmask = 0; if (mask_p == NULL) return (-1); /* * Perform the actual check of the masks against the event. */ if (sorf & AU_PRS_SUCCESS) effmask |= (mask_p->am_success & class); if (sorf & AU_PRS_FAILURE) effmask |= (mask_p->am_failure & class); if (effmask) return (1); else return (0); } /* * Convert sysctl names and present arguments to events. */ au_event_t audit_ctlname_to_sysctlevent(int name[], uint64_t valid_arg) { /* can't parse it - so return the worst case */ if ((valid_arg & (ARG_CTLNAME | ARG_LEN)) != (ARG_CTLNAME | ARG_LEN)) return (AUE_SYSCTL); switch (name[0]) { /* non-admin "lookups" treat them special */ case KERN_OSTYPE: case KERN_OSRELEASE: case KERN_OSREV: case KERN_VERSION: case KERN_ARGMAX: case KERN_CLOCKRATE: case KERN_BOOTTIME: case KERN_POSIX1: case KERN_NGROUPS: case KERN_JOB_CONTROL: case KERN_SAVED_IDS: case KERN_OSRELDATE: case KERN_DUMMY: return (AUE_SYSCTL_NONADMIN); /* only treat the changeable controls as admin */ case KERN_MAXVNODES: case KERN_MAXPROC: case KERN_MAXFILES: case KERN_MAXPROCPERUID: case KERN_MAXFILESPERPROC: case KERN_HOSTID: case KERN_SECURELVL: case KERN_HOSTNAME: case KERN_VNODE: case KERN_PROC: case KERN_FILE: case KERN_PROF: case KERN_NISDOMAINNAME: case KERN_UPDATEINTERVAL: case KERN_NTP_PLL: case KERN_BOOTFILE: case KERN_DUMPDEV: case KERN_IPC: case KERN_PS_STRINGS: case KERN_USRSTACK: case KERN_LOGSIGEXIT: case KERN_IOV_MAX: return ((valid_arg & ARG_VALUE) ? AUE_SYSCTL : AUE_SYSCTL_NONADMIN); default: return (AUE_SYSCTL); } /* NOTREACHED */ } /* * Convert an open flags specifier into a specific type of open event for * auditing purposes. */ au_event_t audit_flags_and_error_to_openevent(int oflags, int error) { int i; /* * Need to check only those flags we care about. */ oflags = oflags & (O_RDONLY | O_CREAT | O_TRUNC | O_RDWR | O_WRONLY); - for (i = 0; i < aue_open_count; i++) { + for (i = 0; i < nitems(aue_open); i++) { if (aue_open[i].aoe_flags == oflags) return (aue_open[i].aoe_event); } return (AUE_OPEN); } au_event_t audit_flags_and_error_to_openatevent(int oflags, int error) { int i; /* * Need to check only those flags we care about. */ oflags = oflags & (O_RDONLY | O_CREAT | O_TRUNC | O_RDWR | O_WRONLY); - for (i = 0; i < aue_openat_count; i++) { + for (i = 0; i < nitems(aue_openat); i++) { if (aue_openat[i].aoe_flags == oflags) return (aue_openat[i].aoe_event); } return (AUE_OPENAT); } /* * Convert a MSGCTL command to a specific event. */ au_event_t audit_msgctl_to_event(int cmd) { switch (cmd) { case IPC_RMID: return (AUE_MSGCTL_RMID); case IPC_SET: return (AUE_MSGCTL_SET); case IPC_STAT: return (AUE_MSGCTL_STAT); default: /* We will audit a bad command. */ return (AUE_MSGCTL); } } /* * Convert a SEMCTL command to a specific event. */ au_event_t audit_semctl_to_event(int cmd) { switch (cmd) { case GETALL: return (AUE_SEMCTL_GETALL); case GETNCNT: return (AUE_SEMCTL_GETNCNT); case GETPID: return (AUE_SEMCTL_GETPID); case GETVAL: return (AUE_SEMCTL_GETVAL); case GETZCNT: return (AUE_SEMCTL_GETZCNT); case IPC_RMID: return (AUE_SEMCTL_RMID); case IPC_SET: return (AUE_SEMCTL_SET); case SETALL: return (AUE_SEMCTL_SETALL); case SETVAL: return (AUE_SEMCTL_SETVAL); case IPC_STAT: return (AUE_SEMCTL_STAT); default: /* We will audit a bad command. */ return (AUE_SEMCTL); } } /* * Convert a command for the auditon() system call to a audit event. */ au_event_t auditon_command_event(int cmd) { switch(cmd) { case A_GETPOLICY: return (AUE_AUDITON_GPOLICY); case A_SETPOLICY: return (AUE_AUDITON_SPOLICY); case A_GETKMASK: return (AUE_AUDITON_GETKMASK); case A_SETKMASK: return (AUE_AUDITON_SETKMASK); case A_GETQCTRL: return (AUE_AUDITON_GQCTRL); case A_SETQCTRL: return (AUE_AUDITON_SQCTRL); case A_GETCWD: return (AUE_AUDITON_GETCWD); case A_GETCAR: return (AUE_AUDITON_GETCAR); case A_GETSTAT: return (AUE_AUDITON_GETSTAT); case A_SETSTAT: return (AUE_AUDITON_SETSTAT); case A_SETUMASK: return (AUE_AUDITON_SETUMASK); case A_SETSMASK: return (AUE_AUDITON_SETSMASK); case A_GETCOND: return (AUE_AUDITON_GETCOND); case A_SETCOND: return (AUE_AUDITON_SETCOND); case A_GETCLASS: return (AUE_AUDITON_GETCLASS); case A_SETCLASS: return (AUE_AUDITON_SETCLASS); case A_GETPINFO: case A_SETPMASK: case A_SETFSIZE: case A_GETFSIZE: case A_GETPINFO_ADDR: case A_GETKAUDIT: case A_SETKAUDIT: default: return (AUE_AUDITON); /* No special record */ } } /* * Create a canonical path from given path by prefixing either the root * directory, or the current working directory. If the process working * directory is NULL, we could use 'rootvnode' to obtain the root directory, * but this results in a volfs name written to the audit log. So we will * leave the filename starting with '/' in the audit log in this case. */ void audit_canon_path(struct thread *td, int dirfd, char *path, char *cpath) { struct vnode *cvnp, *rvnp; char *rbuf, *fbuf, *copy; struct filedesc *fdp; struct sbuf sbf; cap_rights_t rights; int error, needslash; WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s: at %s:%d", __func__, __FILE__, __LINE__); copy = path; rvnp = cvnp = NULL; fdp = td->td_proc->p_fd; FILEDESC_SLOCK(fdp); /* * Make sure that we handle the chroot(2) case. If there is an * alternate root directory, prepend it to the audited pathname. */ if (fdp->fd_rdir != NULL && fdp->fd_rdir != rootvnode) { rvnp = fdp->fd_rdir; vhold(rvnp); } /* * If the supplied path is relative, make sure we capture the current * working directory so we can prepend it to the supplied relative * path. */ if (*path != '/') { if (dirfd == AT_FDCWD) { cvnp = fdp->fd_cdir; vhold(cvnp); } else { /* XXX: fgetvp() that vhold()s vnode instead of vref()ing it would be better */ error = fgetvp(td, dirfd, cap_rights_init(&rights), &cvnp); if (error) { FILEDESC_SUNLOCK(fdp); cpath[0] = '\0'; if (rvnp != NULL) vdrop(rvnp); return; } vhold(cvnp); vrele(cvnp); } needslash = (fdp->fd_rdir != cvnp); } else { needslash = 1; } FILEDESC_SUNLOCK(fdp); /* * NB: We require that the supplied array be at least MAXPATHLEN bytes * long. If this is not the case, then we can run into serious trouble. */ (void) sbuf_new(&sbf, cpath, MAXPATHLEN, SBUF_FIXEDLEN); /* * Strip leading forward slashes. */ while (*copy == '/') copy++; /* * Make sure we handle chroot(2) and prepend the global path to these * environments. * * NB: vn_fullpath(9) on FreeBSD is less reliable than vn_getpath(9) * on Darwin. As a result, this may need some additional attention * in the future. */ if (rvnp != NULL) { error = vn_fullpath_global(td, rvnp, &rbuf, &fbuf); vdrop(rvnp); if (error) { cpath[0] = '\0'; if (cvnp != NULL) vdrop(cvnp); return; } (void) sbuf_cat(&sbf, rbuf); free(fbuf, M_TEMP); } if (cvnp != NULL) { error = vn_fullpath(td, cvnp, &rbuf, &fbuf); vdrop(cvnp); if (error) { cpath[0] = '\0'; return; } (void) sbuf_cat(&sbf, rbuf); free(fbuf, M_TEMP); } if (needslash) (void) sbuf_putc(&sbf, '/'); /* * Now that we have processed any alternate root and relative path * names, add the supplied pathname. */ (void) sbuf_cat(&sbf, copy); /* * One or more of the previous sbuf operations could have resulted in * the supplied buffer being overflowed. Check to see if this is the * case. */ if (sbuf_error(&sbf) != 0) { cpath[0] = '\0'; return; } sbuf_finish(&sbf); } Index: head/sys/security/audit/bsm_errno.c =================================================================== --- head/sys/security/audit/bsm_errno.c (revision 298410) +++ head/sys/security/audit/bsm_errno.c (revision 298411) @@ -1,773 +1,772 @@ /*- * Copyright (c) 2008 Apple Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of Apple Inc. ("Apple") nor the names of * its contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * Different operating systems use different numeric constants for different * error numbers, and sometimes error numbers don't exist in more than one * operating system. These routines convert between BSM and local error * number spaces, subject to the above realities. BSM error numbers are * stored in a single 8-bit character, so don't have a byte order. * * Don't include string definitions when this code is compiled into a kernel. */ struct bsm_errno { int be_bsm_errno; int be_local_errno; #if !defined(KERNEL) && !defined(_KERNEL) const char *be_strerror; #endif }; #define ERRNO_NO_LOCAL_MAPPING -600 #if !defined(KERNEL) && !defined(_KERNEL) #define ES(x) x #else #define ES(x) #endif /* * Mapping table -- please maintain in numeric sorted order with respect to * the BSM constant. Today we do a linear lookup, but could switch to a * binary search if it makes sense. We only ifdef errors that aren't * generally available, but it does make the table a lot more ugly. * * XXXRW: It would be nice to have a similar ordered table mapping to BSM * constant from local constant, but the order of local constants varies by * OS. Really we need to build that table at compile-time but don't do that * yet. * * XXXRW: We currently embed English-language error strings here, but should * support catalogues; these are only used if the OS doesn't have an error * string using strerror(3). */ static const struct bsm_errno bsm_errnos[] = { { BSM_ERRNO_ESUCCESS, 0, ES("Success") }, { BSM_ERRNO_EPERM, EPERM, ES("Operation not permitted") }, { BSM_ERRNO_ENOENT, ENOENT, ES("No such file or directory") }, { BSM_ERRNO_ESRCH, ESRCH, ES("No such process") }, { BSM_ERRNO_EINTR, EINTR, ES("Interrupted system call") }, { BSM_ERRNO_EIO, EIO, ES("Input/output error") }, { BSM_ERRNO_ENXIO, ENXIO, ES("Device not configured") }, { BSM_ERRNO_E2BIG, E2BIG, ES("Argument list too long") }, { BSM_ERRNO_ENOEXEC, ENOEXEC, ES("Exec format error") }, { BSM_ERRNO_EBADF, EBADF, ES("Bad file descriptor") }, { BSM_ERRNO_ECHILD, ECHILD, ES("No child processes") }, { BSM_ERRNO_EAGAIN, EAGAIN, ES("Resource temporarily unavailable") }, { BSM_ERRNO_ENOMEM, ENOMEM, ES("Cannot allocate memory") }, { BSM_ERRNO_EACCES, EACCES, ES("Permission denied") }, { BSM_ERRNO_EFAULT, EFAULT, ES("Bad address") }, { BSM_ERRNO_ENOTBLK, ENOTBLK, ES("Block device required") }, { BSM_ERRNO_EBUSY, EBUSY, ES("Device busy") }, { BSM_ERRNO_EEXIST, EEXIST, ES("File exists") }, { BSM_ERRNO_EXDEV, EXDEV, ES("Cross-device link") }, { BSM_ERRNO_ENODEV, ENODEV, ES("Operation not supported by device") }, { BSM_ERRNO_ENOTDIR, ENOTDIR, ES("Not a directory") }, { BSM_ERRNO_EISDIR, EISDIR, ES("Is a directory") }, { BSM_ERRNO_EINVAL, EINVAL, ES("Invalid argument") }, { BSM_ERRNO_ENFILE, ENFILE, ES("Too many open files in system") }, { BSM_ERRNO_EMFILE, EMFILE, ES("Too many open files") }, { BSM_ERRNO_ENOTTY, ENOTTY, ES("Inappropriate ioctl for device") }, { BSM_ERRNO_ETXTBSY, ETXTBSY, ES("Text file busy") }, { BSM_ERRNO_EFBIG, EFBIG, ES("File too large") }, { BSM_ERRNO_ENOSPC, ENOSPC, ES("No space left on device") }, { BSM_ERRNO_ESPIPE, ESPIPE, ES("Illegal seek") }, { BSM_ERRNO_EROFS, EROFS, ES("Read-only file system") }, { BSM_ERRNO_EMLINK, EMLINK, ES("Too many links") }, { BSM_ERRNO_EPIPE, EPIPE, ES("Broken pipe") }, { BSM_ERRNO_EDOM, EDOM, ES("Numerical argument out of domain") }, { BSM_ERRNO_ERANGE, ERANGE, ES("Result too large") }, { BSM_ERRNO_ENOMSG, ENOMSG, ES("No message of desired type") }, { BSM_ERRNO_EIDRM, EIDRM, ES("Identifier removed") }, { BSM_ERRNO_ECHRNG, #ifdef ECHRNG ECHRNG, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Channel number out of range") }, { BSM_ERRNO_EL2NSYNC, #ifdef EL2NSYNC EL2NSYNC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Level 2 not synchronized") }, { BSM_ERRNO_EL3HLT, #ifdef EL3HLT EL3HLT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Level 3 halted") }, { BSM_ERRNO_EL3RST, #ifdef EL3RST EL3RST, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Level 3 reset") }, { BSM_ERRNO_ELNRNG, #ifdef ELNRNG ELNRNG, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Link number out of range") }, { BSM_ERRNO_EUNATCH, #ifdef EUNATCH EUNATCH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Protocol driver not attached") }, { BSM_ERRNO_ENOCSI, #ifdef ENOCSI ENOCSI, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("No CSI structure available") }, { BSM_ERRNO_EL2HLT, #ifdef EL2HLT EL2HLT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Level 2 halted") }, { BSM_ERRNO_EDEADLK, EDEADLK, ES("Resource deadlock avoided") }, { BSM_ERRNO_ENOLCK, ENOLCK, ES("No locks available") }, { BSM_ERRNO_ECANCELED, ECANCELED, ES("Operation canceled") }, { BSM_ERRNO_ENOTSUP, ENOTSUP, ES("Operation not supported") }, { BSM_ERRNO_EDQUOT, EDQUOT, ES("Disc quota exceeded") }, { BSM_ERRNO_EBADE, #ifdef EBADE EBADE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Invalid exchange") }, { BSM_ERRNO_EBADR, #ifdef EBADR EBADR, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Invalid request descriptor") }, { BSM_ERRNO_EXFULL, #ifdef EXFULL EXFULL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Exchange full") }, { BSM_ERRNO_ENOANO, #ifdef ENOANO ENOANO, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("No anode") }, { BSM_ERRNO_EBADRQC, #ifdef EBADRQC EBADRQC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Invalid request descriptor") }, { BSM_ERRNO_EBADSLT, #ifdef EBADSLT EBADSLT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Invalid slot") }, { BSM_ERRNO_EDEADLOCK, #ifdef EDEADLOCK EDEADLOCK, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Resource deadlock avoided") }, { BSM_ERRNO_EBFONT, #ifdef EBFONT EBFONT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Bad font file format") }, { BSM_ERRNO_EOWNERDEAD, #ifdef EOWNERDEAD EOWNERDEAD, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Process died with the lock") }, { BSM_ERRNO_ENOTRECOVERABLE, #ifdef ENOTRECOVERABLE ENOTRECOVERABLE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Lock is not recoverable") }, { BSM_ERRNO_ENOSTR, #ifdef ENOSTR ENOSTR, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Device not a stream") }, { BSM_ERRNO_ENONET, #ifdef ENONET ENONET, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Machine is not on the network") }, { BSM_ERRNO_ENOPKG, #ifdef ENOPKG ENOPKG, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Package not installed") }, { BSM_ERRNO_EREMOTE, EREMOTE, ES("Too many levels of remote in path") }, { BSM_ERRNO_ENOLINK, #ifdef ENOLINK ENOLINK, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Link has been severed") }, { BSM_ERRNO_EADV, #ifdef EADV EADV, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Advertise error") }, { BSM_ERRNO_ESRMNT, #ifdef ESRMNT ESRMNT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("srmount error") }, { BSM_ERRNO_ECOMM, #ifdef ECOMM ECOMM, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Communication error on send") }, { BSM_ERRNO_EPROTO, #ifdef EPROTO EPROTO, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Protocol error") }, { BSM_ERRNO_ELOCKUNMAPPED, #ifdef ELOCKUNMAPPED ELOCKUNMAPPED, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Locked lock was unmapped") }, { BSM_ERRNO_ENOTACTIVE, #ifdef ENOTACTIVE ENOTACTIVE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Facility is not active") }, { BSM_ERRNO_EMULTIHOP, #ifdef EMULTIHOP EMULTIHOP, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Multihop attempted") }, { BSM_ERRNO_EBADMSG, #ifdef EBADMSG EBADMSG, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Bad message") }, { BSM_ERRNO_ENAMETOOLONG, ENAMETOOLONG, ES("File name too long") }, { BSM_ERRNO_EOVERFLOW, EOVERFLOW, ES("Value too large to be stored in data type") }, { BSM_ERRNO_ENOTUNIQ, #ifdef ENOTUNIQ ENOTUNIQ, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Given log name not unique") }, { BSM_ERRNO_EBADFD, #ifdef EBADFD EBADFD, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Given f.d. invalid for this operation") }, { BSM_ERRNO_EREMCHG, #ifdef EREMCHG EREMCHG, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Remote address changed") }, { BSM_ERRNO_ELIBACC, #ifdef ELIBACC ELIBACC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Can't access a needed shared lib") }, { BSM_ERRNO_ELIBBAD, #ifdef ELIBBAD ELIBBAD, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Accessing a corrupted shared lib") }, { BSM_ERRNO_ELIBSCN, #ifdef ELIBSCN ELIBSCN, #else ERRNO_NO_LOCAL_MAPPING, #endif ES(".lib section in a.out corrupted") }, { BSM_ERRNO_ELIBMAX, #ifdef ELIBMAX ELIBMAX, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Attempting to link in too many libs") }, { BSM_ERRNO_ELIBEXEC, #ifdef ELIBEXEC ELIBEXEC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Attempting to exec a shared library") }, { BSM_ERRNO_EILSEQ, EILSEQ, ES("Illegal byte sequence") }, { BSM_ERRNO_ENOSYS, ENOSYS, ES("Function not implemented") }, { BSM_ERRNO_ELOOP, ELOOP, ES("Too many levels of symbolic links") }, { BSM_ERRNO_ERESTART, #ifdef ERESTART ERESTART, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Restart syscall") }, { BSM_ERRNO_ESTRPIPE, #ifdef ESTRPIPE ESTRPIPE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("If pipe/FIFO, don't sleep in stream head") }, { BSM_ERRNO_ENOTEMPTY, ENOTEMPTY, ES("Directory not empty") }, { BSM_ERRNO_EUSERS, EUSERS, ES("Too many users") }, { BSM_ERRNO_ENOTSOCK, ENOTSOCK, ES("Socket operation on non-socket") }, { BSM_ERRNO_EDESTADDRREQ, EDESTADDRREQ, ES("Destination address required") }, { BSM_ERRNO_EMSGSIZE, EMSGSIZE, ES("Message too long") }, { BSM_ERRNO_EPROTOTYPE, EPROTOTYPE, ES("Protocol wrong type for socket") }, { BSM_ERRNO_ENOPROTOOPT, ENOPROTOOPT, ES("Protocol not available") }, { BSM_ERRNO_EPROTONOSUPPORT, EPROTONOSUPPORT, ES("Protocol not supported") }, { BSM_ERRNO_ESOCKTNOSUPPORT, ESOCKTNOSUPPORT, ES("Socket type not supported") }, { BSM_ERRNO_EOPNOTSUPP, EOPNOTSUPP, ES("Operation not supported") }, { BSM_ERRNO_EPFNOSUPPORT, EPFNOSUPPORT, ES("Protocol family not supported") }, { BSM_ERRNO_EAFNOSUPPORT, EAFNOSUPPORT, ES("Address family not supported by protocol family") }, { BSM_ERRNO_EADDRINUSE, EADDRINUSE, ES("Address already in use") }, { BSM_ERRNO_EADDRNOTAVAIL, EADDRNOTAVAIL, ES("Can't assign requested address") }, { BSM_ERRNO_ENETDOWN, ENETDOWN, ES("Network is down") }, { BSM_ERRNO_ENETRESET, ENETRESET, ES("Network dropped connection on reset") }, { BSM_ERRNO_ECONNABORTED, ECONNABORTED, ES("Software caused connection abort") }, { BSM_ERRNO_ECONNRESET, ECONNRESET, ES("Connection reset by peer") }, { BSM_ERRNO_ENOBUFS, ENOBUFS, ES("No buffer space available") }, { BSM_ERRNO_EISCONN, EISCONN, ES("Socket is already connected") }, { BSM_ERRNO_ENOTCONN, ENOTCONN, ES("Socket is not connected") }, { BSM_ERRNO_ESHUTDOWN, ESHUTDOWN, ES("Can't send after socket shutdown") }, { BSM_ERRNO_ETOOMANYREFS, ETOOMANYREFS, ES("Too many references: can't splice") }, { BSM_ERRNO_ETIMEDOUT, ETIMEDOUT, ES("Operation timed out") }, { BSM_ERRNO_ECONNREFUSED, ECONNREFUSED, ES("Connection refused") }, { BSM_ERRNO_EHOSTDOWN, EHOSTDOWN, ES("Host is down") }, { BSM_ERRNO_EHOSTUNREACH, EHOSTUNREACH, ES("No route to host") }, { BSM_ERRNO_EALREADY, EALREADY, ES("Operation already in progress") }, { BSM_ERRNO_EINPROGRESS, EINPROGRESS, ES("Operation now in progress") }, { BSM_ERRNO_ESTALE, ESTALE, ES("Stale NFS file handle") }, { BSM_ERRNO_EPROCLIM, #ifdef EPROCLIM EPROCLIM, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Too many processes") }, { BSM_ERRNO_EBADRPC, #ifdef EBADRPC EBADRPC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("RPC struct is bad") }, { BSM_ERRNO_ERPCMISMATCH, #ifdef ERPCMISMATCH ERPCMISMATCH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("RPC version wrong") }, { BSM_ERRNO_EPROGUNAVAIL, #ifdef EPROGUNAVAIL EPROGUNAVAIL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("RPC prog. not avail") }, { BSM_ERRNO_EPROGMISMATCH, #ifdef EPROGMISMATCH EPROGMISMATCH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("RPC version wrong") }, { BSM_ERRNO_EPROCUNAVAIL, #ifdef EPROCUNAVAIL EPROCUNAVAIL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Bad procedure for program") }, { BSM_ERRNO_EFTYPE, #ifdef EFTYPE EFTYPE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Inappropriate file type or format") }, { BSM_ERRNO_EAUTH, #ifdef EAUTH EAUTH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Authenticateion error") }, { BSM_ERRNO_ENEEDAUTH, #ifdef ENEEDAUTH ENEEDAUTH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Need authenticator") }, { BSM_ERRNO_ENOATTR, #ifdef ENOATTR ENOATTR, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Attribute not found") }, { BSM_ERRNO_EDOOFUS, #ifdef EDOOFUS EDOOFUS, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Programming error") }, { BSM_ERRNO_EJUSTRETURN, #ifdef EJUSTRETURN EJUSTRETURN, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Just return") }, { BSM_ERRNO_ENOIOCTL, #ifdef ENOIOCTL ENOIOCTL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("ioctl not handled by this layer") }, { BSM_ERRNO_EDIRIOCTL, #ifdef EDIRIOCTL EDIRIOCTL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("do direct ioctl in GEOM") }, { BSM_ERRNO_EPWROFF, #ifdef EPWROFF EPWROFF, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Device power is off") }, { BSM_ERRNO_EDEVERR, #ifdef EDEVERR EDEVERR, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Device error") }, { BSM_ERRNO_EBADEXEC, #ifdef EBADEXEC EBADEXEC, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Bad executable") }, { BSM_ERRNO_EBADARCH, #ifdef EBADARCH EBADARCH, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Bad CPU type in executable") }, { BSM_ERRNO_ESHLIBVERS, #ifdef ESHLIBVERS ESHLIBVERS, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Shared library version mismatch") }, { BSM_ERRNO_EBADMACHO, #ifdef EBADMACHO EBADMACHO, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Malformed Macho file") }, { BSM_ERRNO_EPOLICY, #ifdef EPOLICY EPOLICY, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Operation failed by policy") }, { BSM_ERRNO_EDOTDOT, #ifdef EDOTDOT EDOTDOT, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("RFS specific error") }, { BSM_ERRNO_EUCLEAN, #ifdef EUCLEAN EUCLEAN, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Structure needs cleaning") }, { BSM_ERRNO_ENOTNAM, #ifdef ENOTNAM ENOTNAM, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Not a XENIX named type file") }, { BSM_ERRNO_ENAVAIL, #ifdef ENAVAIL ENAVAIL, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("No XENIX semaphores available") }, { BSM_ERRNO_EISNAM, #ifdef EISNAM EISNAM, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Is a named type file") }, { BSM_ERRNO_EREMOTEIO, #ifdef EREMOTEIO EREMOTEIO, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Remote I/O error") }, { BSM_ERRNO_ENOMEDIUM, #ifdef ENOMEDIUM ENOMEDIUM, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("No medium found") }, { BSM_ERRNO_EMEDIUMTYPE, #ifdef EMEDIUMTYPE EMEDIUMTYPE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Wrong medium type") }, { BSM_ERRNO_ENOKEY, #ifdef ENOKEY ENOKEY, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Required key not available") }, { BSM_ERRNO_EKEYEXPIRED, #ifdef EKEYEXPIRED EKEYEXPIRED, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Key has expired") }, { BSM_ERRNO_EKEYREVOKED, #ifdef EKEYREVOKED EKEYREVOKED, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Key has been revoked") }, { BSM_ERRNO_EKEYREJECTED, #ifdef EKEYREJECTED EKEYREJECTED, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Key was rejected by service") }, { BSM_ERRNO_ENOTCAPABLE, #ifdef ENOTCAPABLE ENOTCAPABLE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Capabilities insufficient") }, { BSM_ERRNO_ECAPMODE, #ifdef ECAPMODE ECAPMODE, #else ERRNO_NO_LOCAL_MAPPING, #endif ES("Not permitted in capability mode") }, }; -static const int bsm_errnos_count = sizeof(bsm_errnos) / sizeof(bsm_errnos[0]); static const struct bsm_errno * bsm_lookup_errno_local(int local_errno) { int i; - for (i = 0; i < bsm_errnos_count; i++) { + for (i = 0; i < nitems(bsm_errnos); i++) { if (bsm_errnos[i].be_local_errno == local_errno) return (&bsm_errnos[i]); } return (NULL); } /* * Conversion to the BSM errno space isn't allowed to fail; we simply map to * BSM_ERRNO_UNKNOWN and let the remote endpoint deal with it. */ u_char au_errno_to_bsm(int local_errno) { const struct bsm_errno *bsme; bsme = bsm_lookup_errno_local(local_errno); if (bsme == NULL) return (BSM_ERRNO_UNKNOWN); return (bsme->be_bsm_errno); } static const struct bsm_errno * bsm_lookup_errno_bsm(u_char bsm_errno) { int i; - for (i = 0; i < bsm_errnos_count; i++) { + for (i = 0; i < nitems(bsm_errnos); i++) { if (bsm_errnos[i].be_bsm_errno == bsm_errno) return (&bsm_errnos[i]); } return (NULL); } /* * Converstion from a BSM error to a local error number may fail if either * OpenBSM doesn't recognize the error on the wire, or because there is no * appropriate local mapping. */ int au_bsm_to_errno(u_char bsm_errno, int *errorp) { const struct bsm_errno *bsme; bsme = bsm_lookup_errno_bsm(bsm_errno); if (bsme == NULL || bsme->be_local_errno == ERRNO_NO_LOCAL_MAPPING) return (-1); *errorp = bsme->be_local_errno; return (0); } #if !defined(KERNEL) && !defined(_KERNEL) const char * au_strerror(u_char bsm_errno) { const struct bsm_errno *bsme; bsme = bsm_lookup_errno_bsm(bsm_errno); if (bsme == NULL) return ("Unrecognized BSM error"); if (bsme->be_local_errno != ERRNO_NO_LOCAL_MAPPING) return (strerror(bsme->be_local_errno)); return (bsme->be_strerror); } #endif Index: head/sys/vm/vm_pager.c =================================================================== --- head/sys/vm/vm_pager.c (revision 298410) +++ head/sys/vm/vm_pager.c (revision 298411) @@ -1,562 +1,560 @@ /*- * Copyright (c) 1991, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * The Mach Operating System project at Carnegie-Mellon University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)vm_pager.c 8.6 (Berkeley) 1/12/94 * * * Copyright (c) 1987, 1990 Carnegie-Mellon University. * All rights reserved. * * Authors: Avadis Tevanian, Jr., Michael Wayne Young * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /* * Paging space routine stubs. Emulates a matchmaker-like interface * for builtin pagers. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include int cluster_pbuf_freecnt = -1; /* unlimited to begin with */ struct buf *swbuf; static int dead_pager_getpages(vm_object_t, vm_page_t *, int, int *, int *); static vm_object_t dead_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *); static void dead_pager_putpages(vm_object_t, vm_page_t *, int, int, int *); static boolean_t dead_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); static void dead_pager_dealloc(vm_object_t); static int dead_pager_getpages(vm_object_t obj, vm_page_t *ma, int count, int *rbehind, int *rahead) { return (VM_PAGER_FAIL); } static vm_object_t dead_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t off, struct ucred *cred) { return NULL; } static void dead_pager_putpages(object, m, count, flags, rtvals) vm_object_t object; vm_page_t *m; int count; int flags; int *rtvals; { int i; for (i = 0; i < count; i++) { rtvals[i] = VM_PAGER_AGAIN; } } static int dead_pager_haspage(object, pindex, prev, next) vm_object_t object; vm_pindex_t pindex; int *prev; int *next; { if (prev) *prev = 0; if (next) *next = 0; return FALSE; } static void dead_pager_dealloc(object) vm_object_t object; { return; } static struct pagerops deadpagerops = { .pgo_alloc = dead_pager_alloc, .pgo_dealloc = dead_pager_dealloc, .pgo_getpages = dead_pager_getpages, .pgo_putpages = dead_pager_putpages, .pgo_haspage = dead_pager_haspage, }; struct pagerops *pagertab[] = { &defaultpagerops, /* OBJT_DEFAULT */ &swappagerops, /* OBJT_SWAP */ &vnodepagerops, /* OBJT_VNODE */ &devicepagerops, /* OBJT_DEVICE */ &physpagerops, /* OBJT_PHYS */ &deadpagerops, /* OBJT_DEAD */ &sgpagerops, /* OBJT_SG */ &mgtdevicepagerops, /* OBJT_MGTDEVICE */ }; -static const int npagers = sizeof(pagertab) / sizeof(pagertab[0]); - /* * Kernel address space for mapping pages. * Used by pagers where KVAs are needed for IO. * * XXX needs to be large enough to support the number of pending async * cleaning requests (NPENDINGIO == 64) * the maximum swap cluster size * (MAXPHYS == 64k) if you want to get the most efficiency. */ struct mtx_padalign pbuf_mtx; static TAILQ_HEAD(swqueue, buf) bswlist; static int bswneeded; vm_offset_t swapbkva; /* swap buffers kva */ void vm_pager_init() { struct pagerops **pgops; TAILQ_INIT(&bswlist); /* * Initialize known pagers */ - for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++) + for (pgops = pagertab; pgops < &pagertab[nitems(pagertab)]; pgops++) if ((*pgops)->pgo_init != NULL) (*(*pgops)->pgo_init) (); } void vm_pager_bufferinit() { struct buf *bp; int i; mtx_init(&pbuf_mtx, "pbuf mutex", NULL, MTX_DEF); bp = swbuf; /* * Now set up swap and physical I/O buffer headers. */ for (i = 0; i < nswbuf; i++, bp++) { TAILQ_INSERT_HEAD(&bswlist, bp, b_freelist); BUF_LOCKINIT(bp); LIST_INIT(&bp->b_dep); bp->b_rcred = bp->b_wcred = NOCRED; bp->b_xflags = 0; } cluster_pbuf_freecnt = nswbuf / 2; vnode_pbuf_freecnt = nswbuf / 2 + 1; vnode_async_pbuf_freecnt = nswbuf / 2; } /* * Allocate an instance of a pager of the given type. * Size, protection and offset parameters are passed in for pagers that * need to perform page-level validation (e.g. the device pager). */ vm_object_t vm_pager_allocate(objtype_t type, void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t off, struct ucred *cred) { vm_object_t ret; struct pagerops *ops; ops = pagertab[type]; if (ops) ret = (*ops->pgo_alloc) (handle, size, prot, off, cred); else ret = NULL; return (ret); } /* * The object must be locked. */ void vm_pager_deallocate(object) vm_object_t object; { VM_OBJECT_ASSERT_WLOCKED(object); (*pagertab[object->type]->pgo_dealloc) (object); } static void vm_pager_assert_in(vm_object_t object, vm_page_t *m, int count) { #ifdef INVARIANTS VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(count > 0, ("%s: 0 count", __func__)); /* * All pages must be busied, not mapped, not fully valid, * not dirty and belong to the proper object. */ for (int i = 0 ; i < count; i++) { vm_page_assert_xbusied(m[i]); KASSERT(!pmap_page_is_mapped(m[i]), ("%s: page %p is mapped", __func__, m[i])); KASSERT(m[i]->valid != VM_PAGE_BITS_ALL, ("%s: request for a valid page %p", __func__, m[i])); KASSERT(m[i]->dirty == 0, ("%s: page %p is dirty", __func__, m[i])); KASSERT(m[i]->object == object, ("%s: wrong object %p/%p", __func__, object, m[i]->object)); } #endif } /* * Page in the pages for the object using its associated pager. * The requested page must be fully valid on successful return. */ int vm_pager_get_pages(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead) { #ifdef INVARIANTS vm_pindex_t pindex = m[0]->pindex; #endif int r; vm_pager_assert_in(object, m, count); r = (*pagertab[object->type]->pgo_getpages)(object, m, count, rbehind, rahead); if (r != VM_PAGER_OK) return (r); for (int i = 0; i < count; i++) { /* * If pager has replaced a page, assert that it had * updated the array. */ KASSERT(m[i] == vm_page_lookup(object, pindex++), ("%s: mismatch page %p pindex %ju", __func__, m[i], (uintmax_t )pindex - 1)); /* * Zero out partially filled data. */ if (m[i]->valid != VM_PAGE_BITS_ALL) vm_page_zero_invalid(m[i], TRUE); } return (VM_PAGER_OK); } int vm_pager_get_pages_async(vm_object_t object, vm_page_t *m, int count, int *rbehind, int *rahead, pgo_getpages_iodone_t iodone, void *arg) { vm_pager_assert_in(object, m, count); return ((*pagertab[object->type]->pgo_getpages_async)(object, m, count, rbehind, rahead, iodone, arg)); } /* * vm_pager_put_pages() - inline, see vm/vm_pager.h * vm_pager_has_page() - inline, see vm/vm_pager.h */ /* * Search the specified pager object list for an object with the * specified handle. If an object with the specified handle is found, * increase its reference count and return it. Otherwise, return NULL. * * The pager object list must be locked. */ vm_object_t vm_pager_object_lookup(struct pagerlst *pg_list, void *handle) { vm_object_t object; TAILQ_FOREACH(object, pg_list, pager_object_list) { if (object->handle == handle) { VM_OBJECT_WLOCK(object); if ((object->flags & OBJ_DEAD) == 0) { vm_object_reference_locked(object); VM_OBJECT_WUNLOCK(object); break; } VM_OBJECT_WUNLOCK(object); } } return (object); } /* * initialize a physical buffer */ /* * XXX This probably belongs in vfs_bio.c */ static void initpbuf(struct buf *bp) { KASSERT(bp->b_bufobj == NULL, ("initpbuf with bufobj")); KASSERT(bp->b_vp == NULL, ("initpbuf with vp")); bp->b_rcred = NOCRED; bp->b_wcred = NOCRED; bp->b_qindex = 0; /* On no queue (QUEUE_NONE) */ bp->b_kvabase = (caddr_t) (MAXPHYS * (bp - swbuf)) + swapbkva; bp->b_data = bp->b_kvabase; bp->b_kvasize = MAXPHYS; bp->b_flags = 0; bp->b_xflags = 0; bp->b_ioflags = 0; bp->b_iodone = NULL; bp->b_error = 0; BUF_LOCK(bp, LK_EXCLUSIVE, NULL); } /* * allocate a physical buffer * * There are a limited number (nswbuf) of physical buffers. We need * to make sure that no single subsystem is able to hog all of them, * so each subsystem implements a counter which is typically initialized * to 1/2 nswbuf. getpbuf() decrements this counter in allocation and * increments it on release, and blocks if the counter hits zero. A * subsystem may initialize the counter to -1 to disable the feature, * but it must still be sure to match up all uses of getpbuf() with * relpbuf() using the same variable. * * NOTE: pfreecnt can be NULL, but this 'feature' will be removed * relatively soon when the rest of the subsystems get smart about it. XXX */ struct buf * getpbuf(int *pfreecnt) { struct buf *bp; mtx_lock(&pbuf_mtx); for (;;) { if (pfreecnt) { while (*pfreecnt == 0) { msleep(pfreecnt, &pbuf_mtx, PVM, "wswbuf0", 0); } } /* get a bp from the swap buffer header pool */ if ((bp = TAILQ_FIRST(&bswlist)) != NULL) break; bswneeded = 1; msleep(&bswneeded, &pbuf_mtx, PVM, "wswbuf1", 0); /* loop in case someone else grabbed one */ } TAILQ_REMOVE(&bswlist, bp, b_freelist); if (pfreecnt) --*pfreecnt; mtx_unlock(&pbuf_mtx); initpbuf(bp); return bp; } /* * allocate a physical buffer, if one is available. * * Note that there is no NULL hack here - all subsystems using this * call understand how to use pfreecnt. */ struct buf * trypbuf(int *pfreecnt) { struct buf *bp; mtx_lock(&pbuf_mtx); if (*pfreecnt == 0 || (bp = TAILQ_FIRST(&bswlist)) == NULL) { mtx_unlock(&pbuf_mtx); return NULL; } TAILQ_REMOVE(&bswlist, bp, b_freelist); --*pfreecnt; mtx_unlock(&pbuf_mtx); initpbuf(bp); return bp; } /* * release a physical buffer * * NOTE: pfreecnt can be NULL, but this 'feature' will be removed * relatively soon when the rest of the subsystems get smart about it. XXX */ void relpbuf(struct buf *bp, int *pfreecnt) { if (bp->b_rcred != NOCRED) { crfree(bp->b_rcred); bp->b_rcred = NOCRED; } if (bp->b_wcred != NOCRED) { crfree(bp->b_wcred); bp->b_wcred = NOCRED; } KASSERT(bp->b_vp == NULL, ("relpbuf with vp")); KASSERT(bp->b_bufobj == NULL, ("relpbuf with bufobj")); BUF_UNLOCK(bp); mtx_lock(&pbuf_mtx); TAILQ_INSERT_HEAD(&bswlist, bp, b_freelist); if (bswneeded) { bswneeded = 0; wakeup(&bswneeded); } if (pfreecnt) { if (++*pfreecnt == 1) wakeup(pfreecnt); } mtx_unlock(&pbuf_mtx); } /* * Associate a p-buffer with a vnode. * * Also sets B_PAGING flag to indicate that vnode is not fully associated * with the buffer. i.e. the bp has not been linked into the vnode or * ref-counted. */ void pbgetvp(struct vnode *vp, struct buf *bp) { KASSERT(bp->b_vp == NULL, ("pbgetvp: not free")); KASSERT(bp->b_bufobj == NULL, ("pbgetvp: not free (bufobj)")); bp->b_vp = vp; bp->b_flags |= B_PAGING; bp->b_bufobj = &vp->v_bufobj; } /* * Associate a p-buffer with a vnode. * * Also sets B_PAGING flag to indicate that vnode is not fully associated * with the buffer. i.e. the bp has not been linked into the vnode or * ref-counted. */ void pbgetbo(struct bufobj *bo, struct buf *bp) { KASSERT(bp->b_vp == NULL, ("pbgetbo: not free (vnode)")); KASSERT(bp->b_bufobj == NULL, ("pbgetbo: not free (bufobj)")); bp->b_flags |= B_PAGING; bp->b_bufobj = bo; } /* * Disassociate a p-buffer from a vnode. */ void pbrelvp(struct buf *bp) { KASSERT(bp->b_vp != NULL, ("pbrelvp: NULL")); KASSERT(bp->b_bufobj != NULL, ("pbrelvp: NULL bufobj")); KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == 0, ("pbrelvp: pager buf on vnode list.")); bp->b_vp = NULL; bp->b_bufobj = NULL; bp->b_flags &= ~B_PAGING; } /* * Disassociate a p-buffer from a bufobj. */ void pbrelbo(struct buf *bp) { KASSERT(bp->b_vp == NULL, ("pbrelbo: vnode")); KASSERT(bp->b_bufobj != NULL, ("pbrelbo: NULL bufobj")); KASSERT((bp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN)) == 0, ("pbrelbo: pager buf on vnode list.")); bp->b_bufobj = NULL; bp->b_flags &= ~B_PAGING; }