Index: sys/cam/ata/ata_da.c =================================================================== --- sys/cam/ata/ata_da.c +++ sys/cam/ata/ata_da.c @@ -297,6 +297,8 @@ char announce_buffer[ADA_ANNOUNCE_SZ]; }; +static uma_zone_t ada_ccb_zone; + struct ada_quirk_entry { struct scsi_inquiry_pattern inq_pat; ada_quirks quirks; @@ -902,6 +904,7 @@ static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD; static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE; static int ada_enable_biospeedup = 1; +static int ada_enable_small_ccbs = 1; static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -921,6 +924,8 @@ &ada_write_cache, 0, "Enable disk write cache"); SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_small_ccbs, CTLFLAG_RWTUN, + &ada_enable_small_ccbs, 0, "Use small CCBs"); /* * ADA_ORDEREDTAG_INTERVAL determines how often, relative @@ -1199,6 +1204,12 @@ NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) printf("adainit: shutdown event registration failed!\n"); } + + if (ada_ccb_zone == NULL) { + ada_ccb_zone = uma_zcreate("ada_ccb", + sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + } } /* @@ -1327,6 +1338,7 @@ case AC_GETDEV_CHANGED: { softc = (struct ada_softc *)periph->softc; + memset(&cgd, 0, sizeof(cgd)); xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL); cgd.ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)&cgd); @@ -1362,6 +1374,7 @@ cam_periph_async(periph, code, path, arg); if (softc->state != ADA_STATE_NORMAL) break; + memset(&cgd, 0, sizeof(cgd)); xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL); cgd.ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)&cgd); @@ -1853,6 +1866,24 @@ "kern.cam.ada.%d.write_cache", periph->unit_number); TUNABLE_INT_FETCH(announce_buf, &softc->write_cache); + /* + * XXX: This shouldn't be needed; for some reason this runs + * _before_ adainit(). + */ + if (ada_ccb_zone == NULL) { + ada_ccb_zone = uma_zcreate("ada_ccb", + sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + } + + /* + * Let XPT know we can use small CCBs. + */ + if (ada_enable_small_ccbs) { + periph->small_ccb_len = sizeof(struct ccb_ataio); + periph->small_ccb_zone = ada_ccb_zone; + } + /* * Set support flags based on the Identify data and quirks. */ Index: sys/cam/ata/ata_xpt.c =================================================================== --- sys/cam/ata/ata_xpt.c +++ sys/cam/ata/ata_xpt.c @@ -1793,6 +1793,12 @@ ata_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_ATA_IO && + start_ccb->ccb_h.path->periph != NULL && + start_ccb->ccb_h.path->periph->small_ccb_len != 0) { + xpt_assert_ccb_is_large(start_ccb); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: { Index: sys/cam/cam_ccb.h =================================================================== --- sys/cam/cam_ccb.h +++ sys/cam/cam_ccb.h @@ -341,7 +341,8 @@ camq_entry xpt_links; /* For chaining in the XPT layer */ camq_entry sim_links; /* For chaining in the SIM layer */ camq_entry periph_links; /* For chaining in the type driver */ - u_int32_t retry_count; + u_int16_t retry_count; + u_int16_t allocated_len; void (*cbfcnp)(struct cam_periph *, union ccb *); /* Callback on completion function */ xpt_opcode func_code; /* XPT function code */ Index: sys/cam/cam_periph.h =================================================================== --- sys/cam/cam_periph.h +++ sys/cam/cam_periph.h @@ -42,6 +42,8 @@ #include #include +#include + #include struct devstat; @@ -147,6 +149,8 @@ ac_callback_t *deferred_callback; ac_code deferred_ac; struct task periph_run_task; + size_t small_ccb_len; + uma_zone_t small_ccb_zone; }; #define CAM_PERIPH_MAXMAPS 2 Index: sys/cam/cam_periph.c =================================================================== --- sys/cam/cam_periph.c +++ sys/cam/cam_periph.c @@ -746,6 +746,7 @@ union ccb ccb; void *arg; + memset(&ccb, 0, sizeof(ccb)); switch (periph->deferred_ac) { case AC_FOUND_DEVICE: ccb.ccb_h.func_code = XPT_GDEV_TYPE; @@ -1318,6 +1319,7 @@ struct ccb_hdr ccb_h; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_freeze_devq\n")); + memset(&ccb_h, 0, sizeof(ccb_h)); xpt_setup_ccb(&ccb_h, path, /*priority*/1); ccb_h.func_code = XPT_NOOP; ccb_h.flags = CAM_DEV_QFREEZE; @@ -1333,6 +1335,7 @@ CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_release_devq(%u, %u, %u, %d)\n", relsim_flags, openings, arg, getcount_only)); + memset(&crs, 0, sizeof(crs)); xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL); crs.ccb_h.func_code = XPT_REL_SIMQ; crs.ccb_h.flags = getcount_only ? CAM_DEV_QFREEZE : 0; @@ -1456,6 +1459,7 @@ { struct ccb_getdevstats cgds; + memset(&cgds, 0, sizeof(cgds)); xpt_setup_ccb(&cgds.ccb_h, periph->path, CAM_PRIORITY_NORMAL); cgds.ccb_h.func_code = XPT_GDEV_STATS; xpt_action((union ccb *)&cgds); @@ -1527,6 +1531,7 @@ * First off, find out what the current * transaction counts are. */ + memset(&cgds, 0, sizeof(cgds)); xpt_setup_ccb(&cgds.ccb_h, ccb->ccb_h.path, CAM_PRIORITY_NORMAL); @@ -1645,6 +1650,7 @@ /* * Grab the inquiry data for this device. */ + memset(&cgd, 0, sizeof(cgd)); xpt_setup_ccb(&cgd.ccb_h, ccb->ccb_h.path, CAM_PRIORITY_NORMAL); cgd.ccb_h.func_code = XPT_GDEV_TYPE; xpt_action((union ccb *)&cgd); Index: sys/cam/cam_xpt.h =================================================================== --- sys/cam/cam_xpt.h +++ sys/cam/cam_xpt.h @@ -146,6 +146,7 @@ void xpt_pollwait(union ccb *start_ccb, uint32_t timeout); uint32_t xpt_poll_setup(union ccb *start_ccb); void xpt_sim_poll(struct cam_sim *sim); +void xpt_assert_ccb_is_large(union ccb *ccb); /* * Perform a path inquiry at the request priority. The bzero may be Index: sys/cam/cam_xpt.c =================================================================== --- sys/cam/cam_xpt.c +++ sys/cam/cam_xpt.c @@ -499,6 +499,7 @@ * This is an immediate CCB, so it's okay to * allocate it on the stack. */ + memset(&ccb, 0, sizeof(ccb)); /* * Create a path using the bus, target, and lun the @@ -2595,6 +2596,7 @@ if ((device->flags & CAM_DEV_UNCONFIGURED) != 0) return (1); + memset(&cgd, 0, sizeof(cgd)); xpt_compile_path(&path, NULL, device->target->bus->path_id, @@ -4676,6 +4678,7 @@ union ccb *new_ccb; new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + new_ccb->ccb_h.allocated_len = sizeof(*new_ccb); return (new_ccb); } @@ -4685,13 +4688,26 @@ union ccb *new_ccb; new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + if (new_ccb != NULL) + new_ccb->ccb_h.allocated_len = sizeof(*new_ccb); return (new_ccb); } void xpt_free_ccb(union ccb *free_ccb) { - free(free_ccb, M_CAMCCB); + struct cam_periph *periph; + + if (free_ccb->ccb_h.allocated_len != 0 && + free_ccb->ccb_h.allocated_len != sizeof(*free_ccb)) { + /* + * Looks like a small CCB, allocated from a periph UMA zone. + */ + periph = free_ccb->ccb_h.path->periph; + uma_zfree(periph->small_ccb_zone, free_ccb); + } else { + free(free_ccb, M_CAMCCB); + } } /* Private XPT functions */ @@ -4705,10 +4721,24 @@ xpt_get_ccb_nowait(struct cam_periph *periph) { union ccb *new_ccb; - - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + size_t len; + + if (periph->small_ccb_zone != NULL) { + KASSERT(periph->small_ccb_len > 0, + ("%s: periph %p with %p small_ccb_zone, but %zd small_ccb_len", + __func__, periph, periph->small_ccb_zone, periph->small_ccb_len)); + len = periph->small_ccb_len; + new_ccb = uma_zalloc(periph->small_ccb_zone, M_ZERO|M_NOWAIT); + } else { + KASSERT(periph->small_ccb_len == 0, + ("%s: periph %p with NULL small_ccb_zone, but %zd small_ccb_len", + __func__, periph, periph->small_ccb_len)); + len = sizeof(*new_ccb); + new_ccb = malloc(len, M_CAMCCB, M_ZERO|M_NOWAIT); + } if (new_ccb == NULL) return (NULL); + new_ccb->ccb_h.allocated_len = len; periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); return (new_ccb); @@ -4718,15 +4748,42 @@ xpt_get_ccb(struct cam_periph *periph) { union ccb *new_ccb; + size_t len; cam_periph_unlock(periph); - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + if (periph->small_ccb_zone != NULL) { + KASSERT(periph->small_ccb_len > 0, + ("%s: periph %p with %p small_ccb_zone, but %zd small_ccb_len", + __func__, periph, periph->small_ccb_zone, periph->small_ccb_len)); + len = periph->small_ccb_len; + new_ccb = uma_zalloc(periph->small_ccb_zone, M_ZERO|M_WAITOK); + } else { + KASSERT(periph->small_ccb_len == 0, + ("%s: periph %p with NULL small_ccb_zone, but %zd small_ccb_len", + __func__, periph, periph->small_ccb_len)); + len = sizeof(*new_ccb); + new_ccb = malloc(len, M_CAMCCB, M_ZERO|M_WAITOK); + } + new_ccb->ccb_h.allocated_len = len; cam_periph_lock(periph); periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); return (new_ccb); } +void +xpt_assert_ccb_is_large(union ccb *ccb) +{ + + if (ccb->ccb_h.allocated_len != 0) { + KASSERT(ccb->ccb_h.allocated_len == sizeof(*ccb), + ("%s: ccb %p: func %#x, allocated_len %zd != %zd\n", + __func__, ccb, ccb->ccb_h.func_code, + (size_t)ccb->ccb_h.allocated_len, + sizeof(*ccb))); + } +} + union ccb * cam_periph_getccb(struct cam_periph *periph, u_int32_t priority) { @@ -5078,6 +5135,7 @@ sim->max_tagged_dev_openings); xpt_dev_ccbq_resize(path, newopenings); xpt_async(AC_GETDEV_CHANGED, path, NULL); + memset(&crs, 0, sizeof(crs)); xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL); crs.ccb_h.func_code = XPT_REL_SIMQ; crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY; @@ -5103,6 +5161,7 @@ device->inq_flags &= ~SID_CmdQue; xpt_dev_ccbq_resize(path, sim->max_dev_openings); xpt_async(AC_GETDEV_CHANGED, path, NULL); + memset(&crs, 0, sizeof(crs)); xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL); crs.ccb_h.func_code = XPT_REL_SIMQ; crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY; @@ -5260,6 +5319,7 @@ xptpath = 1; } + memset(&csa, 0, sizeof(csa)); xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = event; Index: sys/cam/scsi/scsi_da.c =================================================================== --- sys/cam/scsi/scsi_da.c +++ sys/cam/scsi/scsi_da.c @@ -403,6 +403,8 @@ softc->delete_available &= ~(1 << delete_method); \ } +static uma_zone_t da_ccb_zone; + struct da_quirk_entry { struct scsi_inquiry_pattern inq_pat; da_quirks quirks; @@ -1557,6 +1559,7 @@ static int da_send_ordered = DA_DEFAULT_SEND_ORDERED; static int da_disable_wp_detection = 0; static int da_enable_biospeedup = 1; +static int da_enable_small_ccbs = 1; static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -1573,6 +1576,8 @@ "Disable detection of write-protected disks"); SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_small_ccbs, CTLFLAG_RWTUN, + &da_enable_small_ccbs, 0, "Use small CCBs"); SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout, CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0, @@ -2011,6 +2016,10 @@ NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) printf("dainit: shutdown event registration failed!\n"); } + + da_ccb_zone = uma_zcreate("da_ccb", + sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); } /* @@ -2848,6 +2857,14 @@ TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph); + /* + * Let XPT know we can use small CCBs. + */ + if (da_enable_small_ccbs) { + periph->small_ccb_len = sizeof(struct ccb_scsiio); + periph->small_ccb_zone = da_ccb_zone; + } + /* * Take an exclusive section lock on the periph while dastart is called * to finish the probe. The lock will be dropped in dadone at the end @@ -4879,6 +4896,7 @@ /*timeout*/0, /*getcount_only*/0); + memset(&cgd, 0, sizeof(cgd)); xpt_setup_ccb(&cgd.ccb_h, done_ccb->ccb_h.path, CAM_PRIORITY_NORMAL); cgd.ccb_h.func_code = XPT_GDEV_TYPE; @@ -6125,6 +6143,7 @@ * up with something that will make this a bootable * device. */ + memset(&ccg, 0, sizeof(ccg)); xpt_setup_ccb(&ccg.ccb_h, periph->path, CAM_PRIORITY_NORMAL); ccg.ccb_h.func_code = XPT_CALC_GEOMETRY; ccg.block_size = dp->secsize; @@ -6162,6 +6181,7 @@ min(sizeof(softc->rcaplong), rcap_len)) != 0)) { struct ccb_dev_advinfo cdai; + memset(&cdai, 0, sizeof(cdai)); xpt_setup_ccb(&cdai.ccb_h, periph->path, CAM_PRIORITY_NORMAL); cdai.ccb_h.func_code = XPT_DEV_ADVINFO; cdai.buftype = CDAI_TYPE_RCAPLONG; Index: sys/cam/scsi/scsi_xpt.c =================================================================== --- sys/cam/scsi/scsi_xpt.c +++ sys/cam/scsi/scsi_xpt.c @@ -1036,6 +1036,7 @@ { struct ccb_trans_settings cts; + memset(&cts, 0, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cts.type = CTS_TYPE_USER_SETTINGS; @@ -2306,6 +2307,7 @@ CAM_DEBUG(path, CAM_DEBUG_TRACE, ("scsi_scan_lun\n")); + memset(&cpi, 0, sizeof(cpi)); xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); @@ -2432,6 +2434,7 @@ struct scsi_inquiry_data *inq_buf; /* Get transport information from the SIM */ + memset(&cpi, 0, sizeof(cpi)); xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); @@ -2492,6 +2495,7 @@ */ /* Tell the controller what we think */ + memset(&cts, 0, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS; cts.type = CTS_TYPE_CURRENT_SETTINGS; @@ -2625,6 +2629,12 @@ scsi_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_SCSI_IO && + start_ccb->ccb_h.path->periph != NULL && + start_ccb->ccb_h.path->periph->small_ccb_len != 0) { + xpt_assert_ccb_is_large(start_ccb); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: { @@ -2754,6 +2764,7 @@ * Perform sanity checking against what the * controller and device can do. */ + memset(&cur_cts, 0, sizeof(cur_cts)); xpt_setup_ccb(&cur_cts.ccb_h, path, CAM_PRIORITY_NONE); cur_cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; cur_cts.type = cts->type; @@ -2937,6 +2948,7 @@ && (dev->inq_flags & (SID_Sync|SID_WBus16|SID_WBus32)) != 0)) { struct ccb_trans_settings cts; + memset(&cts, 0, sizeof(cts)); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE); cts.protocol = PROTO_SCSI; cts.protocol_version = PROTO_VERSION_UNSPECIFIED; @@ -3044,6 +3056,7 @@ return; /* Ask the SIM for its base transfer speed */ + memset(&cpi, 0, sizeof(cpi)); xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi);