diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c --- a/sys/cam/ata/ata_da.c +++ b/sys/cam/ata/ata_da.c @@ -297,6 +297,8 @@ char announce_buffer[ADA_ANNOUNCE_SZ]; }; +static uma_zone_t ada_ccb_zone; + struct ada_quirk_entry { struct scsi_inquiry_pattern inq_pat; ada_quirks quirks; @@ -902,6 +904,7 @@ static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD; static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE; static int ada_enable_biospeedup = 1; +static int ada_enable_uma_ccbs = 0; static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -921,6 +924,8 @@ &ada_write_cache, 0, "Enable disk write cache"); SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN, + &ada_enable_uma_ccbs, 0, "Use UMA for CCBs"); /* * ADA_ORDEREDTAG_INTERVAL determines how often, relative @@ -1178,6 +1183,10 @@ { cam_status status; + ada_ccb_zone = uma_zcreate("ada_ccb", + sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); + /* * Install a global async callback. This callback will * receive async callbacks like "new device found". @@ -1855,6 +1864,15 @@ "kern.cam.ada.%d.write_cache", periph->unit_number); TUNABLE_INT_FETCH(announce_buf, &softc->write_cache); + /* + * Let XPT know we can use UMA-allocated CCBs. + */ + if (ada_enable_uma_ccbs) { + KASSERT(ada_ccb_zone != NULL, + ("%s: NULL ada_ccb_zone", __func__)); + periph->ccb_zone = ada_ccb_zone; + } + /* * Set support flags based on the Identify data and quirks. */ diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c --- a/sys/cam/ata/ata_xpt.c +++ b/sys/cam/ata/ata_xpt.c @@ -1795,6 +1795,13 @@ ata_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_ATA_IO) { + KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0, + ("%s: ccb %p, func_code %#x should not be allocated " + "from UMA zone\n", + __func__, start_ccb, start_ccb->ccb_h.func_code)); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: { diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h --- a/sys/cam/cam_ccb.h +++ b/sys/cam/cam_ccb.h @@ -58,6 +58,12 @@ /* Struct definitions for CAM control blocks */ /* Common CCB header */ + +/* CCB memory allocation flags */ +typedef enum { + CAM_CCB_FROM_UMA = 0x00000001,/* CCB from a periph UMA zone */ +} ccb_alloc_flags; + /* CAM CCB flags */ typedef enum { CAM_CDB_POINTER = 0x00000001,/* The CDB field is a pointer */ @@ -341,7 +347,13 @@ camq_entry xpt_links; /* For chaining in the XPT layer */ camq_entry sim_links; /* For chaining in the SIM layer */ camq_entry periph_links; /* For chaining in the type driver */ - u_int32_t retry_count; +#if BYTE_ORDER == LITTLE_ENDIAN + u_int16_t retry_count; + u_int16_t alloc_flags; /* ccb_alloc_flags */ +#else + u_int16_t alloc_flags; /* ccb_alloc_flags */ + u_int16_t retry_count; +#endif void (*cbfcnp)(struct cam_periph *, union ccb *); /* Callback on completion function */ xpt_opcode func_code; /* XPT function code */ diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h --- a/sys/cam/cam_periph.h +++ b/sys/cam/cam_periph.h @@ -42,6 +42,8 @@ #include #include +#include + #include struct devstat; @@ -147,6 +149,7 @@ ac_callback_t *deferred_callback; ac_code deferred_ac; struct task periph_run_task; + uma_zone_t ccb_zone; }; #define CAM_PERIPH_MAXMAPS 2 diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c --- a/sys/cam/cam_xpt.c +++ b/sys/cam/cam_xpt.c @@ -4693,7 +4693,17 @@ void xpt_free_ccb(union ccb *free_ccb) { - free(free_ccb, M_CAMCCB); + struct cam_periph *periph; + + if (free_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) { + /* + * Looks like a CCB allocated from a periph UMA zone. + */ + periph = free_ccb->ccb_h.path->periph; + uma_zfree(periph->ccb_zone, free_ccb); + } else { + free(free_ccb, M_CAMCCB); + } } /* Private XPT functions */ @@ -4707,10 +4717,18 @@ xpt_get_ccb_nowait(struct cam_periph *periph) { union ccb *new_ccb; + int alloc_flags; - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + if (periph->ccb_zone != NULL) { + alloc_flags = CAM_CCB_FROM_UMA; + new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_NOWAIT); + } else { + alloc_flags = 0; + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT); + } if (new_ccb == NULL) return (NULL); + new_ccb->ccb_h.alloc_flags = alloc_flags; periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); return (new_ccb); @@ -4720,9 +4738,17 @@ xpt_get_ccb(struct cam_periph *periph) { union ccb *new_ccb; + int alloc_flags; cam_periph_unlock(periph); - new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + if (periph->ccb_zone != NULL) { + alloc_flags = CAM_CCB_FROM_UMA; + new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_WAITOK); + } else { + alloc_flags = 0; + new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK); + } + new_ccb->ccb_h.alloc_flags = alloc_flags; cam_periph_lock(periph); periph->periph_allocated++; cam_ccbq_take_opening(&periph->path->device->ccbq); diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c --- a/sys/cam/scsi/scsi_da.c +++ b/sys/cam/scsi/scsi_da.c @@ -403,6 +403,8 @@ softc->delete_available &= ~(1 << delete_method); \ } +static uma_zone_t da_ccb_zone; + struct da_quirk_entry { struct scsi_inquiry_pattern inq_pat; da_quirks quirks; @@ -1557,6 +1559,7 @@ static int da_send_ordered = DA_DEFAULT_SEND_ORDERED; static int da_disable_wp_detection = 0; static int da_enable_biospeedup = 1; +static int da_enable_uma_ccbs = 0; static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "CAM Direct Access Disk driver"); @@ -1573,6 +1576,8 @@ "Disable detection of write-protected disks"); SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN, &da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing"); +SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN, + &da_enable_uma_ccbs, 0, "Use UMA for CCBs"); SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout, CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0, @@ -2011,6 +2016,10 @@ NULL, SHUTDOWN_PRI_DEFAULT)) == NULL) printf("dainit: shutdown event registration failed!\n"); } + + da_ccb_zone = uma_zcreate("da_ccb", + sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL, + UMA_ALIGN_PTR, 0); } /* @@ -2848,6 +2857,15 @@ TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph); + /* + * Let XPT know we can use UMA-allocated CCBs. + */ + if (da_enable_uma_ccbs) { + KASSERT(da_ccb_zone != NULL, + ("%s: NULL da_ccb_zone", __func__)); + periph->ccb_zone = da_ccb_zone; + } + /* * Take an exclusive section lock on the periph while dastart is called * to finish the probe. The lock will be dropped in dadone at the end diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c --- a/sys/cam/scsi/scsi_xpt.c +++ b/sys/cam/scsi/scsi_xpt.c @@ -2625,6 +2625,13 @@ scsi_action(union ccb *start_ccb) { + if (start_ccb->ccb_h.func_code != XPT_SCSI_IO) { + KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0, + ("%s: ccb %p, func_code %#x should not be allocated " + "from UMA zone\n", + __func__, start_ccb, start_ccb->ccb_h.func_code)); + } + switch (start_ccb->ccb_h.func_code) { case XPT_SET_TRAN_SETTINGS: {