Index: sys/cam/cam_sim.h =================================================================== --- sys/cam/cam_sim.h +++ sys/cam/cam_sim.h @@ -47,6 +47,8 @@ struct cam_sim; struct cam_devq; +#include + typedef void (*sim_action_func)(struct cam_sim *sim, union ccb *ccb); typedef void (*sim_poll_func)(struct cam_sim *sim); @@ -73,7 +75,10 @@ struct cam_devq *queue); void cam_sim_free(struct cam_sim *sim, int free_devq); void cam_sim_hold(struct cam_sim *sim); +void cam_sim_init(void); void cam_sim_release(struct cam_sim *sim); +void cam_sim_enter_smr(void); +void cam_sim_exit_smr(void); /* Optional sim attributes may be set with these. */ void cam_sim_set_path(struct cam_sim *sim, u_int32_t path_id); @@ -95,15 +100,17 @@ void *softc; struct mtx *mtx; TAILQ_HEAD(, ccb_hdr) sim_doneq; - TAILQ_ENTRY(cam_sim) links; + CK_SLIST_ENTRY(cam_sim) sim_link; u_int32_t path_id;/* The Boot device may set this to 0? */ u_int32_t unit_number; u_int32_t bus_id; int max_tagged_dev_openings; int max_dev_openings; u_int32_t flags; +#define CAM_SIM_FREE_DEVQ 0x02 +#define CAM_SIM_DYING 0x04 struct cam_devq *devq; /* Device Queue to use for this SIM */ - int refcount; /* References to the SIM. */ + volatile u_int refcount; /* References to the SIM. */ device_t sim_dev; /* For attached peripherals. */ }; @@ -143,5 +150,16 @@ return (sim->sim_poll != NULL); } +#define CAM_SIM_SMR() cam_sim_smr +#define CAM_SIM_SMR_ASSERT_ENTERED() SMR_ASSERT_ENTERED(CAM_SIM_SMR()) +#define CAM_SIM_SMR_ASSERT_NOT_ENTERED() SMR_ASSERT_NOT_ENTERED(CAM_SIM_SMR()) +#define CAM_SIM_SMR_ENTER() cam_sim_smr_enter() +#define CAM_SIM_SMR_EXIT() cam_sim_smr_exit() +#define CAM_SIM_LIST_LOCK() mtx_lock(&cam_sim_list_mtx); +#define CAM_SIM_LIST_UNLOCK() mtx_unlock(&cam_sim_list_mtx); + +void cam_sim_smr_enter(void); +void cam_sim_smr_exit(void); + #endif /* _KERNEL */ #endif /* _CAM_CAM_SIM_H */ Index: sys/cam/cam_sim.c =================================================================== --- sys/cam/cam_sim.c +++ sys/cam/cam_sim.c @@ -33,11 +33,15 @@ #include #include -#include +#include #include #include +#include #include -#include +#include +#include +#include +#include #include #include @@ -47,10 +51,12 @@ #define CAM_PATH_ANY (u_int32_t)-1 -static MALLOC_DEFINE(M_CAMSIM, "CAM SIM", "CAM SIM buffers"); - -static struct mtx cam_sim_free_mtx; -MTX_SYSINIT(cam_sim_free_init, &cam_sim_free_mtx, "CAM SIM free lock", MTX_DEF); +smr_t cam_sim_smr; +uma_zone_t cam_sim_zone; +static struct mtx cam_sim_list_mtx; +MTX_SYSINIT(cam_sim_list_init, &cam_sim_list_mtx, "CAM SIM listlock", MTX_DEF); +typedef CK_SLIST_HEAD(, cam_sim) sim_list_t; +static sim_list_t sim_list = CK_SLIST_HEAD_INITIALIZER(sim_list); struct cam_devq * cam_simq_alloc(u_int32_t max_sim_transactions) @@ -64,8 +70,6 @@ cam_devq_free(devq); } - - /** * @brief allocate a new sim and fill in the details * @@ -108,7 +112,7 @@ { struct cam_sim *sim; - sim = malloc(sizeof(struct cam_sim), M_CAMSIM, M_ZERO | M_NOWAIT); + sim = uma_zalloc_smr(cam_sim_zone, M_ZERO | M_NOWAIT); if (sim == NULL) return (NULL); @@ -123,9 +127,16 @@ sim->max_tagged_dev_openings = max_tagged_dev_transactions; sim->max_dev_openings = max_dev_transactions; sim->flags = 0; - sim->refcount = 1; + refcount_init(&sim->refcount, 1); sim->devq = queue; sim->mtx = mtx; + /* + * We have to have the sim fully constructed before we make + * it visible to other threads. Insert it into the list last. + */ + CAM_SIM_LIST_LOCK(); + CK_SLIST_INSERT_HEAD(&sim_list, sim, sim_link); + CAM_SIM_LIST_UNLOCK(); return (sim); } @@ -156,72 +167,84 @@ return (sim); } +static void +cam_sim_free_dtor(void *mem, int size, void *arg) +{ + struct cam_sim *sim = mem; + + if (atomic_load_acq_int(&sim->flags) & CAM_SIM_FREE_DEVQ) + cam_simq_free(sim->devq); +} + +static void +cam_sim_action_gone(struct cam_sim *sim, union ccb *ccb) +{ + ccb->ccb_h.status = CAM_DEV_NOT_THERE; + xpt_done(ccb); +} + +static void +nvme_sim_poll(struct cam_sim *sim) +{ +} + +/** + * @brief frees up the sim + * + * Frees up the CAM @c sim and optionally the devq. After this call returns, no + * further @c sim_action or @c sim_poll requests will make it to the driver. The + * driver is responsible for cleaning up all pending requests it may has before + * calling this function. + * + * This function asserts that the mtx associated with this sim, if any, is locked + * when this function is called. It will remain locked. Once this function returns, + * CAM will cease using that mutex. + * + * If there are no more references to this cam, then it will be freed. If there + * are references still, the free will be deferred util those references are + * released. All new calls to @c sim_action or @c sim_poll will return + * immediately with an error (without calling the registered @c sim_action or @c + * sim_poll). The @c devq will be freed once all references have been released. + * + * @param sim The sim to free + * @param free_devq Free the devq associated with the sim at creation. + */ void cam_sim_free(struct cam_sim *sim, int free_devq) { - struct mtx *mtx; - int error; - - if (sim->mtx == NULL) { - mtx = &cam_sim_free_mtx; - mtx_lock(mtx); - } else { - mtx = sim->mtx; - mtx_assert(mtx, MA_OWNED); - } - KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); - sim->refcount--; - if (sim->refcount > 0) { - error = msleep(sim, mtx, PRIBIO, "simfree", 0); - KASSERT(error == 0, ("invalid error value for msleep(9)")); - } - KASSERT(sim->refcount == 0, ("sim->refcount == 0")); - if (mtx == &cam_sim_free_mtx) /* sim->mtx == NULL */ - mtx_unlock(mtx); + int flags; + if (sim->mtx != NULL) + mtx_assert(sim->mtx, MA_OWNED); + CAM_SIM_LIST_LOCK(); + CK_SLIST_REMOVE(&sim_list, sim, cam_sim, sim_link); + CAM_SIM_LIST_UNLOCK(); + flags = CAM_SIM_DYING; if (free_devq) - cam_simq_free(sim->devq); - free(sim, M_CAMSIM); + flags |= CAM_SIM_FREE_DEVQ; + atomic_set_rel_int(&sim->flags, flags); + sim->sim_action = cam_sim_action_gone; + sim->sim_poll = cam_sim_poll_gone; + sim->mtx = NULL; + cam_sim_release(sim); } void cam_sim_release(struct cam_sim *sim) { - struct mtx *mtx; - - if (sim->mtx == NULL) - mtx = &cam_sim_free_mtx; - else if (!mtx_owned(sim->mtx)) - mtx = sim->mtx; - else - mtx = NULL; /* We hold the lock. */ - if (mtx) - mtx_lock(mtx); - KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); - sim->refcount--; - if (sim->refcount == 0) - wakeup(sim); - if (mtx) - mtx_unlock(mtx); + if (refcount_release(&sim->refcount)) + uma_zfree_smr(cam_sim_zone, sim); } void cam_sim_hold(struct cam_sim *sim) { - struct mtx *mtx; - - if (sim->mtx == NULL) - mtx = &cam_sim_free_mtx; - else if (!mtx_owned(sim->mtx)) - mtx = sim->mtx; - else - mtx = NULL; /* We hold the lock. */ - if (mtx) - mtx_lock(mtx); - KASSERT(sim->refcount >= 1, ("sim->refcount >= 1")); - sim->refcount++; - if (mtx) - mtx_unlock(mtx); +#ifdef INVARIANTS + u_int old = +#endif + refcount_acquire(&sim->refcount); + KASSERT(old > 0, ("cam_sim_hold: %s%d tried to hold refcount 0", + sim->sim_name, sim->unit_unmber)); } void @@ -229,3 +252,28 @@ { sim->path_id = path_id; } + +void +cam_sim_init(void) +{ + + cam_sim_zone = uma_zcreate("cam_sim", sizeof(struct cam_sim), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_SMR); + cam_sim_smr = uma_zone_get_smr(cam_sim_zone); +} + +void +cam_sim_smr_enter(void) +{ + CAM_SIM_SMR_ASSERT_NOT_ENTERED(); + + smr_enter(CAM_SIM_SMR()); +} + +void +cam_sim_smr_exit(void) +{ + CAM_SIM_SMR_ASSERT_ENTERED(); + + smr_exit(CAM_SIM_SMR()); +} Index: sys/cam/cam_xpt.c =================================================================== --- sys/cam/cam_xpt.c +++ sys/cam/cam_xpt.c @@ -903,6 +903,8 @@ xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK, taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq); + cam_sim_init(); + #ifdef CAM_BOOT_DELAY /* * Override this value at compile time to assist our users