Index: sys/kern/subr_epoch.c =================================================================== --- sys/kern/subr_epoch.c +++ sys/kern/subr_epoch.c @@ -58,8 +58,6 @@ #include -static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); - #ifdef __amd64__ #define EPOCH_ALIGN CACHE_LINE_SIZE*2 #else @@ -79,7 +77,6 @@ struct epoch { struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); epoch_record_t e_pcpu_record; - int e_idx; int e_flags; struct sx e_drain_sx; struct mtx e_drain_mtx; @@ -128,7 +125,7 @@ CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, ck_epoch_entry_container) -epoch_t allepochs[MAX_EPOCHS]; +static struct epoch epoch_array[MAX_EPOCHS]; DPCPU_DEFINE(struct grouptask, epoch_cb_task); DPCPU_DEFINE(int, epoch_cb_count); @@ -141,6 +138,11 @@ static void epoch_call_task(void *context __unused); static uma_zone_t pcpu_zone_record; +static struct sx epoch_sx; + +#define EPOCH_LOCK() sx_xlock(&epoch_sx) +#define EPOCH_UNLOCK() sx_xunlock(&epoch_sx) + #ifdef EPOCH_TRACE struct stackentry { RB_ENTRY(stackentry) se_node; @@ -281,6 +283,7 @@ #ifdef EPOCH_TRACE SLIST_INIT(&thread0.td_epochs); #endif + sx_init(&epoch_sx, "epoch-sx"); inited = 1; global_epoch = epoch_alloc("Global", 0); global_epoch_preempt = epoch_alloc("Global preemptible", EPOCH_PREEMPT); @@ -326,19 +329,40 @@ epoch_alloc(const char *name, int flags) { epoch_t epoch; + int i; if (__predict_false(!inited)) panic("%s called too early in boot", __func__); - epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO | M_WAITOK); + + MPASS(name != NULL); + + EPOCH_LOCK(); + + /* look for free epoch entry */ + for (i = 0; i != epoch_count; i++) { + if (epoch_array[i].e_name == NULL) + break; + } + + /* check if too many epochs are allocated */ + if (i >= MAX_EPOCHS) { + epoch = NULL; + goto done; + } + + epoch = epoch_array + i; ck_epoch_init(&epoch->e_epoch); epoch_ctor(epoch); - MPASS(epoch_count < MAX_EPOCHS - 2); epoch->e_flags = flags; - epoch->e_idx = epoch_count; epoch->e_name = name; sx_init(&epoch->e_drain_sx, "epoch-drain-sx"); mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF); - allepochs[epoch_count++] = epoch; + + /* track upper epoch index */ + if (i > epoch_count) + epoch_count = i; +done: + EPOCH_UNLOCK(); return (epoch); } @@ -346,13 +370,18 @@ epoch_free(epoch_t epoch) { + EPOCH_LOCK(); + + MPASS(epoch->e_name != NULL); + epoch_drain_callbacks(epoch); - allepochs[epoch->e_idx] = NULL; epoch_wait(global_epoch); uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record); mtx_destroy(&epoch->e_drain_mtx); sx_destroy(&epoch->e_drain_sx); - free(epoch, M_EPOCH); + memset(epoch, 0, sizeof(*epoch)); + + EPOCH_UNLOCK(); } static epoch_record_t @@ -706,7 +735,8 @@ critical_enter(); epoch_enter(global_epoch); for (total = i = 0; i < epoch_count; i++) { - if (__predict_false((epoch = allepochs[i]) == NULL)) + epoch = epoch_array + i; + if (__predict_false(epoch->e_name == NULL)) continue; er = epoch_currecord(epoch); record = &er->er_record;