Changeset View
Changeset View
Standalone View
Standalone View
head/sys/kern/subr_epoch.c
Show First 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | |||||
#endif | #endif | ||||
#include <vm/vm.h> | #include <vm/vm.h> | ||||
#include <vm/vm_extern.h> | #include <vm/vm_extern.h> | ||||
#include <vm/vm_kern.h> | #include <vm/vm_kern.h> | ||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
#include <ck_epoch.h> | #include <ck_epoch.h> | ||||
static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); | |||||
#ifdef __amd64__ | #ifdef __amd64__ | ||||
#define EPOCH_ALIGN CACHE_LINE_SIZE*2 | #define EPOCH_ALIGN CACHE_LINE_SIZE*2 | ||||
#else | #else | ||||
#define EPOCH_ALIGN CACHE_LINE_SIZE | #define EPOCH_ALIGN CACHE_LINE_SIZE | ||||
#endif | #endif | ||||
TAILQ_HEAD (epoch_tdlist, epoch_tracker); | TAILQ_HEAD (epoch_tdlist, epoch_tracker); | ||||
typedef struct epoch_record { | typedef struct epoch_record { | ||||
ck_epoch_record_t er_record; | ck_epoch_record_t er_record; | ||||
struct epoch_context er_drain_ctx; | struct epoch_context er_drain_ctx; | ||||
struct epoch *er_parent; | struct epoch *er_parent; | ||||
volatile struct epoch_tdlist er_tdlist; | volatile struct epoch_tdlist er_tdlist; | ||||
volatile uint32_t er_gen; | volatile uint32_t er_gen; | ||||
uint32_t er_cpuid; | uint32_t er_cpuid; | ||||
} __aligned(EPOCH_ALIGN) *epoch_record_t; | } __aligned(EPOCH_ALIGN) *epoch_record_t; | ||||
struct epoch { | struct epoch { | ||||
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); | struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); | ||||
epoch_record_t e_pcpu_record; | epoch_record_t e_pcpu_record; | ||||
int e_idx; | int e_in_use; | ||||
int e_flags; | int e_flags; | ||||
struct sx e_drain_sx; | struct sx e_drain_sx; | ||||
struct mtx e_drain_mtx; | struct mtx e_drain_mtx; | ||||
volatile int e_drain_count; | volatile int e_drain_count; | ||||
const char *e_name; | const char *e_name; | ||||
}; | }; | ||||
/* arbitrary --- needs benchmarking */ | /* arbitrary --- needs benchmarking */ | ||||
Show All 32 Lines | |||||
SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW, | SYSCTL_COUNTER_U64(_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW, | ||||
&epoch_call_task_count, "# of times a callback task was run"); | &epoch_call_task_count, "# of times a callback task was run"); | ||||
TAILQ_HEAD (threadlist, thread); | TAILQ_HEAD (threadlist, thread); | ||||
CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, | CK_STACK_CONTAINER(struct ck_epoch_entry, stack_entry, | ||||
ck_epoch_entry_container) | ck_epoch_entry_container) | ||||
epoch_t allepochs[MAX_EPOCHS]; | static struct epoch epoch_array[MAX_EPOCHS]; | ||||
DPCPU_DEFINE(struct grouptask, epoch_cb_task); | DPCPU_DEFINE(struct grouptask, epoch_cb_task); | ||||
DPCPU_DEFINE(int, epoch_cb_count); | DPCPU_DEFINE(int, epoch_cb_count); | ||||
static __read_mostly int inited; | static __read_mostly int inited; | ||||
static __read_mostly int epoch_count; | |||||
__read_mostly epoch_t global_epoch; | __read_mostly epoch_t global_epoch; | ||||
__read_mostly epoch_t global_epoch_preempt; | __read_mostly epoch_t global_epoch_preempt; | ||||
static void epoch_call_task(void *context __unused); | static void epoch_call_task(void *context __unused); | ||||
static uma_zone_t pcpu_zone_record; | static uma_zone_t pcpu_zone_record; | ||||
static struct sx epoch_sx; | |||||
#define EPOCH_LOCK() sx_xlock(&epoch_sx) | |||||
#define EPOCH_UNLOCK() sx_xunlock(&epoch_sx) | |||||
#ifdef EPOCH_TRACE | #ifdef EPOCH_TRACE | ||||
struct stackentry { | struct stackentry { | ||||
RB_ENTRY(stackentry) se_node; | RB_ENTRY(stackentry) se_node; | ||||
struct stack se_stack; | struct stack se_stack; | ||||
}; | }; | ||||
static int | static int | ||||
stackentry_compare(struct stackentry *a, struct stackentry *b) | stackentry_compare(struct stackentry *a, struct stackentry *b) | ||||
▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | GROUPTASK_INIT(DPCPU_ID_PTR(cpu, epoch_cb_task), 0, | ||||
epoch_call_task, NULL); | epoch_call_task, NULL); | ||||
taskqgroup_attach_cpu(qgroup_softirq, | taskqgroup_attach_cpu(qgroup_softirq, | ||||
DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, NULL, NULL, | DPCPU_ID_PTR(cpu, epoch_cb_task), NULL, cpu, NULL, NULL, | ||||
"epoch call task"); | "epoch call task"); | ||||
} | } | ||||
#ifdef EPOCH_TRACE | #ifdef EPOCH_TRACE | ||||
SLIST_INIT(&thread0.td_epochs); | SLIST_INIT(&thread0.td_epochs); | ||||
#endif | #endif | ||||
sx_init(&epoch_sx, "epoch-sx"); | |||||
inited = 1; | inited = 1; | ||||
global_epoch = epoch_alloc("Global", 0); | global_epoch = epoch_alloc("Global", 0); | ||||
global_epoch_preempt = epoch_alloc("Global preemptible", EPOCH_PREEMPT); | global_epoch_preempt = epoch_alloc("Global preemptible", EPOCH_PREEMPT); | ||||
} | } | ||||
SYSINIT(epoch, SI_SUB_EPOCH, SI_ORDER_FIRST, epoch_init, NULL); | SYSINIT(epoch, SI_SUB_EPOCH, SI_ORDER_FIRST, epoch_init, NULL); | ||||
#if !defined(EARLY_AP_STARTUP) | #if !defined(EARLY_AP_STARTUP) | ||||
static void | static void | ||||
Show All 29 Lines | epoch_adjust_prio(struct thread *td, u_char prio) | ||||
sched_prio(td, prio); | sched_prio(td, prio); | ||||
thread_unlock(td); | thread_unlock(td); | ||||
} | } | ||||
epoch_t | epoch_t | ||||
epoch_alloc(const char *name, int flags) | epoch_alloc(const char *name, int flags) | ||||
{ | { | ||||
epoch_t epoch; | epoch_t epoch; | ||||
int i; | |||||
MPASS(name != NULL); | |||||
if (__predict_false(!inited)) | if (__predict_false(!inited)) | ||||
panic("%s called too early in boot", __func__); | panic("%s called too early in boot", __func__); | ||||
epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO | M_WAITOK); | |||||
EPOCH_LOCK(); | |||||
/* | |||||
* Find a free index in the epoch array. If no free index is | |||||
* found, try to use the index after the last one. | |||||
*/ | |||||
for (i = 0;; i++) { | |||||
/* | |||||
* If too many epochs are currently allocated, | |||||
* return NULL. | |||||
*/ | |||||
if (i == MAX_EPOCHS) { | |||||
epoch = NULL; | |||||
goto done; | |||||
} | |||||
if (epoch_array[i].e_in_use == 0) | |||||
break; | |||||
} | |||||
epoch = epoch_array + i; | |||||
ck_epoch_init(&epoch->e_epoch); | ck_epoch_init(&epoch->e_epoch); | ||||
epoch_ctor(epoch); | epoch_ctor(epoch); | ||||
MPASS(epoch_count < MAX_EPOCHS - 2); | |||||
epoch->e_flags = flags; | epoch->e_flags = flags; | ||||
epoch->e_idx = epoch_count; | |||||
epoch->e_name = name; | epoch->e_name = name; | ||||
sx_init(&epoch->e_drain_sx, "epoch-drain-sx"); | sx_init(&epoch->e_drain_sx, "epoch-drain-sx"); | ||||
mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF); | mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF); | ||||
allepochs[epoch_count++] = epoch; | |||||
/* | |||||
* Set e_in_use last, because when this field is set the | |||||
* epoch_call_task() function will start scanning this epoch | |||||
* structure. | |||||
*/ | |||||
atomic_store_rel_int(&epoch->e_in_use, 1); | |||||
done: | |||||
EPOCH_UNLOCK(); | |||||
return (epoch); | return (epoch); | ||||
} | } | ||||
void | void | ||||
epoch_free(epoch_t epoch) | epoch_free(epoch_t epoch) | ||||
{ | { | ||||
EPOCH_LOCK(); | |||||
MPASS(epoch->e_in_use != 0); | |||||
epoch_drain_callbacks(epoch); | epoch_drain_callbacks(epoch); | ||||
allepochs[epoch->e_idx] = NULL; | |||||
atomic_store_rel_int(&epoch->e_in_use, 0); | |||||
/* | |||||
* Make sure the epoch_call_task() function see e_in_use equal | |||||
* to zero, by calling epoch_wait() on the global_epoch: | |||||
*/ | |||||
epoch_wait(global_epoch); | epoch_wait(global_epoch); | ||||
uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record); | uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record); | ||||
mtx_destroy(&epoch->e_drain_mtx); | mtx_destroy(&epoch->e_drain_mtx); | ||||
sx_destroy(&epoch->e_drain_sx); | sx_destroy(&epoch->e_drain_sx); | ||||
free(epoch, M_EPOCH); | memset(epoch, 0, sizeof(*epoch)); | ||||
EPOCH_UNLOCK(); | |||||
} | } | ||||
static epoch_record_t | static epoch_record_t | ||||
epoch_currecord(epoch_t epoch) | epoch_currecord(epoch_t epoch) | ||||
{ | { | ||||
return (zpcpu_get(epoch->e_pcpu_record)); | return (zpcpu_get(epoch->e_pcpu_record)); | ||||
} | } | ||||
▲ Show 20 Lines • Show All 336 Lines • ▼ Show 20 Lines | epoch_call_task(void *arg __unused) | ||||
epoch_record_t er; | epoch_record_t er; | ||||
epoch_t epoch; | epoch_t epoch; | ||||
ck_stack_t cb_stack; | ck_stack_t cb_stack; | ||||
int i, npending, total; | int i, npending, total; | ||||
ck_stack_init(&cb_stack); | ck_stack_init(&cb_stack); | ||||
critical_enter(); | critical_enter(); | ||||
epoch_enter(global_epoch); | epoch_enter(global_epoch); | ||||
for (total = i = 0; i < epoch_count; i++) { | for (total = i = 0; i != MAX_EPOCHS; i++) { | ||||
if (__predict_false((epoch = allepochs[i]) == NULL)) | epoch = epoch_array + i; | ||||
if (__predict_false( | |||||
atomic_load_acq_int(&epoch->e_in_use) == 0)) | |||||
continue; | continue; | ||||
er = epoch_currecord(epoch); | er = epoch_currecord(epoch); | ||||
record = &er->er_record; | record = &er->er_record; | ||||
if ((npending = record->n_pending) == 0) | if ((npending = record->n_pending) == 0) | ||||
continue; | continue; | ||||
ck_epoch_poll_deferred(record, &cb_stack); | ck_epoch_poll_deferred(record, &cb_stack); | ||||
total += npending - record->n_pending; | total += npending - record->n_pending; | ||||
} | } | ||||
▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines |