Differential D20109 Diff 59152 head/sys/kern/subr_epoch.c

Changeset View

Standalone View

head/sys/kern/subr_epoch.c

Show All 37 Lines
#include <sys/kernel.h>		#include <sys/kernel.h>
#include <sys/limits.h>		#include <sys/limits.h>
#include <sys/lock.h>		#include <sys/lock.h>
#include <sys/malloc.h>		#include <sys/malloc.h>
#include <sys/mutex.h>		#include <sys/mutex.h>
#include <sys/pcpu.h>		#include <sys/pcpu.h>
#include <sys/proc.h>		#include <sys/proc.h>
#include <sys/sched.h>		#include <sys/sched.h>
		#include <sys/sx.h>
#include <sys/smp.h>		#include <sys/smp.h>
#include <sys/sysctl.h>		#include <sys/sysctl.h>
#include <sys/turnstile.h>		#include <sys/turnstile.h>
#include <vm/vm.h>		#include <vm/vm.h>
#include <vm/vm_extern.h>		#include <vm/vm_extern.h>
#include <vm/vm_kern.h>		#include <vm/vm_kern.h>
#include <vm/uma.h>		#include <vm/uma.h>

#include <ck_epoch.h>		#include <ck_epoch.h>

static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");		static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");

#ifdef __amd64__		#ifdef __amd64__
#define EPOCH_ALIGN CACHE_LINE_SIZE*2		#define EPOCH_ALIGN CACHE_LINE_SIZE*2
#else		#else
#define EPOCH_ALIGN CACHE_LINE_SIZE		#define EPOCH_ALIGN CACHE_LINE_SIZE
#endif		#endif

TAILQ_HEAD (epoch_tdlist, epoch_tracker);		TAILQ_HEAD (epoch_tdlist, epoch_tracker);
typedef struct epoch_record {		typedef struct epoch_record {
ck_epoch_record_t er_record;		ck_epoch_record_t er_record;
		struct epoch_context er_drain_ctx;
		struct epoch *er_parent;
volatile struct epoch_tdlist er_tdlist;		volatile struct epoch_tdlist er_tdlist;
volatile uint32_t er_gen;		volatile uint32_t er_gen;
uint32_t er_cpuid;		uint32_t er_cpuid;
} __aligned(EPOCH_ALIGN) *epoch_record_t;		} __aligned(EPOCH_ALIGN) *epoch_record_t;

struct epoch {		struct epoch {
struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);		struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
epoch_record_t e_pcpu_record;		epoch_record_t e_pcpu_record;
int e_idx;		int e_idx;
int e_flags;		int e_flags;
		struct sx e_drain_sx;
		struct mtx e_drain_mtx;
		volatile int e_drain_count;
};		};

/* arbitrary --- needs benchmarking */		/* arbitrary --- needs benchmarking */
#define MAX_ADAPTIVE_SPIN 100		#define MAX_ADAPTIVE_SPIN 100
#define MAX_EPOCHS 64		#define MAX_EPOCHS 64

CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context));		CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context));
SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information");		SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information");
▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines	epoch_ctor(epoch_t epoch)

epoch->e_pcpu_record = uma_zalloc_pcpu(pcpu_zone_record, M_WAITOK);		epoch->e_pcpu_record = uma_zalloc_pcpu(pcpu_zone_record, M_WAITOK);
CPU_FOREACH(cpu) {		CPU_FOREACH(cpu) {
er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);		er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);
bzero(er, sizeof(*er));		bzero(er, sizeof(*er));
ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);		ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);		TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
er->er_cpuid = cpu;		er->er_cpuid = cpu;
		er->er_parent = epoch;
}		}
}		}

static void		static void
epoch_adjust_prio(struct thread *td, u_char prio)		epoch_adjust_prio(struct thread *td, u_char prio)
{		{

thread_lock(td);		thread_lock(td);
Show All 9 Lines	epoch_alloc(int flags)
if (__predict_false(!inited))		if (__predict_false(!inited))
panic("%s called too early in boot", __func__);		panic("%s called too early in boot", __func__);
epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO \| M_WAITOK);		epoch = malloc(sizeof(struct epoch), M_EPOCH, M_ZERO \| M_WAITOK);
ck_epoch_init(&epoch->e_epoch);		ck_epoch_init(&epoch->e_epoch);
epoch_ctor(epoch);		epoch_ctor(epoch);
MPASS(epoch_count < MAX_EPOCHS - 2);		MPASS(epoch_count < MAX_EPOCHS - 2);
epoch->e_flags = flags;		epoch->e_flags = flags;
epoch->e_idx = epoch_count;		epoch->e_idx = epoch_count;
		sx_init(&epoch->e_drain_sx, "epoch-drain-sx");
		mtx_init(&epoch->e_drain_mtx, "epoch-drain-mtx", NULL, MTX_DEF);
allepochs[epoch_count++] = epoch;		allepochs[epoch_count++] = epoch;
return (epoch);		return (epoch);
}		}

void		void
epoch_free(epoch_t epoch)		epoch_free(epoch_t epoch)
{		{
#ifdef INVARIANTS
struct epoch_record *er;
int cpu;

CPU_FOREACH(cpu) {		epoch_drain_callbacks(epoch);
er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);
MPASS(TAILQ_EMPTY(&er->er_tdlist));
}
#endif
allepochs[epoch->e_idx] = NULL;		allepochs[epoch->e_idx] = NULL;
epoch_wait(global_epoch);		epoch_wait(global_epoch);
uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record);		uma_zfree_pcpu(pcpu_zone_record, epoch->e_pcpu_record);
		mtx_destroy(&epoch->e_drain_mtx);
		sx_destroy(&epoch->e_drain_sx);
free(epoch, M_EPOCH);		free(epoch, M_EPOCH);
}		}

static epoch_record_t		static epoch_record_t
epoch_currecord(epoch_t epoch)		epoch_currecord(epoch_t epoch)
{		{

return (zpcpu_get_cpu(epoch->e_pcpu_record, curcpu));		return (zpcpu_get_cpu(epoch->e_pcpu_record, curcpu));
▲ Show 20 Lines • Show All 417 Lines • ▼ Show 20 Lines	#endif
critical_exit();		critical_exit();
return (0);		return (0);
}		}

int		int
in_epoch(epoch_t epoch)		in_epoch(epoch_t epoch)
{		{
return (in_epoch_verbose(epoch, 0));		return (in_epoch_verbose(epoch, 0));
		}

		static void
		epoch_drain_cb(struct epoch_context *ctx)
		{
		struct epoch *epoch =
		__containerof(ctx, struct epoch_record, er_drain_ctx)->er_parent;

		if (atomic_fetchadd_int(&epoch->e_drain_count, -1) == 1) {
		mtx_lock(&epoch->e_drain_mtx);
		wakeup(epoch);
		mtx_unlock(&epoch->e_drain_mtx);
		}
		}

		void
		epoch_drain_callbacks(epoch_t epoch)
		{
		epoch_record_t er;
		struct thread *td;
		int was_bound;
		int old_pinned;
		int old_cpu;
		int cpu;

		WITNESS_WARN(WARN_GIANTOK \| WARN_SLEEPOK, NULL,
		"epoch_drain_callbacks() may sleep!");

		/* too early in boot to have epoch set up */
		if (__predict_false(epoch == NULL))
		return;
		#if !defined(EARLY_AP_STARTUP)
		if (__predict_false(inited < 2))
		return;
		#endif
		DROP_GIANT();

		sx_xlock(&epoch->e_drain_sx);
		mtx_lock(&epoch->e_drain_mtx);

		td = curthread;
		thread_lock(td);
		old_cpu = PCPU_GET(cpuid);
		old_pinned = td->td_pinned;
		was_bound = sched_is_bound(td);
		sched_unbind(td);
		td->td_pinned = 0;

		CPU_FOREACH(cpu)
		epoch->e_drain_count++;
		CPU_FOREACH(cpu) {
		er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);
		sched_bind(td, cpu);
		epoch_call(epoch, &er->er_drain_ctx, &epoch_drain_cb);
		}

		/* restore CPU binding, if any */
		if (was_bound != 0) {
		sched_bind(td, old_cpu);
		} else {
		/* get thread back to initial CPU, if any */
		if (old_pinned != 0)
		sched_bind(td, old_cpu);
		sched_unbind(td);
		}
		/* restore pinned after bind */
		td->td_pinned = old_pinned;

		thread_unlock(td);

		while (epoch->e_drain_count != 0)
		msleep(epoch, &epoch->e_drain_mtx, PZERO, "EDRAIN", 0);

		mtx_unlock(&epoch->e_drain_mtx);
		sx_xunlock(&epoch->e_drain_sx);

		PICKUP_GIANT();
}		}

void		void
epoch_thread_init(struct thread *td)		epoch_thread_init(struct thread *td)
{		{

td->td_et = malloc(sizeof(struct epoch_tracker), M_EPOCH, M_WAITOK);		td->td_et = malloc(sizeof(struct epoch_tracker), M_EPOCH, M_WAITOK);
}		}

void		void
epoch_thread_fini(struct thread *td)		epoch_thread_fini(struct thread *td)
{		{

free(td->td_et, M_EPOCH);		free(td->td_et, M_EPOCH);
}		}