D17787.id.diff
No OneTemporary
Actions

Size

23 KB

Referenced Files

None

Subscribers

None

D17787.id.diff
View Options

	Index: sys/kern/genoffset.c
	===================================================================
	--- sys/kern/genoffset.c
	+++ sys/kern/genoffset.c
	@@ -36,7 +36,6 @@
	#include <sys/assym.h>
	#include <sys/proc.h>

	-OFFSYM(td_pre_epoch_prio, thread, u_char);
	OFFSYM(td_priority, thread, u_char);
	OFFSYM(td_epochnest, thread, u_char);
	OFFSYM(td_critnest, thread, u_int);
	Index: sys/kern/kern_thread.c
	===================================================================
	--- sys/kern/kern_thread.c
	+++ sys/kern/kern_thread.c
	@@ -272,6 +272,7 @@
	td->td_rlqe = NULL;
	EVENTHANDLER_DIRECT_INVOKE(thread_init, td);
	umtx_thread_init(td);
	+ epoch_thread_init(td);
	td->td_kstack = 0;
	td->td_sel = NULL;
	return (0);
	@@ -291,6 +292,7 @@
	turnstile_free(td->td_turnstile);
	sleepq_free(td->td_sleepqueue);
	umtx_thread_fini(td);
	+ epoch_thread_fini(td);
	seltdfini(td);
	}

	Index: sys/kern/subr_epoch.c
	===================================================================
	--- sys/kern/subr_epoch.c
	+++ sys/kern/subr_epoch.c
	@@ -55,11 +55,32 @@

	static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation");

	+#ifdef __amd64__
	+#define EPOCH_ALIGN CACHE_LINE_SIZE*2
	+#else
	+#define EPOCH_ALIGN CACHE_LINE_SIZE
	+#endif
	+
	+TAILQ_HEAD (epoch_tdlist, epoch_tracker);
	+typedef struct epoch_record {
	+ ck_epoch_record_t er_record;
	+ volatile struct epoch_tdlist er_tdlist;
	+ volatile uint32_t er_gen;
	+ uint32_t er_cpuid;
	+} __aligned(EPOCH_ALIGN) *epoch_record_t;
	+
	+struct epoch {
	+ struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
	+ epoch_record_t e_pcpu_record;
	+ int e_idx;
	+ int e_flags;
	+};
	+
	/* arbitrary --- needs benchmarking */
	#define MAX_ADAPTIVE_SPIN 100
	#define MAX_EPOCHS 64

	-CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context));
	+CTASSERT(sizeof(epoch_context_t) == sizeof(struct epoch_context_user));
	SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information");
	SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats");

	@@ -151,11 +172,19 @@
	er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu);
	bzero(er, sizeof(*er));
	ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL);
	- TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist);
	+ TAILQ_INIT(&er->er_tdlist);
	er->er_cpuid = cpu;
	}
	}

	+static void
	+epoch_adjust_prio(struct thread *td, u_char prio)
	+{
	+ thread_lock(td);
	+ sched_prio(td, prio);
	+ thread_unlock(td);
	+}
	+
	epoch_t
	epoch_alloc(int flags)
	{
	@@ -191,32 +220,107 @@
	free(epoch, M_EPOCH);
	}

	+static epoch_record_t
	+epoch_currecord(epoch_t epoch)
	+{
	+
	+ return zpcpu_get_cpu(epoch->e_pcpu_record, curcpu);
	+}
	+
	+#define INIT_CHECK(epoch) \
	+ do { \
	+ if (__predict_false((epoch) == NULL)) \
	+ return; \
	+ } while (0)
	+
	void
	-epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et)
	+epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et)
	{
	+ struct epoch_record *er;
	+ struct thread *td;
	+
	+ MPASS(cold \|\| epoch != NULL);
	+ INIT_CHECK(epoch);
	+ MPASS(epoch->e_flags & EPOCH_PREEMPT);
	+#ifdef EPOCH_TRACKER_DEBUG
	+ et->et_magic_pre = EPOCH_MAGIC0;
	+ et->et_magic_post = EPOCH_MAGIC1;
	+#endif
	+ td = curthread;
	+ et->et_td = td;
	+ td->td_epochnest++;
	+ critical_enter();
	+ sched_pin();

	- epoch_enter_preempt(epoch, et);
	+ td->td_pre_epoch_prio = td->td_priority;
	+ er = epoch_currecord(epoch);
	+ TAILQ_INSERT_TAIL(&er->er_tdlist, et, et_link);
	+ ck_epoch_begin(&er->er_record, &et->et_section);
	+ critical_exit();
	}

	void
	-epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et)
	+epoch_enter(epoch_t epoch)
	{
	+ struct thread *td;
	+ epoch_record_t er;

	- epoch_exit_preempt(epoch, et);
	+ MPASS(cold \|\| epoch != NULL);
	+ INIT_CHECK(epoch);
	+ td = curthread;
	+
	+ td->td_epochnest++;
	+ critical_enter();
	+ er = epoch_currecord(epoch);
	+ ck_epoch_begin(&er->er_record, NULL);
	}

	void
	-epoch_enter_KBI(epoch_t epoch)
	+epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et)
	{
	+ struct epoch_record *er;
	+ struct thread *td;

	- epoch_enter(epoch);
	+ INIT_CHECK(epoch);
	+ td = curthread;
	+ critical_enter();
	+ sched_unpin();
	+ MPASS(td->td_epochnest);
	+ td->td_epochnest--;
	+ er = epoch_currecord(epoch);
	+ MPASS(epoch->e_flags & EPOCH_PREEMPT);
	+ MPASS(et != NULL);
	+ MPASS(et->et_td == td);
	+#ifdef EPOCH_TRACKER_DEBUG
	+ MPASS(et->et_magic_pre == EPOCH_MAGIC0);
	+ MPASS(et->et_magic_post == EPOCH_MAGIC1);
	+ et->et_magic_pre = 0;
	+ et->et_magic_post = 0;
	+#endif
	+#ifdef INVARIANTS
	+ et->et_td = (void*)0xDEADBEEF;
	+#endif
	+ ck_epoch_end(&er->er_record, &et->et_section);
	+ TAILQ_REMOVE(&er->er_tdlist, et, et_link);
	+ er->er_gen++;
	+ if (__predict_false(td->td_pre_epoch_prio != td->td_priority))
	+ epoch_adjust_prio(td, td->td_pre_epoch_prio);
	+ critical_exit();
	}

	void
	-epoch_exit_KBI(epoch_t epoch)
	+epoch_exit(epoch_t epoch)
	{
	+ struct thread *td;
	+ epoch_record_t er;

	- epoch_exit(epoch);
	+ INIT_CHECK(epoch);
	+ td = curthread;
	+ MPASS(td->td_epochnest);
	+ td->td_epochnest--;
	+ er = epoch_currecord(epoch);
	+ ck_epoch_end(&er->er_record, NULL);
	+ critical_exit();
	}

	/*
	@@ -229,7 +333,7 @@
	{
	epoch_record_t record;
	struct thread td, owner, *curwaittd;
	- struct epoch_thread *tdwait;
	+ struct epoch_tracker *tdwait;
	struct turnstile *ts;
	struct lock_object *lock;
	int spincount, gen;
	@@ -442,12 +546,10 @@
	}

	void
	-epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t))
	+epoch_call(epoch_t epoch, epoch_context_t *ctx, epoch_cb_t callback)
	{
	epoch_record_t er;
	- ck_epoch_entry_t *cb;
	-
	- cb = (void *)ctx;
	+ ck_epoch_entry_t *cb = ctx;

	MPASS(callback);
	/* too early in boot to have epoch set up */
	@@ -511,7 +613,7 @@
	int
	in_epoch_verbose(epoch_t epoch, int dump_onfail)
	{
	- struct epoch_thread *tdwait;
	+ struct epoch_tracker *tdwait;
	struct thread *td;
	epoch_record_t er;

	@@ -547,9 +649,15 @@
	}

	void
	-epoch_adjust_prio(struct thread *td, u_char prio)
	+epoch_thread_init(struct thread *td)
	{
	- thread_lock(td);
	- sched_prio(td, prio);
	- thread_unlock(td);
	+
	+ td->td_et = malloc(sizeof(struct epoch_tracker), M_EPOCH, M_WAITOK);
	+}
	+
	+void
	+epoch_thread_fini(struct thread *td)
	+{
	+
	+ free(td->td_et, M_EPOCH);
	}
	Index: sys/net/if.c
	===================================================================
	--- sys/net/if.c
	+++ sys/net/if.c
	@@ -610,7 +610,7 @@
	}

	static void
	-if_destroy(epoch_context_t ctx)
	+if_destroy(epoch_context_t *ctx)
	{
	struct ifnet *ifp;

	@@ -1764,35 +1764,29 @@
	void
	if_addr_rlock(struct ifnet *ifp)
	{
	- MPASS((uint64_t )&ifp->if_addr_et == 0);
	- epoch_enter_preempt(net_epoch_preempt, &ifp->if_addr_et);
	+
	+ epoch_enter_preempt(net_epoch_preempt, curthread->td_et);
	}

	void
	if_addr_runlock(struct ifnet *ifp)
	{
	- epoch_exit_preempt(net_epoch_preempt, &ifp->if_addr_et);
	-#ifdef INVARIANTS
	- bzero(&ifp->if_addr_et, sizeof(struct epoch_tracker));
	-#endif
	+
	+ epoch_exit_preempt(net_epoch_preempt, curthread->td_et);
	}

	void
	if_maddr_rlock(if_t ifp)
	{

	- MPASS((uint64_t )&ifp->if_maddr_et == 0);
	- epoch_enter_preempt(net_epoch_preempt, &ifp->if_maddr_et);
	+ epoch_enter_preempt(net_epoch_preempt, curthread->td_et);
	}

	void
	if_maddr_runlock(if_t ifp)
	{

	- epoch_exit_preempt(net_epoch_preempt, &ifp->if_maddr_et);
	-#ifdef INVARIANTS
	- bzero(&ifp->if_maddr_et, sizeof(struct epoch_tracker));
	-#endif
	+ epoch_exit_preempt(net_epoch_preempt, curthread->td_et);
	}

	/*
	@@ -1842,7 +1836,7 @@
	}

	static void
	-ifa_destroy(epoch_context_t ctx)
	+ifa_destroy(epoch_context_t *ctx)
	{
	struct ifaddr *ifa;

	@@ -3439,7 +3433,7 @@
	}

	static void
	-if_destroymulti(epoch_context_t ctx)
	+if_destroymulti(epoch_context_t *ctx)
	{
	struct ifmultiaddr *ifma;

	Index: sys/net/if_lagg.h
	===================================================================
	--- sys/net/if_lagg.h
	+++ sys/net/if_lagg.h
	@@ -251,8 +251,8 @@
	const struct sockaddr , struct route );
	struct lagg_counters port_counters; /* ifp counters copy */

	- CK_SLIST_ENTRY(lagg_port) lp_entries;
	- struct epoch_context lp_epoch_ctx;
	+ CK_SLIST_ENTRY(lagg_port) lp_entries;
	+ epoch_context_t lp_epoch_ctx;
	};

	extern struct mbuf (lagg_input_p)(struct ifnet , struct mbuf );
	Index: sys/net/if_lagg.c
	===================================================================
	--- sys/net/if_lagg.c
	+++ sys/net/if_lagg.c
	@@ -805,7 +805,7 @@
	#endif

	static void
	-lagg_port_destroy_cb(epoch_context_t ec)
	+lagg_port_destroy_cb(epoch_context_t *ec)
	{
	struct lagg_port *lp;
	struct ifnet *ifp;
	Index: sys/net/if_llatbl.h
	===================================================================
	--- sys/net/if_llatbl.h
	+++ sys/net/if_llatbl.h
	@@ -82,7 +82,7 @@
	struct callout lle_timer;
	struct rwlock lle_lock;
	struct mtx req_mtx;
	- struct epoch_context lle_epoch_ctx;
	+ epoch_context_t lle_epoch_ctx;
	};

	#define LLE_WLOCK(lle) rw_wlock(&(lle)->lle_lock)
	Index: sys/net/if_var.h
	===================================================================
	--- sys/net/if_var.h
	+++ sys/net/if_var.h
	@@ -380,9 +380,7 @@
	* Netdump hooks to be called while dumping.
	*/
	struct netdump_methods *if_netdump_methods;
	- struct epoch_context if_epoch_ctx;
	- struct epoch_tracker if_addr_et;
	- struct epoch_tracker if_maddr_et;
	+ epoch_context_t if_epoch_ctx;

	/*
	* Spare fields to be added before branching a stable branch, so
	@@ -543,7 +541,7 @@
	counter_u64_t ifa_opackets;
	counter_u64_t ifa_ibytes;
	counter_u64_t ifa_obytes;
	- struct epoch_context ifa_epoch_ctx;
	+ epoch_context_t ifa_epoch_ctx;
	};

	struct ifaddr * ifa_alloc(size_t size, int flags);
	@@ -564,7 +562,7 @@
	int ifma_flags;
	void ifma_protospec; / protocol-specific state, if any */
	struct ifmultiaddr ifma_llifma; / pointer to ifma for ifma_lladdr */
	- struct epoch_context ifma_epoch_ctx;
	+ epoch_context_t ifma_epoch_ctx;
	};

	extern struct rwlock ifnet_rwlock;
	Index: sys/net/if_vlan.c
	===================================================================
	--- sys/net/if_vlan.c
	+++ sys/net/if_vlan.c
	@@ -153,7 +153,7 @@
	struct vlan_mc_entry {
	struct sockaddr_dl mc_addr;
	CK_SLIST_ENTRY(vlan_mc_entry) mc_entries;
	- struct epoch_context mc_epoch_ctx;
	+ epoch_context_t mc_epoch_ctx;
	};

	struct ifvlan {
	@@ -318,7 +318,7 @@
	#define HASH(n, m) ((((n) >> 8) ^ ((n) >> 4) ^ (n)) & (m))

	static void
	-vlan_mc_free(struct epoch_context *ctx)
	+vlan_mc_free(epoch_context_t *ctx)
	{
	struct vlan_mc_entry *mc = __containerof(ctx, struct vlan_mc_entry, mc_epoch_ctx);
	free(mc, M_VLAN);
	Index: sys/netinet/in.c
	===================================================================
	--- sys/netinet/in.c
	+++ sys/netinet/in.c
	@@ -1060,7 +1060,7 @@
	* Do actual deallocation of @lle.
	*/
	static void
	-in_lltable_destroy_lle_unlocked(epoch_context_t ctx)
	+in_lltable_destroy_lle_unlocked(epoch_context_t *ctx)
	{
	struct llentry *lle;

	Index: sys/netinet/in_pcb.h
	===================================================================
	--- sys/netinet/in_pcb.h
	+++ sys/netinet/in_pcb.h
	@@ -330,7 +330,7 @@
	CK_LIST_ENTRY(inpcb) inp_list; /* (p/l) list for all PCBs for proto */
	/* (e[r]) for list iteration */
	/* (p[w]/l) for addition/removal */
	- struct epoch_context inp_epoch_ctx;
	+ epoch_context_t inp_epoch_ctx;
	};
	#endif /* _KERNEL */

	@@ -405,7 +405,7 @@
	#endif /* _SYS_SOCKETVAR_H_ */

	struct inpcbport {
	- struct epoch_context phd_epoch_ctx;
	+ epoch_context_t phd_epoch_ctx;
	CK_LIST_ENTRY(inpcbport) phd_hash;
	struct inpcbhead phd_pcblist;
	u_short phd_port;
	@@ -413,7 +413,7 @@

	struct in_pcblist {
	int il_count;
	- struct epoch_context il_epoch_ctx;
	+ epoch_context_t il_epoch_ctx;
	struct inpcbinfo *il_pcbinfo;
	struct inpcb *il_inp_list[0];
	};
	@@ -568,7 +568,7 @@
	*/
	struct inpcblbgroup {
	CK_LIST_ENTRY(inpcblbgroup) il_list;
	- struct epoch_context il_epoch_ctx;
	+ epoch_context_t il_epoch_ctx;
	uint16_t il_lport; /* (c) */
	u_char il_vflag; /* (c) */
	u_char il_pad;
	@@ -874,7 +874,7 @@
	int in_pcbrele(struct inpcb *);
	int in_pcbrele_rlocked(struct inpcb *);
	int in_pcbrele_wlocked(struct inpcb *);
	-void in_pcblist_rele_rlocked(epoch_context_t ctx);
	+void in_pcblist_rele_rlocked(epoch_context_t *ctx);
	void in_losing(struct inpcb *);
	void in_pcbsetsolabel(struct socket *so);
	int in_getpeeraddr(struct socket so, struct sockaddr *nam);
	Index: sys/netinet/in_pcb.c
	===================================================================
	--- sys/netinet/in_pcb.c
	+++ sys/netinet/in_pcb.c
	@@ -240,7 +240,7 @@
	}

	static void
	-in_pcblbgroup_free_deferred(epoch_context_t ctx)
	+in_pcblbgroup_free_deferred(epoch_context_t *ctx)
	{
	struct inpcblbgroup *grp;

	@@ -1547,7 +1547,7 @@
	}

	void
	-in_pcblist_rele_rlocked(epoch_context_t ctx)
	+in_pcblist_rele_rlocked(epoch_context_t *ctx)
	{
	struct in_pcblist *il;
	struct inpcb *inp;
	@@ -1569,7 +1569,7 @@
	}

	static void
	-inpcbport_free(epoch_context_t ctx)
	+inpcbport_free(epoch_context_t *ctx)
	{
	struct inpcbport *phd;

	@@ -1578,7 +1578,7 @@
	}

	static void
	-in_pcbfree_deferred(epoch_context_t ctx)
	+in_pcbfree_deferred(epoch_context_t *ctx)
	{
	struct inpcb *inp;
	int released __unused;
	@@ -2559,7 +2559,7 @@
	if (phd == NULL) {
	return (ENOBUFS); /* XXX */
	}
	- bzero(&phd->phd_epoch_ctx, sizeof(struct epoch_context));
	+ bzero(&phd->phd_epoch_ctx, sizeof(phd->phd_epoch_ctx));
	phd->phd_port = inp->inp_lport;
	CK_LIST_INIT(&phd->phd_pcblist);
	CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
	Index: sys/netinet/ip_var.h
	===================================================================
	--- sys/netinet/ip_var.h
	+++ sys/netinet/ip_var.h
	@@ -96,7 +96,7 @@
	u_short imo_max_memberships; /* max memberships this socket */
	struct in_multi *imo_membership; / group memberships */
	struct in_mfilter imo_mfilters; / source filters */
	- struct epoch_context imo_epoch_ctx;
	+ epoch_context_t imo_epoch_ctx;
	};

	struct ipstat {
	Index: sys/netinet6/in6.c
	===================================================================
	--- sys/netinet6/in6.c
	+++ sys/netinet6/in6.c
	@@ -2049,7 +2049,7 @@
	* Do actual deallocation of @lle.
	*/
	static void
	-in6_lltable_destroy_lle_unlocked(epoch_context_t ctx)
	+in6_lltable_destroy_lle_unlocked(epoch_context_t *ctx)
	{
	struct llentry *lle;

	Index: sys/netinet6/ip6_var.h
	===================================================================
	--- sys/netinet6/ip6_var.h
	+++ sys/netinet6/ip6_var.h
	@@ -123,7 +123,7 @@
	u_short im6o_max_memberships; /* max memberships this socket */
	struct in6_multi *im6o_membership; / group memberships */
	struct in6_mfilter im6o_mfilters; / source filters */
	- struct epoch_context imo6_epoch_ctx;
	+ epoch_context_t imo6_epoch_ctx;
	};

	/*
	Index: sys/sys/epoch.h
	===================================================================
	--- sys/sys/epoch.h
	+++ sys/sys/epoch.h
	@@ -29,10 +29,24 @@

	#ifndef _SYS_EPOCH_H_
	#define _SYS_EPOCH_H_
	+
	+/*
	+ * For applications that use kvm(3) and see kernel structures
	+ * define a placeholder for epoch_context, and protect its
	+ * size with CTASSERT in subr_epoch.c.
	+ */
	+struct epoch_context_user {
	+ void *data[2];
	+};
	+
	+#ifndef _KERNEL
	+typedef struct epoch_context_user epoch_context_t;
	+#endif
	+
	#ifdef _KERNEL
	#include <sys/lock.h>
	#include <sys/pcpu.h>
	-#endif
	+#include <ck_epoch.h>

	struct epoch;
	typedef struct epoch *epoch_t;
	@@ -43,51 +57,41 @@
	extern epoch_t global_epoch;
	extern epoch_t global_epoch_preempt;

	-struct epoch_context {
	- void *data[2];
	-} __aligned(sizeof(void *));
	-
	-typedef struct epoch_context *epoch_context_t;
	-
	-
	struct epoch_tracker {
	- void *datap[3];
	#ifdef EPOCH_TRACKER_DEBUG
	- int datai[5];
	-#else
	- int datai[1];
	+#define EPOCH_MAGIC0 0xFADECAFEF00DD00D
	+#define EPOCH_MAGIC1 0xBADDBABEDEEDFEED
	+ uint64_t et_magic_pre;
	+#endif
	+ TAILQ_ENTRY(epoch_tracker) et_link;
	+ struct thread *et_td;
	+ ck_epoch_section_t et_section;
	+#ifdef EPOCH_TRACKER_DEBUG
	+ uint64_t et_magic_post;
	#endif
	} __aligned(sizeof(void *));
	-
	typedef struct epoch_tracker *epoch_tracker_t;

	+typedef struct ck_epoch_entry epoch_context_t;
	+typedef void epoch_cb_t(epoch_context_t *);
	+
	epoch_t epoch_alloc(int flags);
	void epoch_free(epoch_t epoch);
	void epoch_wait(epoch_t epoch);
	void epoch_wait_preempt(epoch_t epoch);
	-void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t));
	+void epoch_call(epoch_t epoch, epoch_context_t *ctx, epoch_cb_t callback);
	int in_epoch(epoch_t epoch);
	int in_epoch_verbose(epoch_t epoch, int dump_onfail);
	-#ifdef _KERNEL
	DPCPU_DECLARE(int, epoch_cb_count);
	DPCPU_DECLARE(struct grouptask, epoch_cb_task);
	-#define EPOCH_MAGIC0 0xFADECAFEF00DD00D
	-#define EPOCH_MAGIC1 0xBADDBABEDEEDFEED

	-void epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et);
	-void epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et);
	-void epoch_enter_KBI(epoch_t epoch);
	-void epoch_exit_KBI(epoch_t epoch);
	+void epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et);
	+void epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et);
	+void epoch_enter(epoch_t epoch);
	+void epoch_exit(epoch_t epoch);

	-
	-#if defined(KLD_MODULE) && !defined(KLD_TIED)
	-#define epoch_enter_preempt(e, t) epoch_enter_preempt_KBI((e), (t))
	-#define epoch_exit_preempt(e, t) epoch_exit_preempt_KBI((e), (t))
	-#define epoch_enter(e) epoch_enter_KBI((e))
	-#define epoch_exit(e) epoch_exit_KBI((e))
	-#else
	-#include <sys/epoch_private.h>
	-#endif /* KLD_MODULE */
	+void epoch_thread_init(struct thread *);
	+void epoch_thread_fini(struct thread *);

	#endif /* _KERNEL */
	-#endif
	+#endif /* _SYS_EPOCH_H_ */
	Index: sys/sys/epoch_private.h
	===================================================================
	--- sys/sys/epoch_private.h
	+++ /dev/null
	@@ -1,210 +0,0 @@
	-/*-
	- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
	- *
	- * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
	- *
	- * Redistribution and use in source and binary forms, with or without
	- * modification, are permitted provided that the following conditions
	- * are met:
	- * 1. Redistributions of source code must retain the above copyright
	- * notice, this list of conditions and the following disclaimer.
	- * 2. Redistributions in binary form must reproduce the above copyright
	- * notice, this list of conditions and the following disclaimer in the
	- * documentation and/or other materials provided with the distribution.
	- *
	- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
	- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
	- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	- * SUCH DAMAGE.
	- *
	- * $FreeBSD$
	- */
	-
	-#ifndef _SYS_EPOCH_PRIVATE_H_
	-#define _SYS_EPOCH_PRIVATE_H_
	-#ifndef _KERNEL
	-#error "no user serviceable parts"
	-#else
	-#include <ck_epoch.h>
	-#include <sys/kpilite.h>
	-
	-#include <sys/mutex.h>
	-
	-extern void epoch_adjust_prio(struct thread *td, u_char prio);
	-#ifndef _SYS_SYSTM_H_
	-extern void critical_exit_preempt(void);
	-#endif
	-
	-#ifdef __amd64__
	-#define EPOCH_ALIGN CACHE_LINE_SIZE*2
	-#else
	-#define EPOCH_ALIGN CACHE_LINE_SIZE
	-#endif
	-
	-/*
	- * Standalone (_sa) routines for thread state manipulation
	- */
	-static __inline void
	-critical_enter_sa(void *tdarg)
	-{
	- struct thread_lite *td;
	-
	- td = tdarg;
	- td->td_critnest++;
	- __compiler_membar();
	-}
	-
	-static __inline void
	-critical_exit_sa(void *tdarg)
	-{
	- struct thread_lite *td;
	-
	- td = tdarg;
	- MPASS(td->td_critnest > 0);
	- __compiler_membar();
	- td->td_critnest--;
	- __compiler_membar();
	- if (__predict_false(td->td_owepreempt != 0))
	- critical_exit_preempt();
	-}
	-
	-typedef struct epoch_thread {
	-#ifdef EPOCH_TRACKER_DEBUG
	- uint64_t et_magic_pre;
	-#endif
	- TAILQ_ENTRY(epoch_thread) et_link; /* Epoch queue. */
	- struct thread et_td; / pointer to thread in section */
	- ck_epoch_section_t et_section; /* epoch section object */
	-#ifdef EPOCH_TRACKER_DEBUG
	- uint64_t et_magic_post;
	-#endif
	-} *epoch_thread_t;
	-TAILQ_HEAD (epoch_tdlist, epoch_thread);
	-
	-typedef struct epoch_record {
	- ck_epoch_record_t er_record;
	- volatile struct epoch_tdlist er_tdlist;
	- volatile uint32_t er_gen;
	- uint32_t er_cpuid;
	-} __aligned(EPOCH_ALIGN) *epoch_record_t;
	-
	-struct epoch {
	- struct ck_epoch e_epoch __aligned(EPOCH_ALIGN);
	- epoch_record_t e_pcpu_record;
	- int e_idx;
	- int e_flags;
	-};
	-
	-static epoch_record_t
	-epoch_currecord(epoch_t epoch)
	-{
	- return zpcpu_get_cpu(epoch->e_pcpu_record, curcpu);
	-}
	-
	-#define INIT_CHECK(epoch) \
	- do { \
	- if (__predict_false((epoch) == NULL)) \
	- return; \
	- } while (0)
	-
	-static __inline void
	-epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et)
	-{
	- struct epoch_record *er;
	- struct epoch_thread *etd;
	- struct thread_lite *td;
	-
	- MPASS(cold \|\| epoch != NULL);
	- INIT_CHECK(epoch);
	- etd = (void *)et;
	- MPASS(epoch->e_flags & EPOCH_PREEMPT);
	-#ifdef EPOCH_TRACKER_DEBUG
	- etd->et_magic_pre = EPOCH_MAGIC0;
	- etd->et_magic_post = EPOCH_MAGIC1;
	-#endif
	- td = (struct thread_lite *)curthread;
	- etd->et_td = (void*)td;
	- td->td_epochnest++;
	- critical_enter_sa(td);
	- sched_pin_lite(td);
	-
	- td->td_pre_epoch_prio = td->td_priority;
	- er = epoch_currecord(epoch);
	- TAILQ_INSERT_TAIL(&er->er_tdlist, etd, et_link);
	- ck_epoch_begin(&er->er_record, (ck_epoch_section_t *)&etd->et_section);
	- critical_exit_sa(td);
	-}
	-
	-static __inline void
	-epoch_enter(epoch_t epoch)
	-{
	- struct thread_lite *td;
	- epoch_record_t er;
	-
	- MPASS(cold \|\| epoch != NULL);
	- INIT_CHECK(epoch);
	- td = (struct thread_lite *)curthread;
	-
	- td->td_epochnest++;
	- critical_enter_sa(td);
	- er = epoch_currecord(epoch);
	- ck_epoch_begin(&er->er_record, NULL);
	-}
	-
	-static __inline void
	-epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et)
	-{
	- struct epoch_record *er;
	- struct epoch_thread *etd;
	- struct thread_lite *td;
	-
	- INIT_CHECK(epoch);
	- td = (struct thread_lite *)curthread;
	- critical_enter_sa(td);
	- sched_unpin_lite(td);
	- MPASS(td->td_epochnest);
	- td->td_epochnest--;
	- er = epoch_currecord(epoch);
	- MPASS(epoch->e_flags & EPOCH_PREEMPT);
	- etd = (void *)et;
	- MPASS(etd != NULL);
	- MPASS(etd->et_td == (struct thread *)td);
	-#ifdef EPOCH_TRACKER_DEBUG
	- MPASS(etd->et_magic_pre == EPOCH_MAGIC0);
	- MPASS(etd->et_magic_post == EPOCH_MAGIC1);
	- etd->et_magic_pre = 0;
	- etd->et_magic_post = 0;
	-#endif
	- etd->et_td = (void*)0xDEADBEEF;
	- ck_epoch_end(&er->er_record,
	- (ck_epoch_section_t *)&etd->et_section);
	- TAILQ_REMOVE(&er->er_tdlist, etd, et_link);
	- er->er_gen++;
	- if (__predict_false(td->td_pre_epoch_prio != td->td_priority))
	- epoch_adjust_prio((struct thread *)td, td->td_pre_epoch_prio);
	- critical_exit_sa(td);
	-}
	-
	-static __inline void
	-epoch_exit(epoch_t epoch)
	-{
	- struct thread_lite *td;
	- epoch_record_t er;
	-
	- INIT_CHECK(epoch);
	- td = (struct thread_lite *)curthread;
	- MPASS(td->td_epochnest);
	- td->td_epochnest--;
	- er = epoch_currecord(epoch);
	- ck_epoch_end(&er->er_record, NULL);
	- critical_exit_sa(td);
	-}
	-#endif /* _KERNEL */
	-#endif /* _SYS_EPOCH_PRIVATE_H_ */
	Index: sys/sys/proc.h
	===================================================================
	--- sys/sys/proc.h
	+++ sys/sys/proc.h
	@@ -193,6 +193,7 @@
	struct turnstile;
	struct vm_map;
	struct vm_map_entry;
	+struct epoch_tracker;

	/*
	* XXX: Does this belong in resource.h or resourcevar.h instead?
	@@ -360,6 +361,7 @@
	int td_lastcpu; /* (t) Last cpu we were on. */
	int td_oncpu; /* (t) Which cpu we are on. */
	void td_lkpi_task; / LinuxKPI task struct pointer */
	+ struct epoch_tracker td_et; / (k) compat KPI spare tracker */
	int td_pmcpend;
	};

File Metadata

Mime Type: text/plain
Expires: Fri, Aug 28, 3:17 AM (18 h, 1 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 37263710
Default Alt Text: D17787.id.diff (23 KB)

D17787.id.diffNo OneTemporaryActions

D17787.id.diffView Options

File Metadata

Event Timeline

D17787.id.diff
No OneTemporary
Actions

D17787.id.diff
View Options