Index: sys/kern/genoffset.c =================================================================== --- sys/kern/genoffset.c +++ sys/kern/genoffset.c @@ -36,7 +36,6 @@ #include #include -OFFSYM(td_pre_epoch_prio, thread, u_char); OFFSYM(td_priority, thread, u_char); OFFSYM(td_epochnest, thread, u_char); OFFSYM(td_critnest, thread, u_int); Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -272,6 +272,7 @@ td->td_rlqe = NULL; EVENTHANDLER_DIRECT_INVOKE(thread_init, td); umtx_thread_init(td); + epoch_thread_init(td); td->td_kstack = 0; td->td_sel = NULL; return (0); @@ -291,6 +292,7 @@ turnstile_free(td->td_turnstile); sleepq_free(td->td_sleepqueue); umtx_thread_fini(td); + epoch_thread_fini(td); seltdfini(td); } Index: sys/kern/subr_epoch.c =================================================================== --- sys/kern/subr_epoch.c +++ sys/kern/subr_epoch.c @@ -55,11 +55,32 @@ static MALLOC_DEFINE(M_EPOCH, "epoch", "epoch based reclamation"); +#ifdef __amd64__ +#define EPOCH_ALIGN CACHE_LINE_SIZE*2 +#else +#define EPOCH_ALIGN CACHE_LINE_SIZE +#endif + +TAILQ_HEAD (epoch_tdlist, epoch_tracker); +typedef struct epoch_record { + ck_epoch_record_t er_record; + volatile struct epoch_tdlist er_tdlist; + volatile uint32_t er_gen; + uint32_t er_cpuid; +} __aligned(EPOCH_ALIGN) *epoch_record_t; + +struct epoch { + struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); + epoch_record_t e_pcpu_record; + int e_idx; + int e_flags; +}; + /* arbitrary --- needs benchmarking */ #define MAX_ADAPTIVE_SPIN 100 #define MAX_EPOCHS 64 -CTASSERT(sizeof(ck_epoch_entry_t) == sizeof(struct epoch_context)); +CTASSERT(sizeof(epoch_context_t) == sizeof(struct epoch_context_user)); SYSCTL_NODE(_kern, OID_AUTO, epoch, CTLFLAG_RW, 0, "epoch information"); SYSCTL_NODE(_kern_epoch, OID_AUTO, stats, CTLFLAG_RW, 0, "epoch stats"); @@ -151,11 +172,19 @@ er = zpcpu_get_cpu(epoch->e_pcpu_record, cpu); bzero(er, sizeof(*er)); ck_epoch_register(&epoch->e_epoch, &er->er_record, NULL); - TAILQ_INIT((struct threadlist *)(uintptr_t)&er->er_tdlist); + TAILQ_INIT(&er->er_tdlist); er->er_cpuid = cpu; } } +static void +epoch_adjust_prio(struct thread *td, u_char prio) +{ + thread_lock(td); + sched_prio(td, prio); + thread_unlock(td); +} + epoch_t epoch_alloc(int flags) { @@ -191,32 +220,107 @@ free(epoch, M_EPOCH); } +static epoch_record_t +epoch_currecord(epoch_t epoch) +{ + + return zpcpu_get_cpu(epoch->e_pcpu_record, curcpu); +} + +#define INIT_CHECK(epoch) \ + do { \ + if (__predict_false((epoch) == NULL)) \ + return; \ + } while (0) + void -epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et) +epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et) { + struct epoch_record *er; + struct thread *td; + + MPASS(cold || epoch != NULL); + INIT_CHECK(epoch); + MPASS(epoch->e_flags & EPOCH_PREEMPT); +#ifdef EPOCH_TRACKER_DEBUG + et->et_magic_pre = EPOCH_MAGIC0; + et->et_magic_post = EPOCH_MAGIC1; +#endif + td = curthread; + et->et_td = td; + td->td_epochnest++; + critical_enter(); + sched_pin(); - epoch_enter_preempt(epoch, et); + td->td_pre_epoch_prio = td->td_priority; + er = epoch_currecord(epoch); + TAILQ_INSERT_TAIL(&er->er_tdlist, et, et_link); + ck_epoch_begin(&er->er_record, &et->et_section); + critical_exit(); } void -epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et) +epoch_enter(epoch_t epoch) { + struct thread *td; + epoch_record_t er; - epoch_exit_preempt(epoch, et); + MPASS(cold || epoch != NULL); + INIT_CHECK(epoch); + td = curthread; + + td->td_epochnest++; + critical_enter(); + er = epoch_currecord(epoch); + ck_epoch_begin(&er->er_record, NULL); } void -epoch_enter_KBI(epoch_t epoch) +epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et) { + struct epoch_record *er; + struct thread *td; - epoch_enter(epoch); + INIT_CHECK(epoch); + td = curthread; + critical_enter(); + sched_unpin(); + MPASS(td->td_epochnest); + td->td_epochnest--; + er = epoch_currecord(epoch); + MPASS(epoch->e_flags & EPOCH_PREEMPT); + MPASS(et != NULL); + MPASS(et->et_td == td); +#ifdef EPOCH_TRACKER_DEBUG + MPASS(et->et_magic_pre == EPOCH_MAGIC0); + MPASS(et->et_magic_post == EPOCH_MAGIC1); + et->et_magic_pre = 0; + et->et_magic_post = 0; +#endif +#ifdef INVARIANTS + et->et_td = (void*)0xDEADBEEF; +#endif + ck_epoch_end(&er->er_record, &et->et_section); + TAILQ_REMOVE(&er->er_tdlist, et, et_link); + er->er_gen++; + if (__predict_false(td->td_pre_epoch_prio != td->td_priority)) + epoch_adjust_prio(td, td->td_pre_epoch_prio); + critical_exit(); } void -epoch_exit_KBI(epoch_t epoch) +epoch_exit(epoch_t epoch) { + struct thread *td; + epoch_record_t er; - epoch_exit(epoch); + INIT_CHECK(epoch); + td = curthread; + MPASS(td->td_epochnest); + td->td_epochnest--; + er = epoch_currecord(epoch); + ck_epoch_end(&er->er_record, NULL); + critical_exit(); } /* @@ -229,7 +333,7 @@ { epoch_record_t record; struct thread *td, *owner, *curwaittd; - struct epoch_thread *tdwait; + struct epoch_tracker *tdwait; struct turnstile *ts; struct lock_object *lock; int spincount, gen; @@ -442,12 +546,10 @@ } void -epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)) +epoch_call(epoch_t epoch, epoch_context_t *ctx, epoch_cb_t callback) { epoch_record_t er; - ck_epoch_entry_t *cb; - - cb = (void *)ctx; + ck_epoch_entry_t *cb = ctx; MPASS(callback); /* too early in boot to have epoch set up */ @@ -511,7 +613,7 @@ int in_epoch_verbose(epoch_t epoch, int dump_onfail) { - struct epoch_thread *tdwait; + struct epoch_tracker *tdwait; struct thread *td; epoch_record_t er; @@ -547,9 +649,15 @@ } void -epoch_adjust_prio(struct thread *td, u_char prio) +epoch_thread_init(struct thread *td) { - thread_lock(td); - sched_prio(td, prio); - thread_unlock(td); + + td->td_et = malloc(sizeof(struct epoch_tracker), M_EPOCH, M_WAITOK); +} + +void +epoch_thread_fini(struct thread *td) +{ + + free(td->td_et, M_EPOCH); } Index: sys/net/if.c =================================================================== --- sys/net/if.c +++ sys/net/if.c @@ -610,7 +610,7 @@ } static void -if_destroy(epoch_context_t ctx) +if_destroy(epoch_context_t *ctx) { struct ifnet *ifp; @@ -1764,35 +1764,29 @@ void if_addr_rlock(struct ifnet *ifp) { - MPASS(*(uint64_t *)&ifp->if_addr_et == 0); - epoch_enter_preempt(net_epoch_preempt, &ifp->if_addr_et); + + epoch_enter_preempt(net_epoch_preempt, curthread->td_et); } void if_addr_runlock(struct ifnet *ifp) { - epoch_exit_preempt(net_epoch_preempt, &ifp->if_addr_et); -#ifdef INVARIANTS - bzero(&ifp->if_addr_et, sizeof(struct epoch_tracker)); -#endif + + epoch_exit_preempt(net_epoch_preempt, curthread->td_et); } void if_maddr_rlock(if_t ifp) { - MPASS(*(uint64_t *)&ifp->if_maddr_et == 0); - epoch_enter_preempt(net_epoch_preempt, &ifp->if_maddr_et); + epoch_enter_preempt(net_epoch_preempt, curthread->td_et); } void if_maddr_runlock(if_t ifp) { - epoch_exit_preempt(net_epoch_preempt, &ifp->if_maddr_et); -#ifdef INVARIANTS - bzero(&ifp->if_maddr_et, sizeof(struct epoch_tracker)); -#endif + epoch_exit_preempt(net_epoch_preempt, curthread->td_et); } /* @@ -1842,7 +1836,7 @@ } static void -ifa_destroy(epoch_context_t ctx) +ifa_destroy(epoch_context_t *ctx) { struct ifaddr *ifa; @@ -3439,7 +3433,7 @@ } static void -if_destroymulti(epoch_context_t ctx) +if_destroymulti(epoch_context_t *ctx) { struct ifmultiaddr *ifma; Index: sys/net/if_lagg.h =================================================================== --- sys/net/if_lagg.h +++ sys/net/if_lagg.h @@ -251,8 +251,8 @@ const struct sockaddr *, struct route *); struct lagg_counters port_counters; /* ifp counters copy */ - CK_SLIST_ENTRY(lagg_port) lp_entries; - struct epoch_context lp_epoch_ctx; + CK_SLIST_ENTRY(lagg_port) lp_entries; + epoch_context_t lp_epoch_ctx; }; extern struct mbuf *(*lagg_input_p)(struct ifnet *, struct mbuf *); Index: sys/net/if_lagg.c =================================================================== --- sys/net/if_lagg.c +++ sys/net/if_lagg.c @@ -805,7 +805,7 @@ #endif static void -lagg_port_destroy_cb(epoch_context_t ec) +lagg_port_destroy_cb(epoch_context_t *ec) { struct lagg_port *lp; struct ifnet *ifp; Index: sys/net/if_llatbl.h =================================================================== --- sys/net/if_llatbl.h +++ sys/net/if_llatbl.h @@ -82,7 +82,7 @@ struct callout lle_timer; struct rwlock lle_lock; struct mtx req_mtx; - struct epoch_context lle_epoch_ctx; + epoch_context_t lle_epoch_ctx; }; #define LLE_WLOCK(lle) rw_wlock(&(lle)->lle_lock) Index: sys/net/if_var.h =================================================================== --- sys/net/if_var.h +++ sys/net/if_var.h @@ -380,9 +380,7 @@ * Netdump hooks to be called while dumping. */ struct netdump_methods *if_netdump_methods; - struct epoch_context if_epoch_ctx; - struct epoch_tracker if_addr_et; - struct epoch_tracker if_maddr_et; + epoch_context_t if_epoch_ctx; /* * Spare fields to be added before branching a stable branch, so @@ -543,7 +541,7 @@ counter_u64_t ifa_opackets; counter_u64_t ifa_ibytes; counter_u64_t ifa_obytes; - struct epoch_context ifa_epoch_ctx; + epoch_context_t ifa_epoch_ctx; }; struct ifaddr * ifa_alloc(size_t size, int flags); @@ -564,7 +562,7 @@ int ifma_flags; void *ifma_protospec; /* protocol-specific state, if any */ struct ifmultiaddr *ifma_llifma; /* pointer to ifma for ifma_lladdr */ - struct epoch_context ifma_epoch_ctx; + epoch_context_t ifma_epoch_ctx; }; extern struct rwlock ifnet_rwlock; Index: sys/net/if_vlan.c =================================================================== --- sys/net/if_vlan.c +++ sys/net/if_vlan.c @@ -153,7 +153,7 @@ struct vlan_mc_entry { struct sockaddr_dl mc_addr; CK_SLIST_ENTRY(vlan_mc_entry) mc_entries; - struct epoch_context mc_epoch_ctx; + epoch_context_t mc_epoch_ctx; }; struct ifvlan { @@ -318,7 +318,7 @@ #define HASH(n, m) ((((n) >> 8) ^ ((n) >> 4) ^ (n)) & (m)) static void -vlan_mc_free(struct epoch_context *ctx) +vlan_mc_free(epoch_context_t *ctx) { struct vlan_mc_entry *mc = __containerof(ctx, struct vlan_mc_entry, mc_epoch_ctx); free(mc, M_VLAN); Index: sys/netinet/in.c =================================================================== --- sys/netinet/in.c +++ sys/netinet/in.c @@ -1060,7 +1060,7 @@ * Do actual deallocation of @lle. */ static void -in_lltable_destroy_lle_unlocked(epoch_context_t ctx) +in_lltable_destroy_lle_unlocked(epoch_context_t *ctx) { struct llentry *lle; Index: sys/netinet/in_pcb.h =================================================================== --- sys/netinet/in_pcb.h +++ sys/netinet/in_pcb.h @@ -330,7 +330,7 @@ CK_LIST_ENTRY(inpcb) inp_list; /* (p/l) list for all PCBs for proto */ /* (e[r]) for list iteration */ /* (p[w]/l) for addition/removal */ - struct epoch_context inp_epoch_ctx; + epoch_context_t inp_epoch_ctx; }; #endif /* _KERNEL */ @@ -405,7 +405,7 @@ #endif /* _SYS_SOCKETVAR_H_ */ struct inpcbport { - struct epoch_context phd_epoch_ctx; + epoch_context_t phd_epoch_ctx; CK_LIST_ENTRY(inpcbport) phd_hash; struct inpcbhead phd_pcblist; u_short phd_port; @@ -413,7 +413,7 @@ struct in_pcblist { int il_count; - struct epoch_context il_epoch_ctx; + epoch_context_t il_epoch_ctx; struct inpcbinfo *il_pcbinfo; struct inpcb *il_inp_list[0]; }; @@ -568,7 +568,7 @@ */ struct inpcblbgroup { CK_LIST_ENTRY(inpcblbgroup) il_list; - struct epoch_context il_epoch_ctx; + epoch_context_t il_epoch_ctx; uint16_t il_lport; /* (c) */ u_char il_vflag; /* (c) */ u_char il_pad; @@ -874,7 +874,7 @@ int in_pcbrele(struct inpcb *); int in_pcbrele_rlocked(struct inpcb *); int in_pcbrele_wlocked(struct inpcb *); -void in_pcblist_rele_rlocked(epoch_context_t ctx); +void in_pcblist_rele_rlocked(epoch_context_t *ctx); void in_losing(struct inpcb *); void in_pcbsetsolabel(struct socket *so); int in_getpeeraddr(struct socket *so, struct sockaddr **nam); Index: sys/netinet/in_pcb.c =================================================================== --- sys/netinet/in_pcb.c +++ sys/netinet/in_pcb.c @@ -240,7 +240,7 @@ } static void -in_pcblbgroup_free_deferred(epoch_context_t ctx) +in_pcblbgroup_free_deferred(epoch_context_t *ctx) { struct inpcblbgroup *grp; @@ -1547,7 +1547,7 @@ } void -in_pcblist_rele_rlocked(epoch_context_t ctx) +in_pcblist_rele_rlocked(epoch_context_t *ctx) { struct in_pcblist *il; struct inpcb *inp; @@ -1569,7 +1569,7 @@ } static void -inpcbport_free(epoch_context_t ctx) +inpcbport_free(epoch_context_t *ctx) { struct inpcbport *phd; @@ -1578,7 +1578,7 @@ } static void -in_pcbfree_deferred(epoch_context_t ctx) +in_pcbfree_deferred(epoch_context_t *ctx) { struct inpcb *inp; int released __unused; @@ -2559,7 +2559,7 @@ if (phd == NULL) { return (ENOBUFS); /* XXX */ } - bzero(&phd->phd_epoch_ctx, sizeof(struct epoch_context)); + bzero(&phd->phd_epoch_ctx, sizeof(phd->phd_epoch_ctx)); phd->phd_port = inp->inp_lport; CK_LIST_INIT(&phd->phd_pcblist); CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash); Index: sys/netinet/ip_var.h =================================================================== --- sys/netinet/ip_var.h +++ sys/netinet/ip_var.h @@ -96,7 +96,7 @@ u_short imo_max_memberships; /* max memberships this socket */ struct in_multi **imo_membership; /* group memberships */ struct in_mfilter *imo_mfilters; /* source filters */ - struct epoch_context imo_epoch_ctx; + epoch_context_t imo_epoch_ctx; }; struct ipstat { Index: sys/netinet6/in6.c =================================================================== --- sys/netinet6/in6.c +++ sys/netinet6/in6.c @@ -2049,7 +2049,7 @@ * Do actual deallocation of @lle. */ static void -in6_lltable_destroy_lle_unlocked(epoch_context_t ctx) +in6_lltable_destroy_lle_unlocked(epoch_context_t *ctx) { struct llentry *lle; Index: sys/netinet6/ip6_var.h =================================================================== --- sys/netinet6/ip6_var.h +++ sys/netinet6/ip6_var.h @@ -123,7 +123,7 @@ u_short im6o_max_memberships; /* max memberships this socket */ struct in6_multi **im6o_membership; /* group memberships */ struct in6_mfilter *im6o_mfilters; /* source filters */ - struct epoch_context imo6_epoch_ctx; + epoch_context_t imo6_epoch_ctx; }; /* Index: sys/sys/epoch.h =================================================================== --- sys/sys/epoch.h +++ sys/sys/epoch.h @@ -29,10 +29,24 @@ #ifndef _SYS_EPOCH_H_ #define _SYS_EPOCH_H_ + +/* + * For applications that use kvm(3) and see kernel structures + * define a placeholder for epoch_context, and protect its + * size with CTASSERT in subr_epoch.c. + */ +struct epoch_context_user { + void *data[2]; +}; + +#ifndef _KERNEL +typedef struct epoch_context_user epoch_context_t; +#endif + #ifdef _KERNEL #include #include -#endif +#include struct epoch; typedef struct epoch *epoch_t; @@ -43,51 +57,41 @@ extern epoch_t global_epoch; extern epoch_t global_epoch_preempt; -struct epoch_context { - void *data[2]; -} __aligned(sizeof(void *)); - -typedef struct epoch_context *epoch_context_t; - - struct epoch_tracker { - void *datap[3]; #ifdef EPOCH_TRACKER_DEBUG - int datai[5]; -#else - int datai[1]; +#define EPOCH_MAGIC0 0xFADECAFEF00DD00D +#define EPOCH_MAGIC1 0xBADDBABEDEEDFEED + uint64_t et_magic_pre; +#endif + TAILQ_ENTRY(epoch_tracker) et_link; + struct thread *et_td; + ck_epoch_section_t et_section; +#ifdef EPOCH_TRACKER_DEBUG + uint64_t et_magic_post; #endif } __aligned(sizeof(void *)); - typedef struct epoch_tracker *epoch_tracker_t; +typedef struct ck_epoch_entry epoch_context_t; +typedef void epoch_cb_t(epoch_context_t *); + epoch_t epoch_alloc(int flags); void epoch_free(epoch_t epoch); void epoch_wait(epoch_t epoch); void epoch_wait_preempt(epoch_t epoch); -void epoch_call(epoch_t epoch, epoch_context_t ctx, void (*callback) (epoch_context_t)); +void epoch_call(epoch_t epoch, epoch_context_t *ctx, epoch_cb_t callback); int in_epoch(epoch_t epoch); int in_epoch_verbose(epoch_t epoch, int dump_onfail); -#ifdef _KERNEL DPCPU_DECLARE(int, epoch_cb_count); DPCPU_DECLARE(struct grouptask, epoch_cb_task); -#define EPOCH_MAGIC0 0xFADECAFEF00DD00D -#define EPOCH_MAGIC1 0xBADDBABEDEEDFEED -void epoch_enter_preempt_KBI(epoch_t epoch, epoch_tracker_t et); -void epoch_exit_preempt_KBI(epoch_t epoch, epoch_tracker_t et); -void epoch_enter_KBI(epoch_t epoch); -void epoch_exit_KBI(epoch_t epoch); +void epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et); +void epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et); +void epoch_enter(epoch_t epoch); +void epoch_exit(epoch_t epoch); - -#if defined(KLD_MODULE) && !defined(KLD_TIED) -#define epoch_enter_preempt(e, t) epoch_enter_preempt_KBI((e), (t)) -#define epoch_exit_preempt(e, t) epoch_exit_preempt_KBI((e), (t)) -#define epoch_enter(e) epoch_enter_KBI((e)) -#define epoch_exit(e) epoch_exit_KBI((e)) -#else -#include -#endif /* KLD_MODULE */ +void epoch_thread_init(struct thread *); +void epoch_thread_fini(struct thread *); #endif /* _KERNEL */ -#endif +#endif /* _SYS_EPOCH_H_ */ Index: sys/sys/epoch_private.h =================================================================== --- sys/sys/epoch_private.h +++ /dev/null @@ -1,210 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD - * - * Copyright (c) 2018, Matthew Macy - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -#ifndef _SYS_EPOCH_PRIVATE_H_ -#define _SYS_EPOCH_PRIVATE_H_ -#ifndef _KERNEL -#error "no user serviceable parts" -#else -#include -#include - -#include - -extern void epoch_adjust_prio(struct thread *td, u_char prio); -#ifndef _SYS_SYSTM_H_ -extern void critical_exit_preempt(void); -#endif - -#ifdef __amd64__ -#define EPOCH_ALIGN CACHE_LINE_SIZE*2 -#else -#define EPOCH_ALIGN CACHE_LINE_SIZE -#endif - -/* - * Standalone (_sa) routines for thread state manipulation - */ -static __inline void -critical_enter_sa(void *tdarg) -{ - struct thread_lite *td; - - td = tdarg; - td->td_critnest++; - __compiler_membar(); -} - -static __inline void -critical_exit_sa(void *tdarg) -{ - struct thread_lite *td; - - td = tdarg; - MPASS(td->td_critnest > 0); - __compiler_membar(); - td->td_critnest--; - __compiler_membar(); - if (__predict_false(td->td_owepreempt != 0)) - critical_exit_preempt(); -} - -typedef struct epoch_thread { -#ifdef EPOCH_TRACKER_DEBUG - uint64_t et_magic_pre; -#endif - TAILQ_ENTRY(epoch_thread) et_link; /* Epoch queue. */ - struct thread *et_td; /* pointer to thread in section */ - ck_epoch_section_t et_section; /* epoch section object */ -#ifdef EPOCH_TRACKER_DEBUG - uint64_t et_magic_post; -#endif -} *epoch_thread_t; -TAILQ_HEAD (epoch_tdlist, epoch_thread); - -typedef struct epoch_record { - ck_epoch_record_t er_record; - volatile struct epoch_tdlist er_tdlist; - volatile uint32_t er_gen; - uint32_t er_cpuid; -} __aligned(EPOCH_ALIGN) *epoch_record_t; - -struct epoch { - struct ck_epoch e_epoch __aligned(EPOCH_ALIGN); - epoch_record_t e_pcpu_record; - int e_idx; - int e_flags; -}; - -static epoch_record_t -epoch_currecord(epoch_t epoch) -{ - return zpcpu_get_cpu(epoch->e_pcpu_record, curcpu); -} - -#define INIT_CHECK(epoch) \ - do { \ - if (__predict_false((epoch) == NULL)) \ - return; \ - } while (0) - -static __inline void -epoch_enter_preempt(epoch_t epoch, epoch_tracker_t et) -{ - struct epoch_record *er; - struct epoch_thread *etd; - struct thread_lite *td; - - MPASS(cold || epoch != NULL); - INIT_CHECK(epoch); - etd = (void *)et; - MPASS(epoch->e_flags & EPOCH_PREEMPT); -#ifdef EPOCH_TRACKER_DEBUG - etd->et_magic_pre = EPOCH_MAGIC0; - etd->et_magic_post = EPOCH_MAGIC1; -#endif - td = (struct thread_lite *)curthread; - etd->et_td = (void*)td; - td->td_epochnest++; - critical_enter_sa(td); - sched_pin_lite(td); - - td->td_pre_epoch_prio = td->td_priority; - er = epoch_currecord(epoch); - TAILQ_INSERT_TAIL(&er->er_tdlist, etd, et_link); - ck_epoch_begin(&er->er_record, (ck_epoch_section_t *)&etd->et_section); - critical_exit_sa(td); -} - -static __inline void -epoch_enter(epoch_t epoch) -{ - struct thread_lite *td; - epoch_record_t er; - - MPASS(cold || epoch != NULL); - INIT_CHECK(epoch); - td = (struct thread_lite *)curthread; - - td->td_epochnest++; - critical_enter_sa(td); - er = epoch_currecord(epoch); - ck_epoch_begin(&er->er_record, NULL); -} - -static __inline void -epoch_exit_preempt(epoch_t epoch, epoch_tracker_t et) -{ - struct epoch_record *er; - struct epoch_thread *etd; - struct thread_lite *td; - - INIT_CHECK(epoch); - td = (struct thread_lite *)curthread; - critical_enter_sa(td); - sched_unpin_lite(td); - MPASS(td->td_epochnest); - td->td_epochnest--; - er = epoch_currecord(epoch); - MPASS(epoch->e_flags & EPOCH_PREEMPT); - etd = (void *)et; - MPASS(etd != NULL); - MPASS(etd->et_td == (struct thread *)td); -#ifdef EPOCH_TRACKER_DEBUG - MPASS(etd->et_magic_pre == EPOCH_MAGIC0); - MPASS(etd->et_magic_post == EPOCH_MAGIC1); - etd->et_magic_pre = 0; - etd->et_magic_post = 0; -#endif - etd->et_td = (void*)0xDEADBEEF; - ck_epoch_end(&er->er_record, - (ck_epoch_section_t *)&etd->et_section); - TAILQ_REMOVE(&er->er_tdlist, etd, et_link); - er->er_gen++; - if (__predict_false(td->td_pre_epoch_prio != td->td_priority)) - epoch_adjust_prio((struct thread *)td, td->td_pre_epoch_prio); - critical_exit_sa(td); -} - -static __inline void -epoch_exit(epoch_t epoch) -{ - struct thread_lite *td; - epoch_record_t er; - - INIT_CHECK(epoch); - td = (struct thread_lite *)curthread; - MPASS(td->td_epochnest); - td->td_epochnest--; - er = epoch_currecord(epoch); - ck_epoch_end(&er->er_record, NULL); - critical_exit_sa(td); -} -#endif /* _KERNEL */ -#endif /* _SYS_EPOCH_PRIVATE_H_ */ Index: sys/sys/proc.h =================================================================== --- sys/sys/proc.h +++ sys/sys/proc.h @@ -193,6 +193,7 @@ struct turnstile; struct vm_map; struct vm_map_entry; +struct epoch_tracker; /* * XXX: Does this belong in resource.h or resourcevar.h instead? @@ -360,6 +361,7 @@ int td_lastcpu; /* (t) Last cpu we were on. */ int td_oncpu; /* (t) Which cpu we are on. */ void *td_lkpi_task; /* LinuxKPI task struct pointer */ + struct epoch_tracker *td_et; /* (k) compat KPI spare tracker */ int td_pmcpend; };