Index: sys/kern/uipc_domain.c =================================================================== --- sys/kern/uipc_domain.c +++ sys/kern/uipc_domain.c @@ -81,6 +81,11 @@ static struct mtx dom_mtx; /* domain list lock */ MTX_SYSINIT(domain, &dom_mtx, "domain list", MTX_DEF); +static CK_LIST_HEAD(, protosw) fasttimo_protos = + CK_LIST_HEAD_INITIALIZER(fasttimo_protos); +static CK_LIST_HEAD(, protosw) slowtimo_protos = + CK_LIST_HEAD_INITIALIZER(slowtimo_protos); + /* * Dummy protocol specific user requests function pointer array. * All functions return EOPNOTSUPP. @@ -192,8 +197,18 @@ max_datalen = MHLEN - max_hdr; if (max_datalen < 1) panic("%s: max_datalen < 1", __func__); - if (IS_DEFAULT_VNET(curvnet)) + if (IS_DEFAULT_VNET(curvnet)) { atomic_set_rel_int(&dp->dom_flags, DOMF_INITED); + for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) { + if (pr->pr_fasttimo != NULL) + CK_LIST_INSERT_HEAD(&fasttimo_protos, pr, + pr_fasttimos); + if (pr->pr_slowtimo != NULL) + CK_LIST_INSERT_HEAD(&slowtimo_protos, pr, + pr_slowtimos); + } + + } } #ifdef VIMAGE @@ -374,8 +389,11 @@ return (EEXIST); /* XXX: Check only protocol? */ } /* While here, remember the first free spacer. */ - if ((fpr == NULL) && (pr->pr_protocol == PROTO_SPACER)) + if ((fpr == NULL) && (pr->pr_protocol == PROTO_SPACER)) { + /* See pf_proto_unregister_cb(). */ + atomic_thread_fence_acq(); fpr = pr; + } } /* If no free spacer is found we can't add the new protocol. */ @@ -387,6 +405,12 @@ /* Copy the new struct protosw over the spacer. */ bcopy(npr, fpr, sizeof(*fpr)); + if (fpr->pr_fasttimo != NULL) + CK_LIST_INSERT_HEAD(&fasttimo_protos, pr, pr_fasttimos); + + if (fpr->pr_slowtimo != NULL) + CK_LIST_INSERT_HEAD(&slowtimo_protos, pr, pr_slowtimos); + /* Job is done, no more protection required. */ mtx_unlock(&dom_mtx); @@ -402,6 +426,41 @@ return (0); } +static void +pf_proto_unregister_cb(struct epoch_context *ctx) +{ + struct protosw *pr; + + pr = __containerof(ctx, struct protosw, pr_epoch_ctx); + + KASSERT((pr->pr_flags & PR_REMOVING) != 0, + ("Removing unmarked spacer from %s", pr->pr_domain->dom_name)); + + /* De-orbit the protocol and make the slot available again. */ + pr->pr_type = 0; + pr->pr_flags = 0; + pr->pr_input = NULL; + pr->pr_output = NULL; + pr->pr_ctlinput = NULL; + pr->pr_ctloutput = NULL; + pr->pr_init = NULL; + pr->pr_fasttimo = NULL; + pr->pr_slowtimo = NULL; + pr->pr_drain = NULL; + pr->pr_usrreqs = &nousrreqs; + + /* + * This could race with pf_proto_register() calls since we'll take the + * dom_mtx there but not here. We just need to make sure that it + * doesn't proceed on a protocol marked PROTO_SPACER before all of the + * previous writes have completed. The main problem here is that + * protocol registration will copy the dynamically registered protocol + * into this spacer, so we really need to avoid clobbering that. + */ + atomic_thread_fence_rel(); + pr->pr_protocol = PROTO_SPACER; +} + /* * The caller must make sure the protocol and its functions correctly shut down * all sockets and release all locks and memory references. @@ -447,24 +506,22 @@ return (EPROTONOSUPPORT); } - /* De-orbit the protocol and make the slot available again. */ - dpr->pr_type = 0; - dpr->pr_domain = dp; - dpr->pr_protocol = PROTO_SPACER; - dpr->pr_flags = 0; - dpr->pr_input = NULL; - dpr->pr_output = NULL; - dpr->pr_ctlinput = NULL; - dpr->pr_ctloutput = NULL; - dpr->pr_init = NULL; - dpr->pr_fasttimo = NULL; - dpr->pr_slowtimo = NULL; - dpr->pr_drain = NULL; - dpr->pr_usrreqs = &nousrreqs; - - /* Job is done, not more protection required. */ + KASSERT(dpr->pr_domain == dp, ("Unexpected domain mismatch")); + KASSERT((dpr->pr_flags & PR_REMOVING) == 0, + ("Scheduled removal of marked spacer")); + + dpr->pr_flags |= PR_REMOVING; + if (dpr->pr_fasttimo != NULL) + CK_LIST_REMOVE(dpr, pr_fasttimos); + + if (dpr->pr_slowtimo != NULL) + CK_LIST_REMOVE(dpr, pr_slowtimos); + + /* Job is done, no more protection required. */ mtx_unlock(&dom_mtx); + NET_EPOCH_CALL(pf_proto_unregister_cb, &dpr->pr_epoch_ctx); + return (0); } @@ -484,17 +541,11 @@ pfslowtimo(void *arg) { struct epoch_tracker et; - struct domain *dp; struct protosw *pr; NET_EPOCH_ENTER(et); - for (dp = domains; dp; dp = dp->dom_next) { - if ((atomic_load_int(&dp->dom_flags) & DOMF_INITED) == 0) - continue; - atomic_thread_fence_acq(); - for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) - if (pr->pr_slowtimo) - (*pr->pr_slowtimo)(); + CK_LIST_FOREACH(pr, &slowtimo_protos, pr_slowtimos) { + (*pr->pr_slowtimo)(); } NET_EPOCH_EXIT(et); callout_reset(&pfslow_callout, hz/2, pfslowtimo, NULL); @@ -504,17 +555,11 @@ pffasttimo(void *arg) { struct epoch_tracker et; - struct domain *dp; struct protosw *pr; NET_EPOCH_ENTER(et); - for (dp = domains; dp; dp = dp->dom_next) { - if ((atomic_load_int(&dp->dom_flags) & DOMF_INITED) == 0) - continue; - atomic_thread_fence_acq(); - for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) - if (pr->pr_fasttimo) - (*pr->pr_fasttimo)(); + CK_LIST_FOREACH(pr, &fasttimo_protos, pr_fasttimos) { + (*pr->pr_fasttimo)(); } NET_EPOCH_EXIT(et); callout_reset(&pffast_callout, hz/5, pffasttimo, NULL); Index: sys/sys/protosw.h =================================================================== --- sys/sys/protosw.h +++ sys/sys/protosw.h @@ -35,6 +35,9 @@ #ifndef _SYS_PROTOSW_H_ #define _SYS_PROTOSW_H_ +#include +#include + /* Forward declare these structures referenced from prototypes below. */ struct kaiocb; struct mbuf; @@ -93,6 +96,10 @@ pr_drain_t *pr_drain; /* flush any excess space possible */ struct pr_usrreqs *pr_usrreqs; /* user-protocol hook */ + + struct epoch_context pr_epoch_ctx; + CK_LIST_ENTRY(protosw) pr_fasttimos; + CK_LIST_ENTRY(protosw) pr_slowtimos; }; /*#endif*/ @@ -121,6 +128,7 @@ #define PR_RIGHTS 0x10 /* passes capabilities */ #define PR_IMPLOPCL 0x20 /* implied open/close */ #define PR_LASTHDR 0x40 /* enforce ipsec policy; last header */ +#define PR_REMOVING 0x80 /* spacer queued for removal */ /* * In earlier BSD network stacks, a single pr_usrreq() function pointer was