Index: sys/kern/kern_prot.c =================================================================== --- sys/kern/kern_prot.c +++ sys/kern/kern_prot.c @@ -46,7 +46,6 @@ #include __FBSDID("$FreeBSD$"); -#include "opt_compat.h" #include "opt_inet.h" #include "opt_inet6.h" @@ -67,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -88,6 +88,72 @@ static void crsetgroups_locked(struct ucred *cr, int ngrp, gid_t *groups); +static uma_zone_t zone_ucred; + +static int +ucred_ctor(void *mem, int size, void *arg, int flags) +{ + struct ucred *cr; + + cr = mem; + bzero(cr, sizeof(*cr)); + refcount_init(&cr->cr_ref, 1); +#ifdef AUDIT + audit_cred_init(cr); +#endif +#ifdef MAC + mac_cred_init(cr); +#endif + cr->cr_groups = cr->cr_smallgroups; + cr->cr_agroups = + sizeof(cr->cr_smallgroups) / sizeof(cr->cr_smallgroups[0]); + return (0); +} + +static void +ucred_dtor(void *mem, int size, void *arg) +{ + struct ucred *cr; + + cr = mem; + /* + * Some callers of crget(), such as nfs_statfs(), + * allocate a temporary credential, but don't + * allocate a uidinfo structure. + */ + if (cr->cr_uidinfo != NULL) + uifree(cr->cr_uidinfo); + if (cr->cr_ruidinfo != NULL) + uifree(cr->cr_ruidinfo); + /* + * Free a prison, if any. + */ + if (cr->cr_prison != NULL) + prison_free(cr->cr_prison); + if (cr->cr_loginclass != NULL) + loginclass_free(cr->cr_loginclass); +#ifdef AUDIT + audit_cred_destroy(cr); +#endif +#ifdef MAC + mac_cred_destroy(cr); +#endif + if (cr->cr_groups != cr->cr_smallgroups) + free(cr->cr_groups, M_CRED); +} + + +static void +ucred_init(void *dummy __unused) +{ + zone_ucred = uma_zcreate("ucred_zone", sizeof(struct ucred), + ucred_ctor, ucred_dtor, NULL, NULL, + UMA_ALIGN_CACHE, UMA_ZONE_NOFREE); +} + +SYSINIT(mbuf, SI_SUB_CPU+1, SI_ORDER_ANY, ucred_init, NULL); + + #ifndef _SYS_SYSPROTO_H_ struct getpid_args { int dummy; @@ -576,9 +642,14 @@ setsugid(p); } proc_set_cred(p, newcred); - PROC_UNLOCK(p); #ifdef RACCT racct_proc_ucred_changed(p, oldcred, newcred); + crhold(newcred); +#endif + PROC_UNLOCK(p); +#ifdef RCTL + rctl_proc_ucred_changed(p, newcred); + crfree(newcred); #endif uifree(uip); crfree(oldcred); @@ -924,9 +995,14 @@ setsugid(p); } proc_set_cred(p, newcred); - PROC_UNLOCK(p); #ifdef RACCT racct_proc_ucred_changed(p, oldcred, newcred); + crhold(newcred); +#endif + PROC_UNLOCK(p); +#ifdef RCTL + rctl_proc_ucred_changed(p, newcred); + crfree(newcred); #endif uifree(ruip); uifree(euip); @@ -1065,9 +1141,14 @@ setsugid(p); } proc_set_cred(p, newcred); - PROC_UNLOCK(p); #ifdef RACCT racct_proc_ucred_changed(p, oldcred, newcred); + crhold(newcred); +#endif + PROC_UNLOCK(p); +#ifdef RCTL + rctl_proc_ucred_changed(p, newcred); + crfree(newcred); #endif uifree(ruip); uifree(euip); @@ -1809,24 +1890,19 @@ * Allocate a zeroed cred structure. */ struct ucred * -crget(void) +crget_arg(int flags) { - struct ucred *cr; + return (uma_zalloc(zone_ucred, flags)); +} - cr = malloc(sizeof(*cr), M_CRED, M_WAITOK | M_ZERO); - refcount_init(&cr->cr_ref, 1); -#ifdef AUDIT - audit_cred_init(cr); -#endif -#ifdef MAC - mac_cred_init(cr); -#endif - cr->cr_groups = cr->cr_smallgroups; - cr->cr_agroups = - sizeof(cr->cr_smallgroups) / sizeof(cr->cr_smallgroups[0]); - return (cr); + +struct ucred * +crget(void) +{ + return (crget_arg(M_WAITOK)); } + /* * Claim another reference to a ucred structure. */ @@ -1847,33 +1923,8 @@ KASSERT(cr->cr_ref > 0, ("bad ucred refcount: %d", cr->cr_ref)); KASSERT(cr->cr_ref != 0xdeadc0de, ("dangling reference to ucred")); - if (refcount_release(&cr->cr_ref)) { - /* - * Some callers of crget(), such as nfs_statfs(), - * allocate a temporary credential, but don't - * allocate a uidinfo structure. - */ - if (cr->cr_uidinfo != NULL) - uifree(cr->cr_uidinfo); - if (cr->cr_ruidinfo != NULL) - uifree(cr->cr_ruidinfo); - /* - * Free a prison, if any. - */ - if (cr->cr_prison != NULL) - prison_free(cr->cr_prison); - if (cr->cr_loginclass != NULL) - loginclass_free(cr->cr_loginclass); -#ifdef AUDIT - audit_cred_destroy(cr); -#endif -#ifdef MAC - mac_cred_destroy(cr); -#endif - if (cr->cr_groups != cr->cr_smallgroups) - free(cr->cr_groups, M_CRED); - free(cr, M_CRED); - } + if (refcount_release(&cr->cr_ref)) + uma_zfree(zone_ucred, cr); } /* @@ -1908,7 +1959,18 @@ { struct ucred *newcr; - newcr = crget(); + newcr = crget_arg(M_WAITOK); + crcopy(newcr, cr); + return (newcr); +} + +struct ucred * +crdup_arg(struct ucred *cr, int flags) +{ + struct ucred *newcr; + + if ((newcr = crget_arg(flags)) == NULL) + return (NULL); crcopy(newcr, cr); return (newcr); } Index: sys/kern/kern_thread.c =================================================================== --- sys/kern/kern_thread.c +++ sys/kern/kern_thread.c @@ -81,9 +81,9 @@ "struct thread KBI td_flags"); _Static_assert(offsetof(struct thread, td_pflags) == 0x104, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x468, +_Static_assert(offsetof(struct thread, td_frame) == 0x470, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x510, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x518, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0xb0, "struct proc KBI p_flag"); @@ -91,7 +91,7 @@ "struct proc KBI p_pid"); _Static_assert(offsetof(struct proc, p_filemon) == 0x3d0, "struct proc KBI p_filemon"); -_Static_assert(offsetof(struct proc, p_comm) == 0x3e0, +_Static_assert(offsetof(struct proc, p_comm) == 0x3e4, "struct proc KBI p_comm"); _Static_assert(offsetof(struct proc, p_emuldata) == 0x4b8, "struct proc KBI p_emuldata"); @@ -101,9 +101,9 @@ "struct thread KBI td_flags"); _Static_assert(offsetof(struct thread, td_pflags) == 0xa0, "struct thread KBI td_pflags"); -_Static_assert(offsetof(struct thread, td_frame) == 0x2e4, +_Static_assert(offsetof(struct thread, td_frame) == 0x2e8, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x330, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x334, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0x68, "struct proc KBI p_flag"); @@ -111,9 +111,9 @@ "struct proc KBI p_pid"); _Static_assert(offsetof(struct proc, p_filemon) == 0x27c, "struct proc KBI p_filemon"); -_Static_assert(offsetof(struct proc, p_comm) == 0x288, +_Static_assert(offsetof(struct proc, p_comm) == 0x28c, "struct proc KBI p_comm"); -_Static_assert(offsetof(struct proc, p_emuldata) == 0x314, +_Static_assert(offsetof(struct proc, p_emuldata) == 0x318, "struct proc KBI p_emuldata"); #endif @@ -448,9 +448,12 @@ void thread_cow_get_proc(struct thread *newtd, struct proc *p) { + struct ucred *newcred; PROC_LOCK_ASSERT(p, MA_OWNED); - newtd->td_ucred = crhold(p->p_ucred); + if ((newcred = crdup_arg(p->p_ucred, M_NOWAIT)) == NULL) + newcred = crhold(p->p_ucred); + newtd->td_ucred = newcred; newtd->td_limit = lim_hold(p->p_limit); newtd->td_cowgen = p->p_cowgen; } @@ -459,7 +462,7 @@ thread_cow_get(struct thread *newtd, struct thread *td) { - newtd->td_ucred = crhold(td->td_ucred); + newtd->td_ucred = crdup(td->td_ucred); newtd->td_limit = lim_hold(td->td_limit); newtd->td_cowgen = td->td_cowgen; } @@ -478,7 +481,7 @@ thread_cow_update(struct thread *td) { struct proc *p; - struct ucred *oldcred; + struct ucred *oldcred, *newcred; struct plimit *oldlimit; p = td->td_proc; @@ -487,7 +490,9 @@ PROC_LOCK(p); if (td->td_ucred != p->p_ucred) { oldcred = td->td_ucred; - td->td_ucred = crhold(p->p_ucred); + if ((newcred = crdup_arg(p->p_ucred, M_NOWAIT)) == NULL) + newcred = crhold(p->p_ucred); + td->td_ucred = newcred; } if (td->td_limit != p->p_limit) { oldlimit = td->td_limit; Index: sys/sys/ucred.h =================================================================== --- sys/sys/ucred.h +++ sys/sys/ucred.h @@ -107,11 +107,13 @@ void crcopy(struct ucred *dest, struct ucred *src); struct ucred *crcopysafe(struct proc *p, struct ucred *cr); struct ucred *crdup(struct ucred *cr); +struct ucred *crdup_arg(struct ucred *cr, int flags); void crextend(struct ucred *cr, int n); void proc_set_cred_init(struct proc *p, struct ucred *cr); struct ucred *proc_set_cred(struct proc *p, struct ucred *cr); void crfree(struct ucred *cr); struct ucred *crget(void); +struct ucred *crget_arg(int flags); struct ucred *crhold(struct ucred *cr); void cru2x(struct ucred *cr, struct xucred *xcr); void crsetgroups(struct ucred *cr, int n, gid_t *groups);