Index: sys/kern/kern_racct.c =================================================================== --- sys/kern/kern_racct.c +++ sys/kern/kern_racct.c @@ -91,13 +91,9 @@ */ #define RACCT_PCPU_SECS 3 -static struct mtx racct_lock; +struct mtx racct_lock; MTX_SYSINIT(racct_lock, &racct_lock, "racct lock", MTX_DEF); -#define RACCT_LOCK() mtx_lock(&racct_lock) -#define RACCT_UNLOCK() mtx_unlock(&racct_lock) -#define RACCT_LOCK_ASSERT() mtx_assert(&racct_lock, MA_OWNED) - static uma_zone_t racct_zone; static void racct_sub_racct(struct racct *dest, const struct racct *src); @@ -111,6 +107,8 @@ "struct proc *", "int", "uint64_t"); SDT_PROBE_DEFINE3(racct, , rusage, add__failure, "struct proc *", "int", "uint64_t"); +SDT_PROBE_DEFINE3(racct, , rusage, add__buf, + "struct proc *", "const struct buf *", "int"); SDT_PROBE_DEFINE3(racct, , rusage, add__cred, "struct ucred *", "int", "uint64_t"); SDT_PROBE_DEFINE3(racct, , rusage, add__force, @@ -618,8 +616,6 @@ ASSERT_RACCT_ENABLED(); - SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount); - racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, amount); for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent) racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource, @@ -638,6 +634,8 @@ if (!racct_enable) return; + SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount); + RACCT_LOCK(); racct_add_cred_locked(cred, resource, amount); RACCT_UNLOCK(); @@ -654,6 +652,8 @@ ASSERT_RACCT_ENABLED(); PROC_LOCK_ASSERT(p, MA_OWNED); + SDT_PROBE3(racct, , rusage, add__buf, p, bp, is_write); + RACCT_LOCK(); if (is_write) { racct_add_locked(curproc, RACCT_WRITEBPS, bp->b_bcount, 1); @@ -767,13 +767,19 @@ uint64_t racct_get_limit(struct proc *p, int resource) { +#ifdef RCTL + uint64_t available; if (!racct_enable) return (UINT64_MAX); -#ifdef RCTL - return (rctl_get_limit(p, resource)); + RACCT_LOCK(); + available = rctl_get_limit(p, resource); + RACCT_UNLOCK(); + + return (available); #else + return (UINT64_MAX); #endif } @@ -787,13 +793,19 @@ uint64_t racct_get_available(struct proc *p, int resource) { +#ifdef RCTL + uint64_t available; if (!racct_enable) return (UINT64_MAX); -#ifdef RCTL - return (rctl_get_available(p, resource)); + RACCT_LOCK(); + available = rctl_get_available(p, resource); + RACCT_UNLOCK(); + + return (available); #else + return (UINT64_MAX); #endif } @@ -806,12 +818,18 @@ static int64_t racct_pcpu_available(struct proc *p) { +#ifdef RCTL + uint64_t available; ASSERT_RACCT_ENABLED(); -#ifdef RCTL - return (rctl_pcpu_available(p)); + RACCT_LOCK(); + available = rctl_pcpu_available(p); + RACCT_UNLOCK(); + + return (available); #else + return (INT64_MAX); #endif } @@ -853,14 +871,6 @@ ASSERT_RACCT_ENABLED(); - SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount); - -#ifdef notyet - KASSERT(RACCT_CAN_DROP(resource), - ("%s: called for resource %d which can not drop", __func__, - resource)); -#endif - racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, -amount); for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent) racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource, @@ -878,6 +888,14 @@ if (!racct_enable) return; + SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount); + +#ifdef notyet + KASSERT(RACCT_CAN_DROP(resource), + ("%s: called for resource %d which can not drop", __func__, + resource)); +#endif + RACCT_LOCK(); racct_sub_cred_locked(cred, resource, amount); RACCT_UNLOCK(); @@ -949,11 +967,12 @@ racct_proc_fork_done(struct proc *child) { - PROC_LOCK_ASSERT(child, MA_OWNED); -#ifdef RCTL if (!racct_enable) return; + PROC_LOCK_ASSERT(child, MA_OWNED); + +#ifdef RCTL RACCT_LOCK(); rctl_enforce(child, RACCT_NPROC, 0); rctl_enforce(child, RACCT_NTHR, 0); @@ -1005,13 +1024,12 @@ racct_set_locked(p, i, 0, 0); } - RACCT_UNLOCK(); - PROC_UNLOCK(p); - #ifdef RCTL rctl_racct_release(p->p_racct); #endif - racct_destroy(&p->p_racct); + racct_destroy_locked(&p->p_racct); + RACCT_UNLOCK(); + PROC_UNLOCK(p); } /* Index: sys/kern/kern_rctl.c =================================================================== --- sys/kern/kern_rctl.c +++ sys/kern/kern_rctl.c @@ -212,15 +212,6 @@ static uma_zone_t rctl_rule_link_zone; static uma_zone_t rctl_rule_zone; -static struct rwlock rctl_lock; -RW_SYSINIT(rctl_lock, &rctl_lock, "RCTL lock"); - -#define RCTL_RLOCK() rw_rlock(&rctl_lock) -#define RCTL_RUNLOCK() rw_runlock(&rctl_lock) -#define RCTL_WLOCK() rw_wlock(&rctl_lock) -#define RCTL_WUNLOCK() rw_wunlock(&rctl_lock) -#define RCTL_LOCK_ASSERT() rw_assert(&rctl_lock, RA_LOCKED) -#define RCTL_WLOCK_ASSERT() rw_assert(&rctl_lock, RA_WLOCKED) static int rctl_rule_fully_specified(const struct rctl_rule *rule); static void rctl_rule_to_sbuf(struct sbuf *sb, const struct rctl_rule *rule); @@ -238,9 +229,9 @@ if (val < 1 || val > rctl_throttle_max) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_min = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -256,9 +247,9 @@ if (val < rctl_throttle_min) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_max = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -274,9 +265,9 @@ if (val < 0) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_pct = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -292,9 +283,9 @@ if (val < 0) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_pct2 = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -344,7 +335,7 @@ struct ucred *cred = p->p_ucred; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); switch (rule->rr_per) { case RCTL_SUBJECT_TYPE_PROCESS: @@ -371,7 +362,7 @@ const struct racct *racct; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); racct = rctl_proc_rule_to_racct(p, rule); available = rule->rr_amount - racct->r_resources[rule->rr_resource]; @@ -394,11 +385,10 @@ int64_t minavailable; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); minavailable = INT64_MAX; - RCTL_RLOCK(); - LIST_FOREACH(link, &racct->r_rule_links, rrl_next) { rule = link->rrl_rule; @@ -411,8 +401,6 @@ minavailable = rule->rr_amount; } - RCTL_RUNLOCK(); - if (racct->r_resources[resource] < minavailable) { racct->r_resources[resource] = 0; } else { @@ -440,12 +428,11 @@ int64_t available, minavailable, limit; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); minavailable = INT64_MAX; limit = 0; - RCTL_RLOCK(); - LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { rule = link->rrl_rule; if (rule->rr_resource != RACCT_PCTCPU) @@ -459,8 +446,6 @@ } } - RCTL_RUNLOCK(); - /* * Return slightly less than actual value of the available * %cpu resource. This makes %cpu throttling more agressive @@ -526,10 +511,8 @@ int should_deny = 0; char *buf; - ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -702,8 +685,6 @@ } } - RCTL_RUNLOCK(); - if (should_deny) { /* * Return fake error code; the caller should change it @@ -723,8 +704,7 @@ uint64_t amount = UINT64_MAX; ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -740,8 +720,6 @@ amount = rule->rr_amount; } - RCTL_RUNLOCK(); - return (amount); } @@ -755,8 +733,7 @@ minavailable = INT64_MAX; ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -773,8 +750,6 @@ minavailable = available; } - RCTL_RUNLOCK(); - /* * XXX: Think about this _hard_. */ @@ -783,6 +758,7 @@ minavailable += allocated; if (minavailable < 0) minavailable = 0; + return (minavailable); } @@ -917,9 +893,9 @@ link->rrl_rule = rule; link->rrl_exceeded = 0; - RCTL_WLOCK(); + RACCT_LOCK(); LIST_INSERT_HEAD(&racct->r_rule_links, link, rrl_next); - RCTL_WUNLOCK(); + RACCT_UNLOCK(); } static int @@ -929,7 +905,7 @@ ASSERT_RACCT_ENABLED(); KASSERT(rctl_rule_fully_specified(rule), ("rule not fully specified")); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); link = uma_zalloc(rctl_rule_link_zone, M_NOWAIT); if (link == NULL) @@ -939,6 +915,7 @@ link->rrl_exceeded = 0; LIST_INSERT_HEAD(&racct->r_rule_links, link, rrl_next); + return (0); } @@ -955,7 +932,7 @@ struct rctl_rule_link *link, *linktmp; ASSERT_RACCT_ENABLED(); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); LIST_FOREACH_SAFE(link, &racct->r_rule_links, rrl_next, linktmp) { if (!rctl_rule_matches(link->rrl_rule, filter)) @@ -1426,14 +1403,14 @@ rctl_rule_pre_callback(void) { - RCTL_WLOCK(); + RACCT_LOCK(); } static void rctl_rule_post_callback(void) { - RCTL_WUNLOCK(); + RACCT_UNLOCK(); } static void @@ -1443,7 +1420,7 @@ int found = 0; ASSERT_RACCT_ENABLED(); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); found += rctl_racct_remove_rules(racct, filter); @@ -1464,9 +1441,9 @@ if (filter->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS && filter->rr_subject.rs_proc != NULL) { p = filter->rr_subject.rs_proc; - RCTL_WLOCK(); + RACCT_LOCK(); found = rctl_racct_remove_rules(p->p_racct, filter); - RCTL_WUNLOCK(); + RACCT_UNLOCK(); if (found) return (0); return (ESRCH); @@ -1483,11 +1460,11 @@ filter, (void *)&found); sx_assert(&allproc_lock, SA_LOCKED); - RCTL_WLOCK(); + RACCT_LOCK(); FOREACH_PROC_IN_SYSTEM(p) { found += rctl_racct_remove_rules(p->p_racct, filter); } - RCTL_WUNLOCK(); + RACCT_UNLOCK(); if (found) return (0); @@ -1619,7 +1596,9 @@ for (i = 0; i <= RACCT_MAX; i++) { if (sloppy == 0 && RACCT_IS_SLOPPY(i)) continue; + RACCT_LOCK(); amount = racct->r_resources[i]; + RACCT_UNLOCK(); if (RACCT_IS_IN_MILLIONS(i)) amount /= 1000000; sbuf_printf(sb, "%s=%jd,", rctl_resource_name(i), amount); @@ -1714,7 +1693,7 @@ struct sbuf *sb = (struct sbuf *)arg3; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); LIST_FOREACH(link, &racct->r_rule_links, rrl_next) { if (!rctl_rule_matches(link->rrl_rule, filter)) @@ -1765,7 +1744,7 @@ KASSERT(sb != NULL, ("sbuf_new failed")); FOREACH_PROC_IN_SYSTEM(p) { - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { /* * Non-process rules will be added to the buffer later. @@ -1779,7 +1758,7 @@ rctl_rule_to_sbuf(sb, link->rrl_rule); sbuf_printf(sb, ","); } - RCTL_RUNLOCK(); + RACCT_UNLOCK(); } loginclass_racct_foreach(rctl_get_rules_callback, @@ -1866,13 +1845,13 @@ sb = sbuf_new(NULL, buf, bufsize, SBUF_FIXEDLEN); KASSERT(sb != NULL, ("sbuf_new failed")); - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &filter->rr_subject.rs_proc->p_racct->r_rule_links, rrl_next) { rctl_rule_to_sbuf(sb, link->rrl_rule); sbuf_printf(sb, ","); } - RCTL_RUNLOCK(); + RACCT_UNLOCK(); if (sbuf_error(sb) == ENOMEM) { error = ERANGE; goto out; @@ -1997,7 +1976,7 @@ * credentials. */ rulecnt = 0; - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { if (link->rrl_rule->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS) @@ -2009,7 +1988,7 @@ rulecnt++; LIST_FOREACH(link, &newprr->prr_racct->r_rule_links, rrl_next) rulecnt++; - RCTL_RUNLOCK(); + RACCT_UNLOCK(); /* * Create temporary list. We've dropped the rctl_lock in order @@ -2027,7 +2006,7 @@ /* * Assign rules to the newly allocated list entries. */ - RCTL_WLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { if (link->rrl_rule->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS) { @@ -2095,13 +2074,13 @@ newlink, rrl_next); } - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return; } goaround: - RCTL_WUNLOCK(); + RACCT_UNLOCK(); /* * Rule list changed while we were not holding the rctl_lock. @@ -2128,12 +2107,11 @@ struct rctl_rule_link *link; struct rctl_rule *rule; - LIST_INIT(&child->p_racct->r_rule_links); - ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); KASSERT(parent->p_racct != NULL, ("process without racct; p = %p", parent)); - RCTL_WLOCK(); + LIST_INIT(&child->p_racct->r_rule_links); /* * Go through limits applicable to the parent and assign them @@ -2162,7 +2140,6 @@ } } - RCTL_WUNLOCK(); return (0); fail: @@ -2172,7 +2149,7 @@ rctl_rule_release(link->rrl_rule); uma_zfree(rctl_rule_link_zone, link); } - RCTL_WUNLOCK(); + return (EAGAIN); } @@ -2185,15 +2162,14 @@ struct rctl_rule_link *link; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); - RCTL_WLOCK(); while (!LIST_EMPTY(&racct->r_rule_links)) { link = LIST_FIRST(&racct->r_rule_links); LIST_REMOVE(link, rrl_next); rctl_rule_release(link->rrl_rule); uma_zfree(rctl_rule_link_zone, link); } - RCTL_WUNLOCK(); } static void Index: sys/sys/racct.h =================================================================== --- sys/sys/racct.h +++ sys/sys/racct.h @@ -155,6 +155,12 @@ #ifdef RACCT +extern struct mtx racct_lock; + +#define RACCT_LOCK() mtx_lock(&racct_lock) +#define RACCT_UNLOCK() mtx_unlock(&racct_lock) +#define RACCT_LOCK_ASSERT() mtx_assert(&racct_lock, MA_OWNED) + int racct_add(struct proc *p, int resource, uint64_t amount); void racct_add_cred(struct ucred *cred, int resource, uint64_t amount); void racct_add_force(struct proc *p, int resource, uint64_t amount);