Index: sys/kern/kern_racct.c =================================================================== --- sys/kern/kern_racct.c +++ sys/kern/kern_racct.c @@ -91,13 +91,9 @@ */ #define RACCT_PCPU_SECS 3 -static struct mtx racct_lock; +struct mtx racct_lock; MTX_SYSINIT(racct_lock, &racct_lock, "racct lock", MTX_DEF); -#define RACCT_LOCK() mtx_lock(&racct_lock) -#define RACCT_UNLOCK() mtx_unlock(&racct_lock) -#define RACCT_LOCK_ASSERT() mtx_assert(&racct_lock, MA_OWNED) - static uma_zone_t racct_zone; static void racct_sub_racct(struct racct *dest, const struct racct *src); @@ -111,6 +107,8 @@ "struct proc *", "int", "uint64_t"); SDT_PROBE_DEFINE3(racct, , rusage, add__failure, "struct proc *", "int", "uint64_t"); +SDT_PROBE_DEFINE3(racct, , rusage, add__buf, + "struct proc *", "const struct buf *", "int"); SDT_PROBE_DEFINE3(racct, , rusage, add__cred, "struct ucred *", "int", "uint64_t"); SDT_PROBE_DEFINE3(racct, , rusage, add__force, @@ -618,8 +616,6 @@ ASSERT_RACCT_ENABLED(); - SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount); - racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, amount); for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent) racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource, @@ -638,6 +634,8 @@ if (!racct_enable) return; + SDT_PROBE3(racct, , rusage, add__cred, cred, resource, amount); + RACCT_LOCK(); racct_add_cred_locked(cred, resource, amount); RACCT_UNLOCK(); @@ -654,6 +652,8 @@ ASSERT_RACCT_ENABLED(); PROC_LOCK_ASSERT(p, MA_OWNED); + SDT_PROBE3(racct, , rusage, add__buf, p, bp, is_write); + RACCT_LOCK(); if (is_write) { racct_add_locked(curproc, RACCT_WRITEBPS, bp->b_bcount, 1); @@ -766,13 +766,19 @@ uint64_t racct_get_limit(struct proc *p, int resource) { +#ifdef RCTL + uint64_t available; if (!racct_enable) return (UINT64_MAX); -#ifdef RCTL - return (rctl_get_limit(p, resource)); + RACCT_LOCK(); + available = rctl_get_limit(p, resource); + RACCT_UNLOCK(); + + return (available); #else + return (UINT64_MAX); #endif } @@ -786,13 +792,19 @@ uint64_t racct_get_available(struct proc *p, int resource) { +#ifdef RCTL + uint64_t available; if (!racct_enable) return (UINT64_MAX); -#ifdef RCTL - return (rctl_get_available(p, resource)); + RACCT_LOCK(); + available = rctl_get_available(p, resource); + RACCT_UNLOCK(); + + return (available); #else + return (UINT64_MAX); #endif } @@ -805,12 +817,18 @@ static int64_t racct_pcpu_available(struct proc *p) { +#ifdef RCTL + uint64_t available; ASSERT_RACCT_ENABLED(); -#ifdef RCTL - return (rctl_pcpu_available(p)); + RACCT_LOCK(); + available = rctl_pcpu_available(p); + RACCT_UNLOCK(); + + return (available); #else + return (INT64_MAX); #endif } @@ -852,14 +870,6 @@ ASSERT_RACCT_ENABLED(); - SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount); - -#ifdef notyet - KASSERT(RACCT_CAN_DROP(resource), - ("%s: called for resource %d which can not drop", __func__, - resource)); -#endif - racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, -amount); for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent) racct_adjust_resource(pr->pr_prison_racct->prr_racct, resource, @@ -877,6 +887,14 @@ if (!racct_enable) return; + SDT_PROBE3(racct, , rusage, sub__cred, cred, resource, amount); + +#ifdef notyet + KASSERT(RACCT_CAN_DROP(resource), + ("%s: called for resource %d which can not drop", __func__, + resource)); +#endif + RACCT_LOCK(); racct_sub_cred_locked(cred, resource, amount); RACCT_UNLOCK(); @@ -948,11 +966,12 @@ racct_proc_fork_done(struct proc *child) { - PROC_LOCK_ASSERT(child, MA_OWNED); -#ifdef RCTL if (!racct_enable) return; + PROC_LOCK_ASSERT(child, MA_OWNED); + +#ifdef RCTL RACCT_LOCK(); rctl_enforce(child, RACCT_NPROC, 0); rctl_enforce(child, RACCT_NTHR, 0); @@ -1003,13 +1022,12 @@ racct_set_locked(p, i, 0, 0); } - RACCT_UNLOCK(); - PROC_UNLOCK(p); - #ifdef RCTL rctl_racct_release(p->p_racct); #endif - racct_destroy(&p->p_racct); + racct_destroy_locked(&p->p_racct); + RACCT_UNLOCK(); + PROC_UNLOCK(p); } /* Index: sys/kern/kern_rctl.c =================================================================== --- sys/kern/kern_rctl.c +++ sys/kern/kern_rctl.c @@ -212,15 +212,6 @@ static uma_zone_t rctl_rule_zone; static uma_zone_t rctl_rule_link_zone; -static struct rwlock rctl_lock; -RW_SYSINIT(rctl_lock, &rctl_lock, "RCTL lock"); - -#define RCTL_RLOCK() rw_rlock(&rctl_lock) -#define RCTL_RUNLOCK() rw_runlock(&rctl_lock) -#define RCTL_WLOCK() rw_wlock(&rctl_lock) -#define RCTL_WUNLOCK() rw_wunlock(&rctl_lock) -#define RCTL_LOCK_ASSERT() rw_assert(&rctl_lock, RA_LOCKED) -#define RCTL_WLOCK_ASSERT() rw_assert(&rctl_lock, RA_WLOCKED) static int rctl_rule_fully_specified(const struct rctl_rule *rule); static void rctl_rule_to_sbuf(struct sbuf *sb, const struct rctl_rule *rule); @@ -237,9 +228,9 @@ if (val < 1 || val > rctl_throttle_max) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_min = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -254,9 +245,9 @@ if (val < rctl_throttle_min) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_max = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -271,9 +262,9 @@ if (val < 0) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_pct = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -288,9 +279,9 @@ if (val < 0) return (EINVAL); - RCTL_WLOCK(); + RACCT_LOCK(); rctl_throttle_pct2 = val; - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return (0); } @@ -340,7 +331,7 @@ struct ucred *cred = p->p_ucred; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); switch (rule->rr_per) { case RCTL_SUBJECT_TYPE_PROCESS: @@ -367,7 +358,7 @@ int64_t available; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); racct = rctl_proc_rule_to_racct(p, rule); available = rule->rr_amount - racct->r_resources[rule->rr_resource]; @@ -390,11 +381,10 @@ int64_t minavailable; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); minavailable = INT64_MAX; - RCTL_RLOCK(); - LIST_FOREACH(link, &racct->r_rule_links, rrl_next) { rule = link->rrl_rule; @@ -407,8 +397,6 @@ minavailable = rule->rr_amount; } - RCTL_RUNLOCK(); - if (racct->r_resources[resource] < minavailable) { racct->r_resources[resource] = 0; } else { @@ -436,12 +424,11 @@ int64_t available, minavailable, limit; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); minavailable = INT64_MAX; limit = 0; - RCTL_RLOCK(); - LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { rule = link->rrl_rule; if (rule->rr_resource != RACCT_PCTCPU) @@ -455,8 +442,6 @@ } } - RCTL_RUNLOCK(); - /* * Return slightly less than actual value of the available * %cpu resource. This makes %cpu throttling more agressive @@ -516,10 +501,8 @@ uint64_t sleep_ms, sleep_ratio; int should_deny = 0; - ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -659,7 +642,7 @@ if (sleep_ms > rctl_throttle_max) sleep_ms = rctl_throttle_max; -#if 0 +#if 1 printf("%s: pid %d (%s), %jd of %jd, will sleep for %ld ms (ratio %ld, available %ld)\n", __func__, p->p_pid, p->p_comm, p->p_racct->r_resources[resource], @@ -692,8 +675,6 @@ } } - RCTL_RUNLOCK(); - if (should_deny) { /* * Return fake error code; the caller should change it @@ -713,8 +694,7 @@ uint64_t amount = UINT64_MAX; ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -730,8 +710,6 @@ amount = rule->rr_amount; } - RCTL_RUNLOCK(); - return (amount); } @@ -745,8 +723,7 @@ minavailable = INT64_MAX; ASSERT_RACCT_ENABLED(); - - RCTL_RLOCK(); + RACCT_LOCK_ASSERT(); /* * There may be more than one matching rule; go through all of them. @@ -763,8 +740,6 @@ minavailable = available; } - RCTL_RUNLOCK(); - /* * XXX: Think about this _hard_. */ @@ -773,6 +748,7 @@ minavailable += allocated; if (minavailable < 0) minavailable = 0; + return (minavailable); } @@ -907,9 +883,9 @@ link->rrl_rule = rule; link->rrl_exceeded = 0; - RCTL_WLOCK(); + RACCT_LOCK(); LIST_INSERT_HEAD(&racct->r_rule_links, link, rrl_next); - RCTL_WUNLOCK(); + RACCT_UNLOCK(); } static int @@ -919,7 +895,7 @@ ASSERT_RACCT_ENABLED(); KASSERT(rctl_rule_fully_specified(rule), ("rule not fully specified")); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); link = uma_zalloc(rctl_rule_link_zone, M_NOWAIT); if (link == NULL) @@ -929,6 +905,7 @@ link->rrl_exceeded = 0; LIST_INSERT_HEAD(&racct->r_rule_links, link, rrl_next); + return (0); } @@ -945,7 +922,7 @@ int removed = 0; ASSERT_RACCT_ENABLED(); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); LIST_FOREACH_SAFE(link, &racct->r_rule_links, rrl_next, linktmp) { if (!rctl_rule_matches(link->rrl_rule, filter)) @@ -1416,14 +1393,14 @@ rctl_rule_pre_callback(void) { - RCTL_WLOCK(); + RACCT_LOCK(); } static void rctl_rule_post_callback(void) { - RCTL_WUNLOCK(); + RACCT_UNLOCK(); } static void @@ -1433,7 +1410,7 @@ int found = 0; ASSERT_RACCT_ENABLED(); - RCTL_WLOCK_ASSERT(); + RACCT_LOCK_ASSERT(); found += rctl_racct_remove_rules(racct, filter); @@ -1454,9 +1431,9 @@ if (filter->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS && filter->rr_subject.rs_proc != NULL) { p = filter->rr_subject.rs_proc; - RCTL_WLOCK(); + RACCT_LOCK(); found = rctl_racct_remove_rules(p->p_racct, filter); - RCTL_WUNLOCK(); + RACCT_UNLOCK(); if (found) return (0); return (ESRCH); @@ -1473,11 +1450,11 @@ filter, (void *)&found); sx_assert(&allproc_lock, SA_LOCKED); - RCTL_WLOCK(); + RACCT_LOCK(); FOREACH_PROC_IN_SYSTEM(p) { found += rctl_racct_remove_rules(p->p_racct, filter); } - RCTL_WUNLOCK(); + RACCT_UNLOCK(); if (found) return (0); @@ -1609,7 +1586,9 @@ for (i = 0; i <= RACCT_MAX; i++) { if (sloppy == 0 && RACCT_IS_SLOPPY(i)) continue; + RACCT_LOCK(); amount = racct->r_resources[i]; + RACCT_UNLOCK(); if (RACCT_IS_IN_MILLIONS(i)) amount /= 1000000; sbuf_printf(sb, "%s=%jd,", rctl_resource_name(i), amount); @@ -1704,7 +1683,7 @@ struct sbuf *sb = (struct sbuf *)arg3; ASSERT_RACCT_ENABLED(); - RCTL_LOCK_ASSERT(); + RACCT_LOCK_ASSERT(); LIST_FOREACH(link, &racct->r_rule_links, rrl_next) { if (!rctl_rule_matches(link->rrl_rule, filter)) @@ -1755,7 +1734,7 @@ KASSERT(sb != NULL, ("sbuf_new failed")); FOREACH_PROC_IN_SYSTEM(p) { - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { /* * Non-process rules will be added to the buffer later. @@ -1769,7 +1748,7 @@ rctl_rule_to_sbuf(sb, link->rrl_rule); sbuf_printf(sb, ","); } - RCTL_RUNLOCK(); + RACCT_UNLOCK(); } loginclass_racct_foreach(rctl_get_rules_callback, @@ -1856,13 +1835,13 @@ sb = sbuf_new(NULL, buf, bufsize, SBUF_FIXEDLEN); KASSERT(sb != NULL, ("sbuf_new failed")); - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &filter->rr_subject.rs_proc->p_racct->r_rule_links, rrl_next) { rctl_rule_to_sbuf(sb, link->rrl_rule); sbuf_printf(sb, ","); } - RCTL_RUNLOCK(); + RACCT_UNLOCK(); if (sbuf_error(sb) == ENOMEM) { error = ERANGE; goto out; @@ -1987,7 +1966,7 @@ * credentials. */ rulecnt = 0; - RCTL_RLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { if (link->rrl_rule->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS) @@ -1999,7 +1978,7 @@ rulecnt++; LIST_FOREACH(link, &newprr->prr_racct->r_rule_links, rrl_next) rulecnt++; - RCTL_RUNLOCK(); + RACCT_UNLOCK(); /* * Create temporary list. We've dropped the rctl_lock in order @@ -2017,7 +1996,7 @@ /* * Assign rules to the newly allocated list entries. */ - RCTL_WLOCK(); + RACCT_LOCK(); LIST_FOREACH(link, &p->p_racct->r_rule_links, rrl_next) { if (link->rrl_rule->rr_subject_type == RCTL_SUBJECT_TYPE_PROCESS) { @@ -2085,13 +2064,13 @@ newlink, rrl_next); } - RCTL_WUNLOCK(); + RACCT_UNLOCK(); return; } goaround: - RCTL_WUNLOCK(); + RACCT_UNLOCK(); /* * Rule list changed while we were not holding the rctl_lock. @@ -2118,12 +2097,11 @@ struct rctl_rule_link *link; int error; - LIST_INIT(&child->p_racct->r_rule_links); - ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); KASSERT(parent->p_racct != NULL, ("process without racct; p = %p", parent)); - RCTL_WLOCK(); + LIST_INIT(&child->p_racct->r_rule_links); /* * Go through limits applicable to the parent and assign them @@ -2152,7 +2130,6 @@ } } - RCTL_WUNLOCK(); return (0); fail: @@ -2162,7 +2139,7 @@ rctl_rule_release(link->rrl_rule); uma_zfree(rctl_rule_link_zone, link); } - RCTL_WUNLOCK(); + return (EAGAIN); } @@ -2175,15 +2152,14 @@ struct rctl_rule_link *link; ASSERT_RACCT_ENABLED(); + RACCT_LOCK_ASSERT(); - RCTL_WLOCK(); while (!LIST_EMPTY(&racct->r_rule_links)) { link = LIST_FIRST(&racct->r_rule_links); LIST_REMOVE(link, rrl_next); rctl_rule_release(link->rrl_rule); uma_zfree(rctl_rule_link_zone, link); } - RCTL_WUNLOCK(); } static void Index: sys/sys/racct.h =================================================================== --- sys/sys/racct.h +++ sys/sys/racct.h @@ -155,6 +155,12 @@ #ifdef RACCT +extern struct mtx racct_lock; + +#define RACCT_LOCK() mtx_lock(&racct_lock) +#define RACCT_UNLOCK() mtx_unlock(&racct_lock) +#define RACCT_LOCK_ASSERT() mtx_assert(&racct_lock, MA_OWNED) + int racct_add(struct proc *p, int resource, uint64_t amount); void racct_add_cred(struct ucred *cred, int resource, uint64_t amount); void racct_add_force(struct proc *p, int resource, uint64_t amount);