Page MenuHomeFreeBSD
Differential D49349

uma: Avoid excessive per-CPU draining
ClosedPublic
Actions

Authored by des on Mar 14 2025, 12:56 AM.
Tags
None
Referenced Files
F127238012: D49349.id.diff
Fri, Aug 29, 11:23 AM
Unknown Object (File)
Mon, Aug 11, 9:00 AM
Unknown Object (File)
Mon, Aug 4, 10:43 PM
Unknown Object (File)
Jul 28 2025, 6:59 AM
Unknown Object (File)
Jul 28 2025, 4:58 AM
Unknown Object (File)
Jul 12 2025, 8:46 AM
Unknown Object (File)
Jul 6 2025, 10:20 AM
Unknown Object (File)
Jun 29 2025, 2:31 PM
View All 36 Files
Subscribers
imp
kevans
siderop1_netapp.com

Details

Reviewers
kib
gallatin
siderop1_netapp.com
markj
Commits
rG02324ae8279a: uma: Avoid excessive per-CPU draining
rG910535a82a29: uma: Avoid excessive per-CPU draining
rGf506d5af50fc: uma: Avoid excessive per-CPU draining
Summary

After commit 389a3fa693ef, uma_reclaim_domain(UMA_RECLAIM_DRAIN_CPU)
calls uma_zone_reclaim_domain(UMA_RECLAIM_DRAIN_CPU) twice on each zone
in addition to globally draining per-CPU caches. This was unintended
and is unnecessarily slow; in particular, draining per-CPU caches
requires binding to each CPU.

Stop draining per-CPU caches when visiting each zone, just do it once in
pcpu_cache_drain_safe() to minimize the amount of expensive sched_bind()
calls.

Fixes: 389a3fa693ef ("uma: Add UMA_ZONE_UNMANAGED")

Diff Detail

Repository
rG FreeBSD src repository
Lint
Lint Skipped
Unit
Tests Skipped
Build Status
Buildable 62910
Build 59794: arc lint + arc unit

Event Timeline

markj created this revision.Mar 14 2025, 12:56 AM
Herald added a subscriber: imp. · View Herald TranscriptMar 14 2025, 12:56 AM
markj requested review of this revision.Mar 14 2025, 12:56 AM
Harbormaster completed remote builds in B62910: Diff 152240.Mar 14 2025, 12:56 AM
des accepted this revision.Mar 14 2025, 9:03 AM
This revision is now accepted and ready to land.Mar 14 2025, 9:03 AM
siderop1_netapp.com added a subscriber: siderop1_netapp.com.Mar 14 2025, 1:23 PM
kib accepted this revision.Mar 14 2025, 2:37 PM
gallatin accepted this revision.Mar 14 2025, 5:38 PM
siderop1_netapp.com accepted this revision.Mar 14 2025, 9:21 PM

FWIW, internal testing with an equivalent diff shows the intermittent (~8%) OOM panics seen with 389a3fa693ef are now resolved. Thanks!

kevans added a subscriber: kevans.Mar 15 2025, 4:43 AM
des commandeered this revision.Mar 17 2025, 7:12 PM
des edited reviewers, added: markj; removed: des.

commandeering since @markj is afk

Closed by commit rGf506d5af50fc: uma: Avoid excessive per-CPU draining (authored by markj, committed by des). · Explain WhyMar 17 2025, 7:14 PM
This revision was automatically updated to reflect the committed changes.
des added a commit: rGf506d5af50fc: uma: Avoid excessive per-CPU draining.
des added a commit: rG910535a82a29: uma: Avoid excessive per-CPU draining.Mar 31 2025, 6:36 PM
des added a commit: rG02324ae8279a: uma: Avoid excessive per-CPU draining.

Revision Contents
Changeset List

PathSize
sys/
vm/
7 lines

Diff 152240

View Options

sys/vm/uma_core.c

Show First 20 Lines • Show All 54 Lines • ▼ Show 20 Lines
#include "snd_fxdiv_gen.h" #include "snd_fxdiv_gen.h"
#endif #endif
#define FEEDMATRIX_RESERVOIR (SND_CHN_MAX * PCM_32_BPS) #define FEEDMATRIX_RESERVOIR (SND_CHN_MAX * PCM_32_BPS)
#define SND_CHN_T_EOF 0x00e0fe0f #define SND_CHN_T_EOF 0x00e0fe0f
#define SND_CHN_T_NULL 0x0e0e0e0e #define SND_CHN_T_NULL 0x0e0e0e0e
struct feed_matrix_info;
typedef void (*feed_matrix_t)(struct feed_matrix_info *, uint8_t *,
uint8_t *, uint32_t);
struct feed_matrix_info { struct feed_matrix_info {
uint32_t fmt;
uint32_t bps; uint32_t bps;
uint32_t ialign, oalign; uint32_t ialign, oalign;
uint32_t in, out; uint32_t in, out;
feed_matrix_t apply;
struct { struct {
int chn[SND_CHN_T_MAX + 1]; int chn[SND_CHN_T_MAX + 1];
int mul, shift; int mul, shift;
} matrix[SND_CHN_T_MAX + 1]; } matrix[SND_CHN_T_MAX + 1];
uint8_t reservoir[FEEDMATRIX_RESERVOIR]; uint8_t reservoir[FEEDMATRIX_RESERVOIR];
}; };
static struct pcmchan_matrix feeder_matrix_maps[SND_CHN_MATRIX_MAX] = { static struct pcmchan_matrix feeder_matrix_maps[SND_CHN_MATRIX_MAX] = {
Show All 29 Lines
#else #else
#define FEEDMATRIX_CLIP_CHECK(v, BIT) do { \ #define FEEDMATRIX_CLIP_CHECK(v, BIT) do { \
if ((v) < PCM_S##BIT##_MIN || (v) > PCM_S##BIT##_MAX) \ if ((v) < PCM_S##BIT##_MIN || (v) > PCM_S##BIT##_MAX) \
errx(1, "\n\n%s(): Sample clipping: %jd\n", \ errx(1, "\n\n%s(): Sample clipping: %jd\n", \
__func__, (intmax_t)(v)); \ __func__, (intmax_t)(v)); \
} while (0) } while (0)
#endif #endif
#define FEEDMATRIX_DECLARE(SIGN, BIT, ENDIAN) \ __always_inline static void
static void \ feed_matrix_apply(struct feed_matrix_info *info, uint8_t *src, uint8_t *dst,
feed_matrix_##SIGN##BIT##ENDIAN(struct feed_matrix_info *info, \ uint32_t count, const uint32_t fmt)
uint8_t *src, uint8_t *dst, uint32_t count) \ {
{ \ intpcm64_t accum;
intpcm64_t accum; \ intpcm_t v;
intpcm_t v; \ int i, j;
int i, j; \
\ do {
do { \ for (i = 0; info->matrix[i].chn[0] != SND_CHN_T_EOF; i++) {
for (i = 0; info->matrix[i].chn[0] != SND_CHN_T_EOF; \ if (info->matrix[i].chn[0] == SND_CHN_T_NULL) {
i++) { \ pcm_sample_write(dst, 0, fmt);
if (info->matrix[i].chn[0] == SND_CHN_T_NULL) { \ dst += info->bps;
pcm_sample_write(dst, 0, \ continue;
AFMT_##SIGN##BIT##_##ENDIAN); \ } else if (info->matrix[i].chn[1] == SND_CHN_T_EOF) {
dst += PCM_##BIT##_BPS; \ v = pcm_sample_read(src +
continue; \ info->matrix[i].chn[0], fmt);
} else if (info->matrix[i].chn[1] == \ pcm_sample_write(dst, v, fmt);
SND_CHN_T_EOF) { \ dst += info->bps;
v = pcm_sample_read( \ continue;
src + info->matrix[i].chn[0], \
AFMT_##SIGN##BIT##_##ENDIAN); \
pcm_sample_write(dst, v, \
AFMT_##SIGN##BIT##_##ENDIAN); \
dst += PCM_##BIT##_BPS; \
continue; \
} \
\
accum = 0; \
for (j = 0; \
info->matrix[i].chn[j] != SND_CHN_T_EOF; \
j++) { \
v = pcm_sample_read( \
src + info->matrix[i].chn[j], \
AFMT_##SIGN##BIT##_##ENDIAN); \
accum += v; \
} \
\
accum = (accum * info->matrix[i].mul) >> \
info->matrix[i].shift; \
\
FEEDMATRIX_CLIP_CHECK(accum, BIT); \
\
v = (accum > PCM_S##BIT##_MAX) ? \
PCM_S##BIT##_MAX : \
((accum < PCM_S##BIT##_MIN) ? \
PCM_S##BIT##_MIN : \
accum); \
pcm_sample_write(dst, v, \
AFMT_##SIGN##BIT##_##ENDIAN); \
dst += PCM_##BIT##_BPS; \
} \
src += info->ialign; \
} while (--count != 0); \
} }
#if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT) accum = 0;
FEEDMATRIX_DECLARE(S, 16, LE) for (j = 0; info->matrix[i].chn[j] != SND_CHN_T_EOF;
FEEDMATRIX_DECLARE(S, 32, LE) j++) {
#endif v = pcm_sample_read(src +
#if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT) info->matrix[i].chn[j], fmt);
FEEDMATRIX_DECLARE(S, 16, BE) accum += v;
FEEDMATRIX_DECLARE(S, 32, BE)
#endif
#ifdef SND_FEEDER_MULTIFORMAT
FEEDMATRIX_DECLARE(S, 8, NE)
FEEDMATRIX_DECLARE(S, 24, LE)
FEEDMATRIX_DECLARE(S, 24, BE)
FEEDMATRIX_DECLARE(U, 8, NE)
FEEDMATRIX_DECLARE(U, 16, LE)
FEEDMATRIX_DECLARE(U, 24, LE)
FEEDMATRIX_DECLARE(U, 32, LE)
FEEDMATRIX_DECLARE(U, 16, BE)
FEEDMATRIX_DECLARE(U, 24, BE)
FEEDMATRIX_DECLARE(U, 32, BE)
#endif
#define FEEDMATRIX_ENTRY(SIGN, BIT, ENDIAN) \
{ \
AFMT_##SIGN##BIT##_##ENDIAN, \
feed_matrix_##SIGN##BIT##ENDIAN \
} }
static const struct { accum = (accum * info->matrix[i].mul) >>
uint32_t format; info->matrix[i].shift;
feed_matrix_t apply;
} feed_matrix_tab[] = {
#if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
FEEDMATRIX_ENTRY(S, 16, LE),
FEEDMATRIX_ENTRY(S, 32, LE),
#endif
#if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
FEEDMATRIX_ENTRY(S, 16, BE),
FEEDMATRIX_ENTRY(S, 32, BE),
#endif
#ifdef SND_FEEDER_MULTIFORMAT
FEEDMATRIX_ENTRY(S, 8, NE),
FEEDMATRIX_ENTRY(S, 24, LE),
FEEDMATRIX_ENTRY(S, 24, BE),
FEEDMATRIX_ENTRY(U, 8, NE),
FEEDMATRIX_ENTRY(U, 16, LE),
FEEDMATRIX_ENTRY(U, 24, LE),
FEEDMATRIX_ENTRY(U, 32, LE),
FEEDMATRIX_ENTRY(U, 16, BE),
FEEDMATRIX_ENTRY(U, 24, BE),
FEEDMATRIX_ENTRY(U, 32, BE)
#endif
};
FEEDMATRIX_CLIP_CHECK(accum, AFMT_BIT(fmt));
v = pcm_clamp(accum, fmt);
pcm_sample_write(dst, v, fmt);
dst += info->bps;
}
src += info->ialign;
} while (--count != 0);
}
static void static void
feed_matrix_reset(struct feed_matrix_info *info) feed_matrix_reset(struct feed_matrix_info *info)
{ {
uint32_t i, j; uint32_t i, j;
for (i = 0; i < nitems(info->matrix); i++) { for (i = 0; i < nitems(info->matrix); i++) {
for (j = 0; for (j = 0;
j < (sizeof(info->matrix[i].chn) / j < (sizeof(info->matrix[i].chn) /
▲ Show 20 Lines • Show All 157 Lines • ▼ Show 20 Lines#endif
return (0); return (0);
} }
static int static int
feed_matrix_init(struct pcm_feeder *f) feed_matrix_init(struct pcm_feeder *f)
{ {
struct feed_matrix_info *info; struct feed_matrix_info *info;
struct pcmchan_matrix *m_in, *m_out; struct pcmchan_matrix *m_in, *m_out;
uint32_t i;
int ret; int ret;
if (AFMT_ENCODING(f->desc->in) != AFMT_ENCODING(f->desc->out)) if (AFMT_ENCODING(f->desc->in) != AFMT_ENCODING(f->desc->out))
return (EINVAL); return (EINVAL);
info = malloc(sizeof(*info), M_DEVBUF, M_NOWAIT | M_ZERO); info = malloc(sizeof(*info), M_DEVBUF, M_NOWAIT | M_ZERO);
if (info == NULL) if (info == NULL)
return (ENOMEM); return (ENOMEM);
info->in = f->desc->in; info->in = f->desc->in;
info->out = f->desc->out; info->out = f->desc->out;
info->fmt = AFMT_ENCODING(info->in);
info->bps = AFMT_BPS(info->in); info->bps = AFMT_BPS(info->in);
info->ialign = AFMT_ALIGN(info->in); info->ialign = AFMT_ALIGN(info->in);
info->oalign = AFMT_ALIGN(info->out); info->oalign = AFMT_ALIGN(info->out);
info->apply = NULL;
for (i = 0; info->apply == NULL &&
i < (sizeof(feed_matrix_tab) / sizeof(feed_matrix_tab[0])); i++) {
if (AFMT_ENCODING(info->in) == feed_matrix_tab[i].format)
info->apply = feed_matrix_tab[i].apply;
}
if (info->apply == NULL) {
#ifdef FEEDMATRIX_GENERIC
info->apply = feed_matrix_apply_generic;
#else
free(info, M_DEVBUF);
return (EINVAL);
#endif
}
m_in = feeder_matrix_format_map(info->in); m_in = feeder_matrix_format_map(info->in);
m_out = feeder_matrix_format_map(info->out); m_out = feeder_matrix_format_map(info->out);
ret = feed_matrix_setup(info, m_in, m_out); ret = feed_matrix_setup(info, m_in, m_out);
if (ret != 0) { if (ret != 0) {
free(info, M_DEVBUF); free(info, M_DEVBUF);
return (ret); return (ret);
} }
▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines if (count < inmax) {
src = dst + count - j; src = dst + count - j;
} }
j = SND_FXDIV(FEEDER_FEED(f->source, c, src, j, source), j = SND_FXDIV(FEEDER_FEED(f->source, c, src, j, source),
info->ialign); info->ialign);
if (j == 0) if (j == 0)
break; break;
info->apply(info, src, dst, j); /* Optimize some common formats. */
switch (info->fmt) {
case AFMT_S16_NE:
feed_matrix_apply(info, src, dst, j, AFMT_S16_NE);
break;
case AFMT_S24_NE:
feed_matrix_apply(info, src, dst, j, AFMT_S24_NE);
break;
case AFMT_S32_NE:
feed_matrix_apply(info, src, dst, j, AFMT_S32_NE);
break;
default:
feed_matrix_apply(info, src, dst, j, info->fmt);
break;
}
markjUnsubmitted
Not Done
Inline Actions

Why can't this entire switch statement just be feed_matrix_apply(info, src, dst, j, info->fmt);?

markj: Why can't this entire switch statement just be `feed_matrix_apply(info, src, dst, j, info->fmt)…
christosAuthorUnsubmitted
Done
Inline Actions

There is related discussion in D48032

christos: There is related discussion in D48032
j *= info->oalign; j *= info->oalign;
dst += j; dst += j;
count -= j; count -= j;
} while (count != 0); } while (count != 0);
return (dst - b); return (dst - b);
▲ Show 20 Lines • Show All 305 Lines • Show Last 20 Lines