Index: sys/kern/subr_smp.c =================================================================== --- sys/kern/subr_smp.c +++ sys/kern/subr_smp.c @@ -64,6 +64,14 @@ #endif static int sysctl_kern_smp_active(SYSCTL_HANDLER_ARGS); +static bool topo_analyze_pkg(struct topo_node *pkg_node, int all, + struct topo_analysis *results); +static bool topo_analyze_group(struct topo_node *group_node, int all, + struct topo_analysis *results); +static bool topo_analyze_cachegroup(struct topo_node *cg_node, int all, + struct topo_analysis *results); +static bool topo_analyze_core(struct topo_node *cg_node, int all, + struct topo_analysis *results); /* This is used in modules that need to work in both SMP and UP. */ cpuset_t all_cpus; @@ -993,7 +1001,7 @@ if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); - while ((node = node->parent) != top) + while (node != top && (node = node->parent) != top) if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); @@ -1012,7 +1020,7 @@ if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); - while ((node = node->parent) != top) + while (node != top && (node = node->parent) != top) if ((next = TAILQ_NEXT(node, siblings)) != NULL) return (next); @@ -1054,20 +1062,16 @@ */ int topo_analyze(struct topo_node *topo_root, int all, - int *pkg_count, int *cores_per_pkg, int *thrs_per_core) + struct topo_analysis *results) { struct topo_node *pkg_node; - struct topo_node *core_node; - struct topo_node *pu_node; - int thrs_per_pkg; - int cpp_counter; - int tpc_counter; - int tpp_counter; - *pkg_count = 0; - *cores_per_pkg = -1; - *thrs_per_core = -1; - thrs_per_pkg = -1; + results->pkg_count = 0; + results->cores_per_cachegroup = -1; + results->threads_per_core = -1; + results->groups_per_pkg = -1; + results->cachegroups_per_group = -1; + pkg_node = topo_root; while (pkg_node != NULL) { if (pkg_node->type != TOPO_TYPE_PKG) { @@ -1079,70 +1083,201 @@ continue; } - (*pkg_count)++; - - cpp_counter = 0; - tpp_counter = 0; - core_node = pkg_node; - while (core_node != NULL) { - if (core_node->type == TOPO_TYPE_CORE) { - if (!all && CPU_EMPTY(&core_node->cpuset)) { - core_node = - topo_next_nonchild_node(pkg_node, - core_node); - continue; - } - - cpp_counter++; - - tpc_counter = 0; - pu_node = core_node; - while (pu_node != NULL) { - if (pu_node->type == TOPO_TYPE_PU && - (all || !CPU_EMPTY(&pu_node->cpuset))) - tpc_counter++; - pu_node = topo_next_node(core_node, - pu_node); - } - - if (*thrs_per_core == -1) - *thrs_per_core = tpc_counter; - else if (*thrs_per_core != tpc_counter) - return (0); - - core_node = topo_next_nonchild_node(pkg_node, - core_node); - } else { - /* PU node directly under PKG. */ - if (core_node->type == TOPO_TYPE_PU && - (all || !CPU_EMPTY(&core_node->cpuset))) - tpp_counter++; - core_node = topo_next_node(pkg_node, - core_node); - } - } + results->pkg_count++; - if (*cores_per_pkg == -1) - *cores_per_pkg = cpp_counter; - else if (*cores_per_pkg != cpp_counter) - return (0); - if (thrs_per_pkg == -1) - thrs_per_pkg = tpp_counter; - else if (thrs_per_pkg != tpp_counter) + if (!topo_analyze_pkg(pkg_node, all, results)) return (0); pkg_node = topo_next_nonchild_node(topo_root, pkg_node); } - KASSERT(*pkg_count > 0, + KASSERT(results->pkg_count > 0, ("bug in topology or analysis")); - if (*cores_per_pkg == 0) { - KASSERT(*thrs_per_core == -1 && thrs_per_pkg > 0, - ("bug in topology or analysis")); - *thrs_per_core = thrs_per_pkg; - } return (1); } + +static bool +topo_analyze_pkg(struct topo_node *pkg_node, int all, + struct topo_analysis *results) +{ + struct topo_node *group_node; + int gpp_counter; + + gpp_counter = 0; + group_node = topo_next_node(pkg_node, pkg_node); + + while (group_node != NULL) { + if (group_node->type != TOPO_TYPE_GROUP) { + group_node = topo_next_node(pkg_node, group_node); + continue; + } + if (!all && CPU_EMPTY(&group_node->cpuset)) { + group_node = topo_next_nonchild_node(pkg_node, + group_node); + continue; + } + + gpp_counter++; + + if (!topo_analyze_group(group_node, all, results)) + return (false); + + group_node = topo_next_nonchild_node(pkg_node, group_node); + } + + /* + * Package with no explicit subgroups have essentially a single + * subgroup. + */ + if (gpp_counter == 0) { + gpp_counter = 1; + + if (!topo_analyze_group(pkg_node, all, results)) + return (false); + } + + if (results->groups_per_pkg == -1) + results->groups_per_pkg = gpp_counter; + else if (results->groups_per_pkg != gpp_counter) + return (false); + + return (true); +} + +static bool +topo_analyze_group(struct topo_node *group_node, int all, + struct topo_analysis *results) +{ + struct topo_node *cg_node; + int cgpg_counter; + + cgpg_counter = 0; + cg_node = topo_next_node(group_node, group_node); + + while (cg_node != NULL) { + if (cg_node->type != TOPO_TYPE_CACHE || + cg_node->subtype != CG_SHARE_L3) { + cg_node = topo_next_node(group_node, cg_node); + continue; + } + if (!all && CPU_EMPTY(&cg_node->cpuset)) { + cg_node = topo_next_nonchild_node(group_node, cg_node); + continue; + } + + cgpg_counter++; + + if (!topo_analyze_cachegroup(cg_node, all, results)) + return (false); + + cg_node = topo_next_nonchild_node(group_node, cg_node); + } + + /* + * Groups with no explicit cache groups have essentially a single cache + * group. + */ + if (cgpg_counter == 0) { + cgpg_counter = 1; + + if (!topo_analyze_cachegroup(group_node, all, results)) + return (false); + } + + if (results->cachegroups_per_group == -1) + results->cachegroups_per_group = cgpg_counter; + else if (results->cachegroups_per_group != cgpg_counter) + return (false); + + return (true); +} + +static bool +topo_analyze_cachegroup(struct topo_node *cg_node, int all, + struct topo_analysis *results) +{ + struct topo_node *core_node; + int cpp_counter; + + cpp_counter = 0; + core_node = topo_next_node(cg_node, cg_node); + + while (core_node != NULL) { + if (core_node->type != TOPO_TYPE_CORE) { + core_node = topo_next_node(cg_node, core_node); + continue; + } + + if (!all && CPU_EMPTY(&core_node->cpuset)) { + core_node = topo_next_nonchild_node(cg_node, + core_node); + continue; + } + + cpp_counter++; + + if (!topo_analyze_core(core_node, all, results)) + return (false); + + core_node = topo_next_nonchild_node(cg_node, core_node); + } + + /* + * Cachegroups with no explicit cores have a single core. + */ + if (cpp_counter == 0) { + cpp_counter = 1; + + if (!topo_analyze_core(cg_node, all, results)) + return (false); + } + + if (results->cores_per_cachegroup == -1) + results->cores_per_cachegroup = cpp_counter; + else if (results->cores_per_cachegroup != cpp_counter) + return (false); + + return (true); +} + +static bool +topo_analyze_core(struct topo_node *core_node, int all, + struct topo_analysis *results) +{ + struct topo_node *pu_node; + int tpc_counter; + + tpc_counter = 0; + pu_node = topo_next_node(core_node, core_node); + + while (pu_node != NULL) { + if (pu_node->type != TOPO_TYPE_PU) { + pu_node = topo_next_node(core_node, pu_node); + continue; + } + if (!all && CPU_EMPTY(&pu_node->cpuset)) { + pu_node = topo_next_nonchild_node(core_node, pu_node); + continue; + } + + tpc_counter++; + pu_node = topo_next_nonchild_node(core_node, pu_node); + } + + /* + * Cores with no explicit PUs have one thread. + */ + if (tpc_counter == 0) + tpc_counter = 1; + + if (results->threads_per_core == -1) + results->threads_per_core = tpc_counter; + else if (results->threads_per_core != tpc_counter) + return (false); + + return (true); +} + #endif /* SMP */ Index: sys/sys/smp.h =================================================================== --- sys/sys/smp.h +++ sys/sys/smp.h @@ -120,8 +120,22 @@ struct topo_node * topo_next_nonchild_node(struct topo_node *top, struct topo_node *node); void topo_set_pu_id(struct topo_node *node, cpuid_t id); -int topo_analyze(struct topo_node *topo_root, int all, int *pkg_count, - int *cores_per_pkg, int *thrs_per_core); +struct topo_analysis { + int pkg_count; + + /* + * Some systems have useful sub-package core organizations. On these, + * a package has one or more subgroups. Each subgroup contains one or + * more cache groups (cores that share a last level cache). + */ + int groups_per_pkg; + int cachegroups_per_group; + + int cores_per_cachegroup; + int threads_per_core; +}; +int topo_analyze(struct topo_node *topo_root, int all, + struct topo_analysis *results); #define TOPO_FOREACH(i, root) \ for (i = root; i != NULL; i = topo_next_node(root, i)) Index: sys/x86/x86/mp_x86.c =================================================================== --- sys/x86/x86/mp_x86.c +++ sys/x86/x86/mp_x86.c @@ -656,18 +656,19 @@ { struct topo_node *node; const char *hyperthread; - int pkg_count; - int cores_per_pkg; - int thrs_per_core; + struct topo_analysis topology; printf("FreeBSD/SMP: "); - if (topo_analyze(&topo_root, 1, &pkg_count, - &cores_per_pkg, &thrs_per_core)) { - printf("%d package(s)", pkg_count); - if (cores_per_pkg > 0) - printf(" x %d core(s)", cores_per_pkg); - if (thrs_per_core > 1) - printf(" x %d hardware threads", thrs_per_core); + if (topo_analyze(&topo_root, 1, &topology)) { + printf("%d package(s)", topology.pkg_count); + if (topology.groups_per_pkg > 1) + printf(" x %d groups", topology.groups_per_pkg); + if (topology.cachegroups_per_group > 1) + printf(" x %d cache groups", topology.cachegroups_per_group); + if (topology.cores_per_cachegroup > 0) + printf(" x %d core(s)", topology.cores_per_cachegroup); + if (topology.threads_per_core > 1) + printf(" x %d hardware threads", topology.threads_per_core); } else { printf("Non-uniform topology"); } @@ -675,13 +676,19 @@ if (disabled_cpus) { printf("FreeBSD/SMP Online: "); - if (topo_analyze(&topo_root, 0, &pkg_count, - &cores_per_pkg, &thrs_per_core)) { - printf("%d package(s)", pkg_count); - if (cores_per_pkg > 0) - printf(" x %d core(s)", cores_per_pkg); - if (thrs_per_core > 1) - printf(" x %d hardware threads", thrs_per_core); + if (topo_analyze(&topo_root, 0, &topology)) { + printf("%d package(s)", topology.pkg_count); + if (topology.groups_per_pkg > 1) + printf(" x %d groups", topology.groups_per_pkg); + if (topology.cachegroups_per_group > 1) + printf(" x %d cache groups", + topology.cachegroups_per_group); + if (topology.cores_per_cachegroup > 0) + printf(" x %d core(s)", + topology.cores_per_cachegroup); + if (topology.threads_per_core > 1) + printf(" x %d hardware threads", + topology.threads_per_core); } else { printf("Non-uniform topology"); }