Index: lib/libvmmapi/vmmapi.h =================================================================== --- lib/libvmmapi/vmmapi.h +++ lib/libvmmapi/vmmapi.h @@ -211,6 +211,10 @@ int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus); int vm_activate_cpu(struct vmctx *ctx, int vcpu); +/* Set cpu topology */ +int vm_set_topology(struct vmctx *ctx, uint64_t sockets, uint64_t cores, \ + uint64_t threads, uint64_t maxcpus); + /* * FreeBSD specific APIs */ Index: lib/libvmmapi/vmmapi.c =================================================================== --- lib/libvmmapi/vmmapi.c +++ lib/libvmmapi/vmmapi.c @@ -1420,6 +1420,20 @@ } int +vm_set_topology(struct vmctx *ctx, \ + uint64_t sockets, uint64_t cores, uint64_t threads, uint64_t maxcpus) +{ + struct vm_cpu_topology topology; + + bzero(&topology, sizeof (struct vm_cpu_topology)); + topology.sockets = sockets; + topology.cores = cores; + topology.threads = threads; + topology.maxcpus = maxcpus; + return (ioctl(ctx->fd, VM_SET_TOPOLOGY, &topology)); +} + +int vm_get_device_fd(struct vmctx *ctx) { @@ -1446,7 +1460,7 @@ VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA, VM_ACTIVATE_CPU, VM_GET_CPUS, VM_SET_INTINFO, VM_GET_INTINFO, VM_RTC_WRITE, VM_RTC_READ, VM_RTC_SETTIME, VM_RTC_GETTIME, - VM_RESTART_INSTRUCTION }; + VM_RESTART_INSTRUCTION, VM_SET_TOPOLOGY, VM_GET_TOPOLOGY }; if (len == NULL) { cmds = malloc(sizeof(vm_ioctl_cmds)); Index: sys/amd64/include/vmm.h =================================================================== --- sys/amd64/include/vmm.h +++ sys/amd64/include/vmm.h @@ -181,6 +181,10 @@ void vm_destroy(struct vm *vm); int vm_reinit(struct vm *vm); const char *vm_name(struct vm *vm); +void vm_get_topology(struct vm *vm, uint64_t *sockets, uint64_t *cores, + uint64_t *threads, uint64_t *maxcpus); +int vm_set_topology(struct vm *vm, uint64_t sockets, uint64_t cores, + uint64_t threads, uint64_t maxcpus); /* * APIs that modify the guest memory map require all vcpus to be frozen. Index: sys/amd64/include/vmm_dev.h =================================================================== --- sys/amd64/include/vmm_dev.h +++ sys/amd64/include/vmm_dev.h @@ -218,6 +218,13 @@ uint8_t value; }; +struct vm_cpu_topology { + uint64_t sockets; + uint64_t cores; + uint64_t threads; + uint64_t maxcpus; +}; + enum { /* general routines */ IOCNUM_ABIVERS = 0, @@ -273,6 +280,10 @@ IOCNUM_GET_X2APIC_STATE = 61, IOCNUM_GET_HPET_CAPABILITIES = 62, + /* CPU Topology */ + IOCNUM_SET_TOPOLOGY = 63, + IOCNUM_GET_TOPOLOGY = 64, + /* legacy interrupt injection */ IOCNUM_ISA_ASSERT_IRQ = 80, IOCNUM_ISA_DEASSERT_IRQ = 81, @@ -384,4 +395,8 @@ _IOR('v', IOCNUM_RTC_GETTIME, struct vm_rtc_time) #define VM_RESTART_INSTRUCTION \ _IOW('v', IOCNUM_RESTART_INSTRUCTION, int) +#define VM_SET_TOPOLOGY \ + _IOW('v', IOCNUM_SET_TOPOLOGY, struct vm_cpu_topology) +#define VM_GET_TOPOLOGY \ + _IOR('v', IOCNUM_GET_TOPOLOGY, struct vm_cpu_topology) #endif Index: sys/amd64/vmm/vmm.c =================================================================== --- sys/amd64/vmm/vmm.c +++ sys/amd64/vmm/vmm.c @@ -165,6 +165,11 @@ struct vmspace *vmspace; /* (o) guest's address space */ char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */ struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */ + /* The following describe the vm cpu topology */ + uint64_t sockets; /* (o) num of sockets */ + uint64_t cores; /* (o) num of cores/socket */ + uint64_t threads; /* (o) num of threads/core */ + uint64_t maxcpus; /* (o) max pluggable cpus */ }; static int vmm_initialized; @@ -423,6 +428,9 @@ vcpu_init(vm, i, create); } +extern u_int cores_per_package; +extern u_int threads_per_core; + int vm_create(const char *name, struct vm **retvm) { @@ -448,6 +456,11 @@ vm->vmspace = vmspace; mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); + vm->sockets=1; + vm->cores=cores_per_package; /* XXX backwards compatibility */ + vm->threads=threads_per_core; /* XXX backwards compatibility */ + vm->maxcpus=0; /* XXX not implemented */ + vm_init(vm, true); *retvm = vm; @@ -454,6 +467,30 @@ return (0); } +void +vm_get_topology(struct vm *vm, uint64_t *sockets, uint64_t *cores, + uint64_t *threads, uint64_t *maxcpus) +{ + *sockets = vm->sockets; + *cores = vm->cores; + *threads = vm->threads; + *maxcpus = vm->maxcpus; +} + +int +vm_set_topology(struct vm *vm, uint64_t sockets, uint64_t cores, + uint64_t threads, uint64_t maxcpus) +{ + if (maxcpus != 0) return (EINVAL); /* XXX remove when supported */ + if ((sockets * cores * threads) > VM_MAXCPU) return (EINVAL); + /* XXX need to check sockets * cores * threads == vCPU, how? */ + vm->sockets = sockets; + vm->cores = cores; + vm->threads = threads; + vm->maxcpus = maxcpus; + return(0); +} + static void vm_cleanup(struct vm *vm, bool destroy) { Index: sys/amd64/vmm/vmm_dev.c =================================================================== --- sys/amd64/vmm/vmm_dev.c +++ sys/amd64/vmm/vmm_dev.c @@ -315,6 +315,7 @@ struct vm_rtc_time *rtctime; struct vm_rtc_data *rtcdata; struct vm_memmap *mm; + struct vm_cpu_topology *topology; sc = vmmdev_lookup2(cdev); if (sc == NULL) @@ -642,6 +643,14 @@ case VM_RESTART_INSTRUCTION: error = vm_restart_instruction(sc->vm, vcpu); break; + case VM_SET_TOPOLOGY: + topology = (struct vm_cpu_topology *)data; + error = vm_set_topology(sc->vm, topology->sockets, + topology->cores, topology->threads, topology->maxcpus); + break; + case VM_GET_TOPOLOGY: + break; + default: error = ENOTTY; break; Index: sys/amd64/vmm/x86.c =================================================================== --- sys/amd64/vmm/x86.c +++ sys/amd64/vmm/x86.c @@ -63,12 +63,12 @@ /* * The default CPU topology is a single thread per package. */ -static u_int threads_per_core = 1; -SYSCTL_UINT(_hw_vmm_topology, OID_AUTO, threads_per_core, CTLFLAG_RDTUN, +u_int threads_per_core = 1; +SYSCTL_UINT(_hw_vmm_topology, OID_AUTO, threads_per_core, CTLFLAG_RW, &threads_per_core, 0, NULL); -static u_int cores_per_package = 1; -SYSCTL_UINT(_hw_vmm_topology, OID_AUTO, cores_per_package, CTLFLAG_RDTUN, +u_int cores_per_package = 1; +SYSCTL_UINT(_hw_vmm_topology, OID_AUTO, cores_per_package, CTLFLAG_RW, &cores_per_package, 0, NULL); static int cpuid_leaf_b = 1; @@ -91,7 +91,7 @@ uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { const struct xsave_limits *limits; - uint64_t cr4; + uint64_t cores, cr4, maxcpus, sockets, threads; int error, enable_invpcid, level, width, x2apic_id; unsigned int func, regs[4], logical_cpus; enum x2apic_state x2apic_state; @@ -142,11 +142,12 @@ * * However this matches the logical cpus as * advertised by leaf 0x1 and will work even - * if the 'threads_per_core' tunable is set - * incorrectly on an AMD host. + * if threads is set incorrectly on an AMD host. */ - logical_cpus = threads_per_core * - cores_per_package; + vm_get_topology(vm, &sockets, &cores, &threads, + &maxcpus); + logical_cpus = threads * + cores; regs[2] = logical_cpus - 1; } break; @@ -305,7 +306,9 @@ */ regs[3] |= (CPUID_MCA | CPUID_MCE | CPUID_MTRR); - logical_cpus = threads_per_core * cores_per_package; + vm_get_topology(vm, &sockets, &cores, &threads, + &maxcpus); + logical_cpus = threads * cores; regs[1] &= ~CPUID_HTT_CORES; regs[1] |= (logical_cpus & 0xff) << 16; regs[3] |= CPUID_HTT; @@ -315,8 +318,10 @@ cpuid_count(*eax, *ecx, regs); if (regs[0] || regs[1] || regs[2] || regs[3]) { + vm_get_topology(vm, &sockets, &cores, &threads, + &maxcpus); regs[0] &= 0x3ff; - regs[0] |= (cores_per_package - 1) << 26; + regs[0] |= (cores - 1) << 26; /* * Cache topology: * - L1 and L2 are shared only by the logical @@ -324,10 +329,10 @@ * - L3 and above are shared by all logical * processors in the package. */ - logical_cpus = threads_per_core; + logical_cpus = threads; level = (regs[0] >> 5) & 0x7; if (level >= 3) - logical_cpus *= cores_per_package; + logical_cpus *= cores; regs[0] |= (logical_cpus - 1) << 14; } break; @@ -390,7 +395,9 @@ * Processor topology enumeration */ if (*ecx == 0) { - logical_cpus = threads_per_core; + vm_get_topology(vm, &sockets, &cores, &threads, + &maxcpus); + logical_cpus = threads; width = log2(logical_cpus); level = CPUID_TYPE_SMT; x2apic_id = vcpu_id; @@ -397,8 +404,8 @@ } if (*ecx == 1) { - logical_cpus = threads_per_core * - cores_per_package; + logical_cpus = threads * + cores; width = log2(logical_cpus); level = CPUID_TYPE_CORE; x2apic_id = vcpu_id; Index: usr.sbin/bhyve/bhyve.8 =================================================================== --- usr.sbin/bhyve/bhyve.8 +++ usr.sbin/bhyve/bhyve.8 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd June 2, 2017 +.Dd February 17, 2018 .Dt BHYVE 8 .Os .Sh NAME @@ -33,7 +33,7 @@ .Sh SYNOPSIS .Nm .Op Fl abehuwxACHPSWY -.Op Fl c Ar numcpus +.Op Fl c Oo Ar cpus= Oc Ns Ar numcpus Ns Oo Ar ,sockets=n Oc Ns Oo Ar ,cores=n Oc Ns Op Ar ,threads=n .Op Fl g Ar gdbport .Op Fl l Ar lpcdev Ns Op , Ns Ar conf .Op Fl m Ar memsize Ns Op Ar K|k|M|m|G|g|T|t @@ -77,9 +77,28 @@ kernels compiled with .Cd "device bvmconsole" . This option will be deprecated in a future version. -.It Fl c Ar numcpus -Number of guest virtual CPUs. -The default is 1 and the maximum is 16. +.It Fl c Oo Ar cpus= Oc Ns Ar numcpus Ns Oo Ar ,sockets=n Oc Ns Oo Ar ,cores=n Oc Ns Op Ar ,threads=n +Number of guest virtual CPUs +and optionally the CPU topology. +The optional topology must be consistent in that the +.Ar numcpus +must equal the product of +.Ar sockets +* +.Ar cores +* +.Ar threads . +The default is 1 CPU +implemented as 1 each of +.Ar sockets , +.Ar cores , +and +.Ar threads . +If no topology is specified it is implemented as if +.Ar sockets += +.Ar numcpus . +There is a maximum of 16 virtual CPUs. .It Fl C Include guest memory in core file. .It Fl e Index: usr.sbin/bhyve/bhyverun.c =================================================================== --- usr.sbin/bhyve/bhyverun.c +++ usr.sbin/bhyve/bhyverun.c @@ -93,6 +93,11 @@ char *vmname; int guest_ncpus; +uint64_t sockets; +uint64_t cores; +uint64_t threads; +uint64_t maxcpus; + char *guest_uuid_str; static int guest_vmexit_on_hlt, guest_vmexit_on_pause; @@ -165,6 +170,50 @@ } static int +topology_parse(const char *opt) +{ + char *cp, *str; + int i, tmp; + + str = strdup(opt); + if ((cp = strchr(str, ',')) != NULL) { + for (i = 0; str; i++) { + if (sscanf(str, "cpus=%d", &tmp) == 1) + guest_ncpus = tmp; + else if (sscanf(str, "sockets=%d", &tmp) == 1) + sockets = tmp; + else if (sscanf(str, "cores=%d", &tmp) == 1) + cores = tmp; + else if (sscanf(str, "threads=%d", &tmp) == 1) + threads = tmp; +#ifdef notyet /* Do not expose this until vmm.ko implements it */ + else if (sscanf(str, "maxcpus=%d", &tmp) == 1) + maxcpus = tmp; +#endif + else + return (-1); + if (cp == NULL) + str = NULL; + else { + str = ++cp; + cp = strchr(str, ','); + } + } + } else { + if (sscanf(str, "cpus=%d", &tmp) == 1) + guest_ncpus = tmp; + else if (sscanf(str, "%d", &tmp) == 1) + guest_ncpus = tmp; + else + return (-1); + } + if (guest_ncpus != sockets * cores * threads) + return (-1); + else + return(0); +} + +static int pincpu_parse(const char *opt) { int vcpu, pcpu; @@ -783,6 +832,11 @@ exit(1); } } + error = vm_set_topology(ctx, sockets, cores, threads, maxcpus); + if (error) { + perror("vm_set_topology"); + exit(1); + } return (ctx); } @@ -801,6 +855,8 @@ progname = basename(argv[0]); gdb_port = 0; guest_ncpus = 1; + sockets = cores = threads = 1; + maxcpus = 0; memsize = 256 * MB; mptgen = 1; rtc_localtime = 1; @@ -825,7 +881,10 @@ } break; case 'c': - guest_ncpus = atoi(optarg); + if (topology_parse(optarg) !=0) { + errx(EX_USAGE, "invalid cpu topology " + "'%s'", optarg); + } break; case 'C': memflags |= VM_MEM_F_INCORE;