Index: sys/kern/subr_bus.c =================================================================== --- sys/kern/subr_bus.c +++ sys/kern/subr_bus.c @@ -31,9 +31,11 @@ #include "opt_bus.h" #include "opt_ddb.h" +#include "opt_vm.h" #include #include +#include #include #include #include @@ -52,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -2930,6 +2933,11 @@ return error; } +#if defined(EARLY_AP_STARTUP) && defined(NUMA) && defined(UMA_FIRSTTOUCH) +#define DEVICE_ATTACH_NUMA +static u_int device_attach_numa = 1; +TUNABLE_INT("dev.attach_numa", &device_attach_numa); +#endif /** * @brief Attach a device driver to a device * @@ -2954,6 +2962,9 @@ { uint64_t attachtime; uint16_t attachentropy; +#ifdef DEVICE_ATTACH_NUMA + int domain, prev; +#endif int error; if (resource_disabled(dev->driver->name, dev->unit)) { @@ -2962,6 +2973,14 @@ device_printf(dev, "disabled via hints entry\n"); return (ENXIO); } +#ifdef DEVICE_ATTACH_NUMA + if (device_attach_numa != 0 && bus_get_domain(dev, &domain) == 0) { + thread_lock(curthread); + prev = sched_bind_nested(CPU_FFS(&cpuset_domain[domain])); + thread_unlock(curthread); + } else + domain = -1; +#endif device_sysctl_init(dev); if (!device_is_quiet(dev)) @@ -2977,6 +2996,7 @@ device_sysctl_fini(dev); KASSERT(dev->busy == 0, ("attach failed but busy")); dev->state = DS_NOTPRESENT; + goto out; return (error); } dev->flags |= DF_ATTACHED_ONCE; @@ -2993,8 +3013,19 @@ dev->flags &= ~DF_DONENOMATCH; EVENTHANDLER_DIRECT_INVOKE(device_attach, dev); devadded(dev); - return (0); + error = 0; +out: +#ifdef DEVICE_ATTACH_NUMA + if (domain != -1) { + thread_lock(curthread); + sched_unbind_nested(prev); + thread_unlock(curthread); + } +#endif + + return (error); } +#undef DEVICE_ATTACH_BIND /** * @brief Detach a driver from a device Index: sys/sys/sched.h =================================================================== --- sys/sys/sched.h +++ sys/sys/sched.h @@ -153,6 +153,34 @@ int sched_is_bound(struct thread *td); void sched_affinity(struct thread *td); +/* + * Support nesting CPU binding by returning the previous bound CPU to + * the caller. + */ +static __inline int +sched_bind_nested(int cpu) +{ + int prev; + + if (sched_is_bound(curthread)) + prev = PCPU_GET(cpuid); + else + prev = -1; + sched_bind(curthread, cpu); + + return (prev); +} + +static __inline void +sched_unbind_nested(int cpu) +{ + + if (cpu != -1) + sched_bind(curthread, cpu); + else + sched_unbind(curthread); +} + /* * These procedures tell the process data structure allocation code how * many bytes to actually allocate. Index: sys/x86/x86/local_apic.c =================================================================== --- sys/x86/x86/local_apic.c +++ sys/x86/x86/local_apic.c @@ -1641,6 +1641,7 @@ native_apic_free_vector(u_int apic_id, u_int vector, u_int irq) { struct thread *td; + int prev; KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && vector <= APIC_IO_INTS + APIC_NUM_IOINTS, @@ -1660,9 +1661,7 @@ td = curthread; if (!rebooting) { thread_lock(td); - if (sched_is_bound(td)) - panic("apic_free_vector: Thread already bound.\n"); - sched_bind(td, apic_cpuid(apic_id)); + prev = sched_bind_nested(apic_cpuid(apic_id)); thread_unlock(td); } mtx_lock_spin(&icu_lock); @@ -1670,7 +1669,7 @@ mtx_unlock_spin(&icu_lock); if (!rebooting) { thread_lock(td); - sched_unbind(td); + sched_unbind_nested(prev); thread_unlock(td); } }