Index: head/sys/alpha/alpha/mp_machdep.c =================================================================== --- head/sys/alpha/alpha/mp_machdep.c (revision 123125) +++ head/sys/alpha/alpha/mp_machdep.c (revision 123126) @@ -1,570 +1,569 @@ /*- * Copyright (c) 2000 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready = 0; static struct mtx ap_boot_mtx; u_int boot_cpu_id; static void release_aps(void *dummy); static int smp_cpu_enabled(struct pcs *pcsp); extern void smp_init_secondary_glue(void); static int smp_send_secondary_command(const char *command, int cpuid); static int smp_start_secondary(int cpuid); /* * Communicate with a console running on a secondary processor. * Return 1 on failure. */ static int smp_send_secondary_command(const char *command, int cpuid) { u_int64_t mask = 1L << cpuid; struct pcs *cpu = LOCATE_PCS(hwrpb, cpuid); int i, len; /* * Sanity check. */ len = strlen(command); if (len > sizeof(cpu->pcs_buffer.rxbuf)) { printf("smp_send_secondary_command: command '%s' too long\n", command); return 0; } /* * Wait for the rx bit to clear. */ for (i = 0; i < 100000; i++) { if (!(hwrpb->rpb_rxrdy & mask)) break; DELAY(10); } if (hwrpb->rpb_rxrdy & mask) return 0; /* * Write the command into the processor's buffer. */ bcopy(command, cpu->pcs_buffer.rxbuf, len); cpu->pcs_buffer.rxlen = len; /* * Set the bit in the rxrdy mask and let the secondary try to * handle the command. */ atomic_set_64(&hwrpb->rpb_rxrdy, mask); /* * Wait for the rx bit to clear. */ for (i = 0; i < 100000; i++) { if (!(hwrpb->rpb_rxrdy & mask)) break; DELAY(10); } if (hwrpb->rpb_rxrdy & mask) return 0; return 1; } void smp_init_secondary(void) { struct pcs *cpu; /* spin until all the AP's are ready */ while (!aps_ready) /*spin*/ ; /* * Record the pcpu pointer in the per-cpu system value. */ alpha_pal_wrval((u_int64_t) pcpup); /* Clear userland thread pointer. */ alpha_pal_wrunique(0); /* * Point interrupt/exception vectors to our own. */ alpha_pal_wrent(XentInt, ALPHA_KENTRY_INT); alpha_pal_wrent(XentArith, ALPHA_KENTRY_ARITH); alpha_pal_wrent(XentMM, ALPHA_KENTRY_MM); alpha_pal_wrent(XentIF, ALPHA_KENTRY_IF); alpha_pal_wrent(XentUna, ALPHA_KENTRY_UNA); alpha_pal_wrent(XentSys, ALPHA_KENTRY_SYS); /* lower the ipl and take any pending machine check */ mc_expected = 1; alpha_mb(); alpha_mb(); alpha_pal_wrmces(7); (void)alpha_pal_swpipl(ALPHA_PSL_IPL_HIGH); mc_expected = 0; /* * Set flags in our per-CPU slot in the HWRPB. */ cpu = LOCATE_PCS(hwrpb, PCPU_GET(cpuid)); cpu->pcs_flags &= ~PCS_BIP; cpu->pcs_flags |= PCS_RC; alpha_mb(); /* * XXX: doesn't idleproc already have a pcb from when it was * kthread_create'd? * * cache idleproc's physical address. */ curthread->td_md.md_pcbpaddr = (struct pcb *)PCPU_GET(idlepcbphys); /* * and make idleproc's trapframe pointer point to its * stack pointer for sanity. */ curthread->td_frame = (struct trapframe *)PCPU_PTR(idlepcb)->apcb_ksp; mtx_lock_spin(&ap_boot_mtx); smp_cpus++; CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); /* Build our map of 'other' CPUs. */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); if (smp_cpus == mp_ncpus) { smp_started = 1; smp_active = 1; } mtx_unlock_spin(&ap_boot_mtx); while (smp_started == 0) ; /* nothing */ binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); } static int smp_start_secondary(int cpuid) { struct pcs *cpu = LOCATE_PCS(hwrpb, cpuid); struct pcs *bootcpu = LOCATE_PCS(hwrpb, boot_cpu_id); struct alpha_pcb *pcb = (struct alpha_pcb *) cpu->pcs_hwpcb; struct pcpu *pcpu; int i; size_t sz; if ((cpu->pcs_flags & PCS_PV) == 0) { printf("smp_start_secondary: cpu %d PALcode invalid\n", cpuid); return 0; } if (bootverbose) printf("smp_start_secondary: starting cpu %d\n", cpuid); sz = round_page((UAREA_PAGES + KSTACK_PAGES) * PAGE_SIZE); pcpu = malloc(sz, M_TEMP, M_NOWAIT); if (!pcpu) { printf("smp_start_secondary: can't allocate memory\n"); return 0; } pcpu_init(pcpu, cpuid, sz); /* * Copy the idle pcb and setup the address to start executing. * Use the pcb unique value to point the secondary at its pcpu * structure. */ *pcb = pcpu->pc_idlepcb; pcb->apcb_unique = (u_int64_t)pcpu; hwrpb->rpb_restart = (u_int64_t) smp_init_secondary_glue; hwrpb->rpb_restart_val = (u_int64_t) smp_init_secondary_glue; hwrpb->rpb_checksum = hwrpb_checksum(); /* * Tell the cpu to start with the same PALcode as us. */ bcopy(&bootcpu->pcs_pal_rev, &cpu->pcs_pal_rev, sizeof cpu->pcs_pal_rev); /* * Set flags in cpu structure and push out write buffers to * make sure the secondary sees it. */ cpu->pcs_flags |= PCS_CV|PCS_RC; cpu->pcs_flags &= ~PCS_BIP; alpha_mb(); /* * Fire it up and hope for the best. */ if (!smp_send_secondary_command("START\r\n", cpuid)) { printf("smp_start_secondary: can't send START command\n"); pcpu_destroy(pcpu); free(pcpu, M_TEMP); return 0; } /* * Wait for the secondary to set the BIP flag in its structure. */ for (i = 0; i < 100000; i++) { if (cpu->pcs_flags & PCS_BIP) break; DELAY(10); } if (!(cpu->pcs_flags & PCS_BIP)) { printf("smp_start_secondary: secondary did not respond\n"); pcpu_destroy(pcpu); free(pcpu, M_TEMP); return 0; } /* * It worked (I think). */ if (bootverbose) printf("smp_start_secondary: cpu %d started\n", cpuid); return 1; } /* Other stuff */ static int smp_cpu_enabled(struct pcs *pcsp) { /* Is this CPU present? */ if ((pcsp->pcs_flags & PCS_PP) == 0) return (0); /* Is this CPU available? */ if ((pcsp->pcs_flags & PCS_PA) == 0) /* * The TurboLaser PCS_PA bit doesn't seem to be set * correctly. */ if (hwrpb->rpb_type != ST_DEC_21000) return (0); /* Is this CPU's PALcode valid? */ if ((pcsp->pcs_flags & PCS_PV) == 0) return (0); return (1); } void cpu_mp_setmaxid(void) { int i; mp_maxid = 0; for (i = 0; i < hwrpb->rpb_pcs_cnt && i < MAXCPU; i++) { if (i == PCPU_GET(cpuid)) continue; if (!smp_cpu_enabled(LOCATE_PCS(hwrpb, i))) continue; mp_maxid = i; } - mp_maxid++; } int cpu_mp_probe(void) { int i, cpus; /* XXX: Need to check for valid platforms here. */ boot_cpu_id = PCPU_GET(cpuid); KASSERT(boot_cpu_id == hwrpb->rpb_primary_cpu_id, ("cpu_mp_probe() called on non-primary CPU")); all_cpus = 1 << boot_cpu_id; mp_ncpus = 1; /* Make sure we have at least one secondary CPU. */ cpus = 0; for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { if (i == PCPU_GET(cpuid)) continue; if (!smp_cpu_enabled(LOCATE_PCS(hwrpb, i))) continue; if (i > MAXCPU) continue; cpus++; } return (cpus); } void cpu_mp_start(void) { int i; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { struct pcs *pcsp; if (i == boot_cpu_id) continue; pcsp = LOCATE_PCS(hwrpb, i); if ((pcsp->pcs_flags & PCS_PP) == 0) continue; if ((pcsp->pcs_flags & PCS_PA) == 0) { if (hwrpb->rpb_type == ST_DEC_21000) { printf("Ignoring PA bit for CPU %d.\n", i); } else { if (bootverbose) printf("CPU %d not available.\n", i); continue; } } if ((pcsp->pcs_flags & PCS_PV) == 0) { if (bootverbose) printf("CPU %d does not have valid PALcode.\n", i); continue; } if (i > MAXCPU) { if (bootverbose) { printf("CPU %d not supported.", i); printf(" Only %d CPUs supported.\n", MAXCPU); } continue; } if (resource_disabled("cpu", i)) { printf("CPU %d disabled by loader.\n", i); continue; } all_cpus |= (1 << i); mp_ncpus++; } PCPU_SET(other_cpus, all_cpus & ~(1 << boot_cpu_id)); for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { if (i == boot_cpu_id) continue; if (all_cpus & (1 << i)) smp_start_secondary(i); } } void cpu_mp_announce(void) { } /* * send an IPI to a set of cpus. */ void ipi_selected(u_int32_t cpus, u_int64_t ipi) { struct pcpu *pcpu; CTR2(KTR_SMP, "ipi_selected: cpus: %x ipi: %lx", cpus, ipi); alpha_mb(); while (cpus) { int cpuid = ffs(cpus) - 1; cpus &= ~(1 << cpuid); pcpu = pcpu_find(cpuid); if (pcpu) { atomic_set_64(&pcpu->pc_pending_ipis, ipi); alpha_mb(); CTR1(KTR_SMP, "calling alpha_pal_wripir(%d)", cpuid); alpha_pal_wripir(cpuid); } } } /* * send an IPI INTerrupt containing 'vector' to all CPUs, including myself */ void ipi_all(u_int64_t ipi) { ipi_selected(all_cpus, ipi); } /* * send an IPI to all CPUs EXCEPT myself */ void ipi_all_but_self(u_int64_t ipi) { ipi_selected(PCPU_GET(other_cpus), ipi); } /* * send an IPI to myself */ void ipi_self(u_int64_t ipi) { ipi_selected(1 << PCPU_GET(cpuid), ipi); } /* * Handle an IPI sent to this processor. */ void smp_handle_ipi(struct trapframe *frame) { u_int64_t ipis = atomic_readandclear_64(PCPU_PTR(pending_ipis)); u_int64_t ipi; int cpumask; cpumask = 1 << PCPU_GET(cpuid); CTR1(KTR_SMP, "smp_handle_ipi(), ipis=%lx", ipis); while (ipis) { /* * Find the lowest set bit. */ ipi = ipis & ~(ipis - 1); ipis &= ~ipi; switch (ipi) { case IPI_INVLTLB: CTR0(KTR_SMP, "IPI_NVLTLB"); ALPHA_TBIA(); break; case IPI_RENDEZVOUS: CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); break; case IPI_AST: CTR0(KTR_SMP, "IPI_AST"); break; case IPI_STOP: CTR0(KTR_SMP, "IPI_STOP"); atomic_set_int(&stopped_cpus, cpumask); while ((started_cpus & cpumask) == 0) alpha_mb(); atomic_clear_int(&started_cpus, cpumask); atomic_clear_int(&stopped_cpus, cpumask); break; } } /* * Dump console messages to the console. XXX - we need to handle * requests to provide PALcode to secondaries and to start up new * secondaries that are added to the system on the fly. */ if (PCPU_GET(cpuid) == boot_cpu_id) { u_int cpuid; u_int64_t txrdy; #ifdef DIAGNOSTIC struct pcs *cpu; char buf[81]; #endif alpha_mb(); while (hwrpb->rpb_txrdy != 0) { cpuid = ffs(hwrpb->rpb_txrdy) - 1; #ifdef DIAGNOSTIC cpu = LOCATE_PCS(hwrpb, cpuid); bcopy(&cpu->pcs_buffer.txbuf, buf, cpu->pcs_buffer.txlen); buf[cpu->pcs_buffer.txlen] = '\0'; printf("SMP From CPU%d: %s\n", cpuid, buf); #endif do { txrdy = hwrpb->rpb_txrdy; } while (atomic_cmpset_64(&hwrpb->rpb_txrdy, txrdy, txrdy & ~(1 << cpuid)) == 0); } } } static void release_aps(void *dummy __unused) { if (bootverbose && mp_ncpus > 1) printf("%s: releasing secondary CPUs\n", __func__); atomic_store_rel_int(&aps_ready, 1); while (mp_ncpus > 1 && smp_started == 0) ; /* nothing */ } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); Index: head/sys/amd64/amd64/mp_machdep.c =================================================================== --- head/sys/amd64/amd64/mp_machdep.c (revision 123125) +++ head/sys/amd64/amd64/mp_machdep.c (revision 123126) @@ -1,1036 +1,1036 @@ /*- * Copyright (c) 1996, by Steve Passe * Copyright (c) 2003, by Peter Wemm * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_cpu.h" #include "opt_kstack_pages.h" #include #include #include #ifdef GPROF #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) /* lock region used by kernel profiling */ int mcount_lock; int mp_naps; /* # of Applications processors */ int boot_cpu_id = -1; /* designated BSP */ extern int nkpt; /* * CPU topology map datastructures for HTT. (XXX) */ struct cpu_group mp_groups[MAXCPU]; struct cpu_top mp_top; struct cpu_top *smp_topology; /* AP uses this during bootstrap. Do not staticize. */ char *bootSTK; static int bootAP; /* Free these after use */ void *bootstacks[MAXCPU]; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; /* SMP page table page */ extern pt_entry_t *SMPpt; struct pcb stoppcbs[MAXCPU]; /* Variables needed for SMP tlb shootdown. */ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; struct mtx smp_tlb_mtx; extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); /* * Local data and functions. */ static u_int logical_cpus; static u_int logical_cpus_mask; /* used to hold the AP's until we are ready to release them */ static struct mtx ap_boot_mtx; /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready = 0; /* * Store data from cpu_add() until later in the boot when we actually setup * the APs. */ struct cpu_info { int cpu_present:1; int cpu_bsp:1; } static cpu_info[MAXCPU]; static int cpu_apic_ids[MAXCPU]; static u_int boot_address; static void set_logical_apic_ids(void); static int start_all_aps(void); static int start_ap(int apic_id); static void release_aps(void *dummy); static int hlt_cpus_mask; static int hlt_logical_cpus; static struct sysctl_ctx_list logical_cpu_clist; static u_int bootMP_size; /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { bootMP_size = mptramp_end - mptramp_start; boot_address = trunc_page(basemem * 1024); /* round down to 4k boundary */ if ((basemem - boot_address) < bootMP_size) boot_address -= PAGE_SIZE; /* not enough, lower by 4k */ /* 3 levels of page table pages */ mptramp_pagetables = boot_address - (PAGE_SIZE * 3); return mptramp_pagetables; } void cpu_add(u_int apic_id, char boot_cpu) { if (apic_id > MAXCPU) { printf("SMP: CPU %d exceeds maximum CPU %d, ignoring\n", apic_id, MAXCPU); return; } KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", apic_id)); cpu_info[apic_id].cpu_present = 1; if (boot_cpu) { KASSERT(boot_cpu_id == -1, ("CPU %d claims to be BSP, but CPU %d already is", apic_id, boot_cpu_id)); boot_cpu_id = apic_id; cpu_info[apic_id].cpu_bsp = 1; } mp_ncpus++; - if (apic_id >= mp_maxid) - mp_maxid = apic_id + 1; + if (apic_id > mp_maxid) + mp_maxid = apic_id; if (bootverbose) printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : "AP"); } void cpu_mp_setmaxid(void) { /* * mp_maxid should be already set by calls to cpu_add(). * Just sanity check its value here. */ if (mp_ncpus == 0) KASSERT(mp_maxid == 0, ("%s: mp_ncpus is zero, but mp_maxid is not", __func__)); else if (mp_ncpus == 1) mp_maxid = 0; else KASSERT(mp_maxid >= mp_ncpus - 1, ("%s: counters out of sync: max %d, count %d", __func__, mp_maxid, mp_ncpus)); } int cpu_mp_probe(void) { /* * Always record BSP in CPU map so that the mbuf init code works * correctly. */ all_cpus = 1; if (mp_ncpus == 0) { /* * No CPUs were found, so this must be a UP system. Setup * the variables to represent a system with a single CPU * with an id of 0. */ mp_ncpus = 1; return (0); } /* At least one CPU was found. */ if (mp_ncpus == 1) { /* * One CPU was found, so this must be a UP system with * an I/O APIC. */ mp_maxid = 0; return (0); } /* At least two CPUs were found. */ return (1); } /* * Initialize the IPI handlers and start up the AP's. */ void cpu_mp_start(void) { int i; /* Initialize the logical ID to APIC ID table. */ for (i = 0; i < MAXCPU; i++) cpu_apic_ids[i] = -1; /* Install an inter-CPU IPI for TLB invalidation */ setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYSIGT, SEL_KPL, 0); setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYSIGT, SEL_KPL, 0); setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYSIGT, SEL_KPL, 0); /* Install an inter-CPU IPI for forwarding hardclock() */ setidt(IPI_HARDCLOCK, IDTVEC(hardclock), SDT_SYSIGT, SEL_KPL, 0); /* Install an inter-CPU IPI for forwarding statclock() */ setidt(IPI_STATCLOCK, IDTVEC(statclock), SDT_SYSIGT, SEL_KPL, 0); #ifdef LAZY_SWITCH /* Install an inter-CPU IPI for lazy pmap release */ setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), SDT_SYSIGT, SEL_KPL, 0); #endif /* Install an inter-CPU IPI for all-CPU rendezvous */ setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0); /* Install an inter-CPU IPI for forcing an additional software trap */ setidt(IPI_AST, IDTVEC(cpuast), SDT_SYSIGT, SEL_KPL, 0); /* Install an inter-CPU IPI for CPU stop/restart */ setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0); mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); /* Set boot_cpu_id if needed. */ if (boot_cpu_id == -1) { boot_cpu_id = PCPU_GET(apic_id); cpu_info[boot_cpu_id].cpu_bsp = 1; } else KASSERT(boot_cpu_id == PCPU_GET(apic_id), ("BSP's APIC ID doesn't match boot_cpu_id")); cpu_apic_ids[0] = boot_cpu_id; /* Start each Application Processor */ start_all_aps(); /* Setup the initial logical CPUs info. */ logical_cpus = logical_cpus_mask = 0; if (cpu_feature & CPUID_HTT) logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; set_logical_apic_ids(); } /* * Print various information about the SMP system hardware and setup. */ void cpu_mp_announce(void) { int i, x; /* List CPUs */ printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); for (i = 1, x = 0; x < MAXCPU; x++) { if (cpu_info[x].cpu_present && !cpu_info[x].cpu_bsp) { KASSERT(i < mp_ncpus, ("mp_ncpus and actual cpus are out of whack")); printf(" cpu%d (AP): APIC ID: %2d\n", i++, x); } } } /* * AP CPU's call this to initialize themselves. */ void init_secondary(void) { struct pcpu *pc; u_int64_t msr, cr0; int cpu, gsel_tss; /* Set by the startup code for us to use */ cpu = bootAP; /* Init tss */ common_tss[cpu] = common_tss[0]; common_tss[cpu].tss_rsp0 = 0; /* not used until after switch */ gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu]; ssdtosyssd(&gdt_segs[GPROC0_SEL], (struct system_segment_descriptor *)&gdt[GPROC0_SEL]); lgdt(&r_gdt); /* does magic intra-segment return */ /* Get per-cpu data */ pc = &__pcpu[cpu]; /* prime data page for it to use */ pcpu_init(pc, cpu, sizeof(struct pcpu)); pc->pc_apic_id = cpu_apic_ids[cpu]; pc->pc_prvspace = pc; pc->pc_curthread = 0; pc->pc_tssp = &common_tss[cpu]; pc->pc_rsp0 = 0; wrmsr(MSR_FSBASE, 0); /* User value */ wrmsr(MSR_GSBASE, (u_int64_t)pc); wrmsr(MSR_KGSBASE, (u_int64_t)pc); /* XXX User value while we're in the kernel */ lidt(&r_idt); gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); ltr(gsel_tss); /* * Set to a known state: * Set by mpboot.s: CR0_PG, CR0_PE * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM */ cr0 = rcr0(); cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); load_cr0(cr0); /* Set up the fast syscall stuff */ msr = rdmsr(MSR_EFER) | EFER_SCE; wrmsr(MSR_EFER, msr); wrmsr(MSR_LSTAR, (u_int64_t)IDTVEC(fast_syscall)); wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32)); msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48); wrmsr(MSR_STAR, msr); wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D); /* Disable local apic just to be sure. */ lapic_disable(); /* signal our startup to the BSP. */ mp_naps++; /* Spin until the BSP releases the AP's. */ while (!aps_ready) ia32_pause(); /* set up CPU registers and state */ cpu_setregs(); /* set up FPU state on the AP */ fpuinit(); /* set up SSE registers */ enable_sse(); /* A quick check from sanity claus */ if (PCPU_GET(apic_id) != lapic_id()) { printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); panic("cpuid mismatch! boom!!"); } mtx_lock_spin(&ap_boot_mtx); /* Init local apic for irq's */ lapic_setup(); /* Set memory range attributes for this CPU to match the BSP */ mem_range_AP_init(); smp_cpus++; CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); /* Determine if we are a logical CPU. */ if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) logical_cpus_mask |= PCPU_GET(cpumask); /* Build our map of 'other' CPUs. */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); if (bootverbose) lapic_dump("AP"); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); smp_active = 1; /* historic */ } mtx_unlock_spin(&ap_boot_mtx); /* wait until all the AP's are up */ while (smp_started == 0) ia32_pause(); /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); /* NOTREACHED */ } /******************************************************************* * local functions and data */ /* * Set the APIC logical IDs. * * We want to cluster logical CPU's within the same APIC ID cluster. * Since logical CPU's are aligned simply filling in the clusters in * APIC ID order works fine. Note that this does not try to balance * the number of CPU's in each cluster. (XXX?) */ static void set_logical_apic_ids(void) { u_int apic_id, cluster, cluster_id; /* Force us to allocate cluster 0 at the start. */ cluster = -1; cluster_id = APIC_MAX_INTRACLUSTER_ID; for (apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present) continue; if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { cluster = ioapic_next_logical_cluster(); cluster_id = 0; } else cluster_id++; if (bootverbose) printf("APIC ID: physical %u, logical %u:%u\n", apic_id, cluster, cluster_id); lapic_set_logical_id(apic_id, cluster, cluster_id); } } /* * start each AP in our list */ static int start_all_aps(void) { u_char mpbiosreason; u_int32_t mpbioswarmvec; int apic_id, cpu, i; u_int64_t *pt4, *pt3, *pt2; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* install the AP 1st level boot code */ pmap_kenter(boot_address + KERNBASE, boot_address); bcopy(mptramp_start, (void *)((uintptr_t)boot_address + KERNBASE), bootMP_size); /* Locate the page tables, they'll be below the trampoline */ pt4 = (u_int64_t *)(uintptr_t)(mptramp_pagetables + KERNBASE); pt3 = pt4 + (PAGE_SIZE) / sizeof(u_int64_t); pt2 = pt3 + (PAGE_SIZE) / sizeof(u_int64_t); /* Create the initial 1GB replicated page tables */ for (i = 0; i < 512; i++) { /* Each slot of the level 4 pages points to the same level 3 page */ pt4[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + PAGE_SIZE); pt4[i] |= PG_V | PG_RW | PG_U; /* Each slot of the level 3 pages points to the same level 2 page */ pt3[i] = (u_int64_t)(uintptr_t)(mptramp_pagetables + (2 * PAGE_SIZE)); pt3[i] |= PG_V | PG_RW | PG_U; /* The level 2 page slots are mapped with 2MB pages for 1GB. */ pt2[i] = i * (2 * 1024 * 1024); pt2[i] |= PG_V | PG_RW | PG_PS | PG_U; } /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF); outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ /* start each AP */ cpu = 0; for (apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present || cpu_info[apic_id].cpu_bsp) continue; cpu++; /* save APIC ID for this logical ID */ cpu_apic_ids[cpu] = apic_id; /* allocate and set up an idle stack data page */ bootstacks[cpu] = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); bootSTK = (char *)bootstacks[cpu] + KSTACK_PAGES * PAGE_SIZE - 8; bootAP = cpu; /* attempt to start the Application Processor */ if (!start_ap(apic_id)) { /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; panic("AP #%d (PHY# %d) failed!", cpu, apic_id); } all_cpus |= (1 << cpu); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); /* restore the warmstart vector */ *(u_int32_t *) WARMBOOT_OFF = mpbioswarmvec; outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); /* number of APs actually started */ return mp_naps; } /* * This function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It isn't pretty, * but it seems to work. */ static int start_ap(int apic_id) { int vector, ms; int cpus; /* calculate the vector */ vector = (boot_address >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_naps; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* do an INIT IPI: assert RESET */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); /* wait for pending status end */ lapic_ipi_wait(-1); /* do an INIT IPI: deassert RESET */ lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); /* wait for pending status end */ DELAY(10000); /* wait ~10mS */ lapic_ipi_wait(-1); /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); lapic_ipi_wait(-1); DELAY(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); lapic_ipi_wait(-1); DELAY(200); /* wait ~200uS */ /* Wait up to 5 seconds for it to start. */ for (ms = 0; ms < 50; ms++) { if (mp_naps > cpus) return 1; /* return SUCCESS */ DELAY(100000); } return 0; /* return FAILURE */ } /* * Flush the TLB on all other CPU's */ static void smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) { u_int ncpu; ncpu = mp_ncpus - 1; /* does not shootdown self */ if (ncpu < 1) return; /* no other cpus */ mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); ipi_all_but_self(vector); while (smp_tlb_wait < ncpu) ia32_pause(); } /* * This is about as magic as it gets. fortune(1) has got similar code * for reversing bits in a word. Who thinks up this stuff?? * * Yes, it does appear to be consistently faster than: * while (i = ffs(m)) { * m >>= i; * bits++; * } * and * while (lsb = (m & -m)) { // This is magic too * m &= ~lsb; // or: m ^= lsb * bits++; * } * Both of these latter forms do some very strange things on gcc-3.1 with * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2. * There is probably an SSE or MMX popcnt instruction. * * I wonder if this should be in libkern? * * XXX Stop the presses! Another one: * static __inline u_int32_t * popcnt1(u_int32_t v) * { * v -= ((v >> 1) & 0x55555555); * v = (v & 0x33333333) + ((v >> 2) & 0x33333333); * v = (v + (v >> 4)) & 0x0F0F0F0F; * return (v * 0x01010101) >> 24; * } * The downside is that it has a multiply. With a pentium3 with * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use * an imull, and in that case it is faster. In most other cases * it appears slightly slower. * * Another variant (also from fortune): * #define BITCOUNT(x) (((BX_(x)+(BX_(x)>>4)) & 0x0F0F0F0F) % 255) * #define BX_(x) ((x) - (((x)>>1)&0x77777777) \ * - (((x)>>2)&0x33333333) \ * - (((x)>>3)&0x11111111)) */ static __inline u_int32_t popcnt(u_int32_t m) { m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1); m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2); m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4); m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8); m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16); return m; } static void smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { int ncpu, othercpus; othercpus = mp_ncpus - 1; if (mask == (u_int)-1) { ncpu = othercpus; if (ncpu < 1) return; } else { mask &= ~PCPU_GET(cpumask); if (mask == 0) return; ncpu = popcnt(mask); if (ncpu > othercpus) { /* XXX this should be a panic offence */ printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", ncpu, othercpus); ncpu = othercpus; } /* XXX should be a panic, implied by mask == 0 above */ if (ncpu < 1) return; } mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); if (mask == (u_int)-1) ipi_all_but_self(vector); else ipi_selected(mask, vector); while (smp_tlb_wait < ncpu) ia32_pause(); } void smp_invltlb(void) { if (smp_started) smp_tlb_shootdown(IPI_INVLTLB, 0, 0); } void smp_invlpg(vm_offset_t addr) { if (smp_started) smp_tlb_shootdown(IPI_INVLPG, addr, 0); } void smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); } void smp_masked_invltlb(u_int mask) { if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); } void smp_masked_invlpg(u_int mask, vm_offset_t addr) { if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); } void smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); } /* * For statclock, we send an IPI to all CPU's to have them call this * function. */ void forwarded_statclock(struct clockframe frame) { struct thread *td; CTR0(KTR_SMP, "forwarded_statclock"); td = curthread; td->td_intr_nesting_level++; if (profprocs != 0) profclock(&frame); if (pscnt == psdiv) statclock(&frame); td->td_intr_nesting_level--; } void forward_statclock(void) { int map; CTR0(KTR_SMP, "forward_statclock"); if (!smp_started || cold || panicstr) return; map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); if (map != 0) ipi_selected(map, IPI_STATCLOCK); } /* * For each hardclock(), we send an IPI to all other CPU's to have them * execute this function. It would be nice to reduce contention on * sched_lock if we could simply peek at the CPU to determine the user/kernel * state and call hardclock_process() on the CPU receiving the clock interrupt * and then just use a simple IPI to handle any ast's if needed. */ void forwarded_hardclock(struct clockframe frame) { struct thread *td; CTR0(KTR_SMP, "forwarded_hardclock"); td = curthread; td->td_intr_nesting_level++; hardclock_process(&frame); td->td_intr_nesting_level--; } void forward_hardclock(void) { u_int map; CTR0(KTR_SMP, "forward_hardclock"); if (!smp_started || cold || panicstr) return; map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); if (map != 0) ipi_selected(map, IPI_HARDCLOCK); } /* * send an IPI to a set of cpus. */ void ipi_selected(u_int32_t cpus, u_int ipi) { int cpu; CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); while ((cpu = ffs(cpus)) != 0) { cpu--; KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); cpus &= ~(1 << cpu); } } /* * send an IPI INTerrupt containing 'vector' to all CPUs, including myself */ void ipi_all(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); } /* * send an IPI to all CPUs EXCEPT myself */ void ipi_all_but_self(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } /* * send an IPI to myself */ void ipi_self(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ static void release_aps(void *dummy __unused) { if (mp_ncpus == 1) return; mtx_lock_spin(&sched_lock); atomic_store_rel_int(&aps_ready, 1); while (smp_started == 0) ia32_pause(); mtx_unlock_spin(&sched_lock); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); static int sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) { u_int mask; int error; mask = hlt_cpus_mask; error = sysctl_handle_int(oidp, &mask, 0, req); if (error || !req->newptr) return (error); if (logical_cpus_mask != 0 && (mask & logical_cpus_mask) == logical_cpus_mask) hlt_logical_cpus = 1; else hlt_logical_cpus = 0; if ((mask & all_cpus) == all_cpus) mask &= ~(1<<0); hlt_cpus_mask = mask; return (error); } SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hlt_cpus, "IU", ""); static int sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) { int disable, error; disable = hlt_logical_cpus; error = sysctl_handle_int(oidp, &disable, 0, req); if (error || !req->newptr) return (error); if (disable) hlt_cpus_mask |= logical_cpus_mask; else hlt_cpus_mask &= ~logical_cpus_mask; if ((hlt_cpus_mask & all_cpus) == all_cpus) hlt_cpus_mask &= ~(1<<0); hlt_logical_cpus = disable; return (error); } static void cpu_hlt_setup(void *dummy __unused) { if (logical_cpus_mask != 0) { TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", &hlt_logical_cpus); sysctl_ctx_init(&logical_cpu_clist); SYSCTL_ADD_PROC(&logical_cpu_clist, SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hlt_logical_cpus, "IU", ""); SYSCTL_ADD_UINT(&logical_cpu_clist, SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD, &logical_cpus_mask, 0, ""); if (hlt_logical_cpus) hlt_cpus_mask |= logical_cpus_mask; } } SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL); int mp_grab_cpu_hlt(void) { u_int mask = PCPU_GET(cpumask); int retval; retval = mask & hlt_cpus_mask; while (mask & hlt_cpus_mask) __asm __volatile("sti; hlt" : : : "memory"); return (retval); } Index: head/sys/i386/i386/mp_machdep.c =================================================================== --- head/sys/i386/i386/mp_machdep.c (revision 123125) +++ head/sys/i386/i386/mp_machdep.c (revision 123126) @@ -1,1251 +1,1251 @@ /*- * Copyright (c) 1996, by Steve Passe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. The name of the developer may NOT be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_apic.h" #include "opt_cpu.h" #include "opt_kstack_pages.h" #if !defined(lint) #if !defined(SMP) #error How did you get here? #endif #if defined(I386_CPU) && !defined(COMPILING_LINT) #error SMP not supported with I386_CPU #endif #ifndef DEV_APIC #error The apic device is required for SMP, add "device apic" to your config file. #endif #if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) #error SMP not supported with CPU_DISABLE_CMPXCHG #endif #endif /* not lint */ #include #include #include #include /* cngetc() */ #ifdef GPROF #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** COUNT_XINVLTLB_HITS */ #include #include #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) #define WARMBOOT_SEG (KERNBASE + 0x0469) #define CMOS_REG (0x70) #define CMOS_DATA (0x71) #define BIOS_RESET (0x0f) #define BIOS_WARM (0x0a) /* * this code MUST be enabled here and in mpboot.s. * it follows the very early stages of AP boot by placing values in CMOS ram. * it NORMALLY will never be needed and thus the primitive method for enabling. * #define CHECK_POINTS */ #if defined(CHECK_POINTS) && !defined(PC98) #define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) #define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) #define CHECK_INIT(D); \ CHECK_WRITE(0x34, (D)); \ CHECK_WRITE(0x35, (D)); \ CHECK_WRITE(0x36, (D)); \ CHECK_WRITE(0x37, (D)); \ CHECK_WRITE(0x38, (D)); \ CHECK_WRITE(0x39, (D)); #define CHECK_PRINT(S); \ printf("%s: %d, %d, %d, %d, %d, %d\n", \ (S), \ CHECK_READ(0x34), \ CHECK_READ(0x35), \ CHECK_READ(0x36), \ CHECK_READ(0x37), \ CHECK_READ(0x38), \ CHECK_READ(0x39)); #else /* CHECK_POINTS */ #define CHECK_INIT(D) #define CHECK_PRINT(S) #define CHECK_WRITE(A, D) #endif /* CHECK_POINTS */ /* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 #define MP_PROBE_POST 0x11 #define MPTABLE_PASS1_POST 0x12 #define MP_START_POST 0x13 #define MP_ENABLE_POST 0x14 #define MPTABLE_PASS2_POST 0x15 #define START_ALL_APS_POST 0x16 #define INSTALL_AP_TRAMP_POST 0x17 #define START_AP_POST 0x18 #define MP_ANNOUNCE_POST 0x19 /* lock region used by kernel profiling */ int mcount_lock; /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; int mp_naps; /* # of Applications processors */ int boot_cpu_id = -1; /* designated BSP */ extern int nkpt; /* * CPU topology map datastructures for HTT. (XXX) */ struct cpu_group mp_groups[MAXCPU]; struct cpu_top mp_top; struct cpu_top *smp_topology; /* AP uses this during bootstrap. Do not staticize. */ char *bootSTK; static int bootAP; /* Hotwire a 0->4MB V==P mapping */ extern pt_entry_t *KPTphys; /* SMP page table page */ extern pt_entry_t *SMPpt; struct pcb stoppcbs[MAXCPU]; /* Variables needed for SMP tlb shootdown. */ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; struct mtx smp_tlb_mtx; /* * Local data and functions. */ static u_int logical_cpus; static u_int logical_cpus_mask; /* used to hold the AP's until we are ready to release them */ static struct mtx ap_boot_mtx; /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready = 0; /* * Store data from cpu_add() until later in the boot when we actually setup * the APs. */ struct cpu_info { int cpu_present:1; int cpu_bsp:1; } static cpu_info[MAXCPU]; static int cpu_apic_ids[MAXCPU]; static u_int boot_address; static void set_logical_apic_ids(void); static int start_all_aps(void); static void install_ap_tramp(void); static int start_ap(int apic_id); static void release_aps(void *dummy); static int hlt_cpus_mask; static int hlt_logical_cpus; static struct sysctl_ctx_list logical_cpu_clist; /* * Calculate usable address in base memory for AP trampoline code. */ u_int mp_bootaddress(u_int basemem) { POSTCODE(MP_BOOTADDRESS_POST); boot_address = trunc_page(basemem); /* round down to 4k boundary */ if ((basemem - boot_address) < bootMP_size) boot_address -= PAGE_SIZE; /* not enough, lower by 4k */ return boot_address; } void cpu_add(u_int apic_id, char boot_cpu) { if (apic_id > MAXCPU) { printf("SMP: CPU %d exceeds maximum CPU %d, ignoring\n", apic_id, MAXCPU); return; } KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", apic_id)); cpu_info[apic_id].cpu_present = 1; if (boot_cpu) { KASSERT(boot_cpu_id == -1, ("CPU %d claims to be BSP, but CPU %d already is", apic_id, boot_cpu_id)); boot_cpu_id = apic_id; cpu_info[apic_id].cpu_bsp = 1; } mp_ncpus++; if (bootverbose) printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : "AP"); } void cpu_mp_setmaxid(void) { - mp_maxid = MAXCPU; + mp_maxid = MAXCPU - 1; } int cpu_mp_probe(void) { /* * Always record BSP in CPU map so that the mbuf init code works * correctly. */ all_cpus = 1; if (mp_ncpus == 0) { /* * No CPUs were found, so this must be a UP system. Setup * the variables to represent a system with a single CPU * with an id of 0. */ mp_ncpus = 1; return (0); } /* At least one CPU was found. */ if (mp_ncpus == 1) { /* * One CPU was found, so this must be a UP system with * an I/O APIC. */ return (0); } /* At least two CPUs were found. */ return (1); } /* * Initialize the IPI handlers and start up the AP's. */ void cpu_mp_start(void) { int i; POSTCODE(MP_START_POST); /* Initialize the logical ID to APIC ID table. */ for (i = 0; i < MAXCPU; i++) cpu_apic_ids[i] = -1; /* Install an inter-CPU IPI for TLB invalidation */ setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); setidt(IPI_INVLPG, IDTVEC(invlpg), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); setidt(IPI_INVLRNG, IDTVEC(invlrng), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for forwarding hardclock() */ setidt(IPI_HARDCLOCK, IDTVEC(hardclock), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for forwarding statclock() */ setidt(IPI_STATCLOCK, IDTVEC(statclock), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for lazy pmap release */ setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for all-CPU rendezvous */ setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for forcing an additional software trap */ setidt(IPI_AST, IDTVEC(cpuast), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); /* Install an inter-CPU IPI for CPU stop/restart */ setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); /* Set boot_cpu_id if needed. */ if (boot_cpu_id == -1) { boot_cpu_id = PCPU_GET(apic_id); cpu_info[boot_cpu_id].cpu_bsp = 1; } else KASSERT(boot_cpu_id == PCPU_GET(apic_id), ("BSP's APIC ID doesn't match boot_cpu_id")); cpu_apic_ids[0] = boot_cpu_id; /* Start each Application Processor */ start_all_aps(); /* Setup the initial logical CPUs info. */ logical_cpus = logical_cpus_mask = 0; if (cpu_feature & CPUID_HTT) logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; set_logical_apic_ids(); } /* * Print various information about the SMP system hardware and setup. */ void cpu_mp_announce(void) { int i, x; POSTCODE(MP_ANNOUNCE_POST); /* List CPUs */ printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); for (i = 1, x = 0; x < MAXCPU; x++) { if (cpu_info[x].cpu_present && !cpu_info[x].cpu_bsp) { KASSERT(i < mp_ncpus, ("mp_ncpus and actual cpus are out of whack")); printf(" cpu%d (AP): APIC ID: %2d\n", i++, x); } } } /* * AP CPU's call this to initialize themselves. */ void init_secondary(void) { int gsel_tss; int x, myid; u_int cr0; /* bootAP is set in start_ap() to our ID. */ myid = bootAP; gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; gdt_segs[GPROC0_SEL].ssd_base = (int) &SMP_prvspace[myid].pcpu.pc_common_tss; SMP_prvspace[myid].pcpu.pc_prvspace = &SMP_prvspace[myid].pcpu; for (x = 0; x < NGDT; x++) { ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); } r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; r_gdt.rd_base = (int) &gdt[myid * NGDT]; lgdt(&r_gdt); /* does magic intra-segment return */ lidt(&r_idt); lldt(_default_ldt); PCPU_SET(currentldt, _default_ldt); gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */ PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd); PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); ltr(gsel_tss); /* * Set to a known state: * Set by mpboot.s: CR0_PG, CR0_PE * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM */ cr0 = rcr0(); cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); load_cr0(cr0); CHECK_WRITE(0x38, 5); /* Disable local APIC just to be sure. */ lapic_disable(); /* signal our startup to the BSP. */ mp_naps++; CHECK_WRITE(0x39, 6); /* Spin until the BSP releases the AP's. */ while (!aps_ready) ia32_pause(); /* BSP may have changed PTD while we were waiting */ invltlb(); pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); #if defined(I586_CPU) && !defined(NO_F00F_HACK) lidt(&r_idt); #endif /* set up CPU registers and state */ cpu_setregs(); /* set up FPU state on the AP */ npxinit(__INITIAL_NPXCW__); /* set up SSE registers */ enable_sse(); /* A quick check from sanity claus */ if (PCPU_GET(apic_id) != lapic_id()) { printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); printf("SMP: actual apic_id = %d\n", lapic_id()); printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]); panic("cpuid mismatch! boom!!"); } mtx_lock_spin(&ap_boot_mtx); /* Init local apic for irq's */ lapic_setup(); /* Set memory range attributes for this CPU to match the BSP */ mem_range_AP_init(); smp_cpus++; CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); /* Determine if we are a logical CPU. */ if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) logical_cpus_mask |= PCPU_GET(cpumask); /* Build our map of 'other' CPUs. */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); if (bootverbose) lapic_dump("AP"); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); smp_active = 1; /* historic */ } mtx_unlock_spin(&ap_boot_mtx); /* wait until all the AP's are up */ while (smp_started == 0) ia32_pause(); /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); /* NOTREACHED */ } /******************************************************************* * local functions and data */ /* * Set the APIC logical IDs. * * We want to cluster logical CPU's within the same APIC ID cluster. * Since logical CPU's are aligned simply filling in the clusters in * APIC ID order works fine. Note that this does not try to balance * the number of CPU's in each cluster. (XXX?) */ static void set_logical_apic_ids(void) { u_int apic_id, cluster, cluster_id; /* Force us to allocate cluster 0 at the start. */ cluster = -1; cluster_id = APIC_MAX_INTRACLUSTER_ID; for (apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present) continue; if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { cluster = ioapic_next_logical_cluster(); cluster_id = 0; } else cluster_id++; if (bootverbose) printf("APIC ID: physical %u, logical %u:%u\n", apic_id, cluster, cluster_id); lapic_set_logical_id(apic_id, cluster, cluster_id); } } /* * start each AP in our list */ static int start_all_aps(void) { #ifndef PC98 u_char mpbiosreason; #endif u_long mpbioswarmvec; struct pcpu *pc; char *stack; uintptr_t kptbase; int i, pg, apic_id, cpu; POSTCODE(START_ALL_APS_POST); mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* install the AP 1st level boot code */ install_ap_tramp(); /* save the current value of the warm-start vector */ mpbioswarmvec = *((u_long *) WARMBOOT_OFF); #ifndef PC98 outb(CMOS_REG, BIOS_RESET); mpbiosreason = inb(CMOS_DATA); #endif /* set up temporary P==V mapping for AP boot */ /* XXX this is a hack, we should boot the AP on its own stack/PTD */ kptbase = (uintptr_t)(void *)KPTphys; for (i = 0; i < NKPT; i++) PTD[i] = (pd_entry_t)(PG_V | PG_RW | ((kptbase + i * PAGE_SIZE) & PG_FRAME)); invltlb(); /* start each AP */ for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) { if (!cpu_info[apic_id].cpu_present || cpu_info[apic_id].cpu_bsp) continue; cpu++; /* save APIC ID for this logical ID */ cpu_apic_ids[cpu] = apic_id; /* first page of AP's private space */ pg = cpu * i386_btop(sizeof(struct privatespace)); /* allocate a new private data page */ pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE); /* wire it into the private page table page */ SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(pc)); /* allocate and set up an idle stack data page */ stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ for (i = 0; i < KSTACK_PAGES; i++) SMPpt[pg + 1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); /* prime data page for it to use */ pcpu_init(pc, cpu, sizeof(struct pcpu)); pc->pc_apic_id = apic_id; /* setup a vector to our boot code */ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); #ifndef PC98 outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES * PAGE_SIZE]; bootAP = cpu; /* attempt to start the Application Processor */ CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(apic_id)) { printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } CHECK_PRINT("trace"); /* show checkpoints */ all_cpus |= (1 << cpu); /* record AP in CPU map */ } /* build our map of 'other' CPUs */ PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); /* restore the warmstart vector */ *(u_long *) WARMBOOT_OFF = mpbioswarmvec; #ifndef PC98 outb(CMOS_REG, BIOS_RESET); outb(CMOS_DATA, mpbiosreason); #endif /* * Set up the idle context for the BSP. Similar to above except * that some was done by locore, some by pmap.c and some is implicit * because the BSP is cpu#0 and the page is initially zero and also * because we can refer to variables by name on the BSP.. */ /* Allocate and setup BSP idle stack */ stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); for (i = 0; i < KSTACK_PAGES; i++) SMPpt[1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); for (i = 0; i < NKPT; i++) PTD[i] = 0; pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); /* number of APs actually started */ return mp_naps; } /* * load the 1st level AP boot code into base memory. */ /* targets for relocation */ extern void bigJump(void); extern void bootCodeSeg(void); extern void bootDataSeg(void); extern void MPentry(void); extern u_int MP_GDT; extern u_int mp_gdtbase; static void install_ap_tramp(void) { int x; int size = *(int *) ((u_long) & bootMP_size); u_char *src = (u_char *) ((u_long) bootMP); u_char *dst = (u_char *) boot_address + KERNBASE; u_int boot_base = (u_int) bootMP; u_int8_t *dst8; u_int16_t *dst16; u_int32_t *dst32; POSTCODE(INSTALL_AP_TRAMP_POST); pmap_kenter(boot_address + KERNBASE, boot_address); for (x = 0; x < size; ++x) *dst++ = *src++; /* * modify addresses in code we just moved to basemem. unfortunately we * need fairly detailed info about mpboot.s for this to work. changes * to mpboot.s might require changes here. */ /* boot code is located in KERNEL space */ dst = (u_char *) boot_address + KERNBASE; /* modify the lgdt arg */ dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); *dst32 = boot_address + ((u_int) & MP_GDT - boot_base); /* modify the ljmp target for MPentry() */ dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); *dst32 = ((u_int) MPentry - KERNBASE); /* modify the target for boot code segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_address & 0xffff; *dst8 = ((u_int) boot_address >> 16) & 0xff; /* modify the target for boot data segment */ dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); dst8 = (u_int8_t *) (dst16 + 1); *dst16 = (u_int) boot_address & 0xffff; *dst8 = ((u_int) boot_address >> 16) & 0xff; } /* * This function starts the AP (application processor) identified * by the APIC ID 'physicalCpu'. It does quite a "song and dance" * to accomplish this. This is necessary because of the nuances * of the different hardware we might encounter. It isn't pretty, * but it seems to work. */ static int start_ap(int apic_id) { int vector, ms; int cpus; POSTCODE(START_AP_POST); /* calculate the vector */ vector = (boot_address >> 12) & 0xff; /* used as a watchpoint to signal AP startup */ cpus = mp_naps; /* * first we do an INIT/RESET IPI this INIT IPI might be run, reseting * and running the target CPU. OR this INIT IPI might be latched (P5 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be * ignored. */ /* do an INIT IPI: assert RESET */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); /* wait for pending status end */ lapic_ipi_wait(-1); /* do an INIT IPI: deassert RESET */ lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); /* wait for pending status end */ DELAY(10000); /* wait ~10mS */ lapic_ipi_wait(-1); /* * next we do a STARTUP IPI: the previous INIT IPI might still be * latched, (P5 bug) this 1st STARTUP would then terminate * immediately, and the previously started INIT IPI would continue. OR * the previous INIT IPI has already run. and this STARTUP IPI will * run. OR the previous INIT IPI was ignored. and this STARTUP IPI * will run. */ /* do a STARTUP IPI */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); lapic_ipi_wait(-1); DELAY(200); /* wait ~200uS */ /* * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is * recognized after hardware RESET or INIT IPI. */ lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | vector, apic_id); lapic_ipi_wait(-1); DELAY(200); /* wait ~200uS */ /* Wait up to 5 seconds for it to start. */ for (ms = 0; ms < 5000; ms++) { if (mp_naps > cpus) return 1; /* return SUCCESS */ DELAY(1000); } return 0; /* return FAILURE */ } #ifdef COUNT_XINVLTLB_HITS u_int xhits_gbl[MAXCPU]; u_int xhits_pg[MAXCPU]; u_int xhits_rng[MAXCPU]; SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, sizeof(xhits_gbl), "IU", ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, sizeof(xhits_pg), "IU", ""); SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, sizeof(xhits_rng), "IU", ""); u_int ipi_global; u_int ipi_page; u_int ipi_range; u_int ipi_range_size; SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 0, ""); u_int ipi_masked_global; u_int ipi_masked_page; u_int ipi_masked_range; u_int ipi_masked_range_size; SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, &ipi_masked_global, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, &ipi_masked_page, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, &ipi_masked_range, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, &ipi_masked_range_size, 0, ""); #endif /* COUNT_XINVLTLB_HITS */ /* * Flush the TLB on all other CPU's */ static void smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) { u_int ncpu; ncpu = mp_ncpus - 1; /* does not shootdown self */ if (ncpu < 1) return; /* no other cpus */ mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); ipi_all_but_self(vector); while (smp_tlb_wait < ncpu) ia32_pause(); } /* * This is about as magic as it gets. fortune(1) has got similar code * for reversing bits in a word. Who thinks up this stuff?? * * Yes, it does appear to be consistently faster than: * while (i = ffs(m)) { * m >>= i; * bits++; * } * and * while (lsb = (m & -m)) { // This is magic too * m &= ~lsb; // or: m ^= lsb * bits++; * } * Both of these latter forms do some very strange things on gcc-3.1 with * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2. * There is probably an SSE or MMX popcnt instruction. * * I wonder if this should be in libkern? * * XXX Stop the presses! Another one: * static __inline u_int32_t * popcnt1(u_int32_t v) * { * v -= ((v >> 1) & 0x55555555); * v = (v & 0x33333333) + ((v >> 2) & 0x33333333); * v = (v + (v >> 4)) & 0x0F0F0F0F; * return (v * 0x01010101) >> 24; * } * The downside is that it has a multiply. With a pentium3 with * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use * an imull, and in that case it is faster. In most other cases * it appears slightly slower. * * Another variant (also from fortune): * #define BITCOUNT(x) (((BX_(x)+(BX_(x)>>4)) & 0x0F0F0F0F) % 255) * #define BX_(x) ((x) - (((x)>>1)&0x77777777) \ * - (((x)>>2)&0x33333333) \ * - (((x)>>3)&0x11111111)) */ static __inline u_int32_t popcnt(u_int32_t m) { m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1); m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2); m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4); m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8); m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16); return m; } static void smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { int ncpu, othercpus; othercpus = mp_ncpus - 1; if (mask == (u_int)-1) { ncpu = othercpus; if (ncpu < 1) return; } else { mask &= ~PCPU_GET(cpumask); if (mask == 0) return; ncpu = popcnt(mask); if (ncpu > othercpus) { /* XXX this should be a panic offence */ printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", ncpu, othercpus); ncpu = othercpus; } /* XXX should be a panic, implied by mask == 0 above */ if (ncpu < 1) return; } mtx_assert(&smp_tlb_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); if (mask == (u_int)-1) ipi_all_but_self(vector); else ipi_selected(mask, vector); while (smp_tlb_wait < ncpu) ia32_pause(); } void smp_invltlb(void) { if (smp_started) { smp_tlb_shootdown(IPI_INVLTLB, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_global++; #endif } } void smp_invlpg(vm_offset_t addr) { if (smp_started) { smp_tlb_shootdown(IPI_INVLPG, addr, 0); #ifdef COUNT_XINVLTLB_HITS ipi_page++; #endif } } void smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) { smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); #ifdef COUNT_XINVLTLB_HITS ipi_range++; ipi_range_size += (addr2 - addr1) / PAGE_SIZE; #endif } } void smp_masked_invltlb(u_int mask) { if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_global++; #endif } } void smp_masked_invlpg(u_int mask, vm_offset_t addr) { if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_page++; #endif } } void smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) { if (smp_started) { smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); #ifdef COUNT_XINVLTLB_HITS ipi_masked_range++; ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; #endif } } /* * For statclock, we send an IPI to all CPU's to have them call this * function. */ void forwarded_statclock(struct clockframe frame) { struct thread *td; CTR0(KTR_SMP, "forwarded_statclock"); td = curthread; td->td_intr_nesting_level++; if (profprocs != 0) profclock(&frame); if (pscnt == psdiv) statclock(&frame); td->td_intr_nesting_level--; } void forward_statclock(void) { int map; CTR0(KTR_SMP, "forward_statclock"); if (!smp_started || cold || panicstr) return; map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); if (map != 0) ipi_selected(map, IPI_STATCLOCK); } /* * For each hardclock(), we send an IPI to all other CPU's to have them * execute this function. It would be nice to reduce contention on * sched_lock if we could simply peek at the CPU to determine the user/kernel * state and call hardclock_process() on the CPU receiving the clock interrupt * and then just use a simple IPI to handle any ast's if needed. */ void forwarded_hardclock(struct clockframe frame) { struct thread *td; CTR0(KTR_SMP, "forwarded_hardclock"); td = curthread; td->td_intr_nesting_level++; hardclock_process(&frame); td->td_intr_nesting_level--; } void forward_hardclock(void) { u_int map; CTR0(KTR_SMP, "forward_hardclock"); if (!smp_started || cold || panicstr) return; map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); if (map != 0) ipi_selected(map, IPI_HARDCLOCK); } /* * send an IPI to a set of cpus. */ void ipi_selected(u_int32_t cpus, u_int ipi) { int cpu; CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); while ((cpu = ffs(cpus)) != 0) { cpu--; KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu)); lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); cpus &= ~(1 << cpu); } } /* * send an IPI INTerrupt containing 'vector' to all CPUs, including myself */ void ipi_all(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); } /* * send an IPI to all CPUs EXCEPT myself */ void ipi_all_but_self(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } /* * send an IPI to myself */ void ipi_self(u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } /* * This is called once the rest of the system is up and running and we're * ready to let the AP's out of the pen. */ static void release_aps(void *dummy __unused) { if (mp_ncpus == 1) return; mtx_lock_spin(&sched_lock); atomic_store_rel_int(&aps_ready, 1); while (smp_started == 0) ia32_pause(); mtx_unlock_spin(&sched_lock); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); static int sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) { u_int mask; int error; mask = hlt_cpus_mask; error = sysctl_handle_int(oidp, &mask, 0, req); if (error || !req->newptr) return (error); if (logical_cpus_mask != 0 && (mask & logical_cpus_mask) == logical_cpus_mask) hlt_logical_cpus = 1; else hlt_logical_cpus = 0; if ((mask & all_cpus) == all_cpus) mask &= ~(1<<0); hlt_cpus_mask = mask; return (error); } SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hlt_cpus, "IU", ""); static int sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) { int disable, error; disable = hlt_logical_cpus; error = sysctl_handle_int(oidp, &disable, 0, req); if (error || !req->newptr) return (error); if (disable) hlt_cpus_mask |= logical_cpus_mask; else hlt_cpus_mask &= ~logical_cpus_mask; if ((hlt_cpus_mask & all_cpus) == all_cpus) hlt_cpus_mask &= ~(1<<0); hlt_logical_cpus = disable; return (error); } static void cpu_hlt_setup(void *dummy __unused) { if (logical_cpus_mask != 0) { TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", &hlt_logical_cpus); sysctl_ctx_init(&logical_cpu_clist); SYSCTL_ADD_PROC(&logical_cpu_clist, SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_hlt_logical_cpus, "IU", ""); SYSCTL_ADD_UINT(&logical_cpu_clist, SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD, &logical_cpus_mask, 0, ""); if (hlt_logical_cpus) hlt_cpus_mask |= logical_cpus_mask; } } SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL); int mp_grab_cpu_hlt(void) { u_int mask = PCPU_GET(cpumask); int retval; retval = mask & hlt_cpus_mask; while (mask & hlt_cpus_mask) __asm __volatile("sti; hlt" : : : "memory"); return (retval); } Index: head/sys/kern/sched_ule.c =================================================================== --- head/sys/kern/sched_ule.c (revision 123125) +++ head/sys/kern/sched_ule.c (revision 123126) @@ -1,1499 +1,1499 @@ /*- * Copyright (c) 2002-2003, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #endif #ifdef KTRACE #include #include #endif #include #include #define KTR_ULE KTR_NFS /* decay 95% of `p_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ /* XXX This is bogus compatability crap for ps */ static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, ""); static void sched_setup(void *dummy); SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL) static SYSCTL_NODE(_kern, OID_AUTO, sched, CTLFLAG_RW, 0, "SCHED"); static int sched_strict; SYSCTL_INT(_kern_sched, OID_AUTO, strict, CTLFLAG_RD, &sched_strict, 0, ""); static int slice_min = 1; SYSCTL_INT(_kern_sched, OID_AUTO, slice_min, CTLFLAG_RW, &slice_min, 0, ""); static int slice_max = 10; SYSCTL_INT(_kern_sched, OID_AUTO, slice_max, CTLFLAG_RW, &slice_max, 0, ""); int realstathz; int tickincr = 1; #ifdef SMP /* Callout to handle load balancing SMP systems. */ static struct callout kseq_lb_callout; #endif /* * These datastructures are allocated within their parent datastructure but * are scheduler specific. */ struct ke_sched { int ske_slice; struct runq *ske_runq; /* The following variables are only used for pctcpu calculation */ int ske_ltick; /* Last tick that we were running on */ int ske_ftick; /* First tick that we were running on */ int ske_ticks; /* Tick count */ /* CPU that we have affinity for. */ u_char ske_cpu; }; #define ke_slice ke_sched->ske_slice #define ke_runq ke_sched->ske_runq #define ke_ltick ke_sched->ske_ltick #define ke_ftick ke_sched->ske_ftick #define ke_ticks ke_sched->ske_ticks #define ke_cpu ke_sched->ske_cpu #define ke_assign ke_procq.tqe_next #define KEF_ASSIGNED KEF_SCHED0 /* KSE is being migrated. */ #define KEF_BOUND KEF_SCHED1 /* KSE can not migrate. */ struct kg_sched { int skg_slptime; /* Number of ticks we vol. slept */ int skg_runtime; /* Number of ticks we were running */ }; #define kg_slptime kg_sched->skg_slptime #define kg_runtime kg_sched->skg_runtime struct td_sched { int std_slptime; }; #define td_slptime td_sched->std_slptime struct td_sched td_sched; struct ke_sched ke_sched; struct kg_sched kg_sched; struct ke_sched *kse0_sched = &ke_sched; struct kg_sched *ksegrp0_sched = &kg_sched; struct p_sched *proc0_sched = NULL; struct td_sched *thread0_sched = &td_sched; /* * The priority is primarily determined by the interactivity score. Thus, we * give lower(better) priorities to kse groups that use less CPU. The nice * value is then directly added to this to allow nice to have some effect * on latency. * * PRI_RANGE: Total priority range for timeshare threads. * PRI_NRESV: Number of nice values. * PRI_BASE: The start of the dynamic range. */ #define SCHED_PRI_RANGE (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1) #define SCHED_PRI_NRESV ((PRIO_MAX - PRIO_MIN) + 1) #define SCHED_PRI_NHALF (SCHED_PRI_NRESV / 2) #define SCHED_PRI_BASE (PRI_MIN_TIMESHARE) #define SCHED_PRI_INTERACT(score) \ ((score) * SCHED_PRI_RANGE / SCHED_INTERACT_MAX) /* * These determine the interactivity of a process. * * SLP_RUN_MAX: Maximum amount of sleep time + run time we'll accumulate * before throttling back. * SLP_RUN_FORK: Maximum slp+run time to inherit at fork time. * INTERACT_MAX: Maximum interactivity value. Smaller is better. * INTERACT_THRESH: Threshhold for placement on the current runq. */ #define SCHED_SLP_RUN_MAX ((hz * 5) << 10) #define SCHED_SLP_RUN_FORK ((hz / 2) << 10) #define SCHED_INTERACT_MAX (100) #define SCHED_INTERACT_HALF (SCHED_INTERACT_MAX / 2) #define SCHED_INTERACT_THRESH (30) /* * These parameters and macros determine the size of the time slice that is * granted to each thread. * * SLICE_MIN: Minimum time slice granted, in units of ticks. * SLICE_MAX: Maximum time slice granted. * SLICE_RANGE: Range of available time slices scaled by hz. * SLICE_SCALE: The number slices granted per val in the range of [0, max]. * SLICE_NICE: Determine the amount of slice granted to a scaled nice. * SLICE_NTHRESH: The nice cutoff point for slice assignment. */ #define SCHED_SLICE_MIN (slice_min) #define SCHED_SLICE_MAX (slice_max) #define SCHED_SLICE_NTHRESH (SCHED_PRI_NHALF - 1) #define SCHED_SLICE_RANGE (SCHED_SLICE_MAX - SCHED_SLICE_MIN + 1) #define SCHED_SLICE_SCALE(val, max) (((val) * SCHED_SLICE_RANGE) / (max)) #define SCHED_SLICE_NICE(nice) \ (SCHED_SLICE_MAX - SCHED_SLICE_SCALE((nice), SCHED_SLICE_NTHRESH)) /* * This macro determines whether or not the kse belongs on the current or * next run queue. */ #define SCHED_INTERACTIVE(kg) \ (sched_interact_score(kg) < SCHED_INTERACT_THRESH) #define SCHED_CURR(kg, ke) \ (ke->ke_thread->td_priority != kg->kg_user_pri || \ SCHED_INTERACTIVE(kg)) /* * Cpu percentage computation macros and defines. * * SCHED_CPU_TIME: Number of seconds to average the cpu usage across. * SCHED_CPU_TICKS: Number of hz ticks to average the cpu usage across. */ #define SCHED_CPU_TIME 10 #define SCHED_CPU_TICKS (hz * SCHED_CPU_TIME) /* * kseq - per processor runqs and statistics. */ #define KSEQ_NCLASS (PRI_IDLE + 1) /* Number of run classes. */ struct kseq { struct runq ksq_idle; /* Queue of IDLE threads. */ struct runq ksq_timeshare[2]; /* Run queues for !IDLE. */ struct runq *ksq_next; /* Next timeshare queue. */ struct runq *ksq_curr; /* Current queue. */ int ksq_load_timeshare; /* Load for timeshare. */ int ksq_load; /* Aggregate load. */ short ksq_nice[SCHED_PRI_NRESV]; /* KSEs in each nice bin. */ short ksq_nicemin; /* Least nice. */ #ifdef SMP int ksq_load_transferable; /* kses that may be migrated. */ int ksq_idled; int ksq_cpus; /* Count of CPUs in this kseq. */ volatile struct kse *ksq_assigned; /* assigned by another CPU. */ #endif }; /* * One kse queue per processor. */ #ifdef SMP static int kseq_idle; static struct kseq kseq_cpu[MAXCPU]; static struct kseq *kseq_idmap[MAXCPU]; #define KSEQ_SELF() (kseq_idmap[PCPU_GET(cpuid)]) #define KSEQ_CPU(x) (kseq_idmap[(x)]) #else static struct kseq kseq_cpu; #define KSEQ_SELF() (&kseq_cpu) #define KSEQ_CPU(x) (&kseq_cpu) #endif static void sched_slice(struct kse *ke); static void sched_priority(struct ksegrp *kg); static int sched_interact_score(struct ksegrp *kg); static void sched_interact_update(struct ksegrp *kg); static void sched_interact_fork(struct ksegrp *kg); static void sched_pctcpu_update(struct kse *ke); /* Operations on per processor queues */ static struct kse * kseq_choose(struct kseq *kseq); static void kseq_setup(struct kseq *kseq); static void kseq_load_add(struct kseq *kseq, struct kse *ke); static void kseq_load_rem(struct kseq *kseq, struct kse *ke); static __inline void kseq_runq_add(struct kseq *kseq, struct kse *ke); static __inline void kseq_runq_rem(struct kseq *kseq, struct kse *ke); static void kseq_nice_add(struct kseq *kseq, int nice); static void kseq_nice_rem(struct kseq *kseq, int nice); void kseq_print(int cpu); #ifdef SMP static struct kse *runq_steal(struct runq *rq); static void sched_balance(void *arg); static void kseq_move(struct kseq *from, int cpu); static __inline void kseq_setidle(struct kseq *kseq); static void kseq_notify(struct kse *ke, int cpu); static void kseq_assign(struct kseq *); static struct kse *kseq_steal(struct kseq *kseq); #define KSE_CAN_MIGRATE(ke, class) \ ((class) != PRI_ITHD && (ke)->ke_thread->td_pinned == 0 && \ ((ke)->ke_flags & KEF_BOUND) == 0) #endif void kseq_print(int cpu) { struct kseq *kseq; int i; kseq = KSEQ_CPU(cpu); printf("kseq:\n"); printf("\tload: %d\n", kseq->ksq_load); printf("\tload TIMESHARE: %d\n", kseq->ksq_load_timeshare); #ifdef SMP printf("\tload transferable: %d\n", kseq->ksq_load_transferable); #endif printf("\tnicemin:\t%d\n", kseq->ksq_nicemin); printf("\tnice counts:\n"); for (i = 0; i < SCHED_PRI_NRESV; i++) if (kseq->ksq_nice[i]) printf("\t\t%d = %d\n", i - SCHED_PRI_NHALF, kseq->ksq_nice[i]); } static __inline void kseq_runq_add(struct kseq *kseq, struct kse *ke) { #ifdef SMP if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) kseq->ksq_load_transferable++; #endif runq_add(ke->ke_runq, ke); } static __inline void kseq_runq_rem(struct kseq *kseq, struct kse *ke) { #ifdef SMP if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) kseq->ksq_load_transferable--; #endif runq_remove(ke->ke_runq, ke); } static void kseq_load_add(struct kseq *kseq, struct kse *ke) { int class; mtx_assert(&sched_lock, MA_OWNED); class = PRI_BASE(ke->ke_ksegrp->kg_pri_class); if (class == PRI_TIMESHARE) kseq->ksq_load_timeshare++; kseq->ksq_load++; if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) CTR6(KTR_ULE, "Add kse %p to %p (slice: %d, pri: %d, nice: %d(%d))", ke, ke->ke_runq, ke->ke_slice, ke->ke_thread->td_priority, ke->ke_ksegrp->kg_nice, kseq->ksq_nicemin); if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) kseq_nice_add(kseq, ke->ke_ksegrp->kg_nice); } static void kseq_load_rem(struct kseq *kseq, struct kse *ke) { int class; mtx_assert(&sched_lock, MA_OWNED); class = PRI_BASE(ke->ke_ksegrp->kg_pri_class); if (class == PRI_TIMESHARE) kseq->ksq_load_timeshare--; kseq->ksq_load--; ke->ke_runq = NULL; if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) kseq_nice_rem(kseq, ke->ke_ksegrp->kg_nice); } static void kseq_nice_add(struct kseq *kseq, int nice) { mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ kseq->ksq_nice[nice + SCHED_PRI_NHALF]++; if (nice < kseq->ksq_nicemin || kseq->ksq_load_timeshare == 1) kseq->ksq_nicemin = nice; } static void kseq_nice_rem(struct kseq *kseq, int nice) { int n; mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ n = nice + SCHED_PRI_NHALF; kseq->ksq_nice[n]--; KASSERT(kseq->ksq_nice[n] >= 0, ("Negative nice count.")); /* * If this wasn't the smallest nice value or there are more in * this bucket we can just return. Otherwise we have to recalculate * the smallest nice. */ if (nice != kseq->ksq_nicemin || kseq->ksq_nice[n] != 0 || kseq->ksq_load_timeshare == 0) return; for (; n < SCHED_PRI_NRESV; n++) if (kseq->ksq_nice[n]) { kseq->ksq_nicemin = n - SCHED_PRI_NHALF; return; } } #ifdef SMP /* * sched_balance is a simple CPU load balancing algorithm. It operates by * finding the least loaded and most loaded cpu and equalizing their load * by migrating some processes. * * Dealing only with two CPUs at a time has two advantages. Firstly, most * installations will only have 2 cpus. Secondly, load balancing too much at * once can have an unpleasant effect on the system. The scheduler rarely has * enough information to make perfect decisions. So this algorithm chooses * algorithm simplicity and more gradual effects on load in larger systems. * * It could be improved by considering the priorities and slices assigned to * each task prior to balancing them. There are many pathological cases with * any approach and so the semi random algorithm below may work as well as any. * */ static void sched_balance(void *arg) { struct kseq *kseq; int high_load; int low_load; int high_cpu; int low_cpu; int move; int diff; int i; high_cpu = 0; low_cpu = 0; high_load = 0; low_load = -1; mtx_lock_spin(&sched_lock); if (smp_started == 0) goto out; - for (i = 0; i < mp_maxid; i++) { + for (i = 0; i <= mp_maxid; i++) { if (CPU_ABSENT(i) || (i & stopped_cpus) != 0) continue; kseq = KSEQ_CPU(i); if (kseq->ksq_load_transferable > high_load) { high_load = kseq->ksq_load_transferable; high_cpu = i; } if (low_load == -1 || kseq->ksq_load < low_load) { low_load = kseq->ksq_load; low_cpu = i; } } kseq = KSEQ_CPU(high_cpu); /* * Nothing to do. */ if (high_load == 0 || low_load >= kseq->ksq_load) goto out; /* * Determine what the imbalance is and then adjust that to how many * kses we actually have to give up (load_transferable). */ diff = kseq->ksq_load - low_load; move = diff / 2; if (diff & 0x1) move++; move = min(move, high_load); for (i = 0; i < move; i++) kseq_move(kseq, low_cpu); out: mtx_unlock_spin(&sched_lock); callout_reset(&kseq_lb_callout, hz, sched_balance, NULL); return; } static void kseq_move(struct kseq *from, int cpu) { struct kse *ke; ke = kseq_steal(from); ke->ke_state = KES_THREAD; kseq_runq_rem(from, ke); kseq_load_rem(from, ke); ke->ke_cpu = cpu; kseq_notify(ke, cpu); } static __inline void kseq_setidle(struct kseq *kseq) { if (kseq->ksq_idled) return; kseq->ksq_idled = 1; atomic_set_int(&kseq_idle, PCPU_GET(cpumask)); return; } static void kseq_assign(struct kseq *kseq) { struct kse *nke; struct kse *ke; do { (volatile struct kse *)ke = kseq->ksq_assigned; } while(!atomic_cmpset_ptr(&kseq->ksq_assigned, ke, NULL)); for (; ke != NULL; ke = nke) { nke = ke->ke_assign; ke->ke_flags &= ~KEF_ASSIGNED; sched_add(ke->ke_thread); } } static void kseq_notify(struct kse *ke, int cpu) { struct kseq *kseq; struct thread *td; struct pcpu *pcpu; ke->ke_flags |= KEF_ASSIGNED; kseq = KSEQ_CPU(cpu); /* * Place a KSE on another cpu's queue and force a resched. */ do { (volatile struct kse *)ke->ke_assign = kseq->ksq_assigned; } while(!atomic_cmpset_ptr(&kseq->ksq_assigned, ke->ke_assign, ke)); pcpu = pcpu_find(cpu); td = pcpu->pc_curthread; if (ke->ke_thread->td_priority < td->td_priority || td == pcpu->pc_idlethread) { td->td_flags |= TDF_NEEDRESCHED; ipi_selected(1 << cpu, IPI_AST); } } static struct kse * runq_steal(struct runq *rq) { struct rqhead *rqh; struct rqbits *rqb; struct kse *ke; int word; int bit; mtx_assert(&sched_lock, MA_OWNED); rqb = &rq->rq_status; for (word = 0; word < RQB_LEN; word++) { if (rqb->rqb_bits[word] == 0) continue; for (bit = 0; bit < RQB_BPW; bit++) { if ((rqb->rqb_bits[word] & (1 << bit)) == 0) continue; rqh = &rq->rq_queues[bit + (word << RQB_L2BPW)]; TAILQ_FOREACH(ke, rqh, ke_procq) { if (KSE_CAN_MIGRATE(ke, PRI_BASE(ke->ke_ksegrp->kg_pri_class))) return (ke); } } } return (NULL); } static struct kse * kseq_steal(struct kseq *kseq) { struct kse *ke; if ((ke = runq_steal(kseq->ksq_curr)) != NULL) return (ke); if ((ke = runq_steal(kseq->ksq_next)) != NULL) return (ke); return (runq_steal(&kseq->ksq_idle)); } #endif /* SMP */ /* * Pick the highest priority task we have and return it. */ static struct kse * kseq_choose(struct kseq *kseq) { struct kse *ke; struct runq *swap; mtx_assert(&sched_lock, MA_OWNED); swap = NULL; for (;;) { ke = runq_choose(kseq->ksq_curr); if (ke == NULL) { /* * We already swaped once and didn't get anywhere. */ if (swap) break; swap = kseq->ksq_curr; kseq->ksq_curr = kseq->ksq_next; kseq->ksq_next = swap; continue; } /* * If we encounter a slice of 0 the kse is in a * TIMESHARE kse group and its nice was too far out * of the range that receives slices. */ if (ke->ke_slice == 0) { runq_remove(ke->ke_runq, ke); sched_slice(ke); ke->ke_runq = kseq->ksq_next; runq_add(ke->ke_runq, ke); continue; } return (ke); } return (runq_choose(&kseq->ksq_idle)); } static void kseq_setup(struct kseq *kseq) { runq_init(&kseq->ksq_timeshare[0]); runq_init(&kseq->ksq_timeshare[1]); runq_init(&kseq->ksq_idle); kseq->ksq_curr = &kseq->ksq_timeshare[0]; kseq->ksq_next = &kseq->ksq_timeshare[1]; kseq->ksq_load = 0; kseq->ksq_load_timeshare = 0; #ifdef SMP kseq->ksq_load_transferable = 0; kseq->ksq_idled = 0; kseq->ksq_assigned = NULL; #endif } static void sched_setup(void *dummy) { #ifdef SMP int i; #endif slice_min = (hz/100); /* 10ms */ slice_max = (hz/7); /* ~140ms */ #ifdef SMP /* init kseqs */ /* Create the idmap. */ #ifdef ULE_HTT_EXPERIMENTAL if (smp_topology == NULL) { #else if (1) { #endif for (i = 0; i < MAXCPU; i++) { kseq_setup(&kseq_cpu[i]); kseq_idmap[i] = &kseq_cpu[i]; kseq_cpu[i].ksq_cpus = 1; } } else { int j; for (i = 0; i < smp_topology->ct_count; i++) { struct cpu_group *cg; cg = &smp_topology->ct_group[i]; kseq_setup(&kseq_cpu[i]); for (j = 0; j < MAXCPU; j++) if ((cg->cg_mask & (1 << j)) != 0) kseq_idmap[j] = &kseq_cpu[i]; kseq_cpu[i].ksq_cpus = cg->cg_count; } } callout_init(&kseq_lb_callout, CALLOUT_MPSAFE); sched_balance(NULL); #else kseq_setup(KSEQ_SELF()); #endif mtx_lock_spin(&sched_lock); kseq_load_add(KSEQ_SELF(), &kse0); mtx_unlock_spin(&sched_lock); } /* * Scale the scheduling priority according to the "interactivity" of this * process. */ static void sched_priority(struct ksegrp *kg) { int pri; if (kg->kg_pri_class != PRI_TIMESHARE) return; pri = SCHED_PRI_INTERACT(sched_interact_score(kg)); pri += SCHED_PRI_BASE; pri += kg->kg_nice; if (pri > PRI_MAX_TIMESHARE) pri = PRI_MAX_TIMESHARE; else if (pri < PRI_MIN_TIMESHARE) pri = PRI_MIN_TIMESHARE; kg->kg_user_pri = pri; return; } /* * Calculate a time slice based on the properties of the kseg and the runq * that we're on. This is only for PRI_TIMESHARE ksegrps. */ static void sched_slice(struct kse *ke) { struct kseq *kseq; struct ksegrp *kg; kg = ke->ke_ksegrp; kseq = KSEQ_CPU(ke->ke_cpu); /* * Rationale: * KSEs in interactive ksegs get the minimum slice so that we * quickly notice if it abuses its advantage. * * KSEs in non-interactive ksegs are assigned a slice that is * based on the ksegs nice value relative to the least nice kseg * on the run queue for this cpu. * * If the KSE is less nice than all others it gets the maximum * slice and other KSEs will adjust their slice relative to * this when they first expire. * * There is 20 point window that starts relative to the least * nice kse on the run queue. Slice size is determined by * the kse distance from the last nice ksegrp. * * If the kse is outside of the window it will get no slice * and will be reevaluated each time it is selected on the * run queue. The exception to this is nice 0 ksegs when * a nice -20 is running. They are always granted a minimum * slice. */ if (!SCHED_INTERACTIVE(kg)) { int nice; nice = kg->kg_nice + (0 - kseq->ksq_nicemin); if (kseq->ksq_load_timeshare == 0 || kg->kg_nice < kseq->ksq_nicemin) ke->ke_slice = SCHED_SLICE_MAX; else if (nice <= SCHED_SLICE_NTHRESH) ke->ke_slice = SCHED_SLICE_NICE(nice); else if (kg->kg_nice == 0) ke->ke_slice = SCHED_SLICE_MIN; else ke->ke_slice = 0; } else ke->ke_slice = SCHED_SLICE_MIN; CTR6(KTR_ULE, "Sliced %p(%d) (nice: %d, nicemin: %d, load: %d, interactive: %d)", ke, ke->ke_slice, kg->kg_nice, kseq->ksq_nicemin, kseq->ksq_load_timeshare, SCHED_INTERACTIVE(kg)); return; } /* * This routine enforces a maximum limit on the amount of scheduling history * kept. It is called after either the slptime or runtime is adjusted. * This routine will not operate correctly when slp or run times have been * adjusted to more than double their maximum. */ static void sched_interact_update(struct ksegrp *kg) { int sum; sum = kg->kg_runtime + kg->kg_slptime; if (sum < SCHED_SLP_RUN_MAX) return; /* * If we have exceeded by more than 1/5th then the algorithm below * will not bring us back into range. Dividing by two here forces * us into the range of [3/5 * SCHED_INTERACT_MAX, SCHED_INTERACT_MAX] */ if (sum > (SCHED_INTERACT_MAX / 5) * 6) { kg->kg_runtime /= 2; kg->kg_slptime /= 2; return; } kg->kg_runtime = (kg->kg_runtime / 5) * 4; kg->kg_slptime = (kg->kg_slptime / 5) * 4; } static void sched_interact_fork(struct ksegrp *kg) { int ratio; int sum; sum = kg->kg_runtime + kg->kg_slptime; if (sum > SCHED_SLP_RUN_FORK) { ratio = sum / SCHED_SLP_RUN_FORK; kg->kg_runtime /= ratio; kg->kg_slptime /= ratio; } } static int sched_interact_score(struct ksegrp *kg) { int div; if (kg->kg_runtime > kg->kg_slptime) { div = max(1, kg->kg_runtime / SCHED_INTERACT_HALF); return (SCHED_INTERACT_HALF + (SCHED_INTERACT_HALF - (kg->kg_slptime / div))); } if (kg->kg_slptime > kg->kg_runtime) { div = max(1, kg->kg_slptime / SCHED_INTERACT_HALF); return (kg->kg_runtime / div); } /* * This can happen if slptime and runtime are 0. */ return (0); } /* * This is only somewhat accurate since given many processes of the same * priority they will switch when their slices run out, which will be * at most SCHED_SLICE_MAX. */ int sched_rr_interval(void) { return (SCHED_SLICE_MAX); } static void sched_pctcpu_update(struct kse *ke) { /* * Adjust counters and watermark for pctcpu calc. */ if (ke->ke_ltick > ticks - SCHED_CPU_TICKS) { /* * Shift the tick count out so that the divide doesn't * round away our results. */ ke->ke_ticks <<= 10; ke->ke_ticks = (ke->ke_ticks / (ticks - ke->ke_ftick)) * SCHED_CPU_TICKS; ke->ke_ticks >>= 10; } else ke->ke_ticks = 0; ke->ke_ltick = ticks; ke->ke_ftick = ke->ke_ltick - SCHED_CPU_TICKS; } void sched_prio(struct thread *td, u_char prio) { struct kse *ke; ke = td->td_kse; mtx_assert(&sched_lock, MA_OWNED); if (TD_ON_RUNQ(td)) { /* * If the priority has been elevated due to priority * propagation, we may have to move ourselves to a new * queue. We still call adjustrunqueue below in case kse * needs to fix things up. */ if (prio < td->td_priority && ke && (ke->ke_flags & KEF_ASSIGNED) == 0 && ke->ke_runq != KSEQ_CPU(ke->ke_cpu)->ksq_curr) { runq_remove(ke->ke_runq, ke); ke->ke_runq = KSEQ_CPU(ke->ke_cpu)->ksq_curr; runq_add(ke->ke_runq, ke); } adjustrunqueue(td, prio); } else td->td_priority = prio; } void sched_switch(struct thread *td) { struct thread *newtd; struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; td->td_last_kse = ke; td->td_lastcpu = td->td_oncpu; td->td_oncpu = NOCPU; td->td_flags &= ~TDF_NEEDRESCHED; if (TD_IS_RUNNING(td)) { if (td->td_proc->p_flag & P_SA) { kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); setrunqueue(td); } else { /* * This queue is always correct except for idle threads * which have a higher priority due to priority * propagation. */ if (ke->ke_ksegrp->kg_pri_class == PRI_IDLE) { if (td->td_priority < PRI_MIN_IDLE) ke->ke_runq = KSEQ_SELF()->ksq_curr; else ke->ke_runq = &KSEQ_SELF()->ksq_idle; } kseq_runq_add(KSEQ_SELF(), ke); } } else { if (ke->ke_runq) kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); /* * We will not be on the run queue. So we must be * sleeping or similar. */ if (td->td_proc->p_flag & P_SA) kse_reassign(ke); } newtd = choosethread(); if (td != newtd) cpu_switch(td, newtd); sched_lock.mtx_lock = (uintptr_t)td; td->td_oncpu = PCPU_GET(cpuid); } void sched_nice(struct ksegrp *kg, int nice) { struct kse *ke; struct thread *td; struct kseq *kseq; PROC_LOCK_ASSERT(kg->kg_proc, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); /* * We need to adjust the nice counts for running KSEs. */ if (kg->kg_pri_class == PRI_TIMESHARE) FOREACH_KSE_IN_GROUP(kg, ke) { if (ke->ke_runq == NULL) continue; kseq = KSEQ_CPU(ke->ke_cpu); kseq_nice_rem(kseq, kg->kg_nice); kseq_nice_add(kseq, nice); } kg->kg_nice = nice; sched_priority(kg); FOREACH_THREAD_IN_GROUP(kg, td) td->td_flags |= TDF_NEEDRESCHED; } void sched_sleep(struct thread *td, u_char prio) { mtx_assert(&sched_lock, MA_OWNED); td->td_slptime = ticks; td->td_priority = prio; CTR2(KTR_ULE, "sleep kse %p (tick: %d)", td->td_kse, td->td_slptime); } void sched_wakeup(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); /* * Let the kseg know how long we slept for. This is because process * interactivity behavior is modeled in the kseg. */ if (td->td_slptime) { struct ksegrp *kg; int hzticks; kg = td->td_ksegrp; hzticks = (ticks - td->td_slptime) << 10; if (hzticks >= SCHED_SLP_RUN_MAX) { kg->kg_slptime = SCHED_SLP_RUN_MAX; kg->kg_runtime = 1; } else { kg->kg_slptime += hzticks; sched_interact_update(kg); } sched_priority(kg); if (td->td_kse) sched_slice(td->td_kse); CTR2(KTR_ULE, "wakeup kse %p (%d ticks)", td->td_kse, hzticks); td->td_slptime = 0; } setrunqueue(td); } /* * Penalize the parent for creating a new child and initialize the child's * priority. */ void sched_fork(struct proc *p, struct proc *p1) { mtx_assert(&sched_lock, MA_OWNED); sched_fork_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(p1)); sched_fork_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(p1)); sched_fork_thread(FIRST_THREAD_IN_PROC(p), FIRST_THREAD_IN_PROC(p1)); } void sched_fork_kse(struct kse *ke, struct kse *child) { child->ke_slice = 1; /* Attempt to quickly learn interactivity. */ child->ke_cpu = ke->ke_cpu; child->ke_runq = NULL; /* Grab our parents cpu estimation information. */ child->ke_ticks = ke->ke_ticks; child->ke_ltick = ke->ke_ltick; child->ke_ftick = ke->ke_ftick; } void sched_fork_ksegrp(struct ksegrp *kg, struct ksegrp *child) { PROC_LOCK_ASSERT(child->kg_proc, MA_OWNED); child->kg_slptime = kg->kg_slptime; child->kg_runtime = kg->kg_runtime; child->kg_user_pri = kg->kg_user_pri; child->kg_nice = kg->kg_nice; sched_interact_fork(child); kg->kg_runtime += tickincr << 10; sched_interact_update(kg); CTR6(KTR_ULE, "sched_fork_ksegrp: %d(%d, %d) - %d(%d, %d)", kg->kg_proc->p_pid, kg->kg_slptime, kg->kg_runtime, child->kg_proc->p_pid, child->kg_slptime, child->kg_runtime); } void sched_fork_thread(struct thread *td, struct thread *child) { } void sched_class(struct ksegrp *kg, int class) { struct kseq *kseq; struct kse *ke; int nclass; int oclass; mtx_assert(&sched_lock, MA_OWNED); if (kg->kg_pri_class == class) return; nclass = PRI_BASE(class); oclass = PRI_BASE(kg->kg_pri_class); FOREACH_KSE_IN_GROUP(kg, ke) { if (ke->ke_state != KES_ONRUNQ && ke->ke_state != KES_THREAD) continue; kseq = KSEQ_CPU(ke->ke_cpu); #ifdef SMP /* * On SMP if we're on the RUNQ we must adjust the transferable * count because could be changing to or from an interrupt * class. */ if (ke->ke_state == KES_ONRUNQ) { if (KSE_CAN_MIGRATE(ke, oclass)) kseq->ksq_load_transferable--; if (KSE_CAN_MIGRATE(ke, nclass)) kseq->ksq_load_transferable++; } #endif if (oclass == PRI_TIMESHARE) { kseq->ksq_load_timeshare--; kseq_nice_rem(kseq, kg->kg_nice); } if (nclass == PRI_TIMESHARE) { kseq->ksq_load_timeshare++; kseq_nice_add(kseq, kg->kg_nice); } } kg->kg_pri_class = class; } /* * Return some of the child's priority and interactivity to the parent. */ void sched_exit(struct proc *p, struct proc *child) { mtx_assert(&sched_lock, MA_OWNED); sched_exit_kse(FIRST_KSE_IN_PROC(p), FIRST_KSE_IN_PROC(child)); sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), FIRST_KSEGRP_IN_PROC(child)); } void sched_exit_kse(struct kse *ke, struct kse *child) { kseq_load_rem(KSEQ_CPU(child->ke_cpu), child); } void sched_exit_ksegrp(struct ksegrp *kg, struct ksegrp *child) { /* kg->kg_slptime += child->kg_slptime; */ kg->kg_runtime += child->kg_runtime; sched_interact_update(kg); } void sched_exit_thread(struct thread *td, struct thread *child) { } void sched_clock(struct thread *td) { struct kseq *kseq; struct ksegrp *kg; struct kse *ke; /* * sched_setup() apparently happens prior to stathz being set. We * need to resolve the timers earlier in the boot so we can avoid * calculating this here. */ if (realstathz == 0) { realstathz = stathz ? stathz : hz; tickincr = hz / realstathz; /* * XXX This does not work for values of stathz that are much * larger than hz. */ if (tickincr == 0) tickincr = 1; } ke = td->td_kse; kg = ke->ke_ksegrp; mtx_assert(&sched_lock, MA_OWNED); KASSERT((td != NULL), ("schedclock: null thread pointer")); /* Adjust ticks for pctcpu */ ke->ke_ticks++; ke->ke_ltick = ticks; /* Go up to one second beyond our max and then trim back down */ if (ke->ke_ftick + SCHED_CPU_TICKS + hz < ke->ke_ltick) sched_pctcpu_update(ke); if (td->td_flags & TDF_IDLETD) return; CTR4(KTR_ULE, "Tick kse %p (slice: %d, slptime: %d, runtime: %d)", ke, ke->ke_slice, kg->kg_slptime >> 10, kg->kg_runtime >> 10); /* * We only do slicing code for TIMESHARE ksegrps. */ if (kg->kg_pri_class != PRI_TIMESHARE) return; /* * We used a tick charge it to the ksegrp so that we can compute our * interactivity. */ kg->kg_runtime += tickincr << 10; sched_interact_update(kg); /* * We used up one time slice. */ if (--ke->ke_slice > 0) return; /* * We're out of time, recompute priorities and requeue. */ kseq = KSEQ_SELF(); kseq_load_rem(kseq, ke); sched_priority(kg); sched_slice(ke); if (SCHED_CURR(kg, ke)) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = kseq->ksq_next; kseq_load_add(kseq, ke); td->td_flags |= TDF_NEEDRESCHED; } int sched_runnable(void) { struct kseq *kseq; int load; load = 1; kseq = KSEQ_SELF(); #ifdef SMP if (kseq->ksq_assigned) { mtx_lock_spin(&sched_lock); kseq_assign(kseq); mtx_unlock_spin(&sched_lock); } #endif if ((curthread->td_flags & TDF_IDLETD) != 0) { if (kseq->ksq_load > 0) goto out; } else if (kseq->ksq_load - 1 > 0) goto out; load = 0; out: return (load); } void sched_userret(struct thread *td) { struct ksegrp *kg; kg = td->td_ksegrp; if (td->td_priority != kg->kg_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = kg->kg_user_pri; mtx_unlock_spin(&sched_lock); } } struct kse * sched_choose(void) { struct kseq *kseq; struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); kseq = KSEQ_SELF(); #ifdef SMP if (kseq->ksq_assigned) kseq_assign(kseq); #endif ke = kseq_choose(kseq); if (ke) { #ifdef SMP if (ke->ke_ksegrp->kg_pri_class == PRI_IDLE) kseq_setidle(kseq); #endif kseq_runq_rem(kseq, ke); ke->ke_state = KES_THREAD; if (ke->ke_ksegrp->kg_pri_class == PRI_TIMESHARE) { CTR4(KTR_ULE, "Run kse %p from %p (slice: %d, pri: %d)", ke, ke->ke_runq, ke->ke_slice, ke->ke_thread->td_priority); } return (ke); } #ifdef SMP kseq_setidle(kseq); #endif return (NULL); } void sched_add(struct thread *td) { struct kseq *kseq; struct ksegrp *kg; struct kse *ke; int class; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; kg = td->td_ksegrp; if (ke->ke_flags & KEF_ASSIGNED) return; kseq = KSEQ_SELF(); KASSERT((ke->ke_thread != NULL), ("sched_add: No thread on KSE")); KASSERT((ke->ke_thread->td_kse != NULL), ("sched_add: No KSE on thread")); KASSERT(ke->ke_state != KES_ONRUNQ, ("sched_add: kse %p (%s) already in run queue", ke, ke->ke_proc->p_comm)); KASSERT(ke->ke_proc->p_sflag & PS_INMEM, ("sched_add: process swapped out")); KASSERT(ke->ke_runq == NULL, ("sched_add: KSE %p is still assigned to a run queue", ke)); class = PRI_BASE(kg->kg_pri_class); switch (class) { case PRI_ITHD: case PRI_REALTIME: ke->ke_runq = kseq->ksq_curr; ke->ke_slice = SCHED_SLICE_MAX; ke->ke_cpu = PCPU_GET(cpuid); break; case PRI_TIMESHARE: #ifdef SMP if (ke->ke_cpu != PCPU_GET(cpuid)) { kseq_notify(ke, ke->ke_cpu); return; } #endif if (SCHED_CURR(kg, ke)) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = kseq->ksq_next; break; case PRI_IDLE: #ifdef SMP if (ke->ke_cpu != PCPU_GET(cpuid)) { kseq_notify(ke, ke->ke_cpu); return; } #endif /* * This is for priority prop. */ if (ke->ke_thread->td_priority < PRI_MIN_IDLE) ke->ke_runq = kseq->ksq_curr; else ke->ke_runq = &kseq->ksq_idle; ke->ke_slice = SCHED_SLICE_MIN; break; default: panic("Unknown pri class."); break; } #ifdef SMP /* * If there are any idle processors, give them our extra load. The * threshold at which we start to reassign kses has a large impact * on the overall performance of the system. Tuned too high and * some CPUs may idle. Too low and there will be excess migration * and context swiches. */ if (kseq->ksq_load_transferable > kseq->ksq_cpus && KSE_CAN_MIGRATE(ke, class) && kseq_idle) { int cpu; /* * Multiple cpus could find this bit simultaneously but the * race shouldn't be terrible. */ cpu = ffs(kseq_idle); if (cpu) { cpu--; atomic_clear_int(&kseq_idle, 1 << cpu); ke->ke_cpu = cpu; ke->ke_runq = NULL; kseq_notify(ke, cpu); return; } } if (kseq->ksq_idled && (class == PRI_TIMESHARE || class == PRI_REALTIME)) { atomic_clear_int(&kseq_idle, PCPU_GET(cpumask)); kseq->ksq_idled = 0; } #endif if (td->td_priority < curthread->td_priority) curthread->td_flags |= TDF_NEEDRESCHED; ke->ke_ksegrp->kg_runq_kses++; ke->ke_state = KES_ONRUNQ; kseq_runq_add(kseq, ke); kseq_load_add(kseq, ke); } void sched_rem(struct thread *td) { struct kseq *kseq; struct kse *ke; ke = td->td_kse; /* * It is safe to just return here because sched_rem() is only ever * used in places where we're immediately going to add the * kse back on again. In that case it'll be added with the correct * thread and priority when the caller drops the sched_lock. */ if (ke->ke_flags & KEF_ASSIGNED) return; mtx_assert(&sched_lock, MA_OWNED); KASSERT((ke->ke_state == KES_ONRUNQ), ("KSE not on run queue")); ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; kseq = KSEQ_CPU(ke->ke_cpu); kseq_runq_rem(kseq, ke); kseq_load_rem(kseq, ke); } fixpt_t sched_pctcpu(struct thread *td) { fixpt_t pctcpu; struct kse *ke; pctcpu = 0; ke = td->td_kse; if (ke == NULL) return (0); mtx_lock_spin(&sched_lock); if (ke->ke_ticks) { int rtick; /* * Don't update more frequently than twice a second. Allowing * this causes the cpu usage to decay away too quickly due to * rounding errors. */ if (ke->ke_ltick < (ticks - (hz / 2))) sched_pctcpu_update(ke); /* How many rtick per second ? */ rtick = min(ke->ke_ticks / SCHED_CPU_TIME, SCHED_CPU_TICKS); pctcpu = (FSCALE * ((FSCALE * rtick)/realstathz)) >> FSHIFT; } ke->ke_proc->p_swtime = ke->ke_ltick - ke->ke_ftick; mtx_unlock_spin(&sched_lock); return (pctcpu); } void sched_bind(struct thread *td, int cpu) { struct kse *ke; mtx_assert(&sched_lock, MA_OWNED); ke = td->td_kse; #ifndef SMP ke->ke_flags |= KEF_BOUND; #else if (PCPU_GET(cpuid) == cpu) { ke->ke_flags |= KEF_BOUND; return; } /* sched_rem without the runq_remove */ ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); ke->ke_cpu = cpu; kseq_notify(ke, cpu); /* When we return from mi_switch we'll be on the correct cpu. */ td->td_proc->p_stats->p_ru.ru_nvcsw++; mi_switch(); #endif } void sched_unbind(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); td->td_kse->ke_flags &= ~KEF_BOUND; } int sched_sizeof_kse(void) { return (sizeof(struct kse) + sizeof(struct ke_sched)); } int sched_sizeof_ksegrp(void) { return (sizeof(struct ksegrp) + sizeof(struct kg_sched)); } int sched_sizeof_proc(void) { return (sizeof(struct proc)); } int sched_sizeof_thread(void) { return (sizeof(struct thread) + sizeof(struct td_sched)); } Index: head/sys/sparc64/sparc64/mp_machdep.c =================================================================== --- head/sys/sparc64/sparc64/mp_machdep.c (revision 123125) +++ head/sys/sparc64/sparc64/mp_machdep.c (revision 123126) @@ -1,474 +1,474 @@ /*- * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Berkeley Software Design Inc's name may not be used to endorse or * promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp */ /*- * Copyright (c) 2002 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static ih_func_t cpu_ipi_ast; static ih_func_t cpu_ipi_stop; /* * Argument area used to pass data to non-boot processors as they start up. * This must be statically initialized with a known invalid upa module id, * since the other processors will use it before the boot cpu enters the * kernel. */ struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0 }; struct ipi_cache_args ipi_cache_args; struct ipi_tlb_args ipi_tlb_args; struct mtx ipi_mtx; vm_offset_t mp_tramp; u_int mp_boot_mid; static volatile u_int shutdown_cpus; void cpu_mp_unleash(void *); SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); vm_offset_t mp_tramp_alloc(void) { struct tte *tp; char *v; int i; v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); if (v == NULL) panic("mp_tramp_alloc"); bcopy(mp_tramp_code, v, mp_tramp_code_len); *(u_long *)(v + mp_tramp_tlb_slots) = kernel_tlb_slots; *(u_long *)(v + mp_tramp_func) = (u_long)mp_startup; tp = (struct tte *)(v + mp_tramp_code_len); for (i = 0; i < kernel_tlb_slots; i++) { tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; } for (i = 0; i < PAGE_SIZE; i += sizeof(long)) flush(v + i); return (vm_offset_t)v; } /* * Probe for other cpus. */ void cpu_mp_setmaxid(void) { phandle_t child; phandle_t root; char buf[128]; int cpus; all_cpus = 1 << PCPU_GET(cpuid); mp_boot_mid = PCPU_GET(mid); mp_ncpus = 1; cpus = 0; root = OF_peer(0); for (child = OF_child(root); child != 0; child = OF_peer(child)) { if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 && strcmp(buf, "cpu") == 0) cpus++; } - mp_maxid = cpus; + mp_maxid = cpus - 1; } int cpu_mp_probe(void) { - return (mp_maxid > 1); + return (mp_maxid > 0); } static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg) { static struct { cell_t name; cell_t nargs; cell_t nreturns; cell_t cpu; cell_t func; cell_t arg; } args = { (cell_t)"SUNW,start-cpu", 3, 0, 0, 0, 0 }; args.cpu = cpu; args.func = (cell_t)func; args.arg = (cell_t)arg; openfirmware(&args); } /* * Stop the calling CPU. */ static void sun4u_stopself(void) { static struct { cell_t name; cell_t nargs; cell_t nreturns; } args = { (cell_t)"SUNW,stop-self", 0, 0, }; openfirmware_exit(&args); panic("sun4u_stopself: failed."); } /* * Fire up any non-boot processors. */ void cpu_mp_start(void) { volatile struct cpu_start_args *csa; struct pcpu *pc; phandle_t child; phandle_t root; vm_offset_t va; char buf[128]; u_int clock; int cpuid; u_int mid; u_long s; mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, -1, NULL, NULL); intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); root = OF_peer(0); csa = &cpu_start_args; for (child = OF_child(root); child != 0; child = OF_peer(child)) { if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 || strcmp(buf, "cpu") != 0) continue; if (OF_getprop(child, "upa-portid", &mid, sizeof(mid)) <= 0 && OF_getprop(child, "portid", &mid, sizeof(mid)) <= 0) panic("cpu_mp_start: can't get module id"); if (mid == mp_boot_mid) continue; if (OF_getprop(child, "clock-frequency", &clock, sizeof(clock)) <= 0) panic("cpu_mp_start: can't get clock"); csa->csa_state = 0; sun4u_startcpu(child, (void *)mp_tramp, 0); s = intr_disable(); while (csa->csa_state != CPU_CLKSYNC) ; membar(StoreLoad); csa->csa_tick = rd(tick); while (csa->csa_state != CPU_INIT) ; csa->csa_tick = 0; intr_restore(s); cpuid = mp_ncpus++; cpu_identify(csa->csa_ver, clock, cpuid); va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE); pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; pcpu_init(pc, cpuid, sizeof(*pc)); pc->pc_addr = va; pc->pc_mid = mid; pc->pc_node = child; all_cpus |= 1 << cpuid; } PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid))); smp_active = 1; } void cpu_mp_announce(void) { } void cpu_mp_unleash(void *v) { volatile struct cpu_start_args *csa; struct pcpu *pc; vm_offset_t va; vm_paddr_t pa; u_int ctx_min; u_int ctx_inc; u_long s; int i; ctx_min = TLB_CTX_USER_MIN; ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; csa = &cpu_start_args; csa->csa_count = mp_ncpus; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { pc->pc_tlb_ctx = ctx_min; pc->pc_tlb_ctx_min = ctx_min; pc->pc_tlb_ctx_max = ctx_min + ctx_inc; ctx_min += ctx_inc; if (pc->pc_cpuid == PCPU_GET(cpuid)) continue; KASSERT(pc->pc_idlethread != NULL, ("cpu_mp_unleash: idlethread")); KASSERT(pc->pc_curthread == pc->pc_idlethread, ("cpu_mp_unleash: curthread")); pc->pc_curpcb = pc->pc_curthread->td_pcb; for (i = 0; i < PCPU_PAGES; i++) { va = pc->pc_addr + i * PAGE_SIZE; pa = pmap_kextract(va); if (pa == 0) panic("cpu_mp_unleash: pmap_kextract\n"); csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | TD_L | TD_CP | TD_CV | TD_P | TD_W; } csa->csa_state = 0; csa->csa_pcpu = pc->pc_addr; csa->csa_mid = pc->pc_mid; s = intr_disable(); while (csa->csa_state != CPU_BOOTSTRAP) ; intr_restore(s); } membar(StoreLoad); csa->csa_count = 0; smp_started = 1; } void cpu_mp_bootstrap(struct pcpu *pc) { volatile struct cpu_start_args *csa; csa = &cpu_start_args; pmap_map_tsb(); cpu_setregs(pc); tick_start_ap(); smp_cpus++; PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid))); printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); csa->csa_count--; membar(StoreLoad); csa->csa_state = CPU_BOOTSTRAP; while (csa->csa_count != 0) ; binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); /* ok, now grab sched_lock and enter the scheduler */ mtx_lock_spin(&sched_lock); cpu_throw(NULL, choosethread()); /* doesn't return */ } void cpu_mp_shutdown(void) { int i; critical_enter(); shutdown_cpus = PCPU_GET(other_cpus); if (stopped_cpus != PCPU_GET(other_cpus)) /* XXX */ stop_cpus(stopped_cpus ^ PCPU_GET(other_cpus)); i = 0; while (shutdown_cpus != 0) { if (i++ > 100000) { printf("timeout shutting down CPUs.\n"); break; } } /* XXX: delay a bit to allow the CPUs to actually enter the PROM. */ DELAY(100000); critical_exit(); } static void cpu_ipi_ast(struct trapframe *tf) { } static void cpu_ipi_stop(struct trapframe *tf) { CTR1(KTR_SMP, "cpu_ipi_stop: stopped %d", PCPU_GET(cpuid)); atomic_set_acq_int(&stopped_cpus, PCPU_GET(cpumask)); while ((started_cpus & PCPU_GET(cpumask)) == 0) { if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) { atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask)); sun4u_stopself(); } } atomic_clear_rel_int(&started_cpus, PCPU_GET(cpumask)); atomic_clear_rel_int(&stopped_cpus, PCPU_GET(cpumask)); CTR1(KTR_SMP, "cpu_ipi_stop: restarted %d", PCPU_GET(cpuid)); } void cpu_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) { struct pcpu *pc; u_int cpu; while (cpus) { cpu = ffs(cpus) - 1; cpus &= ~(1 << cpu); pc = pcpu_find(cpu); cpu_ipi_send(pc->pc_mid, d0, d1, d2); } } void cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2) { u_long s; int i; KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, ("ipi_send: outstanding dispatch")); for (i = 0; i < IPI_RETRIES; i++) { s = intr_disable(); stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); stxa(AA_INTR_SEND | (mid << 14), ASI_SDB_INTR_W, 0); membar(Sync); while (ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) ; intr_restore(s); if ((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_NACK) == 0) return; } if ( #ifdef DDB db_active || #endif panicstr != NULL) printf("ipi_send: couldn't send ipi to module %u\n", mid); else panic("ipi_send: couldn't send ipi"); } void ipi_selected(u_int cpus, u_int ipi) { cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); } void ipi_all(u_int ipi) { panic("ipi_all"); } void ipi_all_but_self(u_int ipi) { cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)tl_ipi_level, ipi); } Index: head/sys/sys/smp.h =================================================================== --- head/sys/sys/smp.h (revision 123125) +++ head/sys/sys/smp.h (revision 123126) @@ -1,105 +1,105 @@ /* * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * * $FreeBSD$ */ #ifndef _SYS_SMP_H_ #define _SYS_SMP_H_ #ifdef _KERNEL #ifndef LOCORE #ifdef SMP /* * Topology of a NUMA or HTT system. * * The top level topology is an array of pointers to groups. Each group * contains a bitmask of cpus in its group or subgroups. It may also * contain a pointer to an array of child groups. * * The bitmasks at non leaf groups may be used by consumers who support * a smaller depth than the hardware provides. * * The topology may be omitted by systems where all CPUs are equal. */ struct cpu_group { u_int cg_mask; /* Mask of cpus in this group. */ int cg_count; /* Count of cpus in this group. */ int cg_children; /* Number of children groups. */ struct cpu_group *cg_child; /* Optional child group. */ }; struct cpu_top { int ct_count; /* Count of groups. */ struct cpu_group *ct_group; /* Array of pointers to cpu groups. */ }; extern struct cpu_top *smp_topology; extern void (*cpustop_restartfunc)(void); extern int smp_active; extern int smp_cpus; extern volatile u_int started_cpus; extern volatile u_int stopped_cpus; #endif /* SMP */ extern u_int all_cpus; extern u_int mp_maxid; extern int mp_ncpus; extern volatile int smp_started; /* * Macro allowing us to determine whether a CPU is absent at any given * time, thus permitting us to configure sparse maps of cpuid-dependent * (per-CPU) structures. */ #define CPU_ABSENT(x_cpu) ((all_cpus & (1 << (x_cpu))) == 0) #ifdef SMP /* * Machine dependent functions used to initialize MP support. * * The cpu_mp_probe() should check to see if MP support is present and return * zero if it is not or non-zero if it is. If MP support is present, then * cpu_mp_start() will be called so that MP can be enabled. This function * should do things such as startup secondary processors. It should also * setup mp_ncpus, all_cpus, and smp_cpus. It should also ensure that * smp_active and smp_started are initialized at the appropriate time. * Once cpu_mp_start() returns, machine independent MP startup code will be * executed and a simple message will be output to the console. Finally, * cpu_mp_announce() will be called so that machine dependent messages about * the MP support may be output to the console if desired. * * The cpu_setmaxid() function is called very early during the boot process * so that the MD code may set mp_maxid to provide an upper bound on CPU IDs * that other subsystems may use. If a platform is not able to determine - * the exact maximum ID that early, then it may set mp_maxid to MAXCPU. + * the exact maximum ID that early, then it may set mp_maxid to MAXCPU - 1. */ struct thread; void cpu_mp_announce(void); int cpu_mp_probe(void); void cpu_mp_setmaxid(void); void cpu_mp_start(void); void forward_signal(struct thread *); void forward_roundrobin(void); int restart_cpus(u_int); int stop_cpus(u_int); void smp_rendezvous_action(void); #endif /* SMP */ void smp_rendezvous(void (*)(void *), void (*)(void *), void (*)(void *), void *arg); #endif /* !LOCORE */ #endif /* _KERNEL */ #endif /* _SYS_SMP_H_ */ Index: head/sys/vm/uma_core.c =================================================================== --- head/sys/vm/uma_core.c (revision 123125) +++ head/sys/vm/uma_core.c (revision 123126) @@ -1,2200 +1,2192 @@ /* * Copyright (c) 2002, Jeffrey Roberson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * uma_core.c Implementation of the Universal Memory allocator * * This allocator is intended to replace the multitude of similar object caches * in the standard FreeBSD kernel. The intent is to be flexible as well as * effecient. A primary design goal is to return unused memory to the rest of * the system. This will make the system as a whole more flexible due to the * ability to move memory to subsystems which most need it instead of leaving * pools of reserved memory unused. * * The basic ideas stem from similar slab/zone based allocators whose algorithms * are well known. * */ /* * TODO: * - Improve memory usage for large allocations * - Investigate cache size adjustments */ #include __FBSDID("$FreeBSD$"); /* I should really use ktr.. */ /* #define UMA_DEBUG 1 #define UMA_DEBUG_ALLOC 1 #define UMA_DEBUG_ALLOC_1 1 */ #include "opt_param.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This is the zone from which all zones are spawned. The idea is that even * the zone heads are allocated from the allocator, so we use the bss section * to bootstrap us. */ static struct uma_zone masterzone; static uma_zone_t zones = &masterzone; /* This is the zone from which all of uma_slab_t's are allocated. */ static uma_zone_t slabzone; /* * The initial hash tables come out of this zone so they can be allocated * prior to malloc coming up. */ static uma_zone_t hashzone; static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); /* * Are we allowed to allocate buckets? */ static int bucketdisable = 1; /* Linked list of all zones in the system */ static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); /* This mutex protects the zone list */ static struct mtx uma_mtx; /* These are the pcpu cache locks */ static struct mtx uma_pcpu_mtx[MAXCPU]; /* Linked list of boot time pages */ static LIST_HEAD(,uma_slab) uma_boot_pages = LIST_HEAD_INITIALIZER(&uma_boot_pages); /* Count of free boottime pages */ static int uma_boot_free = 0; /* Is the VM done starting up? */ static int booted = 0; /* - * Rather than #ifdef SMP all over, just give us a bogus definition for - * this on UP. - */ -#ifndef SMP -static int mp_maxid = 1; -#endif - -/* * This is the handle used to schedule events that need to happen * outside of the allocation fast path. */ static struct callout uma_callout; #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ /* * This structure is passed as the zone ctor arg so that I don't have to create * a special allocation function just for zones. */ struct uma_zctor_args { char *name; size_t size; uma_ctor ctor; uma_dtor dtor; uma_init uminit; uma_fini fini; int align; u_int16_t flags; }; struct uma_bucket_zone { uma_zone_t ubz_zone; char *ubz_name; int ubz_entries; }; #define BUCKET_MAX 128 struct uma_bucket_zone bucket_zones[] = { { NULL, "16 Bucket", 16 }, { NULL, "32 Bucket", 32 }, { NULL, "64 Bucket", 64 }, { NULL, "128 Bucket", 128 }, { NULL, NULL, 0} }; #define BUCKET_SHIFT 4 #define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) uint8_t bucket_size[BUCKET_ZONES]; /* Prototypes.. */ static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); static void *page_alloc(uma_zone_t, int, u_int8_t *, int); static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); static void page_free(void *, int, u_int8_t); static uma_slab_t slab_zalloc(uma_zone_t, int); static void cache_drain(uma_zone_t); static void bucket_drain(uma_zone_t, uma_bucket_t); static void zone_ctor(void *, int, void *); static void zone_dtor(void *, int, void *); static void zero_init(void *, int); static void zone_small_init(uma_zone_t zone); static void zone_large_init(uma_zone_t zone); static void zone_foreach(void (*zfunc)(uma_zone_t)); static void zone_timeout(uma_zone_t zone); static int hash_alloc(struct uma_hash *); static int hash_expand(struct uma_hash *, struct uma_hash *); static void hash_free(struct uma_hash *hash); static void uma_timeout(void *); static void uma_startup3(void); static void *uma_zalloc_internal(uma_zone_t, void *, int); static void uma_zfree_internal(uma_zone_t, void *, void *, int); static void bucket_enable(void); static void bucket_init(void); static uma_bucket_t bucket_alloc(int, int); static void bucket_free(uma_bucket_t); static void bucket_zone_drain(void); static int uma_zalloc_bucket(uma_zone_t zone, int flags); static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); static void zone_drain(uma_zone_t); void uma_print_zone(uma_zone_t); void uma_print_stats(void); static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, NULL, 0, sysctl_vm_zone, "A", "Zone Info"); SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); /* * This routine checks to see whether or not it's safe to enable buckets. */ static void bucket_enable(void) { if (cnt.v_free_count < cnt.v_free_min) bucketdisable = 1; else bucketdisable = 0; } static void bucket_init(void) { struct uma_bucket_zone *ubz; int i; int j; for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) { int size; ubz = &bucket_zones[j]; size = roundup(sizeof(struct uma_bucket), sizeof(void *)); size += sizeof(void *) * ubz->ubz_entries; ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT)) bucket_size[i >> BUCKET_SHIFT] = j; } } static uma_bucket_t bucket_alloc(int entries, int bflags) { struct uma_bucket_zone *ubz; uma_bucket_t bucket; int idx; /* * This is to stop us from allocating per cpu buckets while we're * running out of UMA_BOOT_PAGES. Otherwise, we would exhaust the * boot pages. This also prevents us from allocating buckets in * low memory situations. */ if (bucketdisable) return (NULL); idx = howmany(entries, 1 << BUCKET_SHIFT); ubz = &bucket_zones[bucket_size[idx]]; bucket = uma_zalloc_internal(ubz->ubz_zone, NULL, bflags); if (bucket) { #ifdef INVARIANTS bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); #endif bucket->ub_cnt = 0; bucket->ub_entries = ubz->ubz_entries; } return (bucket); } static void bucket_free(uma_bucket_t bucket) { struct uma_bucket_zone *ubz; int idx; idx = howmany(bucket->ub_entries, 1 << BUCKET_SHIFT); ubz = &bucket_zones[bucket_size[idx]]; uma_zfree_internal(ubz->ubz_zone, bucket, NULL, 0); } static void bucket_zone_drain(void) { struct uma_bucket_zone *ubz; for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) zone_drain(ubz->ubz_zone); } /* * Routine called by timeout which is used to fire off some time interval * based calculations. (stats, hash size, etc.) * * Arguments: * arg Unused * * Returns: * Nothing */ static void uma_timeout(void *unused) { bucket_enable(); zone_foreach(zone_timeout); /* Reschedule this event */ callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); } /* * Routine to perform timeout driven calculations. This expands the * hashes and does per cpu statistics aggregation. * * Arguments: * zone The zone to operate on * * Returns: * Nothing */ static void zone_timeout(uma_zone_t zone) { uma_cache_t cache; u_int64_t alloc; int cpu; alloc = 0; /* * Aggregate per cpu cache statistics back to the zone. * * I may rewrite this to set a flag in the per cpu cache instead of * locking. If the flag is not cleared on the next round I will have * to lock and do it here instead so that the statistics don't get too * far out of sync. */ if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { - for (cpu = 0; cpu < mp_maxid; cpu++) { + for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; /* Add them up, and reset */ alloc += cache->uc_allocs; cache->uc_allocs = 0; CPU_UNLOCK(cpu); } } /* Now push these stats back into the zone.. */ ZONE_LOCK(zone); zone->uz_allocs += alloc; /* * Expand the zone hash table. * * This is done if the number of slabs is larger than the hash size. * What I'm trying to do here is completely reduce collisions. This * may be a little aggressive. Should I allow for two collisions max? */ if (zone->uz_flags & UMA_ZONE_HASH && zone->uz_pages / zone->uz_ppera >= zone->uz_hash.uh_hashsize) { struct uma_hash newhash; struct uma_hash oldhash; int ret; /* * This is so involved because allocating and freeing * while the zone lock is held will lead to deadlock. * I have to do everything in stages and check for * races. */ newhash = zone->uz_hash; ZONE_UNLOCK(zone); ret = hash_alloc(&newhash); ZONE_LOCK(zone); if (ret) { if (hash_expand(&zone->uz_hash, &newhash)) { oldhash = zone->uz_hash; zone->uz_hash = newhash; } else oldhash = newhash; ZONE_UNLOCK(zone); hash_free(&oldhash); ZONE_LOCK(zone); } } ZONE_UNLOCK(zone); } /* * Allocate and zero fill the next sized hash table from the appropriate * backing store. * * Arguments: * hash A new hash structure with the old hash size in uh_hashsize * * Returns: * 1 on sucess and 0 on failure. */ static int hash_alloc(struct uma_hash *hash) { int oldsize; int alloc; oldsize = hash->uh_hashsize; /* We're just going to go to a power of two greater */ if (oldsize) { hash->uh_hashsize = oldsize * 2; alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; hash->uh_slab_hash = (struct slabhead *)malloc(alloc, M_UMAHASH, M_NOWAIT); } else { alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; hash->uh_slab_hash = uma_zalloc_internal(hashzone, NULL, M_WAITOK); hash->uh_hashsize = UMA_HASH_SIZE_INIT; } if (hash->uh_slab_hash) { bzero(hash->uh_slab_hash, alloc); hash->uh_hashmask = hash->uh_hashsize - 1; return (1); } return (0); } /* * Expands the hash table for HASH zones. This is done from zone_timeout * to reduce collisions. This must not be done in the regular allocation * path, otherwise, we can recurse on the vm while allocating pages. * * Arguments: * oldhash The hash you want to expand * newhash The hash structure for the new table * * Returns: * Nothing * * Discussion: */ static int hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) { uma_slab_t slab; int hval; int i; if (!newhash->uh_slab_hash) return (0); if (oldhash->uh_hashsize >= newhash->uh_hashsize) return (0); /* * I need to investigate hash algorithms for resizing without a * full rehash. */ for (i = 0; i < oldhash->uh_hashsize; i++) while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); hval = UMA_HASH(newhash, slab->us_data); SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], slab, us_hlink); } return (1); } /* * Free the hash bucket to the appropriate backing store. * * Arguments: * slab_hash The hash bucket we're freeing * hashsize The number of entries in that hash bucket * * Returns: * Nothing */ static void hash_free(struct uma_hash *hash) { if (hash->uh_slab_hash == NULL) return; if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) uma_zfree_internal(hashzone, hash->uh_slab_hash, NULL, 0); else free(hash->uh_slab_hash, M_UMAHASH); } /* * Frees all outstanding items in a bucket * * Arguments: * zone The zone to free to, must be unlocked. * bucket The free/alloc bucket with items, cpu queue must be locked. * * Returns: * Nothing */ static void bucket_drain(uma_zone_t zone, uma_bucket_t bucket) { uma_slab_t slab; int mzone; void *item; if (bucket == NULL) return; slab = NULL; mzone = 0; /* We have to lookup the slab again for malloc.. */ if (zone->uz_flags & UMA_ZONE_MALLOC) mzone = 1; while (bucket->ub_cnt > 0) { bucket->ub_cnt--; item = bucket->ub_bucket[bucket->ub_cnt]; #ifdef INVARIANTS bucket->ub_bucket[bucket->ub_cnt] = NULL; KASSERT(item != NULL, ("bucket_drain: botched ptr, item is NULL")); #endif /* * This is extremely inefficient. The slab pointer was passed * to uma_zfree_arg, but we lost it because the buckets don't * hold them. This will go away when free() gets a size passed * to it. */ if (mzone) slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); uma_zfree_internal(zone, item, slab, 1); } } /* * Drains the per cpu caches for a zone. * * Arguments: * zone The zone to drain, must be unlocked. * * Returns: * Nothing */ static void cache_drain(uma_zone_t zone) { uma_bucket_t bucket; uma_cache_t cache; int cpu; /* * We have to lock each cpu cache before locking the zone */ - for (cpu = 0; cpu < mp_maxid; cpu++) { + for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; bucket_drain(zone, cache->uc_allocbucket); bucket_drain(zone, cache->uc_freebucket); if (cache->uc_allocbucket != NULL) bucket_free(cache->uc_allocbucket); if (cache->uc_freebucket != NULL) bucket_free(cache->uc_freebucket); cache->uc_allocbucket = cache->uc_freebucket = NULL; } /* * Drain the bucket queues and free the buckets, we just keep two per * cpu (alloc/free). */ ZONE_LOCK(zone); while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); ZONE_UNLOCK(zone); bucket_drain(zone, bucket); bucket_free(bucket); ZONE_LOCK(zone); } /* Now we do the free queue.. */ while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); bucket_free(bucket); } - for (cpu = 0; cpu < mp_maxid; cpu++) { + for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; CPU_UNLOCK(cpu); } ZONE_UNLOCK(zone); } /* * Frees pages from a zone back to the system. This is done on demand from * the pageout daemon. * * Arguments: * zone The zone to free pages from * all Should we drain all items? * * Returns: * Nothing. */ static void zone_drain(uma_zone_t zone) { struct slabhead freeslabs = {}; uma_slab_t slab; uma_slab_t n; u_int8_t flags; u_int8_t *mem; int i; /* * We don't want to take pages from staticly allocated zones at this * time */ if (zone->uz_flags & UMA_ZONE_NOFREE || zone->uz_freef == NULL) return; ZONE_LOCK(zone); #ifdef UMA_DEBUG printf("%s free items: %u\n", zone->uz_name, zone->uz_free); #endif if (zone->uz_free == 0) goto finished; slab = LIST_FIRST(&zone->uz_free_slab); while (slab) { n = LIST_NEXT(slab, us_link); /* We have no where to free these to */ if (slab->us_flags & UMA_SLAB_BOOT) { slab = n; continue; } LIST_REMOVE(slab, us_link); zone->uz_pages -= zone->uz_ppera; zone->uz_free -= zone->uz_ipers; if (zone->uz_flags & UMA_ZONE_HASH) UMA_HASH_REMOVE(&zone->uz_hash, slab, slab->us_data); SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); slab = n; } finished: ZONE_UNLOCK(zone); while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); if (zone->uz_fini) for (i = 0; i < zone->uz_ipers; i++) zone->uz_fini( slab->us_data + (zone->uz_rsize * i), zone->uz_size); flags = slab->us_flags; mem = slab->us_data; if (zone->uz_flags & UMA_ZONE_OFFPAGE) uma_zfree_internal(slabzone, slab, NULL, 0); if (zone->uz_flags & UMA_ZONE_MALLOC) { vm_object_t obj; if (flags & UMA_SLAB_KMEM) obj = kmem_object; else obj = NULL; for (i = 0; i < zone->uz_ppera; i++) vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), obj); } #ifdef UMA_DEBUG printf("%s: Returning %d bytes.\n", zone->uz_name, UMA_SLAB_SIZE * zone->uz_ppera); #endif zone->uz_freef(mem, UMA_SLAB_SIZE * zone->uz_ppera, flags); } } /* * Allocate a new slab for a zone. This does not insert the slab onto a list. * * Arguments: * zone The zone to allocate slabs for * wait Shall we wait? * * Returns: * The slab that was allocated or NULL if there is no memory and the * caller specified M_NOWAIT. */ static uma_slab_t slab_zalloc(uma_zone_t zone, int wait) { uma_slab_t slab; /* Starting slab */ u_int8_t *mem; u_int8_t flags; int i; slab = NULL; #ifdef UMA_DEBUG printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); #endif ZONE_UNLOCK(zone); if (zone->uz_flags & UMA_ZONE_OFFPAGE) { slab = uma_zalloc_internal(slabzone, NULL, wait); if (slab == NULL) { ZONE_LOCK(zone); return NULL; } } /* * This reproduces the old vm_zone behavior of zero filling pages the * first time they are added to a zone. * * Malloced items are zeroed in uma_zalloc. */ if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) wait |= M_ZERO; else wait &= ~M_ZERO; mem = zone->uz_allocf(zone, zone->uz_ppera * UMA_SLAB_SIZE, &flags, wait); if (mem == NULL) { ZONE_LOCK(zone); return (NULL); } /* Point the slab into the allocated memory */ if (!(zone->uz_flags & UMA_ZONE_OFFPAGE)) slab = (uma_slab_t )(mem + zone->uz_pgoff); if (zone->uz_flags & UMA_ZONE_MALLOC) for (i = 0; i < zone->uz_ppera; i++) vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); slab->us_zone = zone; slab->us_data = mem; slab->us_freecount = zone->uz_ipers; slab->us_firstfree = 0; slab->us_flags = flags; for (i = 0; i < zone->uz_ipers; i++) slab->us_freelist[i] = i+1; if (zone->uz_init) for (i = 0; i < zone->uz_ipers; i++) zone->uz_init(slab->us_data + (zone->uz_rsize * i), zone->uz_size); ZONE_LOCK(zone); if (zone->uz_flags & UMA_ZONE_HASH) UMA_HASH_INSERT(&zone->uz_hash, slab, mem); zone->uz_pages += zone->uz_ppera; zone->uz_free += zone->uz_ipers; return (slab); } /* * This function is intended to be used early on in place of page_alloc() so * that we may use the boot time page cache to satisfy allocations before * the VM is ready. */ static void * startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) { /* * Check our small startup cache to see if it has pages remaining. */ mtx_lock(&uma_mtx); if (uma_boot_free != 0) { uma_slab_t tmps; tmps = LIST_FIRST(&uma_boot_pages); LIST_REMOVE(tmps, us_link); uma_boot_free--; mtx_unlock(&uma_mtx); *pflag = tmps->us_flags; return (tmps->us_data); } mtx_unlock(&uma_mtx); if (booted == 0) panic("UMA: Increase UMA_BOOT_PAGES"); /* * Now that we've booted reset these users to their real allocator. */ #ifdef UMA_MD_SMALL_ALLOC zone->uz_allocf = uma_small_alloc; #else zone->uz_allocf = page_alloc; #endif return zone->uz_allocf(zone, bytes, pflag, wait); } /* * Allocates a number of pages from the system * * Arguments: * zone Unused * bytes The number of bytes requested * wait Shall we wait? * * Returns: * A pointer to the alloced memory or possibly * NULL if M_NOWAIT is set. */ static void * page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) { void *p; /* Returned page */ *pflag = UMA_SLAB_KMEM; p = (void *) kmem_malloc(kmem_map, bytes, wait); return (p); } /* * Allocates a number of pages from within an object * * Arguments: * zone Unused * bytes The number of bytes requested * wait Shall we wait? * * Returns: * A pointer to the alloced memory or possibly * NULL if M_NOWAIT is set. */ static void * obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) { vm_object_t object; vm_offset_t retkva, zkva; vm_page_t p; int pages, startpages; object = zone->uz_obj; retkva = 0; /* * This looks a little weird since we're getting one page at a time. */ VM_OBJECT_LOCK(object); p = TAILQ_LAST(&object->memq, pglist); pages = p != NULL ? p->pindex + 1 : 0; startpages = pages; zkva = zone->uz_kva + pages * PAGE_SIZE; for (; bytes > 0; bytes -= PAGE_SIZE) { p = vm_page_alloc(object, pages, VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); if (p == NULL) { if (pages != startpages) pmap_qremove(retkva, pages - startpages); while (pages != startpages) { pages--; p = TAILQ_LAST(&object->memq, pglist); vm_page_lock_queues(); vm_page_unwire(p, 0); vm_page_free(p); vm_page_unlock_queues(); } retkva = 0; goto done; } pmap_qenter(zkva, &p, 1); if (retkva == 0) retkva = zkva; zkva += PAGE_SIZE; pages += 1; } done: VM_OBJECT_UNLOCK(object); *flags = UMA_SLAB_PRIV; return ((void *)retkva); } /* * Frees a number of pages to the system * * Arguments: * mem A pointer to the memory to be freed * size The size of the memory being freed * flags The original p->us_flags field * * Returns: * Nothing */ static void page_free(void *mem, int size, u_int8_t flags) { vm_map_t map; if (flags & UMA_SLAB_KMEM) map = kmem_map; else panic("UMA: page_free used with invalid flags %d\n", flags); kmem_free(map, (vm_offset_t)mem, size); } /* * Zero fill initializer * * Arguments/Returns follow uma_init specifications */ static void zero_init(void *mem, int size) { bzero(mem, size); } /* * Finish creating a small uma zone. This calculates ipers, and the zone size. * * Arguments * zone The zone we should initialize * * Returns * Nothing */ static void zone_small_init(uma_zone_t zone) { int rsize; int memused; int ipers; rsize = zone->uz_size; if (rsize < UMA_SMALLEST_UNIT) rsize = UMA_SMALLEST_UNIT; if (rsize & zone->uz_align) rsize = (rsize & ~zone->uz_align) + (zone->uz_align + 1); zone->uz_rsize = rsize; rsize += 1; /* Account for the byte of linkage */ zone->uz_ipers = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / rsize; zone->uz_ppera = 1; KASSERT(zone->uz_ipers != 0, ("zone_small_init: ipers is 0, uh-oh!")); memused = zone->uz_ipers * zone->uz_rsize; /* Can we do any better? */ if ((UMA_SLAB_SIZE - memused) >= UMA_MAX_WASTE) { /* * We can't do this if we're internal or if we've been * asked to not go to the VM for buckets. If we do this we * may end up going to the VM (kmem_map) for slabs which we * do not want to do if we're UMA_ZFLAG_CACHEONLY as a * result of UMA_ZONE_VM, which clearly forbids it. */ if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) || (zone->uz_flags & UMA_ZFLAG_CACHEONLY)) return; ipers = UMA_SLAB_SIZE / zone->uz_rsize; if (ipers > zone->uz_ipers) { zone->uz_flags |= UMA_ZONE_OFFPAGE; if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) zone->uz_flags |= UMA_ZONE_HASH; zone->uz_ipers = ipers; } } } /* * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be * more complicated. * * Arguments * zone The zone we should initialize * * Returns * Nothing */ static void zone_large_init(uma_zone_t zone) { int pages; KASSERT((zone->uz_flags & UMA_ZFLAG_CACHEONLY) == 0, ("zone_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY zone")); pages = zone->uz_size / UMA_SLAB_SIZE; /* Account for remainder */ if ((pages * UMA_SLAB_SIZE) < zone->uz_size) pages++; zone->uz_ppera = pages; zone->uz_ipers = 1; zone->uz_flags |= UMA_ZONE_OFFPAGE; if ((zone->uz_flags & UMA_ZONE_MALLOC) == 0) zone->uz_flags |= UMA_ZONE_HASH; zone->uz_rsize = zone->uz_size; } /* * Zone header ctor. This initializes all fields, locks, etc. And inserts * the zone onto the global zone list. * * Arguments/Returns follow uma_ctor specifications * udata Actually uma_zcreat_args */ static void zone_ctor(void *mem, int size, void *udata) { struct uma_zctor_args *arg = udata; uma_zone_t zone = mem; int privlc; bzero(zone, size); zone->uz_name = arg->name; zone->uz_size = arg->size; zone->uz_ctor = arg->ctor; zone->uz_dtor = arg->dtor; zone->uz_init = arg->uminit; zone->uz_fini = arg->fini; zone->uz_align = arg->align; zone->uz_free = 0; zone->uz_pages = 0; zone->uz_flags = arg->flags; zone->uz_allocf = page_alloc; zone->uz_freef = page_free; if (arg->flags & UMA_ZONE_ZINIT) zone->uz_init = zero_init; if (arg->flags & UMA_ZONE_VM) zone->uz_flags |= UMA_ZFLAG_CACHEONLY; /* * XXX: * The +1 byte added to uz_size is to account for the byte of * linkage that is added to the size in zone_small_init(). If * we don't account for this here then we may end up in * zone_small_init() with a calculated 'ipers' of 0. */ if ((zone->uz_size+1) > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) zone_large_init(zone); else zone_small_init(zone); /* * If we haven't booted yet we need allocations to go through the * startup cache until the vm is ready. */ if (zone->uz_ppera == 1) { #ifdef UMA_MD_SMALL_ALLOC zone->uz_allocf = uma_small_alloc; zone->uz_freef = uma_small_free; #endif if (booted == 0) zone->uz_allocf = startup_alloc; } if (arg->flags & UMA_ZONE_MTXCLASS) privlc = 1; else privlc = 0; /* * If we're putting the slab header in the actual page we need to * figure out where in each page it goes. This calculates a right * justified offset into the memory on an ALIGN_PTR boundary. */ if (!(zone->uz_flags & UMA_ZONE_OFFPAGE)) { int totsize; /* Size of the slab struct and free list */ totsize = sizeof(struct uma_slab) + zone->uz_ipers; if (totsize & UMA_ALIGN_PTR) totsize = (totsize & ~UMA_ALIGN_PTR) + (UMA_ALIGN_PTR + 1); zone->uz_pgoff = UMA_SLAB_SIZE - totsize; totsize = zone->uz_pgoff + sizeof(struct uma_slab) + zone->uz_ipers; /* I don't think it's possible, but I'll make sure anyway */ if (totsize > UMA_SLAB_SIZE) { printf("zone %s ipers %d rsize %d size %d\n", zone->uz_name, zone->uz_ipers, zone->uz_rsize, zone->uz_size); panic("UMA slab won't fit.\n"); } } if (zone->uz_flags & UMA_ZONE_HASH) hash_alloc(&zone->uz_hash); #ifdef UMA_DEBUG printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", zone->uz_name, zone, zone->uz_size, zone->uz_ipers, zone->uz_ppera, zone->uz_pgoff); #endif ZONE_LOCK_INIT(zone, privlc); mtx_lock(&uma_mtx); LIST_INSERT_HEAD(&uma_zones, zone, uz_link); mtx_unlock(&uma_mtx); /* * Some internal zones don't have room allocated for the per cpu * caches. If we're internal, bail out here. */ if (zone->uz_flags & UMA_ZFLAG_INTERNAL) return; if (zone->uz_ipers <= BUCKET_MAX) zone->uz_count = zone->uz_ipers; else zone->uz_count = BUCKET_MAX; } /* * Zone header dtor. This frees all data, destroys locks, frees the hash table * and removes the zone from the global list. * * Arguments/Returns follow uma_dtor specifications * udata unused */ static void zone_dtor(void *arg, int size, void *udata) { uma_zone_t zone; zone = (uma_zone_t)arg; if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) cache_drain(zone); mtx_lock(&uma_mtx); LIST_REMOVE(zone, uz_link); zone_drain(zone); mtx_unlock(&uma_mtx); ZONE_LOCK(zone); if (zone->uz_free != 0) { printf("Zone %s was not empty (%d items). " " Lost %d pages of memory.\n", zone->uz_name, zone->uz_free, zone->uz_pages); uma_print_zone(zone); } ZONE_UNLOCK(zone); if (zone->uz_flags & UMA_ZONE_HASH) hash_free(&zone->uz_hash); ZONE_LOCK_FINI(zone); } /* * Traverses every zone in the system and calls a callback * * Arguments: * zfunc A pointer to a function which accepts a zone * as an argument. * * Returns: * Nothing */ static void zone_foreach(void (*zfunc)(uma_zone_t)) { uma_zone_t zone; mtx_lock(&uma_mtx); LIST_FOREACH(zone, &uma_zones, uz_link) zfunc(zone); mtx_unlock(&uma_mtx); } /* Public functions */ /* See uma.h */ void uma_startup(void *bootmem) { struct uma_zctor_args args; uma_slab_t slab; int slabsize; int i; #ifdef UMA_DEBUG printf("Creating uma zone headers zone.\n"); #endif mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); /* "manually" Create the initial zone */ args.name = "UMA Zones"; args.size = sizeof(struct uma_zone) + - (sizeof(struct uma_cache) * mp_maxid); + (sizeof(struct uma_cache) * (mp_maxid + 1)); args.ctor = zone_ctor; args.dtor = zone_dtor; args.uminit = zero_init; args.fini = NULL; args.align = 32 - 1; args.flags = UMA_ZFLAG_INTERNAL; /* The initial zone has no Per cpu queues so it's smaller */ zone_ctor(zones, sizeof(struct uma_zone), &args); /* Initialize the pcpu cache lock set once and for all */ - for (i = 0; i < mp_maxid; i++) + for (i = 0; i <= mp_maxid; i++) CPU_LOCK_INIT(i); #ifdef UMA_DEBUG printf("Filling boot free list.\n"); #endif for (i = 0; i < UMA_BOOT_PAGES; i++) { slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); slab->us_data = (u_int8_t *)slab; slab->us_flags = UMA_SLAB_BOOT; LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); uma_boot_free++; } #ifdef UMA_DEBUG printf("Creating slab zone.\n"); #endif /* * This is the max number of free list items we'll have with * offpage slabs. */ slabsize = UMA_SLAB_SIZE - sizeof(struct uma_slab); slabsize /= UMA_MAX_WASTE; slabsize++; /* In case there it's rounded */ slabsize += sizeof(struct uma_slab); /* Now make a zone for slab headers */ slabzone = uma_zcreate("UMA Slabs", slabsize, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); hashzone = uma_zcreate("UMA Hash", sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); bucket_init(); #ifdef UMA_MD_SMALL_ALLOC booted = 1; #endif #ifdef UMA_DEBUG printf("UMA startup complete.\n"); #endif } /* see uma.h */ void uma_startup2(void) { booted = 1; bucket_enable(); #ifdef UMA_DEBUG printf("UMA startup2 complete.\n"); #endif } /* * Initialize our callout handle * */ static void uma_startup3(void) { #ifdef UMA_DEBUG printf("Starting callout.\n"); #endif callout_init(&uma_callout, 0); callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); #ifdef UMA_DEBUG printf("UMA startup3 complete.\n"); #endif } /* See uma.h */ uma_zone_t uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, int align, u_int16_t flags) { struct uma_zctor_args args; /* This stuff is essential for the zone ctor */ args.name = name; args.size = size; args.ctor = ctor; args.dtor = dtor; args.uminit = uminit; args.fini = fini; args.align = align; args.flags = flags; return (uma_zalloc_internal(zones, &args, M_WAITOK)); } /* See uma.h */ void uma_zdestroy(uma_zone_t zone) { uma_zfree_internal(zones, zone, NULL, 0); } /* See uma.h */ void * uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) { void *item; uma_cache_t cache; uma_bucket_t bucket; int cpu; /* This is the fast path allocation */ #ifdef UMA_DEBUG_ALLOC_1 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); #endif #ifdef INVARIANTS /* * To make sure that WAITOK or NOWAIT is set, but not more than * one, and check against the API botches that are common. * The uma code implies M_WAITOK if M_NOWAIT is not set, so * we default to waiting if none of the flags is set. */ cpu = flags & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT); if (cpu != M_NOWAIT && cpu != M_WAITOK) { static struct timeval lasterr; static int curerr, once; if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) { printf("Bad uma_zalloc flags: %x\n", cpu); backtrace(); once++; } } #endif if (!(flags & M_NOWAIT)) { KASSERT(curthread->td_intr_nesting_level == 0, ("malloc(M_WAITOK) in interrupt context")); WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "malloc() of \"%s\"", zone->uz_name); } zalloc_restart: cpu = PCPU_GET(cpuid); CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zalloc_start: bucket = cache->uc_allocbucket; if (bucket) { if (bucket->ub_cnt > 0) { bucket->ub_cnt--; item = bucket->ub_bucket[bucket->ub_cnt]; #ifdef INVARIANTS bucket->ub_bucket[bucket->ub_cnt] = NULL; #endif KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); cache->uc_allocs++; #ifdef INVARIANTS ZONE_LOCK(zone); uma_dbg_alloc(zone, NULL, item); ZONE_UNLOCK(zone); #endif CPU_UNLOCK(cpu); if (zone->uz_ctor) zone->uz_ctor(item, zone->uz_size, udata); if (flags & M_ZERO) bzero(item, zone->uz_size); return (item); } else if (cache->uc_freebucket) { /* * We have run out of items in our allocbucket. * See if we can switch with our free bucket. */ if (cache->uc_freebucket->ub_cnt > 0) { #ifdef UMA_DEBUG_ALLOC printf("uma_zalloc: Swapping empty with" " alloc.\n"); #endif bucket = cache->uc_freebucket; cache->uc_freebucket = cache->uc_allocbucket; cache->uc_allocbucket = bucket; goto zalloc_start; } } } ZONE_LOCK(zone); /* Since we have locked the zone we may as well send back our stats */ zone->uz_allocs += cache->uc_allocs; cache->uc_allocs = 0; /* Our old one is now a free bucket */ if (cache->uc_allocbucket) { KASSERT(cache->uc_allocbucket->ub_cnt == 0, ("uma_zalloc_arg: Freeing a non free bucket.")); LIST_INSERT_HEAD(&zone->uz_free_bucket, cache->uc_allocbucket, ub_link); cache->uc_allocbucket = NULL; } /* Check the free list for a new alloc bucket */ if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); LIST_REMOVE(bucket, ub_link); cache->uc_allocbucket = bucket; ZONE_UNLOCK(zone); goto zalloc_start; } /* We are no longer associated with this cpu!!! */ CPU_UNLOCK(cpu); /* Bump up our uz_count so we get here less */ if (zone->uz_count < BUCKET_MAX) zone->uz_count++; /* * Now lets just fill a bucket and put it on the free list. If that * works we'll restart the allocation from the begining. */ if (uma_zalloc_bucket(zone, flags)) { ZONE_UNLOCK(zone); goto zalloc_restart; } ZONE_UNLOCK(zone); /* * We may not be able to get a bucket so return an actual item. */ #ifdef UMA_DEBUG printf("uma_zalloc_arg: Bucketzone returned NULL\n"); #endif return (uma_zalloc_internal(zone, udata, flags)); } static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags) { uma_slab_t slab; /* * This is to prevent us from recursively trying to allocate * buckets. The problem is that if an allocation forces us to * grab a new bucket we will call page_alloc, which will go off * and cause the vm to allocate vm_map_entries. If we need new * buckets there too we will recurse in kmem_alloc and bad * things happen. So instead we return a NULL bucket, and make * the code that allocates buckets smart enough to deal with it */ if (zone->uz_flags & UMA_ZFLAG_INTERNAL && zone->uz_recurse != 0) return (NULL); slab = NULL; for (;;) { /* * Find a slab with some space. Prefer slabs that are partially * used over those that are totally full. This helps to reduce * fragmentation. */ if (zone->uz_free != 0) { if (!LIST_EMPTY(&zone->uz_part_slab)) { slab = LIST_FIRST(&zone->uz_part_slab); } else { slab = LIST_FIRST(&zone->uz_free_slab); LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); } return (slab); } /* * M_NOVM means don't ask at all! */ if (flags & M_NOVM) break; if (zone->uz_maxpages && zone->uz_pages >= zone->uz_maxpages) { zone->uz_flags |= UMA_ZFLAG_FULL; if (flags & M_NOWAIT) break; else msleep(zone, &zone->uz_lock, PVM, "zonelimit", 0); continue; } zone->uz_recurse++; slab = slab_zalloc(zone, flags); zone->uz_recurse--; /* * If we got a slab here it's safe to mark it partially used * and return. We assume that the caller is going to remove * at least one item. */ if (slab) { LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); return (slab); } /* * We might not have been able to get a slab but another cpu * could have while we were unlocked. Check again before we * fail. */ if (flags & M_NOWAIT) flags |= M_NOVM; } return (slab); } static void * uma_slab_alloc(uma_zone_t zone, uma_slab_t slab) { void *item; u_int8_t freei; freei = slab->us_firstfree; slab->us_firstfree = slab->us_freelist[freei]; item = slab->us_data + (zone->uz_rsize * freei); slab->us_freecount--; zone->uz_free--; #ifdef INVARIANTS uma_dbg_alloc(zone, slab, item); #endif /* Move this slab to the full list */ if (slab->us_freecount == 0) { LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&zone->uz_full_slab, slab, us_link); } return (item); } static int uma_zalloc_bucket(uma_zone_t zone, int flags) { uma_bucket_t bucket; uma_slab_t slab; int max; /* * Try this zone's free list first so we don't allocate extra buckets. */ if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { KASSERT(bucket->ub_cnt == 0, ("uma_zalloc_bucket: Bucket on free list is not empty.")); LIST_REMOVE(bucket, ub_link); } else { int bflags; bflags = (flags & ~M_ZERO); if (zone->uz_flags & UMA_ZFLAG_CACHEONLY) bflags |= M_NOVM; ZONE_UNLOCK(zone); bucket = bucket_alloc(zone->uz_count, bflags); ZONE_LOCK(zone); } if (bucket == NULL) return (0); #ifdef SMP /* * This code is here to limit the number of simultaneous bucket fills * for any given zone to the number of per cpu caches in this zone. This * is done so that we don't allocate more memory than we really need. */ if (zone->uz_fills >= mp_ncpus) goto done; #endif zone->uz_fills++; max = MIN(bucket->ub_entries, zone->uz_count); /* Try to keep the buckets totally full */ while (bucket->ub_cnt < max && (slab = uma_zone_slab(zone, flags)) != NULL) { while (slab->us_freecount && bucket->ub_cnt < max) { bucket->ub_bucket[bucket->ub_cnt++] = uma_slab_alloc(zone, slab); } /* Don't block on the next fill */ flags |= M_NOWAIT; } zone->uz_fills--; if (bucket->ub_cnt != 0) { LIST_INSERT_HEAD(&zone->uz_full_bucket, bucket, ub_link); return (1); } #ifdef SMP done: #endif bucket_free(bucket); return (0); } /* * Allocates an item for an internal zone * * Arguments * zone The zone to alloc for. * udata The data to be passed to the constructor. * flags M_WAITOK, M_NOWAIT, M_ZERO. * * Returns * NULL if there is no memory and M_NOWAIT is set * An item if successful */ static void * uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) { uma_slab_t slab; void *item; item = NULL; #ifdef UMA_DEBUG_ALLOC printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); #endif ZONE_LOCK(zone); slab = uma_zone_slab(zone, flags); if (slab == NULL) { ZONE_UNLOCK(zone); return (NULL); } item = uma_slab_alloc(zone, slab); ZONE_UNLOCK(zone); if (zone->uz_ctor != NULL) zone->uz_ctor(item, zone->uz_size, udata); if (flags & M_ZERO) bzero(item, zone->uz_size); return (item); } /* See uma.h */ void uma_zfree_arg(uma_zone_t zone, void *item, void *udata) { uma_cache_t cache; uma_bucket_t bucket; int bflags; int cpu; int skip; /* This is the fast path free */ skip = 0; #ifdef UMA_DEBUG_ALLOC_1 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); #endif /* * The race here is acceptable. If we miss it we'll just have to wait * a little longer for the limits to be reset. */ if (zone->uz_flags & UMA_ZFLAG_FULL) goto zfree_internal; if (zone->uz_dtor) { zone->uz_dtor(item, zone->uz_size, udata); skip = 1; } zfree_restart: cpu = PCPU_GET(cpuid); CPU_LOCK(cpu); cache = &zone->uz_cpu[cpu]; zfree_start: bucket = cache->uc_freebucket; if (bucket) { /* * Do we have room in our bucket? It is OK for this uz count * check to be slightly out of sync. */ if (bucket->ub_cnt < bucket->ub_entries) { KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, ("uma_zfree: Freeing to non free bucket index.")); bucket->ub_bucket[bucket->ub_cnt] = item; bucket->ub_cnt++; #ifdef INVARIANTS ZONE_LOCK(zone); if (zone->uz_flags & UMA_ZONE_MALLOC) uma_dbg_free(zone, udata, item); else uma_dbg_free(zone, NULL, item); ZONE_UNLOCK(zone); #endif CPU_UNLOCK(cpu); return; } else if (cache->uc_allocbucket) { #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Swapping buckets.\n"); #endif /* * We have run out of space in our freebucket. * See if we can switch with our alloc bucket. */ if (cache->uc_allocbucket->ub_cnt < cache->uc_freebucket->ub_cnt) { bucket = cache->uc_freebucket; cache->uc_freebucket = cache->uc_allocbucket; cache->uc_allocbucket = bucket; goto zfree_start; } } } /* * We can get here for two reasons: * * 1) The buckets are NULL * 2) The alloc and free buckets are both somewhat full. */ ZONE_LOCK(zone); bucket = cache->uc_freebucket; cache->uc_freebucket = NULL; /* Can we throw this on the zone full list? */ if (bucket != NULL) { #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Putting old bucket on the free list.\n"); #endif /* ub_cnt is pointing to the last free item */ KASSERT(bucket->ub_cnt != 0, ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); LIST_INSERT_HEAD(&zone->uz_full_bucket, bucket, ub_link); } if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { LIST_REMOVE(bucket, ub_link); ZONE_UNLOCK(zone); cache->uc_freebucket = bucket; goto zfree_start; } /* We're done with this CPU now */ CPU_UNLOCK(cpu); /* And the zone.. */ ZONE_UNLOCK(zone); #ifdef UMA_DEBUG_ALLOC printf("uma_zfree: Allocating new free bucket.\n"); #endif bflags = M_NOWAIT; if (zone->uz_flags & UMA_ZFLAG_CACHEONLY) bflags |= M_NOVM; bucket = bucket_alloc(zone->uz_count, bflags); if (bucket) { ZONE_LOCK(zone); LIST_INSERT_HEAD(&zone->uz_free_bucket, bucket, ub_link); ZONE_UNLOCK(zone); goto zfree_restart; } /* * If nothing else caught this, we'll just do an internal free. */ zfree_internal: #ifdef INVARIANTS /* * If we need to skip the dtor and the uma_dbg_free in * uma_zfree_internal because we've already called the dtor * above, but we ended up here, then we need to make sure * that we take care of the uma_dbg_free immediately. */ if (skip) { ZONE_LOCK(zone); if (zone->uz_flags & UMA_ZONE_MALLOC) uma_dbg_free(zone, udata, item); else uma_dbg_free(zone, NULL, item); ZONE_UNLOCK(zone); } #endif uma_zfree_internal(zone, item, udata, skip); return; } /* * Frees an item to an INTERNAL zone or allocates a free bucket * * Arguments: * zone The zone to free to * item The item we're freeing * udata User supplied data for the dtor * skip Skip the dtor, it was done in uma_zfree_arg */ static void uma_zfree_internal(uma_zone_t zone, void *item, void *udata, int skip) { uma_slab_t slab; u_int8_t *mem; u_int8_t freei; if (!skip && zone->uz_dtor) zone->uz_dtor(item, zone->uz_size, udata); ZONE_LOCK(zone); if (!(zone->uz_flags & UMA_ZONE_MALLOC)) { mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); if (zone->uz_flags & UMA_ZONE_HASH) slab = hash_sfind(&zone->uz_hash, mem); else { mem += zone->uz_pgoff; slab = (uma_slab_t)mem; } } else { slab = (uma_slab_t)udata; } /* Do we need to remove from any lists? */ if (slab->us_freecount+1 == zone->uz_ipers) { LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); } else if (slab->us_freecount == 0) { LIST_REMOVE(slab, us_link); LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); } /* Slab management stuff */ freei = ((unsigned long)item - (unsigned long)slab->us_data) / zone->uz_rsize; #ifdef INVARIANTS if (!skip) uma_dbg_free(zone, slab, item); #endif slab->us_freelist[freei] = slab->us_firstfree; slab->us_firstfree = freei; slab->us_freecount++; /* Zone statistics */ zone->uz_free++; if (zone->uz_flags & UMA_ZFLAG_FULL) { if (zone->uz_pages < zone->uz_maxpages) zone->uz_flags &= ~UMA_ZFLAG_FULL; /* We can handle one more allocation */ wakeup_one(zone); } ZONE_UNLOCK(zone); } /* See uma.h */ void uma_zone_set_max(uma_zone_t zone, int nitems) { ZONE_LOCK(zone); if (zone->uz_ppera > 1) zone->uz_maxpages = nitems * zone->uz_ppera; else zone->uz_maxpages = nitems / zone->uz_ipers; if (zone->uz_maxpages * zone->uz_ipers < nitems) zone->uz_maxpages++; ZONE_UNLOCK(zone); } /* See uma.h */ void uma_zone_set_freef(uma_zone_t zone, uma_free freef) { ZONE_LOCK(zone); zone->uz_freef = freef; ZONE_UNLOCK(zone); } /* See uma.h */ void uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) { ZONE_LOCK(zone); zone->uz_flags |= UMA_ZFLAG_PRIVALLOC; zone->uz_allocf = allocf; ZONE_UNLOCK(zone); } /* See uma.h */ int uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) { int pages; vm_offset_t kva; pages = count / zone->uz_ipers; if (pages * zone->uz_ipers < count) pages++; kva = kmem_alloc_pageable(kernel_map, pages * UMA_SLAB_SIZE); if (kva == 0) return (0); if (obj == NULL) { obj = vm_object_allocate(OBJT_DEFAULT, pages); } else { VM_OBJECT_LOCK_INIT(obj); _vm_object_allocate(OBJT_DEFAULT, pages, obj); } ZONE_LOCK(zone); zone->uz_kva = kva; zone->uz_obj = obj; zone->uz_maxpages = pages; zone->uz_allocf = obj_alloc; zone->uz_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC; ZONE_UNLOCK(zone); return (1); } /* See uma.h */ void uma_prealloc(uma_zone_t zone, int items) { int slabs; uma_slab_t slab; ZONE_LOCK(zone); slabs = items / zone->uz_ipers; if (slabs * zone->uz_ipers < items) slabs++; while (slabs > 0) { slab = slab_zalloc(zone, M_WAITOK); LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); slabs--; } ZONE_UNLOCK(zone); } /* See uma.h */ void uma_reclaim(void) { #ifdef UMA_DEBUG printf("UMA: vm asked us to release pages!\n"); #endif bucket_enable(); zone_foreach(zone_drain); /* * Some slabs may have been freed but this zone will be visited early * we visit again so that we can free pages that are empty once other * zones are drained. We have to do the same for buckets. */ zone_drain(slabzone); bucket_zone_drain(); } void * uma_large_malloc(int size, int wait) { void *mem; uma_slab_t slab; u_int8_t flags; slab = uma_zalloc_internal(slabzone, NULL, wait); if (slab == NULL) return (NULL); mem = page_alloc(NULL, size, &flags, wait); if (mem) { vsetslab((vm_offset_t)mem, slab); slab->us_data = mem; slab->us_flags = flags | UMA_SLAB_MALLOC; slab->us_size = size; } else { uma_zfree_internal(slabzone, slab, NULL, 0); } return (mem); } void uma_large_free(uma_slab_t slab) { vsetobj((vm_offset_t)slab->us_data, kmem_object); /* * XXX: We get a lock order reversal if we don't have Giant: * vm_map_remove (locks system map) -> vm_map_delete -> * vm_map_entry_unwire -> vm_fault_unwire -> mtx_lock(&Giant) */ if (!mtx_owned(&Giant)) { mtx_lock(&Giant); page_free(slab->us_data, slab->us_size, slab->us_flags); mtx_unlock(&Giant); } else page_free(slab->us_data, slab->us_size, slab->us_flags); uma_zfree_internal(slabzone, slab, NULL, 0); } void uma_print_stats(void) { zone_foreach(uma_print_zone); } static void slab_print(uma_slab_t slab) { printf("slab: zone %p, data %p, freecount %d, firstfree %d\n", slab->us_zone, slab->us_data, slab->us_freecount, slab->us_firstfree); } static void cache_print(uma_cache_t cache) { printf("alloc: %p(%d), free: %p(%d)\n", cache->uc_allocbucket, cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, cache->uc_freebucket, cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); } void uma_print_zone(uma_zone_t zone) { uma_cache_t cache; uma_slab_t slab; int i; printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", zone->uz_name, zone, zone->uz_size, zone->uz_rsize, zone->uz_flags, zone->uz_ipers, zone->uz_ppera, (zone->uz_ipers * zone->uz_pages) - zone->uz_free, zone->uz_free); printf("Part slabs:\n"); LIST_FOREACH(slab, &zone->uz_part_slab, us_link) slab_print(slab); printf("Free slabs:\n"); LIST_FOREACH(slab, &zone->uz_free_slab, us_link) slab_print(slab); printf("Full slabs:\n"); LIST_FOREACH(slab, &zone->uz_full_slab, us_link) slab_print(slab); - for (i = 0; i < mp_maxid; i++) { + for (i = 0; i <= mp_maxid; i++) { if (CPU_ABSENT(i)) continue; cache = &zone->uz_cpu[i]; printf("CPU %d Cache:\n", i); cache_print(cache); } } /* * Sysctl handler for vm.zone * * stolen from vm_zone.c */ static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS) { int error, len, cnt; const int linesize = 128; /* conservative */ int totalfree; char *tmpbuf, *offset; uma_zone_t z; char *p; int cpu; int cachefree; uma_bucket_t bucket; uma_cache_t cache; cnt = 0; mtx_lock(&uma_mtx); LIST_FOREACH(z, &uma_zones, uz_link) cnt++; mtx_unlock(&uma_mtx); MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, M_TEMP, M_WAITOK); len = snprintf(tmpbuf, linesize, "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); if (cnt == 0) tmpbuf[len - 1] = '\0'; error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); if (error || cnt == 0) goto out; offset = tmpbuf; mtx_lock(&uma_mtx); LIST_FOREACH(z, &uma_zones, uz_link) { if (cnt == 0) /* list may have changed size */ break; if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) { - for (cpu = 0; cpu < mp_maxid; cpu++) { + for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; CPU_LOCK(cpu); } } ZONE_LOCK(z); cachefree = 0; if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) { - for (cpu = 0; cpu < mp_maxid; cpu++) { + for (cpu = 0; cpu <= mp_maxid; cpu++) { if (CPU_ABSENT(cpu)) continue; cache = &z->uz_cpu[cpu]; if (cache->uc_allocbucket != NULL) cachefree += cache->uc_allocbucket->ub_cnt; if (cache->uc_freebucket != NULL) cachefree += cache->uc_freebucket->ub_cnt; CPU_UNLOCK(cpu); } } LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) { cachefree += bucket->ub_cnt; } totalfree = z->uz_free + cachefree; len = snprintf(offset, linesize, "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", z->uz_name, z->uz_size, z->uz_maxpages * z->uz_ipers, (z->uz_ipers * (z->uz_pages / z->uz_ppera)) - totalfree, totalfree, (unsigned long long)z->uz_allocs); ZONE_UNLOCK(z); for (p = offset + 12; p > offset && *p == ' '; --p) /* nothing */ ; p[1] = ':'; cnt--; offset += len; } mtx_unlock(&uma_mtx); *offset++ = '\0'; error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); out: FREE(tmpbuf, M_TEMP); return (error); }