Index: head/sys/conf/files.powerpc =================================================================== --- head/sys/conf/files.powerpc +++ head/sys/conf/files.powerpc @@ -198,6 +198,7 @@ powerpc/powerpc/copyinout.c standard powerpc/powerpc/copystr.c standard powerpc/powerpc/cpu.c standard +powerpc/powerpc/cpu_subr64.S optional powerpc64 powerpc/powerpc/db_disasm.c optional ddb powerpc/powerpc/db_hwwatch.c optional ddb powerpc/powerpc/db_interface.c optional ddb Index: head/sys/powerpc/aim/locore64.S =================================================================== --- head/sys/powerpc/aim/locore64.S +++ head/sys/powerpc/aim/locore64.S @@ -53,6 +53,8 @@ .llong begin GLOBAL(__endkernel) .llong end +GLOBAL(can_wakeup) + .llong 0x0 .align 4 #define TMPSTKSZ 16384 /* 16K temporary stack */ @@ -60,6 +62,7 @@ .space TMPSTKSZ TOC_ENTRY(tmpstk) +TOC_ENTRY(can_wakeup) /* * Entry point for bootloaders that do not fully implement ELF and start @@ -96,6 +99,10 @@ /* Released */ or 2,2,2 /* unyield */ + + /* Make sure that it will be software reset. Clear SRR1 */ + li %r1,0 + mtsrr1 %r1 ba EXC_RST Index: head/sys/powerpc/aim/mp_cpudep.c =================================================================== --- head/sys/powerpc/aim/mp_cpudep.c +++ head/sys/powerpc/aim/mp_cpudep.c @@ -398,8 +398,11 @@ case IBMPOWER8: case IBMPOWER8E: #ifdef __powerpc64__ - if (mfmsr() & PSL_HV) - mtspr(SPR_LPCR, mfspr(SPR_LPCR) | LPCR_LPES); + if (mfmsr() & PSL_HV) { + mtspr(SPR_LPCR, mfspr(SPR_LPCR) | LPCR_LPES | + LPCR_PECE_WAKESET); + isync(); + } #endif break; default: Index: head/sys/powerpc/aim/trap_subr64.S =================================================================== --- head/sys/powerpc/aim/trap_subr64.S +++ head/sys/powerpc/aim/trap_subr64.S @@ -307,10 +307,24 @@ * once the MMU is turned on. */ .globl CNAME(rstcode), CNAME(rstcodeend), CNAME(cpu_reset_handler) + .globl CNAME(cpu_wakeup_handler) .p2align 3 CNAME(rstcode): + /* + * Check if this is software reset or + * processor is waking up from power saving mode + * It is software reset when 46:47 = 0b00 + */ + mfsrr1 %r9 /* Load SRR1 into r9 */ + andis. %r9,%r9,0x3 /* Logic AND with 46:47 bits */ + beq 2f /* Branch if software reset */ + bl 1f + .llong cpu_wakeup_handler + + /* It is software reset */ + /* Explicitly set MSR[SF] */ - mfmsr %r9 +2: mfmsr %r9 li %r8,1 insrdi %r9,%r8,1,0 mtmsrd %r9 @@ -318,6 +332,7 @@ bl 1f .llong cpu_reset_handler /* Make sure to maintain 8-byte alignment */ + 1: mflr %r9 ld %r9,0(%r9) mtlr %r9 @@ -358,6 +373,59 @@ /* Should not be reached */ 9: b 9b + +cpu_wakeup_handler: + GET_TOCBASE(%r2) + + /* Check for false wake up due to badly SRR1 set (eg. by OPAL) */ + ld %r3,TOC_REF(can_wakeup)(%r2) + ld %r3,0(%r3) + cmpdi %r3,0 + beq cpu_reset_handler + + /* Turn on MMU after return from interrupt */ + mfsrr1 %r3 + ori %r3,%r3,(PSL_IR | PSL_DR) + mtsrr1 %r3 + + /* Turn on MMU (needed to access PCB) */ + mfmsr %r3 + ori %r3,%r3,(PSL_IR | PSL_DR) + mtmsr %r3 + isync + + mfsprg0 %r3 + + ld %r3,PC_CURTHREAD(%r3) /* Get current thread */ + ld %r3,TD_PCB(%r3) /* Get PCB of current thread */ + ld %r12,PCB_CONTEXT(%r3) /* Load the non-volatile GP regs. */ + ld %r13,PCB_CONTEXT+1*8(%r3) + ld %r14,PCB_CONTEXT+2*8(%r3) + ld %r15,PCB_CONTEXT+3*8(%r3) + ld %r16,PCB_CONTEXT+4*8(%r3) + ld %r17,PCB_CONTEXT+5*8(%r3) + ld %r18,PCB_CONTEXT+6*8(%r3) + ld %r19,PCB_CONTEXT+7*8(%r3) + ld %r20,PCB_CONTEXT+8*8(%r3) + ld %r21,PCB_CONTEXT+9*8(%r3) + ld %r22,PCB_CONTEXT+10*8(%r3) + ld %r23,PCB_CONTEXT+11*8(%r3) + ld %r24,PCB_CONTEXT+12*8(%r3) + ld %r25,PCB_CONTEXT+13*8(%r3) + ld %r26,PCB_CONTEXT+14*8(%r3) + ld %r27,PCB_CONTEXT+15*8(%r3) + ld %r28,PCB_CONTEXT+16*8(%r3) + ld %r29,PCB_CONTEXT+17*8(%r3) + ld %r30,PCB_CONTEXT+18*8(%r3) + ld %r31,PCB_CONTEXT+19*8(%r3) + ld %r5,PCB_CR(%r3) /* Load the condition register */ + mtcr %r5 + ld %r5,PCB_LR(%r3) /* Load the link register */ + mtsrr0 %r5 + ld %r1,PCB_SP(%r3) /* Load the stack pointer */ + ld %r2,PCB_TOC(%r3) /* Load the TOC pointer */ + + rfid /* * This code gets copied to all the trap vectors Index: head/sys/powerpc/include/cpu.h =================================================================== --- head/sys/powerpc/include/cpu.h +++ head/sys/powerpc/include/cpu.h @@ -112,6 +112,11 @@ extern char btext[]; extern char etext[]; +#ifdef __powerpc64__ +extern void enter_idle_powerx(void); +extern uint64_t can_wakeup; +#endif + void cpu_halt(void); void cpu_reset(void); void cpu_sleep(void); Index: head/sys/powerpc/include/spr.h =================================================================== --- head/sys/powerpc/include/spr.h +++ head/sys/powerpc/include/spr.h @@ -201,7 +201,14 @@ #define SPR_LPCR 0x13e /* Logical Partitioning Control */ #define LPCR_LPES 0x008 /* Bit 60 */ - +#define LPCR_PECE_DRBL (1ULL << 16) /* Directed Privileged Doorbell */ +#define LPCR_PECE_HDRBL (1ULL << 15) /* Directed Hypervisor Doorbell */ +#define LPCR_PECE_EXT (1ULL << 14) /* External exceptions */ +#define LPCR_PECE_DECR (1ULL << 13) /* Decrementer exceptions */ +#define LPCR_PECE_ME (1ULL << 12) /* Machine Check and Hypervisor */ + /* Maintenance exceptions */ +#define LPCR_PECE_WAKESET (LPCR_PECE_EXT | LPCR_PECE_DECR | LPCR_PECE_ME) + #define SPR_EPCR 0x133 #define EPCR_EXTGS 0x80000000 #define EPCR_DTLBGS 0x40000000 Index: head/sys/powerpc/powernv/platform_powernv.c =================================================================== --- head/sys/powerpc/powernv/platform_powernv.c +++ head/sys/powerpc/powernv/platform_powernv.c @@ -139,7 +139,9 @@ opal_call(OPAL_REINIT_CPUS, 1 /* Big endian */); #endif - cpu_idle_hook = powernv_cpu_idle; + if (cpu_idle_hook == NULL) + cpu_idle_hook = powernv_cpu_idle; + powernv_boot_pir = mfspr(SPR_PIR); /* LPID must not be altered when PSL_DR or PSL_IR is set */ @@ -150,10 +152,10 @@ mtspr(SPR_LPID, 0); isync(); - mtmsr(msr); - mtspr(SPR_LPCR, LPCR_LPES); isync(); + + mtmsr(msr); /* Init CPU bits */ powernv_smp_ap_init(plat); Index: head/sys/powerpc/powerpc/cpu.c =================================================================== --- head/sys/powerpc/powerpc/cpu.c +++ head/sys/powerpc/powerpc/cpu.c @@ -68,6 +68,8 @@ #include #include #include +#include +#include #include #include @@ -81,11 +83,15 @@ static void cpu_6xx_setup(int cpuid, uint16_t vers); static void cpu_970_setup(int cpuid, uint16_t vers); static void cpu_booke_setup(int cpuid, uint16_t vers); +static void cpu_powerx_setup(int cpuid, uint16_t vers); int powerpc_pow_enabled; void (*cpu_idle_hook)(sbintime_t) = NULL; static void cpu_idle_60x(sbintime_t); static void cpu_idle_booke(sbintime_t); +#ifdef __powerpc64__ +static void cpu_idle_powerx(sbintime_t); +#endif struct cputab { const char *name; @@ -156,13 +162,13 @@ PPC_FEATURE_SMT | PPC_FEATURE_ARCH_2_05 | PPC_FEATURE_ARCH_2_06 | PPC_FEATURE_HAS_VSX, PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_HAS_HTM | PPC_FEATURE2_ISEL | - PPC_FEATURE2_HAS_VCRYPTO, NULL }, + PPC_FEATURE2_HAS_VCRYPTO, cpu_powerx_setup }, { "IBM POWER8", IBMPOWER8, REVFMT_MAJMIN, PPC_FEATURE_64 | PPC_FEATURE_HAS_ALTIVEC | PPC_FEATURE_HAS_FPU | PPC_FEATURE_SMT | PPC_FEATURE_ARCH_2_05 | PPC_FEATURE_ARCH_2_06 | PPC_FEATURE_HAS_VSX, PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_HAS_HTM | PPC_FEATURE2_ISEL | - PPC_FEATURE2_HAS_VCRYPTO, NULL }, + PPC_FEATURE2_HAS_VCRYPTO, cpu_powerx_setup }, { "IBM POWER9", IBMPOWER9, REVFMT_MAJMIN, PPC_FEATURE_64 | PPC_FEATURE_HAS_ALTIVEC | PPC_FEATURE_HAS_FPU | PPC_FEATURE_SMT | PPC_FEATURE_ARCH_2_05 | PPC_FEATURE_ARCH_2_06 | @@ -610,6 +616,29 @@ cpu_idle_hook = cpu_idle_60x; } +static void +cpu_powerx_setup(int cpuid, uint16_t vers) +{ + +#ifdef __powerpc64__ + if ((mfmsr() & PSL_HV) == 0) + return; + + /* Configure power-saving */ + switch (vers) { + case IBMPOWER8: + case IBMPOWER8E: + mtspr(SPR_LPCR, mfspr(SPR_LPCR) | LPCR_PECE_WAKESET); + isync(); + break; + default: + return; + } + + cpu_idle_hook = cpu_idle_powerx; +#endif +} + static int cpu_feature_bit(SYSCTL_HANDLER_ARGS) { @@ -696,3 +725,26 @@ #endif } +#ifdef __powerpc64__ +static void +cpu_idle_powerx(sbintime_t sbt) +{ + + /* Sleeping when running on one cpu gives no advantages - avoid it */ + if (smp_started == 0) + return; + + spinlock_enter(); + if (sched_runnable()) { + spinlock_exit(); + return; + } + + if (can_wakeup == 0) + can_wakeup = 1; + mb(); + + enter_idle_powerx(); + spinlock_exit(); +} +#endif Index: head/sys/powerpc/powerpc/cpu_subr64.S =================================================================== --- head/sys/powerpc/powerpc/cpu_subr64.S +++ head/sys/powerpc/powerpc/cpu_subr64.S @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2017-2018 QCM Technologies. + * Copyright (c) 2017-2018 Semihalf. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include "assym.s" + +#include + + .globl CNAME(power_save_sequence) + .p2align 3 +ENTRY(enter_idle_powerx) + mfsprg0 %r3 /* Get the pcpu pointer */ + ld %r3,PC_CURTHREAD(%r3) /* Get current thread */ + ld %r3,TD_PCB(%r3) /* Get PCB of current thread */ + std %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs. */ + std %r13,PCB_CONTEXT+1*8(%r3) + std %r14,PCB_CONTEXT+2*8(%r3) + std %r15,PCB_CONTEXT+3*8(%r3) + std %r16,PCB_CONTEXT+4*8(%r3) + std %r17,PCB_CONTEXT+5*8(%r3) + std %r18,PCB_CONTEXT+6*8(%r3) + std %r19,PCB_CONTEXT+7*8(%r3) + std %r20,PCB_CONTEXT+8*8(%r3) + std %r21,PCB_CONTEXT+9*8(%r3) + std %r22,PCB_CONTEXT+10*8(%r3) + std %r23,PCB_CONTEXT+11*8(%r3) + std %r24,PCB_CONTEXT+12*8(%r3) + std %r25,PCB_CONTEXT+13*8(%r3) + std %r26,PCB_CONTEXT+14*8(%r3) + std %r27,PCB_CONTEXT+15*8(%r3) + std %r28,PCB_CONTEXT+16*8(%r3) + std %r29,PCB_CONTEXT+17*8(%r3) + std %r30,PCB_CONTEXT+18*8(%r3) + std %r31,PCB_CONTEXT+19*8(%r3) + + mfcr %r16 /* Save the condition register */ + std %r16,PCB_CR(%r3) + mflr %r16 /* Save the link register */ + std %r16,PCB_LR(%r3) + std %r1,PCB_SP(%r3) /* Save the stack pointer */ + std %r2,PCB_TOC(%r3) /* Save the TOC pointer */ + + /* Set where we want to jump */ + bl 1f + .llong power_save_sequence /* Remember about 8 byte alignment */ +1: mflr %r3 + ld %r3,0(%r3) + mtsrr0 %r3 + + /* Set MSR */ + li %r3,0 + ori %r3,%r3,(PSL_ME | PSL_RI) + li %r8,0x9 /* PSL_SF and PSL_HV */ + insrdi %r3,%r8,4,0 + mtsrr1 %r3 + + rfid + + .p2align 2 +CNAME(power_save_sequence): + bl 1f + .llong 0x0 /* Playground for power-save sequence */ +1: mflr %r3 + + /* Start power-save sequence */ + std %r2,0(%r3) + ptesync + ld %r2,0(%r3) +2: cmpd %r2,%r2 + bne 2b + nap + b .