diff --git a/sys/amd64/amd64/copyout.c b/sys/amd64/amd64/copyout.c index 36e817635694..d5f94dd39276 100644 --- a/sys/amd64/amd64/copyout.c +++ b/sys/amd64/amd64/copyout.c @@ -1,200 +1,199 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef SAN_NEEDS_INTERCEPTORS #define SAN_RUNTIME #endif #include #include #include #include #include int fubyte_nosmap(volatile const void *base); int fubyte_smap(volatile const void *base); DEFINE_IFUNC(, int, fubyte, (volatile const void *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fubyte_smap : fubyte_nosmap); } int fuword16_nosmap(volatile const void *base); int fuword16_smap(volatile const void *base); DEFINE_IFUNC(, int, fuword16, (volatile const void *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fuword16_smap : fuword16_nosmap); } int fueword_nosmap(volatile const void *base, long *val); int fueword_smap(volatile const void *base, long *val); DEFINE_IFUNC(, int, fueword, (volatile const void *, long *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword_smap : fueword_nosmap); } DEFINE_IFUNC(, int, fueword64, (volatile const void *, int64_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword_smap : fueword_nosmap); } int fueword32_nosmap(volatile const void *base, int32_t *val); int fueword32_smap(volatile const void *base, int32_t *val); DEFINE_IFUNC(, int, fueword32, (volatile const void *, int32_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? fueword32_smap : fueword32_nosmap); } int subyte_nosmap(volatile void *base, int byte); int subyte_smap(volatile void *base, int byte); DEFINE_IFUNC(, int, subyte, (volatile void *, int)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? subyte_smap : subyte_nosmap); } int suword16_nosmap(volatile void *base, int word); int suword16_smap(volatile void *base, int word); DEFINE_IFUNC(, int, suword16, (volatile void *, int)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword16_smap : suword16_nosmap); } int suword32_nosmap(volatile void *base, int32_t word); int suword32_smap(volatile void *base, int32_t word); DEFINE_IFUNC(, int, suword32, (volatile void *, int32_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword32_smap : suword32_nosmap); } int suword_nosmap(volatile void *base, long word); int suword_smap(volatile void *base, long word); DEFINE_IFUNC(, int, suword, (volatile void *, long)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword_smap : suword_nosmap); } DEFINE_IFUNC(, int, suword64, (volatile void *, int64_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? suword_smap : suword_nosmap); } int casueword32_nosmap(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval); int casueword32_smap(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval); DEFINE_IFUNC(, int, casueword32, (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? casueword32_smap : casueword32_nosmap); } int casueword_nosmap(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval); int casueword_smap(volatile u_long *p, u_long oldval, u_long *oldvalp, u_long newval); DEFINE_IFUNC(, int, casueword, (volatile u_long *, u_long, u_long *, u_long)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? casueword_smap : casueword_nosmap); } int copyinstr_nosmap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); int copyinstr_smap(const void *udaddr, void *kaddr, size_t len, size_t *lencopied); DEFINE_IFUNC(, int, copyinstr, (const void *, void *, size_t, size_t *)) { return ((cpu_stdext_feature & CPUID_STDEXT_SMAP) != 0 ? copyinstr_smap : copyinstr_nosmap); } int copyin_nosmap_std(const void *udaddr, void *kaddr, size_t len); int copyin_smap_std(const void *udaddr, void *kaddr, size_t len); int copyin_nosmap_erms(const void *udaddr, void *kaddr, size_t len); int copyin_smap_erms(const void *udaddr, void *kaddr, size_t len); DEFINE_IFUNC(, int, copyin, (const void *, void *, size_t)) { switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { case CPUID_STDEXT_SMAP: return (copyin_smap_std); case CPUID_STDEXT_ERMS: return (copyin_nosmap_erms); case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: return (copyin_smap_erms); default: return (copyin_nosmap_std); } } int copyout_nosmap_std(const void *kaddr, void *udaddr, size_t len); int copyout_smap_std(const void *kaddr, void *udaddr, size_t len); int copyout_nosmap_erms(const void *kaddr, void *udaddr, size_t len); int copyout_smap_erms(const void *kaddr, void *udaddr, size_t len); DEFINE_IFUNC(, int, copyout, (const void *, void *, size_t)) { switch (cpu_stdext_feature & (CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS)) { case CPUID_STDEXT_SMAP: return (copyout_smap_std); case CPUID_STDEXT_ERMS: return (copyout_nosmap_erms); case CPUID_STDEXT_SMAP | CPUID_STDEXT_ERMS: return (copyout_smap_erms); default: return (copyout_nosmap_std); } } diff --git a/sys/amd64/amd64/efirt_support.S b/sys/amd64/amd64/efirt_support.S index b54b13b01fe6..7546a0765204 100644 --- a/sys/amd64/amd64/efirt_support.S +++ b/sys/amd64/amd64/efirt_support.S @@ -1,121 +1,120 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include "assym.inc" .text ENTRY(efi_rt_arch_call) pushq %rbp movq %rsp, %rbp movq %rbx, EC_RBX(%rdi) movq %rsp, EC_RSP(%rdi) movq %rbp, EC_RBP(%rdi) movq %r12, EC_R12(%rdi) movq %r13, EC_R13(%rdi) movq %r14, EC_R14(%rdi) movq %r15, EC_R15(%rdi) pushfq popq %rax movq %rax, EC_RFLAGS(%rdi) movq PCPU(CURTHREAD), %rax movq %rdi, TD_MD+MD_EFIRT_TMP(%rax) movq PCPU(CURPCB), %rsi movl EC_ARGCNT(%rdi), %ecx movl %ecx, %ebx movl $4, %eax cmpl %eax, %ecx cmovbl %eax, %ecx shll $3, %ecx subq %rcx, %rsp cmpl $0, %ebx jz 1f movq EC_ARG1(%rdi), %rcx decl %ebx jz 1f movq EC_ARG2(%rdi), %rdx decl %ebx jz 1f movq EC_ARG3(%rdi), %r8 decl %ebx jz 1f movq EC_ARG4(%rdi), %r9 decl %ebx jz 1f movq EC_ARG5(%rdi), %rax movq %rax, 4*8(%rsp) decl %ebx jz 1f movq $efi_rt_panic_str, %rdi call panic 1: movq EC_FPTR(%rdi), %rax movq $efi_rt_fault, PCB_ONFAULT(%rsi) callq *%rax movq PCPU(CURTHREAD), %rbx movq TD_MD+MD_EFIRT_TMP(%rbx), %rdi movq %rax, EC_EFI_STATUS(%rdi) movq PCPU(CURPCB), %rsi xorl %eax, %eax movq %rax, PCB_ONFAULT(%rsi) efi_rt_arch_call_tail: movq EC_R15(%rdi), %r15 movq EC_R14(%rdi), %r14 movq EC_R13(%rdi), %r13 movq EC_R12(%rdi), %r12 movq EC_RBP(%rdi), %rbp movq EC_RSP(%rdi), %rsp movq EC_RBX(%rdi), %rbx pushq EC_RFLAGS(%rdi) popfq popq %rbp ret END(efi_rt_arch_call) ENTRY(efi_rt_fault) xorl %eax, %eax movq PCPU(CURPCB), %rsi movq %rax, PCB_ONFAULT(%rsi) movl $EFAULT, %eax movq PCPU(CURTHREAD), %rbx movq TD_MD+MD_EFIRT_TMP(%rbx), %rdi jmp efi_rt_arch_call_tail END(efi_rt_fault) efi_rt_panic_str: .asciz "efi_rt_arch_call: too many args" diff --git a/sys/amd64/include/efi.h b/sys/amd64/include/efi.h index 2c24bebfe548..b7bda5388c21 100644 --- a/sys/amd64/include/efi.h +++ b/sys/amd64/include/efi.h @@ -1,79 +1,78 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __AMD64_INCLUDE_EFI_H_ #define __AMD64_INCLUDE_EFI_H_ /* * XXX: from gcc 6.2 manual: * Note, the ms_abi attribute for Microsoft Windows 64-bit targets * currently requires the -maccumulate-outgoing-args option. * * Avoid EFIABI_ATTR declarations for compilers that don't support it. * GCC support began in version 4.4. */ #if defined(__clang__) || defined(__GNUC__) && \ (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4) #define EFIABI_ATTR __attribute__((ms_abi)) #endif #ifdef _KERNEL #include #define ARCH_MAY_USE_EFI #define EFI_TIME_LOCK() mtx_lock(&atrtc_time_lock) #define EFI_TIME_UNLOCK() mtx_unlock(&atrtc_time_lock) #define EFI_TIME_OWNED() mtx_assert(&atrtc_time_lock, MA_OWNED) #define EFI_RT_HANDLE_FAULTS_DEFAULT 1 #endif struct efirt_callinfo { const char *ec_name; register_t ec_efi_status; register_t ec_fptr; register_t ec_argcnt; register_t ec_arg1; register_t ec_arg2; register_t ec_arg3; register_t ec_arg4; register_t ec_arg5; register_t ec_rbx; register_t ec_rsp; register_t ec_rbp; register_t ec_r12; register_t ec_r13; register_t ec_r14; register_t ec_r15; register_t ec_rflags; }; #endif /* __AMD64_INCLUDE_EFI_H_ */ diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h index f0a6fcf5bc9c..fef5ab2b3765 100644 --- a/sys/amd64/include/frame.h +++ b/sys/amd64/include/frame.h @@ -1,50 +1,49 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _AMD64_FRAME_H #define _AMD64_FRAME_H #include struct pti_frame { register_t pti_rdx; register_t pti_rax; register_t pti_err; register_t pti_rip; register_t pti_cs; register_t pti_rflags; register_t pti_rsp; register_t pti_ss; }; #endif diff --git a/sys/arm/arm/gic_common.h b/sys/arm/arm/gic_common.h index 4289146b0086..9e8fb19300ca 100644 --- a/sys/arm/arm/gic_common.h +++ b/sys/arm/arm/gic_common.h @@ -1,104 +1,103 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_COMMON_H_ #define _GIC_COMMON_H_ #define GIC_IVAR_HW_REV 500 #define GIC_IVAR_BUS 501 /* GIC_IVAR_BUS values */ #define GIC_BUS_UNKNOWN 0 #define GIC_BUS_FDT 1 #define GIC_BUS_ACPI 2 #define GIC_BUS_MAX 2 __BUS_ACCESSOR(gic, hw_rev, GIC, HW_REV, u_int); __BUS_ACCESSOR(gic, bus, GIC, BUS, u_int); /* Software Generated Interrupts */ #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ #define GIC_LAST_SGI 15 /* Private Peripheral Interrupts */ #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ /* Shared Peripheral Interrupts */ #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ /* Common register values */ #define GICD_CTLR 0x0000 /* v1 ICDDCR */ #define GICD_TYPER 0x0004 /* v1 ICDICTR */ #define GICD_TYPER_ITLINESNUM_MASK 0x1f #define GICD_TYPER_I_NUM(n) ((((n) & 0x1F) + 1) * 32) #define GICD_IIDR 0x0008 /* v1 ICDIIDR */ #define GICD_IIDR_PROD_SHIFT 24 #define GICD_IIDR_PROD_MASK 0xff000000 #define GICD_IIDR_PROD(x) \ (((x) & GICD_IIDR_PROD_MASK) >> GICD_IIDR_PROD_SHIFT) #define GICD_IIDR_VAR_SHIFT 16 #define GICD_IIDR_VAR_MASK 0x000f0000 #define GICD_IIDR_VAR(x) \ (((x) & GICD_IIDR_VAR_MASK) >> GICD_IIDR_VAR_SHIFT) #define GICD_IIDR_REV_SHIFT 12 #define GICD_IIDR_REV_MASK 0x0000f000 #define GICD_IIDR_REV(x) \ (((x) & GICD_IIDR_REV_MASK) >> GICD_IIDR_REV_SHIFT) #define GICD_IIDR_IMPL_SHIFT 0 #define GICD_IIDR_IMPL_MASK 0x00000fff #define GICD_IIDR_IMPL(x) \ (((x) & GICD_IIDR_IMPL_MASK) >> GICD_IIDR_IMPL_SHIFT) #define GICD_IGROUPR(n) (0x0080 + (((n) >> 5) * 4)) /* v1 ICDISER */ #define GICD_I_PER_IGROUPRn 32 #define GICD_ISENABLER(n) (0x0100 + (((n) >> 5) * 4)) /* v1 ICDISER */ #define GICD_I_MASK(n) (1ul << ((n) & 0x1f)) #define GICD_I_PER_ISENABLERn 32 #define GICD_ICENABLER(n) (0x0180 + (((n) >> 5) * 4)) /* v1 ICDICER */ #define GICD_ISPENDR(n) (0x0200 + (((n) >> 5) * 4)) /* v1 ICDISPR */ #define GICD_ICPENDR(n) (0x0280 + (((n) >> 5) * 4)) /* v1 ICDICPR */ #define GICD_ISACTIVER(n) (0x0300 + (((n) >> 5) * 4)) /* v1 ICDABR */ #define GICD_ICACTIVER(n) (0x0380 + (((n) >> 5) * 4)) #define GICD_IPRIORITYR(n) (0x0400 + (((n) >> 2) * 4)) /* v1 ICDIPR */ #define GICD_I_PER_IPRIORITYn 4 #define GICD_ITARGETSR(n) (0x0800 + (((n) >> 2) * 4)) /* v1 ICDIPTR */ #define GICD_ICFGR(n) (0x0C00 + (((n) >> 4) * 4)) /* v1 ICDICFR */ #define GICD_I_PER_ICFGRn 16 /* First bit is a polarity bit (0 - low, 1 - high) */ #define GICD_ICFGR_POL_LOW (0 << 0) #define GICD_ICFGR_POL_HIGH (1 << 0) #define GICD_ICFGR_POL_MASK 0x1 /* Second bit is a trigger bit (0 - level, 1 - edge) */ #define GICD_ICFGR_TRIG_LVL (0 << 1) #define GICD_ICFGR_TRIG_EDGE (1 << 1) #define GICD_ICFGR_TRIG_MASK 0x2 #define GICD_SGIR 0x0F00 /* v1 ICDSGIR */ #define GICD_SGI_TARGET_SHIFT 16 #endif /* _GIC_COMMON_H_ */ diff --git a/sys/arm/freescale/imx/imx51_sdmareg.h b/sys/arm/freescale/imx/imx51_sdmareg.h index 71c28f6bec14..7c6f14a3e715 100644 --- a/sys/arm/freescale/imx/imx51_sdmareg.h +++ b/sys/arm/freescale/imx/imx51_sdmareg.h @@ -1,144 +1,143 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* Internal Registers definition for Freescale i.MX515 SDMA Core */ /* SDMA Core Instruction Memory Space */ #define SDMA_IBUS_ROM_ADDR_BASE 0x0000 #define SDMA_IBUS_ROM_ADDR_SIZE 0x07ff #define SDMA_IBUS_RAM_ADDR_BASE 0x1000 #define SDMA_IBUS_RAM_ADDR_SIZE 0x1fff /* SDMA Core Internal Registers */ #define SDMA_MC0PTR 0x7000 /* AP (MCU) Channel 0 Pointer R */ #define SDMA_CCPTR 0x7002 /* Current Channel Pointer R */ #define SDMA_ECTL_CCPTR_MASK 0x0000ffff #define SDMA_ECTL_CCPTR_SHIFT 0 #define SDMA_CCR 0x7003 /* Current Channel Register R */ #define SDMA_ECTL_CCR_MASK 0x0000001f #define SDMA_ECTL_CCR_SHIFT 0 #define SDMA_NCR 0x7004 /* Highest Pending Channel Register R */ #define SDMA_ECTL_NCR_MASK 0x0000001f #define SDMA_ECTL_NCR_SHIFT 0 #define SDMA_EVENTS 0x7005 /* External DMA Requests Mirror R */ #define SDMA_CCPRI 0x7006 /* Current Channel Priority R */ #define SDMA_ECTL_CCPRI_MASK 0x00000007 #define SDMA_ECTL_CCPRI_SHIFT 0 #define SDMA_NCPRI 0x7007 /* Next Channel Priority R */ #define SDMA_ECTL_NCPRI_MASK 0x00000007 #define SDMA_ECTL_NCPRI_SHIFT 0 #define SDMA_ECOUNT 0x7009 /* OnCE Event Cell Counter R/W */ #define SDMA_ECTL_ECOUNT_MASK 0x0000ffff #define SDMA_ECTL_ECOUNT_SHIFT 0 #define SDMA_ECTL 0x700A /* OnCE Event Cell Control Register R/W */ #define SDMA_ECTL_EN (1 << 13) #define SDMA_ECTL_CNT (1 << 12) #define SDMA_ECTL_ECTC_MASK 0x00000c00 #define SDMA_ECTL_ECTC_SHIFT 10 #define SDMA_ECTL_DTC_MASK 0x00000300 #define SDMA_ECTL_DTC_SHIFT 8 #define SDMA_ECTL_ATC_MASK 0x000000c0 #define SDMA_ECTL_ATC_SHIFT 6 #define SDMA_ECTL_ABTC_MASK 0x00000030 #define SDMA_ECTL_ABTC_SHIFT 4 #define SDMA_ECTL_AATC_MASK 0x0000000c #define SDMA_ECTL_AATC_SHIFT 2 #define SDMA_ECTL_ATS_MASK 0x00000003 #define SDMA_ECTL_ATS_SHIFT 0 #define SDMA_EAA 0x700B /* OnCE Event Address Register A R/W */ #define SDMA_ECTL_EAA_MASK 0x0000ffff #define SDMA_ECTL_EAA_SHIFT 0 #define SDMA_EAB 0x700C /* OnCE Event Cell Address Register B R/W */ #define SDMA_ECTL_EAB_MASK 0x0000ffff #define SDMA_ECTL_EAB_SHIFT 0 #define SDMA_EAM 0x700D /* OnCE Event Cell Address Mask R/W */ #define SDMA_ECTL_EAM_MASK 0x0000ffff #define SDMA_ECTL_EAM_SHIFT 0 #define SDMA_ED 0x700E /* OnCE Event Cell Data Register R/W */ #define SDMA_EDM 0x700F /* OnCE Event Cell Data Mask R/W */ #define SDMA_RTB 0x7018 /* OnCE Real-Time Buffer R/W */ #define SDMA_TB 0x7019 /* OnCE Trace Buffer R */ #define SDMA_TB_TBF (1 << 28) #define SDMA_TB_TADDR_MASK 0x0fffc000 #define SDMA_TB_TADDR_SHIFT 14 #define SDMA_TB_CHFADDR_MASK 0x00003fff #define SDMA_TB_CHFADDR_SHIFT 0 #define SDMA_OSTAT 0x701A /* OnCE Status R */ #define SDMA_OSTAT_PST_MASK 0x0000f000 #define SDMA_OSTAT_PST_SHIFT 12 #define SDMA_OSTAT_RCV (1 << 11) #define SDMA_OSTAT_EDR (1 << 10) #define SDMA_OSTAT_ODR (1 << 9) #define SDMA_OSTAT_SWB (1 << 8) #define SDMA_OSTAT_MST (1 << 7) #define SDMA_OSTAT_ECDR_MASK 0x00000007 #define SDMA_OSTAT_ECDR_SHIFT 0 #define SDMA_MCHN0ADDR 0x701C /* Channel 0 Boot Address R */ #define SDMA_MCHN0ADDR_SMS_Z (1 << 14) #define SDMA_MCHN0ADDR_CHN0ADDR_MASK 0x00003fff #define SDMA_MCHN0ADDR_CHN0ADDR_SHIFT 0 #define SDMA_MODE 0x701D /* Mode Status Register R */ #define SDMA_MODE_DSPCtrl (1 << 3) #define SDMA_MODE_AP_END (1 << 0) #define SDMA_LOCK 0x701E /* Lock Status Register R */ #define SDMA_LOCK_LOCK (1 << 0) #define SDMA_EVENTS2 0x701F /* External DMA Requests Mirror #2 R */ #define SDMA_HE 0x7020 /* AP Enable Register R */ #define SDMA_PRIV 0x7022 /* Current Channel BP Privilege Register R */ #define SDMA_PRIV_BPPRIV (1 << 0) #define SDMA_PRF_CNT 0x7023 /* Profile Free Running Register R/W */ #define SDMA_PRF_CNT_SEL_MASK 0xc0000000 #define SDMA_PRF_CNT_SEL_SHIFT 30 #define SDMA_PRF_CNT_EN (1 << 29) #define SDMA_PRF_CNT_OFL (1 << 22) #define SDMA_PRF_CNT_COUNTER_MASK 0x003fffff #define SDMA_PRF_CNT_COUNTER_SHIFT 0 diff --git a/sys/arm/freescale/imx/imx51_ssireg.h b/sys/arm/freescale/imx/imx51_ssireg.h index 3846b4950750..05c01a79a449 100644 --- a/sys/arm/freescale/imx/imx51_ssireg.h +++ b/sys/arm/freescale/imx/imx51_ssireg.h @@ -1,182 +1,181 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* Registers definition for Freescale i.MX515 Synchronous Serial Interface */ #define IMX51_SSI_STX0_REG 0x0000 /* SSI TX Data Register 0 */ #define IMX51_SSI_STX1_REG 0x0004 /* SSI TX Data Register 1 */ #define IMX51_SSI_SRX0_REG 0x0008 /* SSI RX Data Register 0 */ #define IMX51_SSI_SRX1_REG 0x000C /* SSI RX Data Register 1 */ #define IMX51_SSI_SCR_REG 0x0010 /* SSI Control Register */ #define SSI_SCR_RFR_CLK_DIS (1 << 11) /* RX FC Disable */ #define SSI_SCR_TFR_CLK_DIS (1 << 10) /* TX FC Disable */ #define SSI_SCR_CLK_IST (1 << 9) /* Clock Idle */ #define SSI_SCR_TCH_EN (1 << 8) /* 2Chan Enable */ #define SSI_SCR_SYS_CLK_EN (1 << 7) /* System Clock En */ #define SSI_SCR_MODE_NORMAL (0 << 5) #define SSI_SCR_MODE_I2S_MASTER (1 << 5) #define SSI_SCR_MODE_I2S_SLAVE (2 << 5) #define SSI_SCR_MODE_MASK (3 << 5) #define SSI_SCR_SYN (1 << 4) /* Sync Mode */ #define SSI_SCR_NET (1 << 3) /* Network Mode */ #define SSI_SCR_RE (1 << 2) /* RX Enable */ #define SSI_SCR_TE (1 << 1) /* TX Enable */ #define SSI_SCR_SSIEN (1 << 0) /* SSI Enable */ #define IMX51_SSI_SISR_REG 0x0014 /* SSI Interrupt Status Register */ #define SSI_SISR_RFRC (1 << 24) /* RX Frame Complete */ #define SSI_SIR_TFRC (1 << 23) /* TX Frame Complete */ #define SSI_SIR_CMDAU (1 << 18) /* Command Address Updated */ #define SSI_SIR_CMDDU (1 << 17) /* Command Data Updated */ #define SSI_SIR_RXT (1 << 16) /* RX Tag Updated */ #define SSI_SIR_RDR1 (1 << 15) /* RX Data Ready 1 */ #define SSI_SIR_RDR0 (1 << 14) /* RX Data Ready 0 */ #define SSI_SIR_TDE1 (1 << 13) /* TX Data Reg Empty 1 */ #define SSI_SIR_TDE0 (1 << 12) /* TX Data Reg Empty 0 */ #define SSI_SIR_ROE1 (1 << 11) /* RXer Overrun Error 1 */ #define SSI_SIR_ROE0 (1 << 10) /* RXer Overrun Error 0 */ #define SSI_SIR_TUE1 (1 << 9) /* TXer Underrun Error 1 */ #define SSI_SIR_TUE0 (1 << 8) /* TXer Underrun Error 0 */ #define SSI_SIR_TFS (1 << 7) /* TX Frame Sync */ #define SSI_SIR_RFS (1 << 6) /* RX Frame Sync */ #define SSI_SIR_TLS (1 << 5) /* TX Last Time Slot */ #define SSI_SIR_RLS (1 << 4) /* RX Last Time Slot */ #define SSI_SIR_RFF1 (1 << 3) /* RX FIFO Full 1 */ #define SSI_SIR_RFF0 (1 << 2) /* RX FIFO Full 0 */ #define SSI_SIR_TFE1 (1 << 1) /* TX FIFO Empty 1 */ #define SSI_SIR_TFE0 (1 << 0) /* TX FIFO Empty 0 */ #define IMX51_SSI_SIER_REG 0x0018 /* SSI Interrupt Enable Register */ /* 24-23 Enable Bit (See SISR) */ #define SSI_SIER_RDMAE (1 << 22) /* RX DMA Enable */ #define SSI_SIER_RIE (1 << 21) /* RX Interrupt Enable */ #define SSI_SIER_TDMAE (1 << 20) /* TX DMA Enable */ #define SSI_SIER_TIE (1 << 19) /* TX Interrupt Enable */ /* 18-0 Enable Bits (See SISR) */ #define IMX51_SSI_STCR_REG 0x001C /* SSI TX Configuration Register */ #define SSI_STCR_TXBIT0 (1 << 9) /* TX Bit 0 */ #define SSI_STCR_TFEN1 (1 << 8) /* TX FIFO Enable 1 */ #define SSI_STCR_TFEN0 (1 << 7) /* TX FIFO Enable 0 */ #define SSI_STCR_TFDIR (1 << 6) /* TX Frame Direction */ #define SSI_STCR_TXDIR (1 << 5) /* TX Clock Direction */ #define SSI_STCR_TSHFD (1 << 4) /* TX Shift Direction */ #define SSI_STCR_TSCKP (1 << 3) /* TX Clock Polarity */ #define SSI_STCR_TFSI (1 << 2) /* TX Frame Sync Invert */ #define SSI_STCR_TFSL (1 << 1) /* TX Frame Sync Length */ #define SSI_STCR_TEFS (1 << 0) /* TX Early Frame Sync */ #define IMX51_SSI_SRCR_REG 0x0020 /* SSI RX Configuration Register */ #define SSI_SRCR_RXEXT (1 << 10) /* RX Data Extension */ #define SSI_SRCR_RXBIT0 (1 << 9) /* RX Bit 0 */ #define SSI_SRCR_RFEN1 (1 << 8) /* RX FIFO Enable 1 */ #define SSI_SRCR_RFEN0 (1 << 7) /* RX FIFO Enable 0 */ #define SSI_SRCR_RFDIR (1 << 6) /* RX Frame Direction */ #define SSI_SRCR_RXDIR (1 << 5) /* RX Clock Direction */ #define SSI_SRCR_RSHFD (1 << 4) /* RX Shift Direction */ #define SSI_SRCR_RSCKP (1 << 3) /* RX Clock Polarity */ #define SSI_SRCR_RFSI (1 << 2) /* RX Frame Sync Invert */ #define SSI_SRCR_RFSL (1 << 1) /* RX Frame Sync Length */ #define SSI_SRCR_REFS (1 << 0) /* RX Early Frame Sync */ #define IMX51_SSI_STCCR_REG 0x0024 /* TX Clock Control */ #define IMX51_SSI_SRCCR_REG 0x0028 /* RX Clock Control */ #define SSI_SXCCR_DIV2 (1 << 18) /* Divide By 2 */ #define SSI_SXCCR_PSR (1 << 17) /* Prescaler Range */ #define SSI_SXCCR_WL_MASK 0x0001e000 #define SSI_SXCCR_WL_SHIFT 13 /* Word Length Control */ #define SSI_SXCCR_DC_MASK 0x00001f00 #define SSI_SXCCR_DC_SHIFT 8 /* Frame Rate Divider */ #define SSI_SXCCR_PM_MASK 0x000000ff #define SSI_SXCCR_PM_SHIFT 0 /* Prescaler Modulus */ #define IMX51_SSI_SFCSR_REG 0x002C /* SSI FIFO Control/Status Register */ #define SSI_SFCSR_RFCNT1_MASK 0xf0000000 #define SSI_SFCSR_RFCNT1_SHIFT 28 /* RX FIFO Counter 1 */ #define SSI_SFCSR_TFCNT1_MASK 0x0f000000 #define SSI_SFCSR_TFCNT1_SHIFT 24 /* TX FIFO Counter 1 */ #define SSI_SFCSR_RFWM1_MASK 0x00f00000 #define SSI_SFCSR_RFWM1_SHIFT 20 /* RX FIFO Full WaterMark 1 */ #define SSI_SFCSR_TFWM1_MASK 0x000f0000 #define SSI_SFCSR_TFWM1_SHIFT 16 /* TX FIFO Empty WaterMark 1 */ #define SSI_SFCSR_RFCNT0_MASK 0x0000f000 #define SSI_SFCSR_RFCNT0_SHIFT 12 /* RX FIFO Counter 0 */ #define SSI_SFCSR_TFCNT0_MASK 0x00000f00 #define SSI_SFCSR_TFCNT0_SHIFT 8 /* TX FIFO Counter 0 */ #define SSI_SFCSR_RFWM0_MASK 0x000000f0 #define SSI_SFCSR_RFWM0_SHIFT 4 /* RX FIFO Full WaterMark 0 */ #define SSI_SFCSR_TFWM0_MASK 0x0000000f #define SSI_SFCSR_TFWM0_SHIFT 0 /* TX FIFO Empty WaterMark 0 */ #define IMX51_SSI_STR_REG 0x0030 /* SSI Test Register1 */ #define SSI_STR_TEST (1 << 15) /* Test Mode */ #define SSI_STR_RCK2TCK (1 << 14) /* RX<->TX Clock Loop Back */ #define SSI_STR_RFS2TFS (1 << 13) /* RX<->TX Frame Loop Back */ #define SSI_STR_RXSTATE_MASK 0x00001f00 #define SSI_STR_RXSTATE_SHIFT 8 /* RXer State Machine Status */ #define SSI_STR_TXD2RXD (1 << 7) /* TX<->RX Data Loop Back */ #define SSI_STR_TCK2RCK (1 << 6) /* TX<->RX Clock Loop Back */ #define SSI_STR_TFS2RFS (1 << 5) /* TX<->RX Frame Loop Back */ #define SSI_STR_TXSTATE_MASK 0x0000001f #define SSI_STR_TXSTATE_SHIFT 0 /* TXer State Machine Status */ #define IMX51_SSI_SOR_REG 0x0034 /* SSI Option Register2 */ #define SSI_SOR_CLKOFF (1 << 6) /* Clock Off */ #define SSI_SOR_RX_CLR (1 << 5) /* RXer Clear */ #define SSI_SOR_TX_CLR (1 << 4) /* TXer Clear */ #define SSI_SOR_INIT (1 << 3) /* Initialize */ #define SSI_SOR_WAIT_MASK 0x00000006 #define SSI_SOR_INIT_SHIFT 1 /* Wait */ #define SSI_SOR_SYNRST (1 << 0) /* Frame Sync Reset */ #define IMX51_SSI_SACNT_REG 0x0038 /* SSI AC97 Control Register */ #define SSI_SACNT_FRDIV_MASK 0x000007e0 #define SSI_SACNT_FRDIV_SHIFT 5 /* Frame Rate Divider */ #define SSI_SACNT_WR (1 << 4) /* Write Command */ #define SSI_SACNT_RD (1 << 3) /* Read Command */ #define SSI_SACNT_TIF (1 << 2) /* Tag in FIFO */ #define SSI_SACNT_FV (1 << 1) /* Fixed/Variable Operation */ #define SSI_SACNT_AC97EN (1 << 0) /* AC97 Mode Enable */ #define IMX51_SSI_SACADD_REG 0x003C /* SSI AC97 Command Address Register */ #define SSI_SACADD_MASK 0x0007ffff #define IMX51_SSI_SACDAT_REG 0x0040 /* SSI AC97 Command Data Register */ #define SSI_SACDAT_MASK 0x000fffff #define IMX51_SSI_SATAG_REG 0x0044 /* SSI AC97 Tag Register */ #define SSI_SATAG_MASK 0x0000ffff #define IMX51_SSI_STMSK_REG 0x0048 /* SSI TX Time Slot Mask Register */ #define IMX51_SSI_SRMSK_REG 0x004C /* SSI RX Time Slot Mask Register */ #define IMX51_SSI_SACCST_REG 0x0050 /* SSI AC97 Channel Status Register */ #define IMX51_SSI_SACCEN_REG 0x0054 /* SSI AC97 Channel Enable Register */ #define IMX51_SSI_SACCDIS_REG 0x0058 /* SSI AC97 Channel Disable Register */ #define SSI_SAC_MASK 0x000003ff /* SACCST,SACCEN,SACCDIS */ diff --git a/sys/arm/freescale/imx/imx_console.c b/sys/arm/freescale/imx/imx_console.c index d5ade2decd7b..b497b9cf3a14 100644 --- a/sys/arm/freescale/imx/imx_console.c +++ b/sys/arm/freescale/imx/imx_console.c @@ -1,176 +1,175 @@ /*- * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Simple UART console driver for Freescale i.MX515 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include /* Allow it to be predefined, to be able to use another UART for console */ #ifndef IMX_UART_BASE #define IMX_UART_BASE 0xe3fbc000 /* imx51 UART1 */ #endif #define IMX_RXD 0x00 #define IMX_TXD 0x40 #define IMX_UFCR 0x90 #define IMX_USR1 0x94 #define IMX_USR1_TRDY (1 << 13) #define IMX_USR2 0x98 #define IMX_USR2_RDR (1 << 0) #define IMX_USR2_TXFE (1 << 14) #define IMX_USR2_TXDC (1 << 3) #define IMX_UTS 0xb4 #define IMX_UTS_TXFULL (1 << 4) /* * The base address of the uart registers. * * This is global so that it can be changed on the fly from the outside. For * example, set imx_uart_base=physaddr and then call cninit() as the first two * lines of initarm() and enjoy printf() availability through the tricky bits of * startup. After initarm() switches from physical to virtual addressing, just * set imx_uart_base=virtaddr and printf keeps working. */ uint32_t imx_uart_base = IMX_UART_BASE; /* * uart related funcs */ static uint32_t ub_getreg(uint32_t off) { return *((volatile uint32_t *)(imx_uart_base + off)); } static void ub_setreg(uint32_t off, uint32_t val) { *((volatile uint32_t *)(imx_uart_base + off)) = val; } static int ub_tstc(void) { return ((ub_getreg(IMX_USR2) & IMX_USR2_RDR) ? 1 : 0); } static int ub_getc(void) { while (!ub_tstc()); __asm __volatile("nop"); return (ub_getreg(IMX_RXD) & 0xff); } static void ub_putc(unsigned char c) { if (c == '\n') ub_putc('\r'); while (ub_getreg(IMX_UTS) & IMX_UTS_TXFULL) __asm __volatile("nop"); ub_setreg(IMX_TXD, c); } static cn_probe_t uart_cnprobe; static cn_init_t uart_cninit; static cn_term_t uart_cnterm; static cn_getc_t uart_cngetc; static cn_putc_t uart_cnputc; static cn_grab_t uart_cngrab; static cn_ungrab_t uart_cnungrab; static void uart_cngrab(struct consdev *cp) { } static void uart_cnungrab(struct consdev *cp) { } static void uart_cnprobe(struct consdev *cp) { sprintf(cp->cn_name, "uart"); cp->cn_pri = CN_NORMAL; } static void uart_cninit(struct consdev *cp) { /* Init fifo trigger levels to 32 bytes, refclock div to 2. */ ub_setreg(IMX_UFCR, 0x00004210); } static void uart_cnputc(struct consdev *cp, int c) { ub_putc(c); } static int uart_cngetc(struct consdev * cp) { return ub_getc(); } static void uart_cnterm(struct consdev * cp) { } CONSOLE_DRIVER(uart); diff --git a/sys/arm/freescale/imx/imx_gpio.c b/sys/arm/freescale/imx/imx_gpio.c index 8324d8daa30c..ef4c7917f342 100644 --- a/sys/arm/freescale/imx/imx_gpio.c +++ b/sys/arm/freescale/imx/imx_gpio.c @@ -1,959 +1,958 @@ /*- * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Freescale i.MX515 GPIO driver. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(EXT_RESOURCES) && defined(__aarch64__) #define IMX_ENABLE_CLOCKS #endif #ifdef IMX_ENABLE_CLOCKS #include #endif #include "gpio_if.h" #ifdef INTRNG #include "pic_if.h" #endif #define WRITE4(_sc, _r, _v) \ bus_space_write_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r), (_v)) #define READ4(_sc, _r) \ bus_space_read_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r)) #define SET4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) | (_m)) #define CLEAR4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) & ~(_m)) /* Registers definition for Freescale i.MX515 GPIO controller */ #define IMX_GPIO_DR_REG 0x000 /* Pin Data */ #define IMX_GPIO_OE_REG 0x004 /* Set Pin Output */ #define IMX_GPIO_PSR_REG 0x008 /* Pad Status */ #define IMX_GPIO_ICR1_REG 0x00C /* Interrupt Configuration */ #define IMX_GPIO_ICR2_REG 0x010 /* Interrupt Configuration */ #define GPIO_ICR_COND_LOW 0 #define GPIO_ICR_COND_HIGH 1 #define GPIO_ICR_COND_RISE 2 #define GPIO_ICR_COND_FALL 3 #define GPIO_ICR_COND_MASK 0x3 #define IMX_GPIO_IMR_REG 0x014 /* Interrupt Mask Register */ #define IMX_GPIO_ISR_REG 0x018 /* Interrupt Status Register */ #define IMX_GPIO_EDGE_REG 0x01C /* Edge Detect Register */ #ifdef INTRNG #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | \ GPIO_INTR_LEVEL_LOW | GPIO_INTR_LEVEL_HIGH | GPIO_INTR_EDGE_RISING | \ GPIO_INTR_EDGE_FALLING | GPIO_INTR_EDGE_BOTH) #else #define DEFAULT_CAPS (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT) #endif #define NGPIO 32 #ifdef INTRNG struct gpio_irqsrc { struct intr_irqsrc gi_isrc; u_int gi_irq; uint32_t gi_mode; }; #endif struct imx51_gpio_softc { device_t dev; device_t sc_busdev; struct mtx sc_mtx; struct resource *sc_res[3]; /* 1 x mem, 2 x IRQ */ void *gpio_ih[2]; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; int gpio_npins; struct gpio_pin gpio_pins[NGPIO]; #ifdef INTRNG struct gpio_irqsrc gpio_pic_irqsrc[NGPIO]; #endif #ifdef IMX_ENABLE_CLOCKS clk_t clk; #endif }; static struct ofw_compat_data compat_data[] = { {"fsl,imx8mq-gpio", 1}, {"fsl,imx6q-gpio", 1}, {"fsl,imx53-gpio", 1}, {"fsl,imx51-gpio", 1}, {NULL, 0} }; static struct resource_spec imx_gpio_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { SYS_RES_IRQ, 1, RF_ACTIVE }, { -1, 0 } }; #define FIRST_IRQRES 1 #define NUM_IRQRES 2 /* * Helpers */ static void imx51_gpio_pin_configure(struct imx51_gpio_softc *, struct gpio_pin *, uint32_t); /* * Driver stuff */ static int imx51_gpio_probe(device_t); static int imx51_gpio_attach(device_t); static int imx51_gpio_detach(device_t); /* * GPIO interface */ static device_t imx51_gpio_get_bus(device_t); static int imx51_gpio_pin_max(device_t, int *); static int imx51_gpio_pin_getcaps(device_t, uint32_t, uint32_t *); static int imx51_gpio_pin_getflags(device_t, uint32_t, uint32_t *); static int imx51_gpio_pin_getname(device_t, uint32_t, char *); static int imx51_gpio_pin_setflags(device_t, uint32_t, uint32_t); static int imx51_gpio_pin_set(device_t, uint32_t, unsigned int); static int imx51_gpio_pin_get(device_t, uint32_t, unsigned int *); static int imx51_gpio_pin_toggle(device_t, uint32_t pin); #ifdef INTRNG static int gpio_pic_map_fdt(struct imx51_gpio_softc *sc, struct intr_map_data_fdt *daf, u_int *irqp, uint32_t *modep) { u_int irq; uint32_t mode; /* * From devicetree/bindings/gpio/fsl-imx-gpio.txt: * #interrupt-cells: 2. The first cell is the GPIO number. The second * cell bits[3:0] is used to specify trigger type and level flags: * 1 = low-to-high edge triggered. * 2 = high-to-low edge triggered. * 4 = active high level-sensitive. * 8 = active low level-sensitive. * We can do any single one of these modes, and also edge low+high * (i.e., trigger on both edges); other combinations are not supported. */ if (daf->ncells != 2) { device_printf(sc->dev, "Invalid #interrupt-cells\n"); return (EINVAL); } irq = daf->cells[0]; if (irq >= sc->gpio_npins) { device_printf(sc->dev, "Invalid interrupt number %u\n", irq); return (EINVAL); } switch (daf->cells[1]) { case 1: mode = GPIO_INTR_EDGE_RISING; break; case 2: mode = GPIO_INTR_EDGE_FALLING; break; case 3: mode = GPIO_INTR_EDGE_BOTH; break; case 4: mode = GPIO_INTR_LEVEL_HIGH; break; case 8: mode = GPIO_INTR_LEVEL_LOW; break; default: device_printf(sc->dev, "Unsupported interrupt mode 0x%2x\n", daf->cells[1]); return (ENOTSUP); } *irqp = irq; if (modep != NULL) *modep = mode; return (0); } static int gpio_pic_map_gpio(struct imx51_gpio_softc *sc, struct intr_map_data_gpio *dag, u_int *irqp, uint32_t *modep) { u_int irq; irq = dag->gpio_pin_num; if (irq >= sc->gpio_npins) { device_printf(sc->dev, "Invalid interrupt number %u\n", irq); return (EINVAL); } switch (dag->gpio_intr_mode) { case GPIO_INTR_LEVEL_LOW: case GPIO_INTR_LEVEL_HIGH: case GPIO_INTR_EDGE_RISING: case GPIO_INTR_EDGE_FALLING: case GPIO_INTR_EDGE_BOTH: break; default: device_printf(sc->dev, "Unsupported interrupt mode 0x%8x\n", dag->gpio_intr_mode); return (EINVAL); } *irqp = irq; if (modep != NULL) *modep = dag->gpio_intr_mode; return (0); } static int gpio_pic_map(struct imx51_gpio_softc *sc, struct intr_map_data *data, u_int *irqp, uint32_t *modep) { switch (data->type) { case INTR_MAP_DATA_FDT: return (gpio_pic_map_fdt(sc, (struct intr_map_data_fdt *)data, irqp, modep)); case INTR_MAP_DATA_GPIO: return (gpio_pic_map_gpio(sc, (struct intr_map_data_gpio *)data, irqp, modep)); default: return (ENOTSUP); } } static int gpio_pic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { int error; u_int irq; struct imx51_gpio_softc *sc; sc = device_get_softc(dev); error = gpio_pic_map(sc, data, &irq, NULL); if (error == 0) *isrcp = &sc->gpio_pic_irqsrc[irq].gi_isrc; return (error); } static int gpio_pic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct imx51_gpio_softc *sc; struct gpio_irqsrc *gi; sc = device_get_softc(dev); if (isrc->isrc_handlers == 0) { gi = (struct gpio_irqsrc *)isrc; gi->gi_mode = GPIO_INTR_CONFORM; // XXX Not sure this is necessary mtx_lock_spin(&sc->sc_mtx); CLEAR4(sc, IMX_GPIO_IMR_REG, (1U << gi->gi_irq)); WRITE4(sc, IMX_GPIO_ISR_REG, (1U << gi->gi_irq)); mtx_unlock_spin(&sc->sc_mtx); } return (0); } static int gpio_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct imx51_gpio_softc *sc; struct gpio_irqsrc *gi; int error; u_int icfg, irq, reg, shift, wrk; uint32_t mode; if (data == NULL) return (ENOTSUP); sc = device_get_softc(dev); gi = (struct gpio_irqsrc *)isrc; /* Get config for interrupt. */ error = gpio_pic_map(sc, data, &irq, &mode); if (error != 0) return (error); if (gi->gi_irq != irq) return (EINVAL); /* Compare config if this is not first setup. */ if (isrc->isrc_handlers != 0) return (gi->gi_mode == mode ? 0 : EINVAL); gi->gi_mode = mode; /* * To interrupt on both edges we have to use the EDGE register. The * manual says it only exists for backwards compatibilty with older imx * chips, but it's also the only way to configure interrupting on both * edges. If the EDGE bit is on, the corresponding ICRn bit is ignored. */ mtx_lock_spin(&sc->sc_mtx); if (mode == GPIO_INTR_EDGE_BOTH) { SET4(sc, IMX_GPIO_EDGE_REG, (1u << irq)); } else { CLEAR4(sc, IMX_GPIO_EDGE_REG, (1u << irq)); switch (mode) { default: /* silence warnings; default can't actually happen. */ /* FALLTHROUGH */ case GPIO_INTR_LEVEL_LOW: icfg = GPIO_ICR_COND_LOW; break; case GPIO_INTR_LEVEL_HIGH: icfg = GPIO_ICR_COND_HIGH; break; case GPIO_INTR_EDGE_RISING: icfg = GPIO_ICR_COND_RISE; break; case GPIO_INTR_EDGE_FALLING: icfg = GPIO_ICR_COND_FALL; break; } if (irq < 16) { reg = IMX_GPIO_ICR1_REG; shift = 2 * irq; } else { reg = IMX_GPIO_ICR2_REG; shift = 2 * (irq - 16); } wrk = READ4(sc, reg); wrk &= ~(GPIO_ICR_COND_MASK << shift); wrk |= icfg << shift; WRITE4(sc, reg, wrk); } WRITE4(sc, IMX_GPIO_ISR_REG, (1u << irq)); SET4(sc, IMX_GPIO_IMR_REG, (1u << irq)); mtx_unlock_spin(&sc->sc_mtx); return (0); } /* * this is mask_intr */ static void gpio_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; sc = device_get_softc(dev); irq = ((struct gpio_irqsrc *)isrc)->gi_irq; mtx_lock_spin(&sc->sc_mtx); CLEAR4(sc, IMX_GPIO_IMR_REG, (1U << irq)); mtx_unlock_spin(&sc->sc_mtx); } /* * this is unmask_intr */ static void gpio_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; sc = device_get_softc(dev); irq = ((struct gpio_irqsrc *)isrc)->gi_irq; mtx_lock_spin(&sc->sc_mtx); SET4(sc, IMX_GPIO_IMR_REG, (1U << irq)); mtx_unlock_spin(&sc->sc_mtx); } static void gpio_pic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; sc = device_get_softc(dev); irq = ((struct gpio_irqsrc *)isrc)->gi_irq; arm_irq_memory_barrier(0); /* EOI. W1C reg so no r-m-w, no locking needed. */ WRITE4(sc, IMX_GPIO_ISR_REG, (1U << irq)); } static void gpio_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; sc = device_get_softc(dev); irq = ((struct gpio_irqsrc *)isrc)->gi_irq; arm_irq_memory_barrier(0); /* EOI. W1C reg so no r-m-w, no locking needed. */ WRITE4(sc, IMX_GPIO_ISR_REG, (1U << irq)); gpio_pic_enable_intr(dev, isrc); } static void gpio_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { gpio_pic_disable_intr(dev, isrc); } static int gpio_pic_filter(void *arg) { struct imx51_gpio_softc *sc; struct intr_irqsrc *isrc; uint32_t i, interrupts; sc = arg; mtx_lock_spin(&sc->sc_mtx); interrupts = READ4(sc, IMX_GPIO_ISR_REG) & READ4(sc, IMX_GPIO_IMR_REG); mtx_unlock_spin(&sc->sc_mtx); for (i = 0; interrupts != 0; i++, interrupts >>= 1) { if ((interrupts & 0x1) == 0) continue; isrc = &sc->gpio_pic_irqsrc[i].gi_isrc; if (intr_isrc_dispatch(isrc, curthread->td_intr_frame) != 0) { gpio_pic_disable_intr(sc->dev, isrc); gpio_pic_post_filter(sc->dev, isrc); device_printf(sc->dev, "Stray irq %u disabled\n", i); } } return (FILTER_HANDLED); } /* * Initialize our isrcs and register them with intrng. */ static int gpio_pic_register_isrcs(struct imx51_gpio_softc *sc) { int error; uint32_t irq; const char *name; name = device_get_nameunit(sc->dev); for (irq = 0; irq < NGPIO; irq++) { sc->gpio_pic_irqsrc[irq].gi_irq = irq; sc->gpio_pic_irqsrc[irq].gi_mode = GPIO_INTR_CONFORM; error = intr_isrc_register(&sc->gpio_pic_irqsrc[irq].gi_isrc, sc->dev, 0, "%s,%u", name, irq); if (error != 0) { /* XXX call intr_isrc_deregister() */ device_printf(sc->dev, "%s failed", __func__); return (error); } } return (0); } #endif /* * */ static void imx51_gpio_pin_configure(struct imx51_gpio_softc *sc, struct gpio_pin *pin, unsigned int flags) { u_int newflags, pad; mtx_lock_spin(&sc->sc_mtx); /* * Manage input/output; other flags not supported yet (maybe not ever, * since we have no connection to the pad config registers from here). * * When setting a pin to output, honor the PRESET_[LOW,HIGH] flags if * present. Otherwise, for glitchless transistions on pins with pulls, * read the current state of the pad and preset the DR register to drive * the current value onto the pin before enabling the pin for output. * * Note that changes to pin->gp_flags must be acccumulated in newflags * and stored with a single writeback to gp_flags at the end, to enable * unlocked reads of that value elsewhere. This is only about unlocked * access to gp_flags from elsewhere; we still use locking in this * function to protect r-m-w access to the hardware registers. */ if (flags & (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT)) { newflags = pin->gp_flags & ~(GPIO_PIN_INPUT | GPIO_PIN_OUTPUT); if (flags & GPIO_PIN_OUTPUT) { if (flags & GPIO_PIN_PRESET_LOW) { pad = 0; } else if (flags & GPIO_PIN_PRESET_HIGH) { pad = 1; } else { if (flags & GPIO_PIN_OPENDRAIN) pad = READ4(sc, IMX_GPIO_PSR_REG); else pad = READ4(sc, IMX_GPIO_DR_REG); pad = (pad >> pin->gp_pin) & 1; } newflags |= GPIO_PIN_OUTPUT; SET4(sc, IMX_GPIO_DR_REG, (pad << pin->gp_pin)); SET4(sc, IMX_GPIO_OE_REG, (1U << pin->gp_pin)); } else { newflags |= GPIO_PIN_INPUT; CLEAR4(sc, IMX_GPIO_OE_REG, (1U << pin->gp_pin)); } pin->gp_flags = newflags; } mtx_unlock_spin(&sc->sc_mtx); } static device_t imx51_gpio_get_bus(device_t dev) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); return (sc->sc_busdev); } static int imx51_gpio_pin_max(device_t dev, int *maxpin) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); *maxpin = sc->gpio_npins - 1; return (0); } static int imx51_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); *caps = sc->gpio_pins[pin].gp_caps; return (0); } static int imx51_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); *flags = sc->gpio_pins[pin].gp_flags; return (0); } static int imx51_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); mtx_lock_spin(&sc->sc_mtx); memcpy(name, sc->gpio_pins[pin].gp_name, GPIOMAXNAME); mtx_unlock_spin(&sc->sc_mtx); return (0); } static int imx51_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); imx51_gpio_pin_configure(sc, &sc->gpio_pins[pin], flags); return (0); } static int imx51_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); mtx_lock_spin(&sc->sc_mtx); if (value) SET4(sc, IMX_GPIO_DR_REG, (1U << pin)); else CLEAR4(sc, IMX_GPIO_DR_REG, (1U << pin)); mtx_unlock_spin(&sc->sc_mtx); return (0); } static int imx51_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); /* * Normally a pin set for output can be read by reading the DR reg which * indicates what value is being driven to that pin. The exception is * pins configured for open-drain mode, in which case we have to read * the pad status register in case the pin is being driven externally. * Doing so requires that the SION bit be configured in pinmux, which * isn't the case for most normal gpio pins, so only try to read via PSR * if the OPENDRAIN flag is set, and it's the user's job to correctly * configure SION along with open-drain output mode for those pins. */ if (sc->gpio_pins[pin].gp_flags & GPIO_PIN_OPENDRAIN) *val = (READ4(sc, IMX_GPIO_PSR_REG) >> pin) & 1; else *val = (READ4(sc, IMX_GPIO_DR_REG) >> pin) & 1; return (0); } static int imx51_gpio_pin_toggle(device_t dev, uint32_t pin) { struct imx51_gpio_softc *sc; sc = device_get_softc(dev); if (pin >= sc->gpio_npins) return (EINVAL); mtx_lock_spin(&sc->sc_mtx); WRITE4(sc, IMX_GPIO_DR_REG, (READ4(sc, IMX_GPIO_DR_REG) ^ (1U << pin))); mtx_unlock_spin(&sc->sc_mtx); return (0); } static int imx51_gpio_pin_access_32(device_t dev, uint32_t first_pin, uint32_t clear_pins, uint32_t change_pins, uint32_t *orig_pins) { struct imx51_gpio_softc *sc; if (first_pin != 0) return (EINVAL); sc = device_get_softc(dev); if (orig_pins != NULL) *orig_pins = READ4(sc, IMX_GPIO_DR_REG); if ((clear_pins | change_pins) != 0) { mtx_lock_spin(&sc->sc_mtx); WRITE4(sc, IMX_GPIO_DR_REG, (READ4(sc, IMX_GPIO_DR_REG) & ~clear_pins) ^ change_pins); mtx_unlock_spin(&sc->sc_mtx); } return (0); } static int imx51_gpio_pin_config_32(device_t dev, uint32_t first_pin, uint32_t num_pins, uint32_t *pin_flags) { struct imx51_gpio_softc *sc; u_int i; uint32_t bit, drclr, drset, flags, oeclr, oeset, pads; sc = device_get_softc(dev); if (first_pin != 0 || num_pins > sc->gpio_npins) return (EINVAL); drclr = drset = oeclr = oeset = 0; pads = READ4(sc, IMX_GPIO_DR_REG); for (i = 0; i < num_pins; ++i) { bit = 1u << i; flags = pin_flags[i]; if (flags & GPIO_PIN_INPUT) { oeclr |= bit; } else if (flags & GPIO_PIN_OUTPUT) { oeset |= bit; if (flags & GPIO_PIN_PRESET_LOW) drclr |= bit; else if (flags & GPIO_PIN_PRESET_HIGH) drset |= bit; else /* Drive whatever it's now pulled to. */ drset |= pads & bit; } } mtx_lock_spin(&sc->sc_mtx); WRITE4(sc, IMX_GPIO_DR_REG, (READ4(sc, IMX_GPIO_DR_REG) & ~drclr) | drset); WRITE4(sc, IMX_GPIO_OE_REG, (READ4(sc, IMX_GPIO_OE_REG) & ~oeclr) | oeset); mtx_unlock_spin(&sc->sc_mtx); return (0); } static int imx51_gpio_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) { device_set_desc(dev, "Freescale i.MX GPIO Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int imx51_gpio_attach(device_t dev) { struct imx51_gpio_softc *sc; int i, irq, unit; #ifdef IMX_ENABLE_CLOCKS int err; #endif sc = device_get_softc(dev); sc->dev = dev; sc->gpio_npins = NGPIO; mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), NULL, MTX_SPIN); #ifdef IMX_ENABLE_CLOCKS if (clk_get_by_ofw_index(sc->dev, 0, 0, &sc->clk) != 0) { device_printf(dev, "could not get clock"); return (ENOENT); } err = clk_enable(sc->clk); if (err != 0) { device_printf(sc->dev, "could not enable ipg clock\n"); return (err); } #endif if (bus_alloc_resources(dev, imx_gpio_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); bus_release_resources(dev, imx_gpio_spec, sc->sc_res); mtx_destroy(&sc->sc_mtx); return (ENXIO); } sc->sc_iot = rman_get_bustag(sc->sc_res[0]); sc->sc_ioh = rman_get_bushandle(sc->sc_res[0]); /* * Mask off all interrupts in hardware, then set up interrupt handling. */ WRITE4(sc, IMX_GPIO_IMR_REG, 0); for (irq = 0; irq < 2; irq++) { #ifdef INTRNG if ((bus_setup_intr(dev, sc->sc_res[1 + irq], INTR_TYPE_CLK, gpio_pic_filter, NULL, sc, &sc->gpio_ih[irq]))) { device_printf(dev, "WARNING: unable to register interrupt handler\n"); imx51_gpio_detach(dev); return (ENXIO); } #endif } unit = device_get_unit(dev); for (i = 0; i < sc->gpio_npins; i++) { sc->gpio_pins[i].gp_pin = i; sc->gpio_pins[i].gp_caps = DEFAULT_CAPS; sc->gpio_pins[i].gp_flags = (READ4(sc, IMX_GPIO_OE_REG) & (1U << i)) ? GPIO_PIN_OUTPUT : GPIO_PIN_INPUT; snprintf(sc->gpio_pins[i].gp_name, GPIOMAXNAME, "GPIO%d_IO%02d", unit + 1, i); } #ifdef INTRNG gpio_pic_register_isrcs(sc); intr_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev))); #endif sc->sc_busdev = gpiobus_attach_bus(dev); if (sc->sc_busdev == NULL) { imx51_gpio_detach(dev); return (ENXIO); } return (0); } static int imx51_gpio_detach(device_t dev) { int irq; struct imx51_gpio_softc *sc; #ifdef IMX_ENABLE_CLOCKS int error; #endif sc = device_get_softc(dev); #ifdef IMX_ENABLE_CLOCKS error = clk_disable(sc->clk); if (error != 0) { device_printf(sc->dev, "could not disable ipg clock\n"); return (error); } #endif gpiobus_detach_bus(dev); for (irq = 0; irq < NUM_IRQRES; irq++) { if (sc->gpio_ih[irq]) bus_teardown_intr(dev, sc->sc_res[irq + FIRST_IRQRES], sc->gpio_ih[irq]); } bus_release_resources(dev, imx_gpio_spec, sc->sc_res); mtx_destroy(&sc->sc_mtx); return(0); } static phandle_t imx51_gpio_get_node(device_t bus, device_t dev) { /* * Share controller node with gpiobus device */ return ofw_bus_get_node(bus); } static device_method_t imx51_gpio_methods[] = { DEVMETHOD(device_probe, imx51_gpio_probe), DEVMETHOD(device_attach, imx51_gpio_attach), DEVMETHOD(device_detach, imx51_gpio_detach), #ifdef INTRNG /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, gpio_pic_disable_intr), DEVMETHOD(pic_enable_intr, gpio_pic_enable_intr), DEVMETHOD(pic_map_intr, gpio_pic_map_intr), DEVMETHOD(pic_setup_intr, gpio_pic_setup_intr), DEVMETHOD(pic_teardown_intr, gpio_pic_teardown_intr), DEVMETHOD(pic_post_filter, gpio_pic_post_filter), DEVMETHOD(pic_post_ithread, gpio_pic_post_ithread), DEVMETHOD(pic_pre_ithread, gpio_pic_pre_ithread), #endif /* OFW methods */ DEVMETHOD(ofw_bus_get_node, imx51_gpio_get_node), /* GPIO protocol */ DEVMETHOD(gpio_get_bus, imx51_gpio_get_bus), DEVMETHOD(gpio_pin_max, imx51_gpio_pin_max), DEVMETHOD(gpio_pin_getname, imx51_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, imx51_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, imx51_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, imx51_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, imx51_gpio_pin_get), DEVMETHOD(gpio_pin_set, imx51_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, imx51_gpio_pin_toggle), DEVMETHOD(gpio_pin_access_32, imx51_gpio_pin_access_32), DEVMETHOD(gpio_pin_config_32, imx51_gpio_pin_config_32), {0, 0}, }; static driver_t imx51_gpio_driver = { "gpio", imx51_gpio_methods, sizeof(struct imx51_gpio_softc), }; static devclass_t imx51_gpio_devclass; EARLY_DRIVER_MODULE(imx51_gpio, simplebus, imx51_gpio_driver, imx51_gpio_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE); diff --git a/sys/arm/freescale/imx/imx_gpt.c b/sys/arm/freescale/imx/imx_gpt.c index 6149ec6c6f04..0605f5ca3558 100644 --- a/sys/arm/freescale/imx/imx_gpt.c +++ b/sys/arm/freescale/imx/imx_gpt.c @@ -1,425 +1,424 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include /* For arm_set_delay */ #include #include #include #include #include #define WRITE4(_sc, _r, _v) \ bus_space_write_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r), (_v)) #define READ4(_sc, _r) \ bus_space_read_4((_sc)->sc_iot, (_sc)->sc_ioh, (_r)) #define SET4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) | (_m)) #define CLEAR4(_sc, _r, _m) \ WRITE4((_sc), (_r), READ4((_sc), (_r)) & ~(_m)) static u_int imx_gpt_get_timecount(struct timecounter *); static int imx_gpt_timer_start(struct eventtimer *, sbintime_t, sbintime_t); static int imx_gpt_timer_stop(struct eventtimer *); static void imx_gpt_do_delay(int, void *); static int imx_gpt_intr(void *); static int imx_gpt_probe(device_t); static int imx_gpt_attach(device_t); static struct timecounter imx_gpt_timecounter = { .tc_name = "iMXGPT", .tc_get_timecount = imx_gpt_get_timecount, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 1000, }; struct imx_gpt_softc { device_t sc_dev; struct resource * res[2]; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; void * sc_ih; /* interrupt handler */ uint32_t sc_period; uint32_t sc_clksrc; uint32_t clkfreq; uint32_t ir_reg; struct eventtimer et; }; /* Try to divide down an available fast clock to this frequency. */ #define TARGET_FREQUENCY 1000000000 static struct resource_spec imx_gpt_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; static struct ofw_compat_data compat_data[] = { {"fsl,imx6dl-gpt", 1}, {"fsl,imx6q-gpt", 1}, {"fsl,imx6ul-gpt", 1}, {"fsl,imx53-gpt", 1}, {"fsl,imx51-gpt", 1}, {"fsl,imx31-gpt", 1}, {"fsl,imx27-gpt", 1}, {"fsl,imx25-gpt", 1}, {NULL, 0} }; static int imx_gpt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); /* * We only support a single unit, because the only thing this driver * does with the complex timer hardware is supply the system * timecounter and eventtimer. There is nothing useful we can do with * the additional device instances that exist in some chips. */ if (device_get_unit(dev) > 0) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) { device_set_desc(dev, "Freescale i.MX GPT timer"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int imx_gpt_attach(device_t dev) { struct imx_gpt_softc *sc; int ctlreg, err; uint32_t basefreq, prescale, setup_ticks, t1, t2; sc = device_get_softc(dev); if (bus_alloc_resources(dev, imx_gpt_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->sc_dev = dev; sc->sc_iot = rman_get_bustag(sc->res[0]); sc->sc_ioh = rman_get_bushandle(sc->res[0]); /* * For now, just automatically choose a good clock for the hardware * we're running on. Eventually we could allow selection from the fdt; * the code in this driver will cope with any clock frequency. */ sc->sc_clksrc = GPT_CR_CLKSRC_IPG; ctlreg = 0; switch (sc->sc_clksrc) { case GPT_CR_CLKSRC_32K: basefreq = 32768; break; case GPT_CR_CLKSRC_IPG: basefreq = imx_ccm_ipg_hz(); break; case GPT_CR_CLKSRC_IPG_HIGH: basefreq = imx_ccm_ipg_hz() * 2; break; case GPT_CR_CLKSRC_24M: ctlreg |= GPT_CR_24MEN; basefreq = 24000000; break; case GPT_CR_CLKSRC_NONE:/* Can't run without a clock. */ case GPT_CR_CLKSRC_EXT: /* No way to get the freq of an ext clock. */ default: device_printf(dev, "Unsupported clock source '%d'\n", sc->sc_clksrc); return (EINVAL); } /* * The following setup sequence is from the I.MX6 reference manual, * "Selecting the clock source". First, disable the clock and * interrupts. This also clears input and output mode bits and in * general completes several of the early steps in the procedure. */ WRITE4(sc, IMX_GPT_CR, 0); WRITE4(sc, IMX_GPT_IR, 0); /* Choose the clock and the power-saving behaviors. */ ctlreg |= sc->sc_clksrc | /* Use selected clock */ GPT_CR_FRR | /* Just count (FreeRunner mode) */ GPT_CR_STOPEN | /* Run in STOP mode */ GPT_CR_DOZEEN | /* Run in DOZE mode */ GPT_CR_WAITEN | /* Run in WAIT mode */ GPT_CR_DBGEN; /* Run in DEBUG mode */ WRITE4(sc, IMX_GPT_CR, ctlreg); /* * The datasheet says to do the software reset after choosing the clock * source. It says nothing about needing to wait for the reset to * complete, but the register description does document the fact that * the reset isn't complete until the SWR bit reads 0, so let's be safe. * The reset also clears all registers except for a few of the bits in * CR, but we'll rewrite all the CR bits when we start the counter. */ WRITE4(sc, IMX_GPT_CR, ctlreg | GPT_CR_SWR); while (READ4(sc, IMX_GPT_CR) & GPT_CR_SWR) continue; /* Set a prescaler value that gets us near the target frequency. */ if (basefreq < TARGET_FREQUENCY) { prescale = 0; sc->clkfreq = basefreq; } else { prescale = basefreq / TARGET_FREQUENCY; sc->clkfreq = basefreq / prescale; prescale -= 1; /* 1..n range is 0..n-1 in hardware. */ } WRITE4(sc, IMX_GPT_PR, prescale); /* Clear the status register. */ WRITE4(sc, IMX_GPT_SR, GPT_IR_ALL); /* Start the counter. */ WRITE4(sc, IMX_GPT_CR, ctlreg | GPT_CR_EN); if (bootverbose) device_printf(dev, "Running on %dKHz clock, base freq %uHz CR=0x%08x, PR=0x%08x\n", sc->clkfreq / 1000, basefreq, READ4(sc, IMX_GPT_CR), READ4(sc, IMX_GPT_PR)); /* Setup the timer interrupt. */ err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_CLK, imx_gpt_intr, NULL, sc, &sc->sc_ih); if (err != 0) { bus_release_resources(dev, imx_gpt_spec, sc->res); device_printf(dev, "Unable to setup the clock irq handler, " "err = %d\n", err); return (ENXIO); } /* * Measure how many clock ticks it takes to setup a one-shot event (it's * longer than you might think, due to wait states in accessing gpt * registers). Scale up the result by a factor of 1.5 to be safe, * and use that to set the minimum eventtimer period we can schedule. In * the real world, the value works out to about 750ns on imx5 hardware. */ t1 = READ4(sc, IMX_GPT_CNT); WRITE4(sc, IMX_GPT_OCR3, 0); t2 = READ4(sc, IMX_GPT_CNT); setup_ticks = ((t2 - t1 + 1) * 3) / 2; /* Register as an eventtimer. */ sc->et.et_name = "iMXGPT"; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERIODIC; sc->et.et_quality = 800; sc->et.et_frequency = sc->clkfreq; sc->et.et_min_period = ((uint64_t)setup_ticks << 32) / sc->clkfreq; sc->et.et_max_period = ((uint64_t)0xfffffffe << 32) / sc->clkfreq; sc->et.et_start = imx_gpt_timer_start; sc->et.et_stop = imx_gpt_timer_stop; sc->et.et_priv = sc; et_register(&sc->et); /* Register as a timecounter. */ imx_gpt_timecounter.tc_frequency = sc->clkfreq; imx_gpt_timecounter.tc_priv = sc; tc_init(&imx_gpt_timecounter); /* If this is the first unit, store the softc for use in DELAY. */ if (device_get_unit(dev) == 0) { arm_set_delay(imx_gpt_do_delay, sc); } return (0); } static int imx_gpt_timer_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { struct imx_gpt_softc *sc; uint32_t ticks; sc = (struct imx_gpt_softc *)et->et_priv; if (period != 0) { sc->sc_period = ((uint32_t)et->et_frequency * period) >> 32; /* Set expected value */ WRITE4(sc, IMX_GPT_OCR2, READ4(sc, IMX_GPT_CNT) + sc->sc_period); /* Enable compare register 2 Interrupt */ sc->ir_reg |= GPT_IR_OF2; WRITE4(sc, IMX_GPT_IR, sc->ir_reg); return (0); } else if (first != 0) { /* Enable compare register 3 interrupt if not already on. */ if ((sc->ir_reg & GPT_IR_OF3) == 0) { sc->ir_reg |= GPT_IR_OF3; WRITE4(sc, IMX_GPT_IR, sc->ir_reg); } ticks = ((uint32_t)et->et_frequency * first) >> 32; /* Do not disturb, otherwise event will be lost */ spinlock_enter(); /* Set expected value */ WRITE4(sc, IMX_GPT_OCR3, READ4(sc, IMX_GPT_CNT) + ticks); /* Now everybody can relax */ spinlock_exit(); return (0); } return (EINVAL); } static int imx_gpt_timer_stop(struct eventtimer *et) { struct imx_gpt_softc *sc; sc = (struct imx_gpt_softc *)et->et_priv; /* Disable interrupts and clear any pending status. */ sc->ir_reg &= ~(GPT_IR_OF2 | GPT_IR_OF3); WRITE4(sc, IMX_GPT_IR, sc->ir_reg); WRITE4(sc, IMX_GPT_SR, GPT_IR_OF2 | GPT_IR_OF3); sc->sc_period = 0; return (0); } static int imx_gpt_intr(void *arg) { struct imx_gpt_softc *sc; uint32_t status; sc = (struct imx_gpt_softc *)arg; status = READ4(sc, IMX_GPT_SR); /* * Clear interrupt status before invoking event callbacks. The callback * often sets up a new one-shot timer event and if the interval is short * enough it can fire before we get out of this function. If we cleared * at the bottom we'd miss the interrupt and hang until the clock wraps. */ WRITE4(sc, IMX_GPT_SR, status); /* Handle one-shot timer events. */ if (status & GPT_IR_OF3) { if (sc->et.et_active) { sc->et.et_event_cb(&sc->et, sc->et.et_arg); } } /* Handle periodic timer events. */ if (status & GPT_IR_OF2) { if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); if (sc->sc_period != 0) WRITE4(sc, IMX_GPT_OCR2, READ4(sc, IMX_GPT_CNT) + sc->sc_period); } return (FILTER_HANDLED); } static u_int imx_gpt_get_timecount(struct timecounter *tc) { struct imx_gpt_softc *sc; sc = tc->tc_priv; return (READ4(sc, IMX_GPT_CNT)); } static device_method_t imx_gpt_methods[] = { DEVMETHOD(device_probe, imx_gpt_probe), DEVMETHOD(device_attach, imx_gpt_attach), DEVMETHOD_END }; static driver_t imx_gpt_driver = { "imx_gpt", imx_gpt_methods, sizeof(struct imx_gpt_softc), }; static devclass_t imx_gpt_devclass; EARLY_DRIVER_MODULE(imx_gpt, simplebus, imx_gpt_driver, imx_gpt_devclass, 0, 0, BUS_PASS_TIMER); static void imx_gpt_do_delay(int usec, void *arg) { struct imx_gpt_softc *sc = arg; uint64_t curcnt, endcnt, startcnt, ticks; /* * Calculate the tick count with 64-bit values so that it works for any * clock frequency. Loop until the hardware count reaches start+ticks. * If the 32-bit hardware count rolls over while we're looping, just * manually do a carry into the high bits after each read; don't worry * that doing this on each loop iteration is inefficient -- we're trying * to waste time here. */ ticks = 1 + ((uint64_t)usec * sc->clkfreq) / 1000000; curcnt = startcnt = READ4(sc, IMX_GPT_CNT); endcnt = startcnt + ticks; while (curcnt < endcnt) { curcnt = READ4(sc, IMX_GPT_CNT); if (curcnt < startcnt) curcnt += 1ULL << 32; } } diff --git a/sys/arm/freescale/imx/imx_gptreg.h b/sys/arm/freescale/imx/imx_gptreg.h index e1c13b7d0d6a..f37f94b88425 100644 --- a/sys/arm/freescale/imx/imx_gptreg.h +++ b/sys/arm/freescale/imx/imx_gptreg.h @@ -1,103 +1,102 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* Registers definition for Freescale i.MX515 Generic Periodic Timer */ #define IMX_GPT_CR 0x0000 /* Control Register R/W */ #define GPT_CR_FO3 (1U << 31) #define GPT_CR_FO2 (1 << 30) #define GPT_CR_FO1 (1 << 29) #define GPT_CR_OM3_SHIFT 26 #define GPT_CR_OM3_MASK 0x1c000000 #define GPT_CR_OM2_SHIFT 23 #define GPT_CR_OM2_MASK 0x03800000 #define GPT_CR_OM1_SHIFT 20 #define GPT_CR_OM1_MASK 0x00700000 #define GPT_CR_OMX_NONE 0 #define GPT_CR_OMX_TOGGLE 1 #define GPT_CR_OMX_CLEAR 2 #define GPT_CR_OMX_SET 3 #define GPT_CR_OMX_PULSE 4 /* Run CLKSRC on output pin */ #define GPT_CR_IM2_SHIFT 18 #define GPT_CR_IM2_MASK 0x000c0000 #define GPT_CR_IM1_SHIFT 16 #define GPT_CR_IM1_MASK 0x00030000 #define GPT_CR_IMX_NONE 0 #define GPT_CR_IMX_REDGE 1 #define GPT_CR_IMX_FEDGE 2 #define GPT_CR_IMX_BOTH 3 #define GPT_CR_SWR (1 << 15) #define GPT_CR_24MEN (1 << 10) #define GPT_CR_FRR (1 << 9) #define GPT_CR_CLKSRC_NONE (0 << 6) #define GPT_CR_CLKSRC_IPG (1 << 6) #define GPT_CR_CLKSRC_IPG_HIGH (2 << 6) #define GPT_CR_CLKSRC_EXT (3 << 6) #define GPT_CR_CLKSRC_32K (4 << 6) #define GPT_CR_CLKSRC_24M (5 << 6) #define GPT_CR_STOPEN (1 << 5) #define GPT_CR_DOZEEN (1 << 4) #define GPT_CR_WAITEN (1 << 3) #define GPT_CR_DBGEN (1 << 2) #define GPT_CR_ENMOD (1 << 1) #define GPT_CR_EN (1 << 0) #define IMX_GPT_PR 0x0004 /* Prescaler Register R/W */ #define GPT_PR_VALUE_SHIFT 0 #define GPT_PR_VALUE_MASK 0x00000fff #define GPT_PR_VALUE_SHIFT_24M 12 #define GPT_PR_VALUE_MASK_24M 0x0000f000 /* Same map for SR and IR */ #define IMX_GPT_SR 0x0008 /* Status Register R/W */ #define IMX_GPT_IR 0x000c /* Interrupt Register R/W */ #define GPT_IR_ROV (1 << 5) #define GPT_IR_IF2 (1 << 4) #define GPT_IR_IF1 (1 << 3) #define GPT_IR_OF3 (1 << 2) #define GPT_IR_OF2 (1 << 1) #define GPT_IR_OF1 (1 << 0) #define GPT_IR_ALL \ (GPT_IR_ROV | \ GPT_IR_IF2 | \ GPT_IR_IF1 | \ GPT_IR_OF3 | \ GPT_IR_OF2 | \ GPT_IR_OF1) #define IMX_GPT_OCR1 0x0010 /* Output Compare Register 1 R/W */ #define IMX_GPT_OCR2 0x0014 /* Output Compare Register 2 R/W */ #define IMX_GPT_OCR3 0x0018 /* Output Compare Register 3 R/W */ #define IMX_GPT_ICR1 0x001c /* Input capture Register 1 RO */ #define IMX_GPT_ICR2 0x0020 /* Input capture Register 2 RO */ #define IMX_GPT_CNT 0x0024 /* Counter Register RO */ diff --git a/sys/arm/freescale/imx/imx_wdog.c b/sys/arm/freescale/imx/imx_wdog.c index df2ae510e5fb..287a8771f791 100644 --- a/sys/arm/freescale/imx/imx_wdog.c +++ b/sys/arm/freescale/imx/imx_wdog.c @@ -1,264 +1,263 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct imx_wdog_softc { struct mtx sc_mtx; device_t sc_dev; struct resource *sc_res[2]; void *sc_ih; uint32_t sc_timeout; bool sc_pde_enabled; }; static struct resource_spec imx_wdog_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, RESOURCE_SPEC_END }; #define MEMRES 0 #define IRQRES 1 static struct ofw_compat_data compat_data[] = { {"fsl,imx6sx-wdt", 1}, {"fsl,imx6sl-wdt", 1}, {"fsl,imx6q-wdt", 1}, {"fsl,imx53-wdt", 1}, {"fsl,imx51-wdt", 1}, {"fsl,imx50-wdt", 1}, {"fsl,imx35-wdt", 1}, {"fsl,imx27-wdt", 1}, {"fsl,imx25-wdt", 1}, {"fsl,imx21-wdt", 1}, {NULL, 0} }; static inline uint16_t RD2(struct imx_wdog_softc *sc, bus_size_t offs) { return (bus_read_2(sc->sc_res[MEMRES], offs)); } static inline void WR2(struct imx_wdog_softc *sc, bus_size_t offs, uint16_t val) { bus_write_2(sc->sc_res[MEMRES], offs, val); } static int imx_wdog_enable(struct imx_wdog_softc *sc, u_int timeout) { uint16_t reg; if (timeout < 1 || timeout > 128) return (EINVAL); mtx_lock(&sc->sc_mtx); if (timeout != sc->sc_timeout) { sc->sc_timeout = timeout; reg = RD2(sc, WDOG_CR_REG); reg &= ~WDOG_CR_WT_MASK; reg |= ((2 * timeout - 1) << WDOG_CR_WT_SHIFT); WR2(sc, WDOG_CR_REG, reg | WDOG_CR_WDE); } /* Refresh counter */ WR2(sc, WDOG_SR_REG, WDOG_SR_STEP1); WR2(sc, WDOG_SR_REG, WDOG_SR_STEP2); /* Watchdog active, can disable rom-boot watchdog. */ if (sc->sc_pde_enabled) { sc->sc_pde_enabled = false; reg = RD2(sc, WDOG_MCR_REG); WR2(sc, WDOG_MCR_REG, reg & ~WDOG_MCR_PDE); } mtx_unlock(&sc->sc_mtx); return (0); } static void imx_watchdog(void *arg, u_int cmd, int *error) { struct imx_wdog_softc *sc; u_int timeout; sc = arg; if (cmd == 0) { if (bootverbose) device_printf(sc->sc_dev, "Can not be disabled.\n"); *error = EOPNOTSUPP; } else { timeout = (u_int)((1ULL << (cmd & WD_INTERVAL)) / 1000000000U); if (imx_wdog_enable(sc, timeout) == 0) *error = 0; } } static int imx_wdog_intr(void *arg) { struct imx_wdog_softc *sc = arg; /* * When configured for external reset, the actual reset is supposed to * happen when some external device responds to the assertion of the * WDOG_B signal by asserting the POR signal to the chip. This * interrupt handler is a backstop mechanism; it is set up to fire * simultaneously with WDOG_B, and if the external reset happens we'll * never actually make it to here. If we do make it here, just trigger * a software reset. That code will see that external reset is * configured, and it will wait for 1 second for it to take effect, then * it will do a software reset as a fallback. */ imx_wdog_cpu_reset(BUS_SPACE_PHYSADDR(sc->sc_res[MEMRES], WDOG_CR_REG)); return (FILTER_HANDLED); /* unreached */ } static int imx_wdog_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) return (ENXIO); device_set_desc(dev, "Freescale i.MX Watchdog"); return (0); } static int imx_wdog_attach(device_t dev) { struct imx_wdog_softc *sc; pcell_t timeout; sc = device_get_softc(dev); sc->sc_dev = dev; if (bus_alloc_resources(dev, imx_wdog_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "imx_wdt", MTX_DEF); /* * If we're configured to assert an external reset signal, set up the * hardware to do so, and install an interrupt handler whose only * purpose is to backstop the external reset. Don't worry if the * interrupt setup fails, since it's only a backstop measure. */ if (ofw_bus_has_prop(sc->sc_dev, "fsl,ext-reset-output")) { WR2(sc, WDOG_CR_REG, WDOG_CR_WDT | RD2(sc, WDOG_CR_REG)); bus_setup_intr(sc->sc_dev, sc->sc_res[IRQRES], INTR_TYPE_MISC | INTR_MPSAFE, imx_wdog_intr, NULL, sc, &sc->sc_ih); WR2(sc, WDOG_ICR_REG, WDOG_ICR_WIE); /* Enable, count is 0. */ } /* * Note whether the rom-boot so-called "power-down" watchdog is active, * so we can disable it when the regular watchdog is first enabled. */ if (RD2(sc, WDOG_MCR_REG) & WDOG_MCR_PDE) sc->sc_pde_enabled = true; EVENTHANDLER_REGISTER(watchdog_list, imx_watchdog, sc, 0); /* If there is a timeout-sec property, activate the watchdog. */ if (OF_getencprop(ofw_bus_get_node(sc->sc_dev), "timeout-sec", &timeout, sizeof(timeout)) == sizeof(timeout)) { if (timeout < 1 || timeout > 128) { device_printf(sc->sc_dev, "ERROR: bad timeout-sec " "property value %u, using 128\n", timeout); timeout = 128; } imx_wdog_enable(sc, timeout); device_printf(sc->sc_dev, "watchdog enabled using " "timeout-sec property value %u\n", timeout); } /* * The watchdog hardware cannot be disabled, so there's little point in * coding up a detach() routine to carefully tear everything down, just * make the device busy so that detach can't happen. */ device_busy(sc->sc_dev); return (0); } static device_method_t imx_wdog_methods[] = { DEVMETHOD(device_probe, imx_wdog_probe), DEVMETHOD(device_attach, imx_wdog_attach), DEVMETHOD_END }; static driver_t imx_wdog_driver = { "imx_wdog", imx_wdog_methods, sizeof(struct imx_wdog_softc), }; static devclass_t imx_wdog_devclass; EARLY_DRIVER_MODULE(imx_wdog, simplebus, imx_wdog_driver, imx_wdog_devclass, 0, 0, BUS_PASS_TIMER); SIMPLEBUS_PNP_INFO(compat_data); diff --git a/sys/arm/freescale/imx/imx_wdogreg.h b/sys/arm/freescale/imx/imx_wdogreg.h index d192a28bbdc5..a66ca5adfc8e 100644 --- a/sys/arm/freescale/imx/imx_wdogreg.h +++ b/sys/arm/freescale/imx/imx_wdogreg.h @@ -1,63 +1,62 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #define WDOG_CLK_FREQ 32768 #define WDOG_CR_REG 0x00 /* Control Register */ #define WDOG_CR_WT_MASK 0xff00 /* Count; 0.5 sec units */ #define WDOG_CR_WT_SHIFT 8 #define WDOG_CR_WDW (1u << 7) /* Suspend when in WAIT mode */ #define WDOG_CR_WDA (1u << 5) /* Don't assert ext reset */ #define WDOG_CR_SRS (1u << 4) /* Don't assert soft reset */ #define WDOG_CR_WDT (1u << 3) /* Assert ext reset on timeout */ #define WDOG_CR_WDE (1u << 2) /* Watchdog Enable */ #define WDOG_CR_WDBG (1u << 1) /* Suspend when DBG mode */ #define WDOG_CR_WDZST (1u << 0) /* Suspend when LP mode */ #define WDOG_SR_REG 0x02 /* Service Register */ #define WDOG_SR_STEP1 0x5555 #define WDOG_SR_STEP2 0xaaaa #define WDOG_RSR_REG 0x04 /* Reset Status Register */ #define WDOG_RSR_POR (1u << 4) /* Due to Power-On Reset */ #define WDOG_RSR_TOUT (1u << 1) /* Due WDog timeout reset */ #define WDOG_RSR_SFTW (1u << 0) /* Due Soft reset */ #define WDOG_ICR_REG 0x06 /* Interrupt Control Register */ #define WDOG_ICR_WIE (1u << 15) /* Enable Interrupt */ #define WDOG_ICR_WTIS (1u << 14) /* Interrupt has occurred */ #define WDOG_ICR_WTCT_MASK 0x00ff /* Interrupt lead time in 0.5s */ #define WDOG_ICR_WTCT_SHIFT 0 /* units before reset occurs */ #define WDOG_MCR_REG 0x08 /* Miscellaneous Control Register */ #define WDOG_MCR_PDE (1u << 0) /* Power-down enable */ diff --git a/sys/arm/freescale/imx/tzic.c b/sys/arm/freescale/imx/tzic.c index 6f3aca14da84..cddbd6ba9919 100644 --- a/sys/arm/freescale/imx/tzic.c +++ b/sys/arm/freescale/imx/tzic.c @@ -1,312 +1,311 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define TZIC_NIRQS 128 struct tzic_irqsrc { struct intr_irqsrc isrc; u_int irq; }; struct tzic_softc { device_t dev; struct resource *tzicregs; struct tzic_irqsrc isrcs[TZIC_NIRQS]; }; static struct tzic_softc *tzic_sc; static inline uint32_t tzic_read_4(struct tzic_softc *sc, int reg) { return (bus_read_4(sc->tzicregs, reg)); } static inline void tzic_write_4(struct tzic_softc *sc, int reg, uint32_t val) { bus_write_4(sc->tzicregs, reg, val); } static inline void tzic_irq_eoi(struct tzic_softc *sc) { tzic_write_4(sc, TZIC_PRIOMASK, 0xff); } static inline void tzic_irq_mask(struct tzic_softc *sc, u_int irq) { tzic_write_4(sc, TZIC_ENCLEAR(irq >> 5), (1u << (irq & 0x1f))); } static inline void tzic_irq_unmask(struct tzic_softc *sc, u_int irq) { tzic_write_4(sc, TZIC_ENSET(irq >> 5), (1u << (irq & 0x1f))); } static int tzic_intr(void *arg) { struct tzic_softc *sc = arg; int b, i, irq; uint32_t pending; /* Get active interrupt */ for (i = 0; i < TZIC_NIRQS / 32; ++i) { pending = tzic_read_4(sc, TZIC_PND(i)); if ((b = 31 - __builtin_clz(pending)) < 0) continue; irq = i * 32 + b; tzic_write_4(sc, TZIC_PRIOMASK, 0); if (intr_isrc_dispatch(&sc->isrcs[irq].isrc, curthread->td_intr_frame) != 0) { tzic_irq_mask(sc, irq); tzic_irq_eoi(sc); arm_irq_memory_barrier(irq); if (bootverbose) { device_printf(sc->dev, "Stray irq %u disabled\n", irq); } } return (FILTER_HANDLED); } if (bootverbose) device_printf(sc->dev, "Spurious interrupt detected\n"); return (FILTER_HANDLED); } static void tzic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq = ((struct tzic_irqsrc *)isrc)->irq; struct tzic_softc *sc = device_get_softc(dev); arm_irq_memory_barrier(irq); tzic_irq_unmask(sc, irq); } static void tzic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq = ((struct tzic_irqsrc *)isrc)->irq; struct tzic_softc *sc = device_get_softc(dev); tzic_irq_mask(sc, irq); } static int tzic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct tzic_softc *sc; if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells != 1 || daf->cells[0] >= TZIC_NIRQS) return (EINVAL); sc = device_get_softc(dev); *isrcp = &sc->isrcs[daf->cells[0]].isrc; return (0); } static void tzic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct tzic_softc *sc = device_get_softc(dev); tzic_irq_mask(sc, ((struct tzic_irqsrc *)isrc)->irq); tzic_irq_eoi(sc); } static void tzic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { tzic_enable_intr(dev, isrc); } static void tzic_post_filter(device_t dev, struct intr_irqsrc *isrc) { tzic_irq_eoi(device_get_softc(dev)); } static int tzic_pic_attach(struct tzic_softc *sc) { struct intr_pic *pic; const char *name; intptr_t xref; int error; u_int irq; name = device_get_nameunit(sc->dev); for (irq = 0; irq < TZIC_NIRQS; irq++) { sc->isrcs[irq].irq = irq; error = intr_isrc_register(&sc->isrcs[irq].isrc, sc->dev, 0, "%s,%u", name, irq); if (error != 0) return (error); } xref = OF_xref_from_node(ofw_bus_get_node(sc->dev)); pic = intr_pic_register(sc->dev, xref); if (pic == NULL) return (ENXIO); return (intr_pic_claim_root(sc->dev, xref, tzic_intr, sc, 0)); } static int tzic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "fsl,tzic")) { device_set_desc(dev, "TrustZone Interrupt Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int tzic_attach(device_t dev) { struct tzic_softc *sc = device_get_softc(dev); int i; if (tzic_sc) return (ENXIO); tzic_sc = sc; sc->dev = dev; i = 0; sc->tzicregs = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i, RF_ACTIVE); if (sc->tzicregs == NULL) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* route all interrupts to IRQ. secure interrupts are for FIQ */ for (i = 0; i < 4; i++) tzic_write_4(sc, TZIC_INTSEC(i), 0xffffffff); /* disable all interrupts */ for (i = 0; i < 4; i++) tzic_write_4(sc, TZIC_ENCLEAR(i), 0xffffffff); /* Set all interrupts to priority 0 (max). */ for (i = 0; i < 128 / 4; ++i) tzic_write_4(sc, TZIC_PRIORITY(i), 0); /* * Set priority mask to lowest (unmasked) prio, set synchronizer to * low-latency mode (as opposed to low-power), enable the controller. */ tzic_write_4(sc, TZIC_PRIOMASK, 0xff); tzic_write_4(sc, TZIC_SYNCCTRL, 0); tzic_write_4(sc, TZIC_INTCNTL, INTCNTL_NSEN_MASK|INTCNTL_NSEN|INTCNTL_EN); /* Register as a root pic. */ if (tzic_pic_attach(sc) != 0) { device_printf(dev, "could not attach PIC\n"); return (ENXIO); } return (0); } static device_method_t tzic_methods[] = { DEVMETHOD(device_probe, tzic_probe), DEVMETHOD(device_attach, tzic_attach), DEVMETHOD(pic_disable_intr, tzic_disable_intr), DEVMETHOD(pic_enable_intr, tzic_enable_intr), DEVMETHOD(pic_map_intr, tzic_map_intr), DEVMETHOD(pic_post_filter, tzic_post_filter), DEVMETHOD(pic_post_ithread, tzic_post_ithread), DEVMETHOD(pic_pre_ithread, tzic_pre_ithread), DEVMETHOD_END }; static driver_t tzic_driver = { "tzic", tzic_methods, sizeof(struct tzic_softc), }; static devclass_t tzic_devclass; EARLY_DRIVER_MODULE(tzic, ofwbus, tzic_driver, tzic_devclass, 0, 0, BUS_PASS_INTERRUPT); diff --git a/sys/arm/include/debug_monitor.h b/sys/arm/include/debug_monitor.h index c160397393ad..1271d9ec052b 100644 --- a/sys/arm/include/debug_monitor.h +++ b/sys/arm/include/debug_monitor.h @@ -1,60 +1,59 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_DEBUG_MONITOR_H_ #define _MACHINE_DEBUG_MONITOR_H_ #ifdef DDB #include enum dbg_access_t { HW_BREAKPOINT_X = 0, HW_WATCHPOINT_R = 1, HW_WATCHPOINT_W = 2, HW_WATCHPOINT_RW = HW_WATCHPOINT_R | HW_WATCHPOINT_W, }; void dbg_monitor_init(void); void dbg_monitor_init_secondary(void); void dbg_show_watchpoint(void); int dbg_setup_watchpoint(db_expr_t, db_expr_t, enum dbg_access_t); int dbg_remove_watchpoint(db_expr_t, db_expr_t); void dbg_resume_dbreg(void); #else /* DDB */ static __inline void dbg_monitor_init(void) { } #endif #endif /* _MACHINE_DEBUG_MONITOR_H_ */ diff --git a/sys/arm/include/fdt.h b/sys/arm/include/fdt.h index e8af11c8c4b0..37849d381372 100644 --- a/sys/arm/include/fdt.h +++ b/sys/arm/include/fdt.h @@ -1,44 +1,43 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_FDT_H_ #define _MACHINE_FDT_H_ #include /* * Bus space tag. XXX endianess info needs to be derived from the blob. */ extern bus_space_tag_t fdtbus_bs_tag; #endif /* _MACHINE_FDT_H_ */ diff --git a/sys/arm/include/ofw_machdep.h b/sys/arm/include/ofw_machdep.h index 76ab31c963cf..1adbb02965c3 100644 --- a/sys/arm/include/ofw_machdep.h +++ b/sys/arm/include/ofw_machdep.h @@ -1,49 +1,48 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_OFW_MACHDEP_H_ #define _MACHINE_OFW_MACHDEP_H_ #include #include #include #include typedef uint32_t cell_t; struct mem_region { uint64_t mr_start; uint64_t mr_size; }; #endif /* _MACHINE_OFW_MACHDEP_H_ */ diff --git a/sys/arm64/acpica/acpi_wakeup.c b/sys/arm64/acpica/acpi_wakeup.c index 7724fddd0fde..8504cac5894f 100644 --- a/sys/arm64/acpica/acpi_wakeup.c +++ b/sys/arm64/acpica/acpi_wakeup.c @@ -1,61 +1,60 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * ARM64TODO: Implement this. */ int acpi_sleep_machdep(struct acpi_softc *sc, int state) { return (-1); } int acpi_wakeup_machdep(struct acpi_softc *sc, int state, int sleep_result, int intr_enabled) { /* ARM64TODO: We will need this with acpi_sleep_machdep */ KASSERT(sleep_result == -1, ("acpi_wakeup_machdep: Invalid sleep result")); return (sleep_result); } diff --git a/sys/arm64/acpica/pci_cfgreg.c b/sys/arm64/acpica/pci_cfgreg.c index e80803fbaa4a..f8e777b65ff4 100644 --- a/sys/arm64/acpica/pci_cfgreg.c +++ b/sys/arm64/acpica/pci_cfgreg.c @@ -1,76 +1,75 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include /* * This file contains stubs for ACPI PCI functions */ /* * Read configuration space register */ uint32_t pci_cfgregread(int bus, int slot, int func, int reg, int bytes) { /* ARM64TODO */ panic("pci_cfgregread not implemented"); return (0); } /* * Write configuration space register */ void pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) { /* ARM64TODO */ panic("pci_cfgregwrite not implemented"); } /* * Initialize access to configuration space */ int pci_cfgregopen(void) { /* ARM64TODO */ panic("pci_cfgregopen not implemented"); return (0); } diff --git a/sys/arm64/arm64/autoconf.c b/sys/arm64/arm64/autoconf.c index 9788c789cfc4..f762a0fe858e 100644 --- a/sys/arm64/arm64/autoconf.c +++ b/sys/arm64/arm64/autoconf.c @@ -1,92 +1,91 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); /* * Setup the system to run on the current machine. * * Configure() is called at boot time and initializes the vba * device tables and the memory controller monitoring. Available * devices are determined (from possibilities mentioned in ioconf.c), * and the drivers are initialized. */ #include #include #include #include #include #include static void configure_first(void *); static void configure(void *); static void configure_final(void *); SYSINIT(configure1, SI_SUB_CONFIGURE, SI_ORDER_FIRST, configure_first, NULL); /* SI_ORDER_SECOND is hookable */ SYSINIT(configure2, SI_SUB_CONFIGURE, SI_ORDER_THIRD, configure, NULL); /* SI_ORDER_MIDDLE is hookable */ SYSINIT(configure3, SI_SUB_CONFIGURE, SI_ORDER_ANY, configure_final, NULL); /* * Determine i/o configuration for a machine. */ static void configure_first(void *dummy) { /* nexus0 is the top of the device tree */ device_add_child(root_bus, "nexus", 0); } static void configure(void *dummy) { /* initialize new bus architecture */ root_bus_configure(); } static void configure_final(void *dummy) { /* Enable interrupt reception on this CPU */ intr_enable(); cninit_finish(); if (bootverbose) printf("Device configuration finished.\n"); cold = 0; } diff --git a/sys/arm64/arm64/copyinout.S b/sys/arm64/arm64/copyinout.S index 45d08d3ac545..86e3c55f9a09 100644 --- a/sys/arm64/arm64/copyinout.S +++ b/sys/arm64/arm64/copyinout.S @@ -1,224 +1,223 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include "assym.inc" .macro check_user_access user_arg, size_arg, bad_access_func adds x6, x\user_arg, x\size_arg b.cs \bad_access_func ldr x7, =VM_MAXUSER_ADDRESS cmp x6, x7 b.hi \bad_access_func .endm /* * Fault handler for the copy{in,out} functions below. */ ENTRY(copyio_fault) SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ EXIT_USER_ACCESS_CHECK(w0, x1) copyio_fault_nopcb: mov x0, #EFAULT ret END(copyio_fault) /* * Copies from a kernel to user address * * int copyout(const void *kaddr, void *udaddr, size_t len) */ ENTRY(copyout) cbz x2, 1f check_user_access 1, 2, copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyout) /* * Copies from a user to kernel address * * int copyin(const void *uaddr, void *kdaddr, size_t len) */ ENTRY(copyin) cbz x2, 1f check_user_access 0, 2, copyio_fault_nopcb b copycommon 1: mov x0, xzr /* return 0 */ ret END(copyin) /* * Copies a string from a user to kernel address * * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) mov x5, xzr /* count = 0 */ mov w4, #1 /* If zero return faulure */ cbz x2, 3f /* If len == 0 then skip loop */ adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ ldr x7, =VM_MAXUSER_ADDRESS 1: cmp x0, x7 b.cs copyio_fault ldtrb w4, [x0] /* Load from uaddr */ add x0, x0, #1 /* Next char */ strb w4, [x1], #1 /* Store in kaddr */ add x5, x5, #1 /* count++ */ cbz w4, 2f /* Break when NUL-terminated */ sub x2, x2, #1 /* len-- */ cbnz x2, 1b 2: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 3: cbz x3, 4f /* Check if done != NULL */ str x5, [x3] /* done = count */ 4: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ cmp w4, #0 /* Check if we saved the NUL-terminator */ csel w0, wzr, w1, eq /* If so return success, else failure */ ret END(copyinstr) /* * Local helper * * x0 - src pointer * x1 - dst pointer * x2 - size * lr - the return address, so jump here instead of calling * * This function is optimized to minimize concurrent memory accesses. In * present form it is suited for cores with a single memory prefetching * unit. * ARM64TODO: * Consider using separate functions for each ARM64 core. Adding memory * access interleaving might increase a total throughput on A57 or A72. */ .text .align 4 .local copycommon .type copycommon,@function copycommon: adr x6, copyio_fault /* Get the handler address */ SET_FAULT_HANDLER(x6, x7) /* Set the handler */ ENTER_USER_ACCESS(w6, x7) /* Check alignment */ orr x3, x0, x1 ands x3, x3, 0x07 b.eq aligned /* Unaligned is byte by byte copy */ byte_by_byte: ldrb w3, [x0], #0x01 strb w3, [x1], #0x01 subs x2, x2, #0x01 b.ne byte_by_byte b ending aligned: cmp x2, #0x10 b.lt lead_out cmp x2, #0x40 b.lt by_dwords_start /* Block copy */ lsr x15, x2, #0x06 by_blocks: ldp x3, x4, [x0], #0x10 ldp x5, x6, [x0], #0x10 ldp x7, x8, [x0], #0x10 ldp x9, x10, [x0], #0x10 stp x3, x4, [x1], #0x10 stp x5, x6, [x1], #0x10 stp x7, x8, [x1], #0x10 stp x9, x10, [x1], #0x10 subs x15, x15, #0x01 b.ne by_blocks and x2, x2, #0x3f by_dwords_start: lsr x15, x2, #0x04 cbz x15, lead_out by_dwords: ldp x3, x4, [x0], #0x10 stp x3, x4, [x1], #0x10 subs x15, x15, #0x01 b.ne by_dwords /* Less than 16 bytes to copy */ lead_out: tbz x2, #0x03, last_word ldr x3, [x0], #0x08 str x3, [x1], #0x08 last_word: tbz x2, #0x02, last_hword ldr w3, [x0], #0x04 str w3, [x1], #0x04 last_hword: tbz x2, #0x01, last_byte ldrh w3, [x0], #0x02 strh w3, [x1], #0x02 last_byte: tbz x2, #0x00, ending ldrb w3, [x0] strb w3, [x1] ending: EXIT_USER_ACCESS_CHECK(w6, x7) SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ mov x0, xzr /* return 0 */ ret .size copycommon, . - copycommon diff --git a/sys/arm64/arm64/db_disasm.c b/sys/arm64/arm64/db_disasm.c index 73efca0bdee9..a7d65845acbf 100644 --- a/sys/arm64/arm64/db_disasm.c +++ b/sys/arm64/arm64/db_disasm.c @@ -1,70 +1,69 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include static u_int db_disasm_read_word(vm_offset_t); static void db_disasm_printaddr(vm_offset_t); /* Glue code to interface db_disasm to the generic ARM disassembler */ static const struct disasm_interface db_disasm_interface = { .di_readword = db_disasm_read_word, .di_printaddr = db_disasm_printaddr, .di_printf = db_printf, }; static u_int db_disasm_read_word(vm_offset_t address) { return (db_get_value(address, INSN_SIZE, 0)); } static void db_disasm_printaddr(vm_offset_t address) { db_printsym((db_addr_t)address, DB_STGY_ANY); } vm_offset_t db_disasm(vm_offset_t loc, bool altfmt) { return (disasm(&db_disasm_interface, loc, altfmt)); } /* End of db_disasm.c */ diff --git a/sys/arm64/arm64/db_interface.c b/sys/arm64/arm64/db_interface.c index 5138bf3f1cab..d8ae79715a19 100644 --- a/sys/arm64/arm64/db_interface.c +++ b/sys/arm64/arm64/db_interface.c @@ -1,194 +1,193 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include static int db_frame(struct db_variable *vp, db_expr_t *valuep, int op) { long *reg; if (kdb_frame == NULL) return (0); reg = (long *)((uintptr_t)kdb_frame + (db_expr_t)vp->valuep); if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; return (1); } #define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x) struct db_variable db_regs[] = { { "spsr", DB_OFFSET(tf_spsr), db_frame }, { "x0", DB_OFFSET(tf_x[0]), db_frame }, { "x1", DB_OFFSET(tf_x[1]), db_frame }, { "x2", DB_OFFSET(tf_x[2]), db_frame }, { "x3", DB_OFFSET(tf_x[3]), db_frame }, { "x4", DB_OFFSET(tf_x[4]), db_frame }, { "x5", DB_OFFSET(tf_x[5]), db_frame }, { "x6", DB_OFFSET(tf_x[6]), db_frame }, { "x7", DB_OFFSET(tf_x[7]), db_frame }, { "x8", DB_OFFSET(tf_x[8]), db_frame }, { "x9", DB_OFFSET(tf_x[9]), db_frame }, { "x10", DB_OFFSET(tf_x[10]), db_frame }, { "x11", DB_OFFSET(tf_x[11]), db_frame }, { "x12", DB_OFFSET(tf_x[12]), db_frame }, { "x13", DB_OFFSET(tf_x[13]), db_frame }, { "x14", DB_OFFSET(tf_x[14]), db_frame }, { "x15", DB_OFFSET(tf_x[15]), db_frame }, { "x16", DB_OFFSET(tf_x[16]), db_frame }, { "x17", DB_OFFSET(tf_x[17]), db_frame }, { "x18", DB_OFFSET(tf_x[18]), db_frame }, { "x19", DB_OFFSET(tf_x[19]), db_frame }, { "x20", DB_OFFSET(tf_x[20]), db_frame }, { "x21", DB_OFFSET(tf_x[21]), db_frame }, { "x22", DB_OFFSET(tf_x[22]), db_frame }, { "x23", DB_OFFSET(tf_x[23]), db_frame }, { "x24", DB_OFFSET(tf_x[24]), db_frame }, { "x25", DB_OFFSET(tf_x[25]), db_frame }, { "x26", DB_OFFSET(tf_x[26]), db_frame }, { "x27", DB_OFFSET(tf_x[27]), db_frame }, { "x28", DB_OFFSET(tf_x[28]), db_frame }, { "x29", DB_OFFSET(tf_x[29]), db_frame }, { "lr", DB_OFFSET(tf_lr), db_frame }, { "elr", DB_OFFSET(tf_elr), db_frame }, { "sp", DB_OFFSET(tf_sp), db_frame }, }; struct db_variable *db_eregs = db_regs + nitems(db_regs); void db_show_mdpcpu(struct pcpu *pc) { } /* * Read bytes from kernel address space for debugger. */ int db_read_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; const char *src; int ret; uint64_t tmp64; uint32_t tmp32; uint16_t tmp16; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { src = (const char *)addr; if (size == 8 && (addr & 7) == 0) { tmp64 = *((const int *)src); src = (const char *)&tmp64; } else if (size == 4 && (addr & 3) == 0) { tmp32 = *((const int *)src); src = (const char *)&tmp32; } else if (size == 2 && (addr & 1) == 0) { tmp16 = *((const short *)src); src = (const char *)&tmp16; } while (size-- > 0) *data++ = *src++; } (void)kdb_jmpbuf(prev_jb); return (ret); } /* * Write bytes to kernel address space for debugger. */ int db_write_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; char *dst; int ret; uint64_t tmp64; uint32_t tmp32; uint16_t tmp16; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { if (size == 8 && (addr & 7) == 0) { dst = (char *)&tmp64; while (size-- > 0) *dst++ = *data++; *((uint64_t *)addr) = tmp64; } else if (size == 4 && (addr & 3) == 0) { dst = (char *)&tmp32; while (size-- > 0) *dst++ = *data++; *((uint32_t *)addr) = tmp32; } else if (size == 2 && (addr & 1) == 0) { dst = (char *)&tmp16; while (size-- > 0) *dst++ = *data++; *((uint32_t *)addr) = tmp16; } else { dst = (char *)addr; while (size-- > 0) *dst++ = *data++; } dsb(ish); /* Clean D-cache and invalidate I-cache */ cpu_dcache_wb_range(addr, (vm_size_t)size); cpu_icache_sync_range(addr, (vm_size_t)size); } (void)kdb_jmpbuf(prev_jb); return (ret); } diff --git a/sys/arm64/arm64/db_trace.c b/sys/arm64/arm64/db_trace.c index 5f018f2133b6..f6edb34e55d5 100644 --- a/sys/arm64/arm64/db_trace.c +++ b/sys/arm64/arm64/db_trace.c @@ -1,164 +1,163 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #define FRAME_NORMAL 0 #define FRAME_SYNC 1 #define FRAME_IRQ 2 #define FRAME_SERROR 3 #define FRAME_UNHANDLED 4 void db_md_list_watchpoints() { dbg_show_watchpoint(); } static void db_stack_trace_cmd(struct thread *td, struct unwind_state *frame) { c_db_sym_t sym; const char *name; db_expr_t value; db_expr_t offset; int frame_type; while (1) { sym = db_search_symbol(frame->pc, DB_STGY_ANY, &offset); if (sym == C_DB_SYM_NULL) { value = 0; name = "(null)"; } else db_symbol_values(sym, &name, &value); db_printf("%s() at ", name); db_printsym(frame->pc, DB_STGY_PROC); db_printf("\n"); if (strcmp(name, "handle_el0_sync") == 0 || strcmp(name, "handle_el1h_sync") == 0) frame_type = FRAME_SYNC; else if (strcmp(name, "handle_el0_irq") == 0 || strcmp(name, "handle_el1h_irq") == 0) frame_type = FRAME_IRQ; else if (strcmp(name, "handle_serror") == 0) frame_type = FRAME_SERROR; else if (strcmp(name, "handle_empty_exception") == 0) frame_type = FRAME_UNHANDLED; else frame_type = FRAME_NORMAL; if (frame_type != FRAME_NORMAL) { struct trapframe *tf; tf = (struct trapframe *)(uintptr_t)frame->fp - 1; if (!kstack_contains(td, (vm_offset_t)tf, sizeof(*tf))) { db_printf("--- invalid trapframe %p\n", tf); break; } switch (frame_type) { case FRAME_SYNC: db_printf("--- exception, esr %#x\n", tf->tf_esr); break; case FRAME_IRQ: db_printf("--- interrupt\n"); break; case FRAME_SERROR: db_printf("--- system error, esr %#x\n", tf->tf_esr); break; case FRAME_UNHANDLED: db_printf("--- unhandled exception, esr %#x\n", tf->tf_esr); break; default: __assert_unreachable(); break; } frame->fp = tf->tf_x[29]; frame->pc = tf->tf_elr; if (!INKERNEL(frame->fp)) break; } else { if (strcmp(name, "fork_trampoline") == 0) break; if (!unwind_frame(td, frame)) break; } } } int db_trace_thread(struct thread *thr, int count) { struct unwind_state frame; struct pcb *ctx; if (thr != curthread) { ctx = kdb_thr_ctx(thr); frame.fp = (uintptr_t)ctx->pcb_x[29]; frame.pc = (uintptr_t)ctx->pcb_lr; db_stack_trace_cmd(thr, &frame); } else db_trace_self(); return (0); } void db_trace_self(void) { struct unwind_state frame; frame.fp = (uintptr_t)__builtin_frame_address(0); frame.pc = (uintptr_t)db_trace_self; db_stack_trace_cmd(curthread, &frame); } diff --git a/sys/arm64/arm64/debug_monitor.c b/sys/arm64/arm64/debug_monitor.c index 8ef2ba360786..83b1f2b8b65b 100644 --- a/sys/arm64/arm64/debug_monitor.c +++ b/sys/arm64/arm64/debug_monitor.c @@ -1,605 +1,604 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_ddb.h" #include "opt_gdb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #include #endif enum dbg_t { DBG_TYPE_BREAKPOINT = 0, DBG_TYPE_WATCHPOINT = 1, }; static int dbg_watchpoint_num; static int dbg_breakpoint_num; static struct debug_monitor_state kernel_monitor = { .dbg_flags = DBGMON_KERNEL }; /* Called from the exception handlers */ void dbg_monitor_enter(struct thread *); void dbg_monitor_exit(struct thread *, struct trapframe *); /* Watchpoints/breakpoints control register bitfields */ #define DBG_WATCH_CTRL_LEN_1 (0x1 << 5) #define DBG_WATCH_CTRL_LEN_2 (0x3 << 5) #define DBG_WATCH_CTRL_LEN_4 (0xf << 5) #define DBG_WATCH_CTRL_LEN_8 (0xff << 5) #define DBG_WATCH_CTRL_LEN_MASK(x) ((x) & (0xff << 5)) #define DBG_WATCH_CTRL_EXEC (0x0 << 3) #define DBG_WATCH_CTRL_LOAD (0x1 << 3) #define DBG_WATCH_CTRL_STORE (0x2 << 3) #define DBG_WATCH_CTRL_ACCESS_MASK(x) ((x) & (0x3 << 3)) /* Common for breakpoint and watchpoint */ #define DBG_WB_CTRL_EL1 (0x1 << 1) #define DBG_WB_CTRL_EL0 (0x2 << 1) #define DBG_WB_CTRL_ELX_MASK(x) ((x) & (0x3 << 1)) #define DBG_WB_CTRL_E (0x1 << 0) #define DBG_REG_BASE_BVR 0 #define DBG_REG_BASE_BCR (DBG_REG_BASE_BVR + 16) #define DBG_REG_BASE_WVR (DBG_REG_BASE_BCR + 16) #define DBG_REG_BASE_WCR (DBG_REG_BASE_WVR + 16) /* Watchpoint/breakpoint helpers */ #define DBG_WB_WVR "wvr" #define DBG_WB_WCR "wcr" #define DBG_WB_BVR "bvr" #define DBG_WB_BCR "bcr" #define DBG_WB_READ(reg, num, val) do { \ __asm __volatile("mrs %0, dbg" reg #num "_el1" : "=r" (val)); \ } while (0) #define DBG_WB_WRITE(reg, num, val) do { \ __asm __volatile("msr dbg" reg #num "_el1, %0" :: "r" (val)); \ } while (0) #define READ_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_READ(reg, num, val); \ break #define WRITE_WB_REG_CASE(reg, num, offset, val) \ case (num + offset): \ DBG_WB_WRITE(reg, num, val); \ break #define SWITCH_CASES_READ_WB_REG(reg, offset, val) \ READ_WB_REG_CASE(reg, 0, offset, val); \ READ_WB_REG_CASE(reg, 1, offset, val); \ READ_WB_REG_CASE(reg, 2, offset, val); \ READ_WB_REG_CASE(reg, 3, offset, val); \ READ_WB_REG_CASE(reg, 4, offset, val); \ READ_WB_REG_CASE(reg, 5, offset, val); \ READ_WB_REG_CASE(reg, 6, offset, val); \ READ_WB_REG_CASE(reg, 7, offset, val); \ READ_WB_REG_CASE(reg, 8, offset, val); \ READ_WB_REG_CASE(reg, 9, offset, val); \ READ_WB_REG_CASE(reg, 10, offset, val); \ READ_WB_REG_CASE(reg, 11, offset, val); \ READ_WB_REG_CASE(reg, 12, offset, val); \ READ_WB_REG_CASE(reg, 13, offset, val); \ READ_WB_REG_CASE(reg, 14, offset, val); \ READ_WB_REG_CASE(reg, 15, offset, val) #define SWITCH_CASES_WRITE_WB_REG(reg, offset, val) \ WRITE_WB_REG_CASE(reg, 0, offset, val); \ WRITE_WB_REG_CASE(reg, 1, offset, val); \ WRITE_WB_REG_CASE(reg, 2, offset, val); \ WRITE_WB_REG_CASE(reg, 3, offset, val); \ WRITE_WB_REG_CASE(reg, 4, offset, val); \ WRITE_WB_REG_CASE(reg, 5, offset, val); \ WRITE_WB_REG_CASE(reg, 6, offset, val); \ WRITE_WB_REG_CASE(reg, 7, offset, val); \ WRITE_WB_REG_CASE(reg, 8, offset, val); \ WRITE_WB_REG_CASE(reg, 9, offset, val); \ WRITE_WB_REG_CASE(reg, 10, offset, val); \ WRITE_WB_REG_CASE(reg, 11, offset, val); \ WRITE_WB_REG_CASE(reg, 12, offset, val); \ WRITE_WB_REG_CASE(reg, 13, offset, val); \ WRITE_WB_REG_CASE(reg, 14, offset, val); \ WRITE_WB_REG_CASE(reg, 15, offset, val) #ifdef DDB static uint64_t dbg_wb_read_reg(int reg, int n) { uint64_t val = 0; switch (reg + n) { SWITCH_CASES_READ_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_READ_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to read from wrong debug register %d\n", n); } return val; } #endif /* DDB */ static void dbg_wb_write_reg(int reg, int n, uint64_t val) { switch (reg + n) { SWITCH_CASES_WRITE_WB_REG(DBG_WB_WVR, DBG_REG_BASE_WVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_WCR, DBG_REG_BASE_WCR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BVR, DBG_REG_BASE_BVR, val); SWITCH_CASES_WRITE_WB_REG(DBG_WB_BCR, DBG_REG_BASE_BCR, val); default: printf("trying to write to wrong debug register %d\n", n); return; } isb(); } #if defined(DDB) || defined(GDB) void kdb_cpu_set_singlestep(void) { KASSERT((READ_SPECIALREG(daif) & PSR_D) == PSR_D, ("%s: debug exceptions are not masked", __func__)); kdb_frame->tf_spsr |= DBG_SPSR_SS; WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_SS | DBG_MDSCR_KDE); /* * Disable breakpoints and watchpoints, e.g. stepping * over watched instruction will trigger break exception instead of * single-step exception and locks CPU on that instruction for ever. */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~DBG_MDSCR_MDE); } } void kdb_cpu_clear_singlestep(void) { KASSERT((READ_SPECIALREG(daif) & PSR_D) == PSR_D, ("%s: debug exceptions are not masked", __func__)); WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_SS | DBG_MDSCR_KDE)); /* Restore breakpoints and watchpoints */ if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_MDE); if ((kernel_monitor.dbg_flags & DBGMON_KERNEL) != 0) { WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) | DBG_MDSCR_KDE); } } } int kdb_cpu_set_watchpoint(vm_offset_t addr, vm_size_t size, int access) { enum dbg_access_t dbg_access; switch (access) { case KDB_DBG_ACCESS_R: dbg_access = HW_BREAKPOINT_R; break; case KDB_DBG_ACCESS_W: dbg_access = HW_BREAKPOINT_W; break; case KDB_DBG_ACCESS_RW: dbg_access = HW_BREAKPOINT_RW; break; default: return (EINVAL); } return (dbg_setup_watchpoint(NULL, addr, size, dbg_access)); } int kdb_cpu_clr_watchpoint(vm_offset_t addr, vm_size_t size) { return (dbg_remove_watchpoint(NULL, addr, size)); } #endif /* DDB || GDB */ #ifdef DDB static const char * dbg_watchtype_str(uint32_t type) { switch (type) { case DBG_WATCH_CTRL_EXEC: return ("execute"); case DBG_WATCH_CTRL_STORE: return ("write"); case DBG_WATCH_CTRL_LOAD: return ("read"); case DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE: return ("read/write"); default: return ("invalid"); } } static int dbg_watchtype_len(uint32_t len) { switch (len) { case DBG_WATCH_CTRL_LEN_1: return (1); case DBG_WATCH_CTRL_LEN_2: return (2); case DBG_WATCH_CTRL_LEN_4: return (4); case DBG_WATCH_CTRL_LEN_8: return (8); default: return (0); } } void dbg_show_watchpoint(void) { uint32_t wcr, len, type; uint64_t addr; int i; db_printf("\nhardware watchpoints:\n"); db_printf(" watch status type len address symbol\n"); db_printf(" ----- -------- ---------- --- ------------------ ------------------\n"); for (i = 0; i < dbg_watchpoint_num; i++) { wcr = dbg_wb_read_reg(DBG_REG_BASE_WCR, i); if ((wcr & DBG_WB_CTRL_E) != 0) { type = DBG_WATCH_CTRL_ACCESS_MASK(wcr); len = DBG_WATCH_CTRL_LEN_MASK(wcr); addr = dbg_wb_read_reg(DBG_REG_BASE_WVR, i); db_printf(" %-5d %-8s %10s %3d 0x%16lx ", i, "enabled", dbg_watchtype_str(type), dbg_watchtype_len(len), addr); db_printsym((db_addr_t)addr, DB_STGY_ANY); db_printf("\n"); } else { db_printf(" %-5d disabled\n", i); } } } #endif /* DDB */ static int dbg_find_free_slot(struct debug_monitor_state *monitor, enum dbg_t type) { uint64_t *reg; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if ((reg[i] & DBG_WB_CTRL_E) == 0) return (i); } return (-1); } static int dbg_find_slot(struct debug_monitor_state *monitor, enum dbg_t type, vm_offset_t addr) { uint64_t *reg_addr, *reg_ctrl; u_int max, i; switch(type) { case DBG_TYPE_BREAKPOINT: max = dbg_breakpoint_num; reg_addr = monitor->dbg_bvr; reg_ctrl = monitor->dbg_bcr; break; case DBG_TYPE_WATCHPOINT: max = dbg_watchpoint_num; reg_addr = monitor->dbg_wvr; reg_ctrl = monitor->dbg_wcr; break; default: printf("Unsupported debug type\n"); return (i); } for (i = 0; i < max; i++) { if (reg_addr[i] == addr && (reg_ctrl[i] & DBG_WB_CTRL_E) != 0) return (i); } return (-1); } int dbg_setup_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size, enum dbg_access_t access) { uint64_t wcr_size, wcr_priv, wcr_access; u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_free_slot(monitor, DBG_TYPE_WATCHPOINT); if (i == -1) { printf("Can not find slot for watchpoint, max %d" " watchpoints supported\n", dbg_watchpoint_num); return (EBUSY); } switch(size) { case 1: wcr_size = DBG_WATCH_CTRL_LEN_1; break; case 2: wcr_size = DBG_WATCH_CTRL_LEN_2; break; case 4: wcr_size = DBG_WATCH_CTRL_LEN_4; break; case 8: wcr_size = DBG_WATCH_CTRL_LEN_8; break; default: printf("Unsupported address size for watchpoint: %zu\n", size); return (EINVAL); } if ((monitor->dbg_flags & DBGMON_KERNEL) == 0) wcr_priv = DBG_WB_CTRL_EL0; else wcr_priv = DBG_WB_CTRL_EL1; switch(access) { case HW_BREAKPOINT_X: wcr_access = DBG_WATCH_CTRL_EXEC; break; case HW_BREAKPOINT_R: wcr_access = DBG_WATCH_CTRL_LOAD; break; case HW_BREAKPOINT_W: wcr_access = DBG_WATCH_CTRL_STORE; break; case HW_BREAKPOINT_RW: wcr_access = DBG_WATCH_CTRL_LOAD | DBG_WATCH_CTRL_STORE; break; default: printf("Unsupported access type for watchpoint: %d\n", access); return (EINVAL); } monitor->dbg_wvr[i] = addr; monitor->dbg_wcr[i] = wcr_size | wcr_access | wcr_priv | DBG_WB_CTRL_E; monitor->dbg_enable_count++; monitor->dbg_flags |= DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } int dbg_remove_watchpoint(struct debug_monitor_state *monitor, vm_offset_t addr, vm_size_t size) { u_int i; if (monitor == NULL) monitor = &kernel_monitor; i = dbg_find_slot(monitor, DBG_TYPE_WATCHPOINT, addr); if (i == -1) { printf("Can not find watchpoint for address 0%lx\n", addr); return (EINVAL); } monitor->dbg_wvr[i] = 0; monitor->dbg_wcr[i] = 0; monitor->dbg_enable_count--; if (monitor->dbg_enable_count == 0) monitor->dbg_flags &= ~DBGMON_ENABLED; dbg_register_sync(monitor); return (0); } void dbg_register_sync(struct debug_monitor_state *monitor) { uint64_t mdscr; int i; if (monitor == NULL) monitor = &kernel_monitor; mdscr = READ_SPECIALREG(mdscr_el1); if ((monitor->dbg_flags & DBGMON_ENABLED) == 0) { mdscr &= ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE); } else { for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, monitor->dbg_bcr[i]); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, monitor->dbg_bvr[i]); } for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, monitor->dbg_wcr[i]); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, monitor->dbg_wvr[i]); } mdscr |= DBG_MDSCR_MDE; if ((monitor->dbg_flags & DBGMON_KERNEL) == DBGMON_KERNEL) mdscr |= DBG_MDSCR_KDE; } WRITE_SPECIALREG(mdscr_el1, mdscr); isb(); } void dbg_monitor_init(void) { uint64_t aa64dfr0; u_int i; /* Find out many breakpoints and watchpoints we can use */ aa64dfr0 = READ_SPECIALREG(id_aa64dfr0_el1); dbg_watchpoint_num = ID_AA64DFR0_WRPs_VAL(aa64dfr0); dbg_breakpoint_num = ID_AA64DFR0_BRPs_VAL(aa64dfr0); if (bootverbose && PCPU_GET(cpuid) == 0) { printf("%d watchpoints and %d breakpoints supported\n", dbg_watchpoint_num, dbg_breakpoint_num); } /* * We have limited number of {watch,break}points, each consists of * two registers: * - wcr/bcr regsiter configurates corresponding {watch,break}point * behaviour * - wvr/bvr register keeps address we are hunting for * * Reset all breakpoints and watchpoints. */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } dbg_enable(); } void dbg_monitor_enter(struct thread *thread) { int i; if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { /* Install the kernel version of the registers */ dbg_register_sync(&kernel_monitor); } else if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) { /* Disable the user breakpoints until we return to userspace */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE)); isb(); } } void dbg_monitor_exit(struct thread *thread, struct trapframe *frame) { int i; /* * PSR_D is an aarch64-only flag. On aarch32, it switches * the processor to big-endian, so avoid setting it for * 32bits binaries. */ if (!(SV_PROC_FLAG(thread->td_proc, SV_ILP32))) frame->tf_spsr |= PSR_D; if ((thread->td_pcb->pcb_dbg_regs.dbg_flags & DBGMON_ENABLED) != 0) { /* Install the thread's version of the registers */ dbg_register_sync(&thread->td_pcb->pcb_dbg_regs); frame->tf_spsr &= ~PSR_D; } else if ((kernel_monitor.dbg_flags & DBGMON_ENABLED) != 0) { /* Disable the kernel breakpoints until we re-enter */ for (i = 0; i < dbg_watchpoint_num; i++) { dbg_wb_write_reg(DBG_REG_BASE_WCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_WVR, i, 0); } for (i = 0; i < dbg_breakpoint_num; ++i) { dbg_wb_write_reg(DBG_REG_BASE_BCR, i, 0); dbg_wb_write_reg(DBG_REG_BASE_BVR, i, 0); } WRITE_SPECIALREG(mdscr_el1, READ_SPECIALREG(mdscr_el1) & ~(DBG_MDSCR_MDE | DBG_MDSCR_KDE)); isb(); } } diff --git a/sys/arm64/arm64/dump_machdep.c b/sys/arm64/arm64/dump_machdep.c index d92777fea051..dc5126bee058 100644 --- a/sys/arm64/arm64/dump_machdep.c +++ b/sys/arm64/arm64/dump_machdep.c @@ -1,73 +1,72 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include int do_minidump = 1; TUNABLE_INT("debug.minidump", &do_minidump); SYSCTL_INT(_debug, OID_AUTO, minidump, CTLFLAG_RW, &do_minidump, 0, "Enable mini crash dumps"); void dumpsys_wbinv_all(void) { printf("dumpsys_wbinv_all\n"); } void dumpsys_map_chunk(vm_paddr_t pa, size_t chunk __unused, void **va) { printf("dumpsys_map_chunk\n"); while(1); } /* * Add a header to be used by libkvm to get the va to pa delta */ int dumpsys_write_aux_headers(struct dumperinfo *di) { printf("dumpsys_map_chunk\n"); return (0); } diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c index 03b6490e7d61..3e7cd30140eb 100644 --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -1,1469 +1,1468 @@ /*- * Copyright (c) 2015-2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * the sponsorship of the FreeBSD Foundation. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #endif #ifdef DEV_ACPI #include #include #endif #include "pic_if.h" #include "msi_if.h" #include #include "gic_v3_reg.h" #include "gic_v3_var.h" static bus_get_domain_t gic_v3_get_domain; static bus_read_ivar_t gic_v3_read_ivar; static pic_disable_intr_t gic_v3_disable_intr; static pic_enable_intr_t gic_v3_enable_intr; static pic_map_intr_t gic_v3_map_intr; static pic_setup_intr_t gic_v3_setup_intr; static pic_teardown_intr_t gic_v3_teardown_intr; static pic_post_filter_t gic_v3_post_filter; static pic_post_ithread_t gic_v3_post_ithread; static pic_pre_ithread_t gic_v3_pre_ithread; static pic_bind_intr_t gic_v3_bind_intr; #ifdef SMP static pic_init_secondary_t gic_v3_init_secondary; static pic_ipi_send_t gic_v3_ipi_send; static pic_ipi_setup_t gic_v3_ipi_setup; #endif static msi_alloc_msi_t gic_v3_alloc_msi; static msi_release_msi_t gic_v3_release_msi; static msi_alloc_msix_t gic_v3_alloc_msix; static msi_release_msix_t gic_v3_release_msix; static msi_map_msi_t gic_v3_map_msi; static u_int gic_irq_cpu; #ifdef SMP static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; static u_int sgi_first_unused = GIC_FIRST_SGI; #endif static device_method_t gic_v3_methods[] = { /* Device interface */ DEVMETHOD(device_detach, gic_v3_detach), /* Bus interface */ DEVMETHOD(bus_get_domain, gic_v3_get_domain), DEVMETHOD(bus_read_ivar, gic_v3_read_ivar), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, gic_v3_disable_intr), DEVMETHOD(pic_enable_intr, gic_v3_enable_intr), DEVMETHOD(pic_map_intr, gic_v3_map_intr), DEVMETHOD(pic_setup_intr, gic_v3_setup_intr), DEVMETHOD(pic_teardown_intr, gic_v3_teardown_intr), DEVMETHOD(pic_post_filter, gic_v3_post_filter), DEVMETHOD(pic_post_ithread, gic_v3_post_ithread), DEVMETHOD(pic_pre_ithread, gic_v3_pre_ithread), #ifdef SMP DEVMETHOD(pic_bind_intr, gic_v3_bind_intr), DEVMETHOD(pic_init_secondary, gic_v3_init_secondary), DEVMETHOD(pic_ipi_send, gic_v3_ipi_send), DEVMETHOD(pic_ipi_setup, gic_v3_ipi_setup), #endif /* MSI/MSI-X */ DEVMETHOD(msi_alloc_msi, gic_v3_alloc_msi), DEVMETHOD(msi_release_msi, gic_v3_release_msi), DEVMETHOD(msi_alloc_msix, gic_v3_alloc_msix), DEVMETHOD(msi_release_msix, gic_v3_release_msix), DEVMETHOD(msi_map_msi, gic_v3_map_msi), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(gic, gic_v3_driver, gic_v3_methods, sizeof(struct gic_v3_softc)); /* * Driver-specific definitions. */ MALLOC_DEFINE(M_GIC_V3, "GICv3", GIC_V3_DEVSTR); /* * Helper functions and definitions. */ /* Destination registers, either Distributor or Re-Distributor */ enum gic_v3_xdist { DIST = 0, REDIST, }; struct gic_v3_irqsrc { struct intr_irqsrc gi_isrc; uint32_t gi_irq; enum intr_polarity gi_pol; enum intr_trigger gi_trig; #define GI_FLAG_MSI (1 << 1) /* This interrupt source should only */ /* be used for MSI/MSI-X interrupts */ #define GI_FLAG_MSI_USED (1 << 2) /* This irq is already allocated */ /* for a MSI/MSI-X interrupt */ u_int gi_flags; }; /* Helper routines starting with gic_v3_ */ static int gic_v3_dist_init(struct gic_v3_softc *); static int gic_v3_redist_alloc(struct gic_v3_softc *); static int gic_v3_redist_find(struct gic_v3_softc *); static int gic_v3_redist_init(struct gic_v3_softc *); static int gic_v3_cpu_init(struct gic_v3_softc *); static void gic_v3_wait_for_rwp(struct gic_v3_softc *, enum gic_v3_xdist); /* A sequence of init functions for primary (boot) CPU */ typedef int (*gic_v3_initseq_t) (struct gic_v3_softc *); /* Primary CPU initialization sequence */ static gic_v3_initseq_t gic_v3_primary_init[] = { gic_v3_dist_init, gic_v3_redist_alloc, gic_v3_redist_init, gic_v3_cpu_init, NULL }; #ifdef SMP /* Secondary CPU initialization sequence */ static gic_v3_initseq_t gic_v3_secondary_init[] = { gic_v3_redist_init, gic_v3_cpu_init, NULL }; #endif uint32_t gic_r_read_4(device_t dev, bus_size_t offset) { struct gic_v3_softc *sc; struct resource *rdist; sc = device_get_softc(dev); rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; return (bus_read_4(rdist, offset)); } uint64_t gic_r_read_8(device_t dev, bus_size_t offset) { struct gic_v3_softc *sc; struct resource *rdist; sc = device_get_softc(dev); rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; return (bus_read_8(rdist, offset)); } void gic_r_write_4(device_t dev, bus_size_t offset, uint32_t val) { struct gic_v3_softc *sc; struct resource *rdist; sc = device_get_softc(dev); rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; bus_write_4(rdist, offset, val); } void gic_r_write_8(device_t dev, bus_size_t offset, uint64_t val) { struct gic_v3_softc *sc; struct resource *rdist; sc = device_get_softc(dev); rdist = &sc->gic_redists.pcpu[PCPU_GET(cpuid)]->res; bus_write_8(rdist, offset, val); } /* * Device interface. */ int gic_v3_attach(device_t dev) { struct gic_v3_softc *sc; gic_v3_initseq_t *init_func; uint32_t typer; int rid; int err; size_t i; u_int irq; const char *name; sc = device_get_softc(dev); sc->gic_registered = FALSE; sc->dev = dev; err = 0; /* Initialize mutex */ mtx_init(&sc->gic_mtx, "GICv3 lock", NULL, MTX_SPIN); /* * Allocate array of struct resource. * One entry for Distributor and all remaining for Re-Distributor. */ sc->gic_res = malloc( sizeof(*sc->gic_res) * (sc->gic_redists.nregions + 1), M_GIC_V3, M_WAITOK); /* Now allocate corresponding resources */ for (i = 0, rid = 0; i < (sc->gic_redists.nregions + 1); i++, rid++) { sc->gic_res[rid] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->gic_res[rid] == NULL) return (ENXIO); } /* * Distributor interface */ sc->gic_dist = sc->gic_res[0]; /* * Re-Dristributor interface */ /* Allocate space under region descriptions */ sc->gic_redists.regions = malloc( sizeof(*sc->gic_redists.regions) * sc->gic_redists.nregions, M_GIC_V3, M_WAITOK); /* Fill-up bus_space information for each region. */ for (i = 0, rid = 1; i < sc->gic_redists.nregions; i++, rid++) sc->gic_redists.regions[i] = sc->gic_res[rid]; /* Get the number of supported SPI interrupts */ typer = gic_d_read(sc, 4, GICD_TYPER); sc->gic_nirqs = GICD_TYPER_I_NUM(typer); if (sc->gic_nirqs > GIC_I_NUM_MAX) sc->gic_nirqs = GIC_I_NUM_MAX; sc->gic_irqs = malloc(sizeof(*sc->gic_irqs) * sc->gic_nirqs, M_GIC_V3, M_WAITOK | M_ZERO); name = device_get_nameunit(dev); for (irq = 0; irq < sc->gic_nirqs; irq++) { struct intr_irqsrc *isrc; sc->gic_irqs[irq].gi_irq = irq; sc->gic_irqs[irq].gi_pol = INTR_POLARITY_CONFORM; sc->gic_irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; isrc = &sc->gic_irqs[irq].gi_isrc; if (irq <= GIC_LAST_SGI) { err = intr_isrc_register(isrc, sc->dev, INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); } else if (irq <= GIC_LAST_PPI) { err = intr_isrc_register(isrc, sc->dev, INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); } else { err = intr_isrc_register(isrc, sc->dev, 0, "%s,s%u", name, irq - GIC_FIRST_SPI); } if (err != 0) { /* XXX call intr_isrc_deregister() */ free(sc->gic_irqs, M_DEVBUF); return (err); } } if (sc->gic_mbi_start > 0) { /* Reserve these interrupts for MSI/MSI-X use */ for (irq = sc->gic_mbi_start; irq <= sc->gic_mbi_end; irq++) { sc->gic_irqs[irq].gi_pol = INTR_POLARITY_HIGH; sc->gic_irqs[irq].gi_trig = INTR_TRIGGER_EDGE; sc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI; } mtx_init(&sc->gic_mbi_mtx, "GICv3 mbi lock", NULL, MTX_DEF); if (bootverbose) { device_printf(dev, "using spi %u to %u\n", sc->gic_mbi_start, sc->gic_mbi_end); } } /* * Read the Peripheral ID2 register. This is an implementation * defined register, but seems to be implemented in all GICv3 * parts and Linux expects it to be there. */ sc->gic_pidr2 = gic_d_read(sc, 4, GICD_PIDR2); /* Get the number of supported interrupt identifier bits */ sc->gic_idbits = GICD_TYPER_IDBITS(typer); if (bootverbose) { device_printf(dev, "SPIs: %u, IDs: %u\n", sc->gic_nirqs, (1 << sc->gic_idbits) - 1); } /* Train init sequence for boot CPU */ for (init_func = gic_v3_primary_init; *init_func != NULL; init_func++) { err = (*init_func)(sc); if (err != 0) return (err); } return (0); } int gic_v3_detach(device_t dev) { struct gic_v3_softc *sc; size_t i; int rid; sc = device_get_softc(dev); if (device_is_attached(dev)) { /* * XXX: We should probably deregister PIC */ if (sc->gic_registered) panic("Trying to detach registered PIC"); } for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); for (i = 0; i <= mp_maxid; i++) free(sc->gic_redists.pcpu[i], M_GIC_V3); free(sc->gic_res, M_GIC_V3); free(sc->gic_redists.regions, M_GIC_V3); return (0); } static int gic_v3_get_domain(device_t dev, device_t child, int *domain) { struct gic_v3_devinfo *di; di = device_get_ivars(child); if (di->gic_domain < 0) return (ENOENT); *domain = di->gic_domain; return (0); } static int gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct gic_v3_softc *sc; sc = device_get_softc(dev); switch (which) { case GICV3_IVAR_NIRQS: *result = (intr_nirq - sc->gic_nirqs) / sc->gic_nchildren; return (0); case GICV3_IVAR_REDIST: *result = (uintptr_t)sc->gic_redists.pcpu[PCPU_GET(cpuid)]; return (0); case GIC_IVAR_HW_REV: KASSERT( GICR_PIDR2_ARCH(sc->gic_pidr2) == GICR_PIDR2_ARCH_GICv3 || GICR_PIDR2_ARCH(sc->gic_pidr2) == GICR_PIDR2_ARCH_GICv4, ("gic_v3_read_ivar: Invalid GIC architecture: %d (%.08X)", GICR_PIDR2_ARCH(sc->gic_pidr2), sc->gic_pidr2)); *result = GICR_PIDR2_ARCH(sc->gic_pidr2); return (0); case GIC_IVAR_BUS: KASSERT(sc->gic_bus != GIC_BUS_UNKNOWN, ("gic_v3_read_ivar: Unknown bus type")); KASSERT(sc->gic_bus <= GIC_BUS_MAX, ("gic_v3_read_ivar: Invalid bus type %u", sc->gic_bus)); *result = sc->gic_bus; return (0); } return (ENOENT); } int arm_gic_v3_intr(void *arg) { struct gic_v3_softc *sc = arg; struct gic_v3_irqsrc *gi; struct intr_pic *pic; uint64_t active_irq; struct trapframe *tf; pic = sc->gic_pic; while (1) { if (CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1) { /* * Hardware: Cavium ThunderX * Chip revision: Pass 1.0 (early version) * Pass 1.1 (production) * ERRATUM: 22978, 23154 */ __asm __volatile( "nop;nop;nop;nop;nop;nop;nop;nop; \n" "mrs %0, ICC_IAR1_EL1 \n" "nop;nop;nop;nop; \n" "dsb sy \n" : "=&r" (active_irq)); } else { active_irq = gic_icc_read(IAR1); } if (active_irq >= GIC_FIRST_LPI) { intr_child_irq_handler(pic, active_irq); continue; } if (__predict_false(active_irq >= sc->gic_nirqs)) return (FILTER_HANDLED); tf = curthread->td_intr_frame; gi = &sc->gic_irqs[active_irq]; if (active_irq <= GIC_LAST_SGI) { /* Call EOI for all IPI before dispatch. */ gic_icc_write(EOIR1, (uint64_t)active_irq); #ifdef SMP intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); #else device_printf(sc->dev, "SGI %ju on UP system detected\n", (uintmax_t)(active_irq - GIC_FIRST_SGI)); #endif } else if (active_irq >= GIC_FIRST_PPI && active_irq <= GIC_LAST_SPI) { if (gi->gi_trig == INTR_TRIGGER_EDGE) gic_icc_write(EOIR1, gi->gi_irq); if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { if (gi->gi_trig != INTR_TRIGGER_EDGE) gic_icc_write(EOIR1, gi->gi_irq); gic_v3_disable_intr(sc->dev, &gi->gi_isrc); device_printf(sc->dev, "Stray irq %lu disabled\n", active_irq); } } } } #ifdef FDT static int gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { u_int irq; if (ncells < 3) return (EINVAL); /* * The 1st cell is the interrupt type: * 0 = SPI * 1 = PPI * The 2nd cell contains the interrupt number: * [0 - 987] for SPI * [0 - 15] for PPI * The 3rd cell is the flags, encoded as follows: * bits[3:0] trigger type and level flags * 1 = edge triggered * 2 = edge triggered (PPI only) * 4 = level-sensitive * 8 = level-sensitive (PPI only) */ switch (cells[0]) { case 0: irq = GIC_FIRST_SPI + cells[1]; /* SPI irq is checked later. */ break; case 1: irq = GIC_FIRST_PPI + cells[1]; if (irq > GIC_LAST_PPI) { device_printf(dev, "unsupported PPI interrupt " "number %u\n", cells[1]); return (EINVAL); } break; default: device_printf(dev, "unsupported interrupt type " "configuration %u\n", cells[0]); return (EINVAL); } switch (cells[2] & FDT_INTR_MASK) { case FDT_INTR_EDGE_RISING: *trigp = INTR_TRIGGER_EDGE; *polp = INTR_POLARITY_HIGH; break; case FDT_INTR_EDGE_FALLING: *trigp = INTR_TRIGGER_EDGE; *polp = INTR_POLARITY_LOW; break; case FDT_INTR_LEVEL_HIGH: *trigp = INTR_TRIGGER_LEVEL; *polp = INTR_POLARITY_HIGH; break; case FDT_INTR_LEVEL_LOW: *trigp = INTR_TRIGGER_LEVEL; *polp = INTR_POLARITY_LOW; break; default: device_printf(dev, "unsupported trigger/polarity " "configuration 0x%02x\n", cells[2]); return (EINVAL); } /* Check the interrupt is valid */ if (irq >= GIC_FIRST_SPI && *polp != INTR_POLARITY_HIGH) return (EINVAL); *irqp = irq; return (0); } #endif static int gic_map_msi(device_t dev, struct intr_map_data_msi *msi_data, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { struct gic_v3_irqsrc *gi; /* SPI-mapped MSI */ gi = (struct gic_v3_irqsrc *)msi_data->isrc; if (gi == NULL) return (ENXIO); *irqp = gi->gi_irq; /* MSI/MSI-X interrupts are always edge triggered with high polarity */ *polp = INTR_POLARITY_HIGH; *trigp = INTR_TRIGGER_EDGE; return (0); } static int do_gic_v3_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { struct gic_v3_softc *sc; enum intr_polarity pol; enum intr_trigger trig; struct intr_map_data_msi *dam; #ifdef FDT struct intr_map_data_fdt *daf; #endif #ifdef DEV_ACPI struct intr_map_data_acpi *daa; #endif u_int irq; sc = device_get_softc(dev); switch (data->type) { #ifdef FDT case INTR_MAP_DATA_FDT: daf = (struct intr_map_data_fdt *)data; if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol, &trig) != 0) return (EINVAL); break; #endif #ifdef DEV_ACPI case INTR_MAP_DATA_ACPI: daa = (struct intr_map_data_acpi *)data; irq = daa->irq; pol = daa->pol; trig = daa->trig; break; #endif case INTR_MAP_DATA_MSI: /* SPI-mapped MSI */ dam = (struct intr_map_data_msi *)data; if (gic_map_msi(dev, dam, &irq, &pol, &trig) != 0) return (EINVAL); break; default: return (EINVAL); } if (irq >= sc->gic_nirqs) return (EINVAL); switch (pol) { case INTR_POLARITY_CONFORM: case INTR_POLARITY_LOW: case INTR_POLARITY_HIGH: break; default: return (EINVAL); } switch (trig) { case INTR_TRIGGER_CONFORM: case INTR_TRIGGER_EDGE: case INTR_TRIGGER_LEVEL: break; default: return (EINVAL); } *irqp = irq; if (polp != NULL) *polp = pol; if (trigp != NULL) *trigp = trig; return (0); } static int gic_v3_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct gic_v3_softc *sc; int error; u_int irq; error = do_gic_v3_map_intr(dev, data, &irq, NULL, NULL); if (error == 0) { sc = device_get_softc(dev); *isrcp = GIC_INTR_ISRC(sc, irq); } return (error); } static int gic_v3_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct gic_v3_softc *sc = device_get_softc(dev); struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; enum intr_trigger trig; enum intr_polarity pol; uint32_t reg; u_int irq; int error; if (data == NULL) return (ENOTSUP); error = do_gic_v3_map_intr(dev, data, &irq, &pol, &trig); if (error != 0) return (error); if (gi->gi_irq != irq || pol == INTR_POLARITY_CONFORM || trig == INTR_TRIGGER_CONFORM) return (EINVAL); /* Compare config if this is not first setup. */ if (isrc->isrc_handlers != 0) { if (pol != gi->gi_pol || trig != gi->gi_trig) return (EINVAL); else return (0); } /* For MSI/MSI-X we should have already configured these */ if ((gi->gi_flags & GI_FLAG_MSI) == 0) { gi->gi_pol = pol; gi->gi_trig = trig; } /* * XXX - In case that per CPU interrupt is going to be enabled in time * when SMP is already started, we need some IPI call which * enables it on others CPUs. Further, it's more complicated as * pic_enable_source() and pic_disable_source() should act on * per CPU basis only. Thus, it should be solved here somehow. */ if (isrc->isrc_flags & INTR_ISRCF_PPI) CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_SPI) { mtx_lock_spin(&sc->gic_mtx); /* Set the trigger and polarity */ if (irq <= GIC_LAST_PPI) reg = gic_r_read(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICFGR(irq)); else reg = gic_d_read(sc, 4, GICD_ICFGR(irq)); if (trig == INTR_TRIGGER_LEVEL) reg &= ~(2 << ((irq % 16) * 2)); else reg |= 2 << ((irq % 16) * 2); if (irq <= GIC_LAST_PPI) { gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICFGR(irq), reg); gic_v3_wait_for_rwp(sc, REDIST); } else { gic_d_write(sc, 4, GICD_ICFGR(irq), reg); gic_v3_wait_for_rwp(sc, DIST); } mtx_unlock_spin(&sc->gic_mtx); gic_v3_bind_intr(dev, isrc); } return (0); } static int gic_v3_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; if (isrc->isrc_handlers == 0 && (gi->gi_flags & GI_FLAG_MSI) == 0) { gi->gi_pol = INTR_POLARITY_CONFORM; gi->gi_trig = INTR_TRIGGER_CONFORM; } return (0); } static void gic_v3_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; u_int irq; sc = device_get_softc(dev); gi = (struct gic_v3_irqsrc *)isrc; irq = gi->gi_irq; if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_d_write(sc, 4, GICD_ICENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, DIST); } else panic("%s: Unsupported IRQ %u", __func__, irq); } static void gic_v3_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; u_int irq; sc = device_get_softc(dev); gi = (struct gic_v3_irqsrc *)isrc; irq = gi->gi_irq; if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); } else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) { /* SPIs in distributor */ gic_d_write(sc, 4, GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, DIST); } else panic("%s: Unsupported IRQ %u", __func__, irq); } static void gic_v3_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; gic_v3_disable_intr(dev, isrc); gic_icc_write(EOIR1, gi->gi_irq); } static void gic_v3_post_ithread(device_t dev, struct intr_irqsrc *isrc) { gic_v3_enable_intr(dev, isrc); } static void gic_v3_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; if (gi->gi_trig == INTR_TRIGGER_EDGE) return; gic_icc_write(EOIR1, gi->gi_irq); } static int gic_v3_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; int cpu; gi = (struct gic_v3_irqsrc *)isrc; if (gi->gi_irq <= GIC_LAST_PPI) return (EINVAL); KASSERT(gi->gi_irq >= GIC_FIRST_SPI && gi->gi_irq <= GIC_LAST_SPI, ("%s: Attempting to bind an invalid IRQ", __func__)); sc = device_get_softc(dev); if (CPU_EMPTY(&isrc->isrc_cpu)) { gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); gic_d_write(sc, 8, GICD_IROUTER(gi->gi_irq), CPU_AFFINITY(gic_irq_cpu)); } else { /* * We can only bind to a single CPU so select * the first CPU found. */ cpu = CPU_FFS(&isrc->isrc_cpu) - 1; gic_d_write(sc, 8, GICD_IROUTER(gi->gi_irq), CPU_AFFINITY(cpu)); } return (0); } #ifdef SMP static void gic_v3_init_secondary(device_t dev) { device_t child; struct gic_v3_softc *sc; gic_v3_initseq_t *init_func; struct intr_irqsrc *isrc; u_int cpu, irq; int err, i; sc = device_get_softc(dev); cpu = PCPU_GET(cpuid); /* Train init sequence for boot CPU */ for (init_func = gic_v3_secondary_init; *init_func != NULL; init_func++) { err = (*init_func)(sc); if (err != 0) { device_printf(dev, "Could not initialize GIC for CPU%u\n", cpu); return; } } /* Unmask attached SGI interrupts. */ for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { isrc = GIC_INTR_ISRC(sc, irq); if (intr_isrc_init_on_cpu(isrc, cpu)) gic_v3_enable_intr(dev, isrc); } /* Unmask attached PPI interrupts. */ for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { isrc = GIC_INTR_ISRC(sc, irq); if (intr_isrc_init_on_cpu(isrc, cpu)) gic_v3_enable_intr(dev, isrc); } for (i = 0; i < sc->gic_nchildren; i++) { child = sc->gic_children[i]; PIC_INIT_SECONDARY(child); } } static void gic_v3_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, u_int ipi) { struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; uint64_t aff, val, irq; int i; #define GIC_AFF_MASK (CPU_AFF3_MASK | CPU_AFF2_MASK | CPU_AFF1_MASK) #define GIC_AFFINITY(i) (CPU_AFFINITY(i) & GIC_AFF_MASK) aff = GIC_AFFINITY(0); irq = gi->gi_irq; val = 0; /* Iterate through all CPUs in set */ for (i = 0; i <= mp_maxid; i++) { /* Move to the next affinity group */ if (aff != GIC_AFFINITY(i)) { /* Send the IPI */ if (val != 0) { gic_icc_write(SGI1R, val); val = 0; } aff = GIC_AFFINITY(i); } /* Send the IPI to this cpu */ if (CPU_ISSET(i, &cpus)) { #define ICC_SGI1R_AFFINITY(aff) \ (((uint64_t)CPU_AFF3(aff) << ICC_SGI1R_EL1_AFF3_SHIFT) | \ ((uint64_t)CPU_AFF2(aff) << ICC_SGI1R_EL1_AFF2_SHIFT) | \ ((uint64_t)CPU_AFF1(aff) << ICC_SGI1R_EL1_AFF1_SHIFT)) /* Set the affinity when the first at this level */ if (val == 0) val = ICC_SGI1R_AFFINITY(aff) | irq << ICC_SGI1R_EL1_SGIID_SHIFT; /* Set the bit to send the IPI to te CPU */ val |= 1 << CPU_AFF0(CPU_AFFINITY(i)); } } /* Send the IPI to the last cpu affinity group */ if (val != 0) gic_icc_write(SGI1R, val); #undef GIC_AFF_MASK #undef GIC_AFFINITY } static int gic_v3_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) { struct intr_irqsrc *isrc; struct gic_v3_softc *sc = device_get_softc(dev); if (sgi_first_unused > GIC_LAST_SGI) return (ENOSPC); isrc = GIC_INTR_ISRC(sc, sgi_first_unused); sgi_to_ipi[sgi_first_unused++] = ipi; CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); *isrcp = isrc; return (0); } #endif /* SMP */ /* * Helper routines */ static void gic_v3_wait_for_rwp(struct gic_v3_softc *sc, enum gic_v3_xdist xdist) { struct resource *res; u_int cpuid; size_t us_left = 1000000; cpuid = PCPU_GET(cpuid); switch (xdist) { case DIST: res = sc->gic_dist; break; case REDIST: res = &sc->gic_redists.pcpu[cpuid]->res; break; default: KASSERT(0, ("%s: Attempt to wait for unknown RWP", __func__)); return; } while ((bus_read_4(res, GICD_CTLR) & GICD_CTLR_RWP) != 0) { DELAY(1); if (us_left-- == 0) panic("GICD Register write pending for too long"); } } /* CPU interface. */ static __inline void gic_v3_cpu_priority(uint64_t mask) { /* Set prority mask */ gic_icc_write(PMR, mask & ICC_PMR_EL1_PRIO_MASK); } static int gic_v3_cpu_enable_sre(struct gic_v3_softc *sc) { uint64_t sre; u_int cpuid; cpuid = PCPU_GET(cpuid); /* * Set the SRE bit to enable access to GIC CPU interface * via system registers. */ sre = READ_SPECIALREG(icc_sre_el1); sre |= ICC_SRE_EL1_SRE; WRITE_SPECIALREG(icc_sre_el1, sre); isb(); /* * Now ensure that the bit is set. */ sre = READ_SPECIALREG(icc_sre_el1); if ((sre & ICC_SRE_EL1_SRE) == 0) { /* We are done. This was disabled in EL2 */ device_printf(sc->dev, "ERROR: CPU%u cannot enable CPU interface " "via system registers\n", cpuid); return (ENXIO); } else if (bootverbose) { device_printf(sc->dev, "CPU%u enabled CPU interface via system registers\n", cpuid); } return (0); } static int gic_v3_cpu_init(struct gic_v3_softc *sc) { int err; /* Enable access to CPU interface via system registers */ err = gic_v3_cpu_enable_sre(sc); if (err != 0) return (err); /* Priority mask to minimum - accept all interrupts */ gic_v3_cpu_priority(GIC_PRIORITY_MIN); /* Disable EOI mode */ gic_icc_clear(CTLR, ICC_CTLR_EL1_EOIMODE); /* Enable group 1 (insecure) interrups */ gic_icc_set(IGRPEN1, ICC_IGRPEN0_EL1_EN); return (0); } /* Distributor */ static int gic_v3_dist_init(struct gic_v3_softc *sc) { uint64_t aff; u_int i; /* * 1. Disable the Distributor */ gic_d_write(sc, 4, GICD_CTLR, 0); gic_v3_wait_for_rwp(sc, DIST); /* * 2. Configure the Distributor */ /* Set all SPIs to be Group 1 Non-secure */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_IGROUPRn) gic_d_write(sc, 4, GICD_IGROUPR(i), 0xFFFFFFFF); /* Set all global interrupts to be level triggered, active low. */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ICFGRn) gic_d_write(sc, 4, GICD_ICFGR(i), 0x00000000); /* Set priority to all shared interrupts */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_IPRIORITYn) { /* Set highest priority */ gic_d_write(sc, 4, GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); } /* * Disable all interrupts. Leave PPI and SGIs as they are enabled in * Re-Distributor registers. */ for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i += GICD_I_PER_ISENABLERn) gic_d_write(sc, 4, GICD_ICENABLER(i), 0xFFFFFFFF); gic_v3_wait_for_rwp(sc, DIST); /* * 3. Enable Distributor */ /* Enable Distributor with ARE, Group 1 */ gic_d_write(sc, 4, GICD_CTLR, GICD_CTLR_ARE_NS | GICD_CTLR_G1A | GICD_CTLR_G1); /* * 4. Route all interrupts to boot CPU. */ aff = CPU_AFFINITY(0); for (i = GIC_FIRST_SPI; i < sc->gic_nirqs; i++) gic_d_write(sc, 8, GICD_IROUTER(i), aff); return (0); } /* Re-Distributor */ static int gic_v3_redist_alloc(struct gic_v3_softc *sc) { u_int cpuid; /* Allocate struct resource for all CPU's Re-Distributor registers */ for (cpuid = 0; cpuid <= mp_maxid; cpuid++) if (CPU_ISSET(cpuid, &all_cpus) != 0) sc->gic_redists.pcpu[cpuid] = malloc(sizeof(*sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); else sc->gic_redists.pcpu[cpuid] = NULL; return (0); } static int gic_v3_redist_find(struct gic_v3_softc *sc) { struct resource r_res; bus_space_handle_t r_bsh; uint64_t aff; uint64_t typer; uint32_t pidr2; u_int cpuid; size_t i; cpuid = PCPU_GET(cpuid); aff = CPU_AFFINITY(cpuid); /* Affinity in format for comparison with typer */ aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) | (CPU_AFF1(aff) << 8) | CPU_AFF0(aff); if (bootverbose) { device_printf(sc->dev, "Start searching for Re-Distributor\n"); } /* Iterate through Re-Distributor regions */ for (i = 0; i < sc->gic_redists.nregions; i++) { /* Take a copy of the region's resource */ r_res = *sc->gic_redists.regions[i]; r_bsh = rman_get_bushandle(&r_res); pidr2 = bus_read_4(&r_res, GICR_PIDR2); switch (GICR_PIDR2_ARCH(pidr2)) { case GICR_PIDR2_ARCH_GICv3: /* fall through */ case GICR_PIDR2_ARCH_GICv4: break; default: device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); return (ENODEV); } do { typer = bus_read_8(&r_res, GICR_TYPER); if ((typer >> GICR_TYPER_AFF_SHIFT) == aff) { KASSERT(sc->gic_redists.pcpu[cpuid] != NULL, ("Invalid pointer to per-CPU redistributor")); /* Copy res contents to its final destination */ sc->gic_redists.pcpu[cpuid]->res = r_res; sc->gic_redists.pcpu[cpuid]->lpi_enabled = false; if (bootverbose) { device_printf(sc->dev, "CPU%u Re-Distributor has been found\n", cpuid); } return (0); } r_bsh += (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); if ((typer & GICR_TYPER_VLPIS) != 0) { r_bsh += (GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE); } rman_set_bushandle(&r_res, r_bsh); } while ((typer & GICR_TYPER_LAST) == 0); } device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); return (ENXIO); } static int gic_v3_redist_wake(struct gic_v3_softc *sc) { uint32_t waker; size_t us_left = 1000000; waker = gic_r_read(sc, 4, GICR_WAKER); /* Wake up Re-Distributor for this CPU */ waker &= ~GICR_WAKER_PS; gic_r_write(sc, 4, GICR_WAKER, waker); /* * When clearing ProcessorSleep bit it is required to wait for * ChildrenAsleep to become zero following the processor power-on. */ while ((gic_r_read(sc, 4, GICR_WAKER) & GICR_WAKER_CA) != 0) { DELAY(1); if (us_left-- == 0) { panic("Could not wake Re-Distributor for CPU%u", PCPU_GET(cpuid)); } } if (bootverbose) { device_printf(sc->dev, "CPU%u Re-Distributor woke up\n", PCPU_GET(cpuid)); } return (0); } static int gic_v3_redist_init(struct gic_v3_softc *sc) { int err; size_t i; err = gic_v3_redist_find(sc); if (err != 0) return (err); err = gic_v3_redist_wake(sc); if (err != 0) return (err); /* Configure SGIs and PPIs to be Group1 Non-secure */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_IGROUPR0, 0xFFFFFFFF); /* Disable SPIs */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ICENABLER0, GICR_I_ENABLER_PPI_MASK); /* Enable SGIs */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICR_ISENABLER0, GICR_I_ENABLER_SGI_MASK); /* Set priority for SGIs and PPIs */ for (i = 0; i <= GIC_LAST_PPI; i += GICR_I_PER_IPRIORITYn) { gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_IPRIORITYR(i), GIC_PRIORITY_MAX); } gic_v3_wait_for_rwp(sc, REDIST); return (0); } /* * SPI-mapped Message Based Interrupts -- a GICv3 MSI/MSI-X controller. */ static int gic_v3_alloc_msi(device_t dev, device_t child, int count, int maxcount, device_t *pic, struct intr_irqsrc **srcs) { struct gic_v3_softc *sc; int i, irq, end_irq; bool found; KASSERT(powerof2(count), ("%s: bad count", __func__)); KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__)); sc = device_get_softc(dev); mtx_lock(&sc->gic_mbi_mtx); found = false; for (irq = sc->gic_mbi_start; irq < sc->gic_mbi_end; irq++) { /* Start on an aligned interrupt */ if ((irq & (maxcount - 1)) != 0) continue; /* Assume we found a valid range until shown otherwise */ found = true; /* Check this range is valid */ for (end_irq = irq; end_irq != irq + count; end_irq++) { /* No free interrupts */ if (end_irq == sc->gic_mbi_end) { found = false; break; } KASSERT((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI)!= 0, ("%s: Non-MSI interrupt found", __func__)); /* This is already used */ if ((sc->gic_irqs[end_irq].gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED) { found = false; break; } } if (found) break; } /* Not enough interrupts were found */ if (!found || irq == sc->gic_mbi_end) { mtx_unlock(&sc->gic_mbi_mtx); return (ENXIO); } for (i = 0; i < count; i++) { /* Mark the interrupt as used */ sc->gic_irqs[irq + i].gi_flags |= GI_FLAG_MSI_USED; } mtx_unlock(&sc->gic_mbi_mtx); for (i = 0; i < count; i++) srcs[i] = (struct intr_irqsrc *)&sc->gic_irqs[irq + i]; *pic = dev; return (0); } static int gic_v3_release_msi(device_t dev, device_t child, int count, struct intr_irqsrc **isrc) { struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; int i; sc = device_get_softc(dev); mtx_lock(&sc->gic_mbi_mtx); for (i = 0; i < count; i++) { gi = (struct gic_v3_irqsrc *)isrc[i]; KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, ("%s: Trying to release an unused MSI-X interrupt", __func__)); gi->gi_flags &= ~GI_FLAG_MSI_USED; } mtx_unlock(&sc->gic_mbi_mtx); return (0); } static int gic_v3_alloc_msix(device_t dev, device_t child, device_t *pic, struct intr_irqsrc **isrcp) { struct gic_v3_softc *sc; int irq; sc = device_get_softc(dev); mtx_lock(&sc->gic_mbi_mtx); /* Find an unused interrupt */ for (irq = sc->gic_mbi_start; irq < sc->gic_mbi_end; irq++) { KASSERT((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) != 0, ("%s: Non-MSI interrupt found", __func__)); if ((sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == 0) break; } /* No free interrupt was found */ if (irq == sc->gic_mbi_end) { mtx_unlock(&sc->gic_mbi_mtx); return (ENXIO); } /* Mark the interrupt as used */ sc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI_USED; mtx_unlock(&sc->gic_mbi_mtx); *isrcp = (struct intr_irqsrc *)&sc->gic_irqs[irq]; *pic = dev; return (0); } static int gic_v3_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) { struct gic_v3_softc *sc; struct gic_v3_irqsrc *gi; sc = device_get_softc(dev); gi = (struct gic_v3_irqsrc *)isrc; KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, ("%s: Trying to release an unused MSI-X interrupt", __func__)); mtx_lock(&sc->gic_mbi_mtx); gi->gi_flags &= ~GI_FLAG_MSI_USED; mtx_unlock(&sc->gic_mbi_mtx); return (0); } static int gic_v3_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, uint64_t *addr, uint32_t *data) { struct gic_v3_softc *sc = device_get_softc(dev); struct gic_v3_irqsrc *gi = (struct gic_v3_irqsrc *)isrc; *addr = vtophys(rman_get_virtual(sc->gic_dist)) + GICD_SETSPI_NSR; *data = gi->gi_irq; return (0); } diff --git a/sys/arm64/arm64/gic_v3_acpi.c b/sys/arm64/arm64/gic_v3_acpi.c index e73e37f8e851..6a090baffacf 100644 --- a/sys/arm64/arm64/gic_v3_acpi.c +++ b/sys/arm64/arm64/gic_v3_acpi.c @@ -1,444 +1,443 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include "gic_v3_reg.h" #include "gic_v3_var.h" struct gic_v3_acpi_devinfo { struct gic_v3_devinfo di_gic_dinfo; struct resource_list di_rl; }; static device_identify_t gic_v3_acpi_identify; static device_probe_t gic_v3_acpi_probe; static device_attach_t gic_v3_acpi_attach; static bus_alloc_resource_t gic_v3_acpi_bus_alloc_res; static void gic_v3_acpi_bus_attach(device_t); static device_method_t gic_v3_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, gic_v3_acpi_identify), DEVMETHOD(device_probe, gic_v3_acpi_probe), DEVMETHOD(device_attach, gic_v3_acpi_attach), /* Bus interface */ DEVMETHOD(bus_alloc_resource, gic_v3_acpi_bus_alloc_res), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gic, gic_v3_acpi_driver, gic_v3_acpi_methods, sizeof(struct gic_v3_softc), gic_v3_driver); static devclass_t gic_v3_acpi_devclass; EARLY_DRIVER_MODULE(gic_v3, acpi, gic_v3_acpi_driver, gic_v3_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); struct madt_table_data { device_t parent; device_t dev; ACPI_MADT_GENERIC_DISTRIBUTOR *dist; int count; bool rdist_use_gicc; }; static void madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct madt_table_data *madt_data; madt_data = (struct madt_table_data *)arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR: if (madt_data->dist != NULL) { if (bootverbose) device_printf(madt_data->parent, "gic: Already have a distributor table"); break; } madt_data->dist = (ACPI_MADT_GENERIC_DISTRIBUTOR *)entry; break; case ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR: break; default: break; } } static void rdist_map(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_REDISTRIBUTOR *redist; ACPI_MADT_GENERIC_INTERRUPT *intr; struct madt_table_data *madt_data; rman_res_t count; madt_data = (struct madt_table_data *)arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR: if (madt_data->rdist_use_gicc) break; redist = (ACPI_MADT_GENERIC_REDISTRIBUTOR *)entry; madt_data->count++; BUS_SET_RESOURCE(madt_data->parent, madt_data->dev, SYS_RES_MEMORY, madt_data->count, redist->BaseAddress, redist->Length); break; case ACPI_MADT_TYPE_GENERIC_INTERRUPT: if (!madt_data->rdist_use_gicc) break; intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; madt_data->count++; /* * Map the two 64k redistributor frames. */ count = GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE; if (madt_data->dist->Version == ACPI_MADT_GIC_VERSION_V4) count += GICR_VLPI_BASE_SIZE + GICR_RESERVED_SIZE; BUS_SET_RESOURCE(madt_data->parent, madt_data->dev, SYS_RES_MEMORY, madt_data->count, intr->GicrBaseAddress, count); default: break; } } static void gic_v3_acpi_identify(driver_t *driver, device_t parent) { struct madt_table_data madt_data; ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; device_t dev; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(parent, "gic: Unable to map the MADT\n"); return; } madt_data.parent = parent; madt_data.dist = NULL; madt_data.count = 0; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_handler, &madt_data); if (madt_data.dist == NULL) { device_printf(parent, "No gic interrupt or distributor table\n"); goto out; } /* Check the GIC version is supported by thiss driver */ switch(madt_data.dist->Version) { case ACPI_MADT_GIC_VERSION_V3: case ACPI_MADT_GIC_VERSION_V4: break; default: goto out; } dev = BUS_ADD_CHILD(parent, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, "gic", -1); if (dev == NULL) { device_printf(parent, "add gic child failed\n"); goto out; } /* Add the MADT data */ BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0, madt_data.dist->BaseAddress, 128 * 1024); madt_data.dev = dev; madt_data.rdist_use_gicc = false; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, rdist_map, &madt_data); if (madt_data.count == 0) { /* * No redistributors found, fall back to use the GICR * address from the GICC sub-table. */ madt_data.rdist_use_gicc = true; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, rdist_map, &madt_data); } acpi_set_private(dev, (void *)(uintptr_t)madt_data.dist->Version); out: acpi_unmap_table(madt); } static int gic_v3_acpi_probe(device_t dev) { switch((uintptr_t)acpi_get_private(dev)) { case ACPI_MADT_GIC_VERSION_V3: case ACPI_MADT_GIC_VERSION_V4: break; default: return (ENXIO); } device_set_desc(dev, GIC_V3_DEVSTR); return (BUS_PROBE_NOWILDCARD); } static void madt_count_redistrib(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct gic_v3_softc *sc = arg; if (entry->Type == ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR) sc->gic_redists.nregions++; } static void madt_count_gicc_redistrib(ACPI_SUBTABLE_HEADER *entry, void *arg) { struct gic_v3_softc *sc = arg; if (entry->Type == ACPI_MADT_TYPE_GENERIC_INTERRUPT) sc->gic_redists.nregions++; } static int gic_v3_acpi_count_regions(device_t dev) { struct gic_v3_softc *sc; ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; sc = device_get_softc(dev); physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return (ENXIO); madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(dev, "Unable to map the MADT\n"); return (ENXIO); } acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_count_redistrib, sc); /* Fall back to use the distributor GICR base address */ if (sc->gic_redists.nregions == 0) { acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_count_gicc_redistrib, sc); } acpi_unmap_table(madt); return (sc->gic_redists.nregions > 0 ? 0 : ENXIO); } static int gic_v3_acpi_attach(device_t dev) { struct gic_v3_softc *sc; int err; sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_ACPI; err = gic_v3_acpi_count_regions(dev); if (err != 0) goto count_error; err = gic_v3_attach(dev); if (err != 0) goto error; sc->gic_pic = intr_pic_register(dev, ACPI_INTR_XREF); if (sc->gic_pic == NULL) { device_printf(dev, "could not register PIC\n"); err = ENXIO; goto error; } if (intr_pic_claim_root(dev, ACPI_INTR_XREF, arm_gic_v3_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { err = ENXIO; goto error; } /* * Try to register the ITS driver to this GIC. The GIC will act as * a bus in that case. Failure here will not affect the main GIC * functionality. */ gic_v3_acpi_bus_attach(dev); if (device_get_children(dev, &sc->gic_children, &sc->gic_nchildren) !=0) sc->gic_nchildren = 0; return (0); error: /* Failure so free resources */ gic_v3_detach(dev); count_error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } return (err); } static void gic_v3_add_children(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_TRANSLATOR *gict; struct gic_v3_acpi_devinfo *di; struct gic_v3_softc *sc; device_t child, dev; u_int xref; int err, pxm; if (entry->Type == ACPI_MADT_TYPE_GENERIC_TRANSLATOR) { /* We have an ITS, add it as a child */ gict = (ACPI_MADT_GENERIC_TRANSLATOR *)entry; dev = arg; sc = device_get_softc(dev); child = device_add_child(dev, "its", -1); if (child == NULL) return; di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); resource_list_init(&di->di_rl); resource_list_add(&di->di_rl, SYS_RES_MEMORY, 0, gict->BaseAddress, gict->BaseAddress + 128 * 1024 - 1, 128 * 1024); err = acpi_iort_its_lookup(gict->TranslationId, &xref, &pxm); if (err == 0) { di->di_gic_dinfo.gic_domain = pxm; di->di_gic_dinfo.msi_xref = xref; } else { di->di_gic_dinfo.gic_domain = -1; di->di_gic_dinfo.msi_xref = ACPI_MSI_XREF; } sc->gic_nchildren++; device_set_ivars(child, di); } } static void gic_v3_acpi_bus_attach(device_t dev) { ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { device_printf(dev, "Unable to map the MADT to add children\n"); return; } acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, gic_v3_add_children, dev); acpi_unmap_table(madt); bus_generic_attach(dev); } static struct resource * gic_v3_acpi_bus_alloc_res(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct gic_v3_acpi_devinfo *di; struct resource_list_entry *rle; /* We only allocate memory */ if (type != SYS_RES_MEMORY) return (NULL); if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); /* Find defaults for this rid */ rle = resource_list_find(&di->di_rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; end = rle->end; count = rle->count; } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c index 483d67ba6deb..d2c0611c9167 100644 --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -1,351 +1,350 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gic_v3_reg.h" #include "gic_v3_var.h" /* * FDT glue. */ static int gic_v3_fdt_probe(device_t); static int gic_v3_fdt_attach(device_t); static int gic_v3_fdt_print_child(device_t, device_t); static struct resource *gic_v3_ofw_bus_alloc_res(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static const struct ofw_bus_devinfo *gic_v3_ofw_get_devinfo(device_t, device_t); static device_method_t gic_v3_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gic_v3_fdt_probe), DEVMETHOD(device_attach, gic_v3_fdt_attach), /* Bus interface */ DEVMETHOD(bus_print_child, gic_v3_fdt_print_child), DEVMETHOD(bus_alloc_resource, gic_v3_ofw_bus_alloc_res), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, gic_v3_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(gic, gic_v3_fdt_driver, gic_v3_fdt_methods, sizeof(struct gic_v3_softc), gic_v3_driver); static devclass_t gic_v3_fdt_devclass; EARLY_DRIVER_MODULE(gic_v3, simplebus, gic_v3_fdt_driver, gic_v3_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic_v3, ofwbus, gic_v3_fdt_driver, gic_v3_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); /* * Helper functions declarations. */ static int gic_v3_ofw_bus_attach(device_t); /* * Device interface. */ static int gic_v3_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "arm,gic-v3")) return (ENXIO); device_set_desc(dev, GIC_V3_DEVSTR); return (BUS_PROBE_DEFAULT); } static int gic_v3_fdt_attach(device_t dev) { struct gic_v3_softc *sc; pcell_t redist_regions; intptr_t xref; int err; uint32_t *mbi_ranges; ssize_t ret; sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_FDT; /* * Recover number of the Re-Distributor regions. */ if (OF_getencprop(ofw_bus_get_node(dev), "#redistributor-regions", &redist_regions, sizeof(redist_regions)) <= 0) sc->gic_redists.nregions = 1; else sc->gic_redists.nregions = redist_regions; /* Add Message Based Interrupts using SPIs. */ ret = OF_getencprop_alloc_multi(ofw_bus_get_node(dev), "mbi-ranges", sizeof(*mbi_ranges), (void **)&mbi_ranges); if (ret > 0) { if (ret % 2 == 0) { /* Limit to a single range for now. */ sc->gic_mbi_start = mbi_ranges[0]; sc->gic_mbi_end = mbi_ranges[0] + mbi_ranges[1] - 1; } else { if (bootverbose) device_printf(dev, "Malformed mbi-ranges property\n"); } free(mbi_ranges, M_OFWPROP); } err = gic_v3_attach(dev); if (err != 0) goto error; xref = OF_xref_from_node(ofw_bus_get_node(dev)); sc->gic_pic = intr_pic_register(dev, xref); if (sc->gic_pic == NULL) { device_printf(dev, "could not register PIC\n"); err = ENXIO; goto error; } if (sc->gic_mbi_start > 0) intr_msi_register(dev, xref); /* Register xref */ OF_device_register_xref(xref, dev); if (intr_pic_claim_root(dev, xref, arm_gic_v3_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { err = ENXIO; goto error; } /* * Try to register ITS to this GIC. * GIC will act as a bus in that case. * Failure here will not affect main GIC functionality. */ if (gic_v3_ofw_bus_attach(dev) != 0) { if (bootverbose) { device_printf(dev, "Failed to attach ITS to this GIC\n"); } } if (device_get_children(dev, &sc->gic_children, &sc->gic_nchildren) != 0) sc->gic_nchildren = 0; return (err); error: if (bootverbose) { device_printf(dev, "Failed to attach. Error %d\n", err); } /* Failure so free resources */ gic_v3_detach(dev); return (err); } /* OFW bus interface */ struct gic_v3_ofw_devinfo { struct gic_v3_devinfo di_gic_dinfo; struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; static int gic_v3_fdt_print_child(device_t bus, device_t child) { struct gic_v3_ofw_devinfo *di = device_get_ivars(child); struct resource_list *rl = &di->di_rl; int retval = 0; retval += bus_print_child_header(bus, child); retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); retval += bus_print_child_footer(bus, child); return (retval); } static const struct ofw_bus_devinfo * gic_v3_ofw_get_devinfo(device_t bus __unused, device_t child) { struct gic_v3_ofw_devinfo *di; di = device_get_ivars(child); return (&di->di_dinfo); } static struct resource * gic_v3_ofw_bus_alloc_res(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct gic_v3_ofw_devinfo *di; struct resource_list_entry *rle; int ranges_len; if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type != SYS_RES_MEMORY) return (NULL); /* Find defaults for this rid */ rle = resource_list_find(&di->di_rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; end = rle->end; count = rle->count; } /* * XXX: No ranges remap! * Absolute address is expected. */ if (ofw_bus_has_prop(bus, "ranges")) { ranges_len = OF_getproplen(ofw_bus_get_node(bus), "ranges"); if (ranges_len != 0) { if (bootverbose) { device_printf(child, "Ranges remap not supported\n"); } return (NULL); } } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } /* Helper functions */ /* * Bus capability support for GICv3. * Collects and configures device informations and finally * adds ITS device as a child of GICv3 in Newbus hierarchy. */ static int gic_v3_ofw_bus_attach(device_t dev) { struct gic_v3_ofw_devinfo *di; struct gic_v3_softc *sc; device_t child; phandle_t parent, node; pcell_t addr_cells, size_cells; sc = device_get_softc(dev); parent = ofw_bus_get_node(dev); if (parent > 0) { addr_cells = 2; OF_getencprop(parent, "#address-cells", &addr_cells, sizeof(addr_cells)); size_cells = 2; OF_getencprop(parent, "#size-cells", &size_cells, sizeof(size_cells)); /* Iterate through all GIC subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_GIC_V3, M_WAITOK | M_ZERO); /* Read the numa node, or -1 if there is none */ if (OF_getencprop(node, "numa-node-id", &di->di_gic_dinfo.gic_domain, sizeof(di->di_gic_dinfo.gic_domain)) <= 0) { di->di_gic_dinfo.gic_domain = -1; } if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node)) { if (bootverbose) { device_printf(dev, "Could not set up devinfo for ITS\n"); } free(di, M_GIC_V3); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, &di->di_rl); /* Should not have any interrupts, so don't add any */ /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (!child) { if (bootverbose) { device_printf(dev, "Could not add child: %s\n", di->di_dinfo.obd_name); } resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_GIC_V3); continue; } sc->gic_nchildren++; device_set_ivars(child, di); } } return (bus_generic_attach(dev)); } diff --git a/sys/arm64/arm64/gic_v3_reg.h b/sys/arm64/arm64/gic_v3_reg.h index 2ba35ec9dab3..94033d28cb01 100644 --- a/sys/arm64/arm64/gic_v3_reg.h +++ b/sys/arm64/arm64/gic_v3_reg.h @@ -1,512 +1,511 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_V3_REG_H_ #define _GIC_V3_REG_H_ /* * Maximum number of interrupts * supported by GIC (including SGIs, PPIs and SPIs) */ #define GIC_I_NUM_MAX (1020) /* * Priority MAX/MIN values */ #define GIC_PRIORITY_MAX (0x00UL) /* Upper value is determined by LPI max priority */ #define GIC_PRIORITY_MIN (0xFCUL) /* Numbers for shared peripheral interrupts */ #define GIC_LAST_SPI (1019) /* Numbers for local peripheral interrupts */ #define GIC_FIRST_LPI (8192) /* * Registers (v2/v3) */ /* GICD_CTLR */ #define GICD_CTLR_G1 (1 << 0) #define GICD_CTLR_G1A (1 << 1) #define GICD_CTLR_ARE_NS (1 << 4) #define GICD_CTLR_RWP (1 << 31) /* GICD_TYPER */ #define GICD_TYPER_SECURITYEXTN (1 << 10) #define GICD_TYPER_MBIS (1 << 16) #define GICD_TYPER_LPIS (1 << 17) #define GICD_TYPER_DVIS (1 << 18) #define GICD_TYPER_IDBITS_SHIFT 19 #define GICD_TYPER_IDBITS(n) ((((n) >> 19) & 0x1F) + 1) /* * Registers (v3) */ #define GICD_STATUSR 0x0010 #define GICD_SETSPI_NSR 0x0040 #define GICD_CLRSPI_NSR 0x0048 #define GICD_SETSPI_SR 0x0050 #define GICD_CLRSPI_SR 0x0058 #define GICD_SPI_INTID_MASK 0x3ff #define GICD_IROUTER(n) (0x6000 + ((n) * 8)) #define GICD_AFF3 (0xfful << 32) #define GICD_IROUTER_IRM (1ul << 31) #define GICD_AFF2 (0xfful << 16) #define GICD_AFF1 (0xfful << 8) #define GICD_AFF0 (0xfful << 0) #define GICD_AFF (GICD_AFF0 | GICD_AFF1 | GICD_AFF1 | GICD_AFF3) #define GICD_PIDR4 0xFFD0 #define GICD_PIDR5 0xFFD4 #define GICD_PIDR6 0xFFD8 #define GICD_PIDR7 0xFFDC #define GICD_PIDR0 0xFFE0 #define GICD_PIDR1 0xFFE4 #define GICD_PIDR2 0xFFE8 #define GICD_SIZE 0x10000 #define GICR_PIDR2_ARCH_SHIFT 4 #define GICR_PIDR2_ARCH_MASK 0xF0 #define GICR_PIDR2_ARCH(x) \ (((x) & GICR_PIDR2_ARCH_MASK) >> GICR_PIDR2_ARCH_SHIFT) #define GICR_PIDR2_ARCH_GICv3 0x3 #define GICR_PIDR2_ARCH_GICv4 0x4 #define GICD_PIDR3 0xFFEC /* * Redistributor registers */ /* RD_base registers */ #define GICR_CTLR 0x0000 #define GICR_CTLR_LPI_ENABLE (1 << 0) #define GICR_CTLR_RWP (1 << 3) #define GICR_CTLR_DPG0 (1 << 24) #define GICR_CTLR_DPG1NS (1 << 25) #define GICR_CTLR_DPG1S (1 << 26) #define GICR_CTLR_UWP (1 << 31) #define GICR_IIDR 0x0004 #define GICR_TYPER 0x0008 #define GICR_TYPER_PLPIS (1 << 0) #define GICR_TYPER_VLPIS (1 << 1) #define GICR_TYPER_LAST (1 << 4) #define GICR_TYPER_CPUNUM_SHIFT (8) #define GICR_TYPER_CPUNUM_MASK (0xFFFUL << GICR_TYPER_CPUNUM_SHIFT) #define GICR_TYPER_CPUNUM(x) \ (((x) & GICR_TYPER_CPUNUM_MASK) >> GICR_TYPER_CPUNUM_SHIFT) #define GICR_TYPER_AFF_SHIFT 32 #define GICR_TYPER_AFF_MASK (0xfffffffful << GICR_TYPER_AFF_SHIFT) #define GICR_TYPER_AFF(x) \ (((x) & GICR_TYPER_AFF_MASK) >> GICR_TYPER_AFF_SHIFT) #define GICR_STATUSR 0x0010 #define GICR_WAKER 0x0014 #define GICR_WAKER_PS (1 << 1) /* Processor sleep */ #define GICR_WAKER_CA (1 << 2) /* Children asleep */ #define GICR_SETLPIR 0x0040 #define GICR_CLRLPIR 0x0048 #define GICR_PROPBASER 0x0070 #define GICR_PROPBASER_IDBITS_MASK 0x1FUL /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GICR_PROPBASER_CACHE_SHIFT 7 #define GICR_PROPBASER_CACHE_DnGnRnE 0x0UL #define GICR_PROPBASER_CACHE_NIN 0x1UL #define GICR_PROPBASER_CACHE_NIRAWT 0x2UL #define GICR_PROPBASER_CACHE_NIRAWB 0x3UL #define GICR_PROPBASER_CACHE_NIWAWT 0x4UL #define GICR_PROPBASER_CACHE_NIWAWB 0x5UL #define GICR_PROPBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PROPBASER_CACHE_NIRAWAWB 0x7UL #define GICR_PROPBASER_CACHE_MASK \ (0x7UL << GICR_PROPBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GICR_PROPBASER_SHARE_SHIFT 10 #define GICR_PROPBASER_SHARE_NS 0x0UL #define GICR_PROPBASER_SHARE_IS 0x1UL #define GICR_PROPBASER_SHARE_OS 0x2UL #define GICR_PROPBASER_SHARE_RES 0x3UL #define GICR_PROPBASER_SHARE_MASK \ (0x3UL << GICR_PROPBASER_SHARE_SHIFT) #define GICR_PROPBASER_OUTER_CACHE_SHIFT 56 #define GICR_PROPBASER_OUTER_CACHE_MASK \ (0x7UL << GICR_PROPBASER_OUTER_CACHE_SHIFT) #define GICR_PENDBASER 0x0078 /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GICR_PENDBASER_CACHE_SHIFT 7 #define GICR_PENDBASER_CACHE_DnGnRnE 0x0UL #define GICR_PENDBASER_CACHE_NIN 0x1UL #define GICR_PENDBASER_CACHE_NIRAWT 0x2UL #define GICR_PENDBASER_CACHE_NIRAWB 0x3UL #define GICR_PENDBASER_CACHE_NIWAWT 0x4UL #define GICR_PENDBASER_CACHE_NIWAWB 0x5UL #define GICR_PENDBASER_CACHE_NIRAWAWT 0x6UL #define GICR_PENDBASER_CACHE_NIRAWAWB 0x7UL #define GICR_PENDBASER_CACHE_MASK \ (0x7UL << GICR_PENDBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GICR_PENDBASER_SHARE_SHIFT 10 #define GICR_PENDBASER_SHARE_NS 0x0UL #define GICR_PENDBASER_SHARE_IS 0x1UL #define GICR_PENDBASER_SHARE_OS 0x2UL #define GICR_PENDBASER_SHARE_RES 0x3UL #define GICR_PENDBASER_SHARE_MASK \ (0x3UL << GICR_PENDBASER_SHARE_SHIFT) #define GICR_PENDBASER_OUTER_CACHE_SHIFT 56 #define GICR_PENDBASER_OUTER_CACHE_MASK \ (0x7UL << GICR_PENDBASER_OUTER_CACHE_SHIFT) #define GICR_INVLPIR 0x00a0 #define GICR_INVALLR 0x00b0 #define GICR_SYNCR 0x00c0 #define GICR_PIDR2 GICD_PIDR2 /* SGI_base registers */ #define GICR_IGROUPR0 (0x0080) #define GICR_ISENABLER0 (0x0100) #define GICR_ICENABLER0 (0x0180) #define GICR_I_ENABLER_SGI_MASK (0x0000FFFF) #define GICR_I_ENABLER_PPI_MASK (0xFFFF0000) #define GICR_I_PER_IPRIORITYn (GICD_I_PER_IPRIORITYn) #define GICR_ISPENDR0 0x0200 #define GICR_ICPENDR0 0x0280 #define GICR_ISACTIVER0 0x0300 #define GICR_ICACTIVER0 0x0380 #define GICR_IPRIORITYR(n) (0x0400 + (((n) >> 2) * 4)) #define GICR_ICFGR0 0x0c00 #define GICR_ICFGR1 0x0c04 #define GICR_IGRPMODR0 0x0d00 #define GICR_NSACR 0x0e00 /* Re-distributor registers for SGIs and PPIs */ #define GICR_RD_BASE 0 #define GICR_RD_BASE_SIZE PAGE_SIZE_64K #define GICR_SGI_BASE (1 * PAGE_SIZE_64K) #define GICR_SGI_BASE_SIZE PAGE_SIZE_64K #define GICR_VLPI_BASE (2 * PAGE_SIZE_64K) #define GICR_VLPI_BASE_SIZE PAGE_SIZE_64K #define GICR_RESERVED_SIZE PAGE_SIZE_64K /* * ITS registers */ #define GITS_PIDR2 GICR_PIDR2 #define GITS_PIDR2_ARCH_MASK GICR_PIDR2_ARCH_MASK #define GITS_PIDR2_ARCH_GICv3 GICR_PIDR2_ARCH_GICv3 #define GITS_PIDR2_ARCH_GICv4 GICR_PIDR2_ARCH_GICv4 #define GITS_CTLR (0x0000) #define GITS_CTLR_EN (1 << 0) #define GITS_IIDR (0x0004) #define GITS_IIDR_PRODUCT_SHIFT 24 #define GITS_IIDR_PRODUCT_MASK (0xff << GITS_IIDR_PRODUCT_SHIFT) #define GITS_IIDR_VARIANT_SHIFT 16 #define GITS_IIDR_VARIANT_MASK (0xf << GITS_IIDR_VARIANT_SHIFT) #define GITS_IIDR_REVISION_SHIFT 12 #define GITS_IIDR_REVISION_MASK (0xf << GITS_IIDR_REVISION_SHIFT) #define GITS_IIDR_IMPLEMENTOR_SHIFT 0 #define GITS_IIDR_IMPLEMENTOR_MASK (0xfff << GITS_IIDR_IMPLEMENTOR_SHIFT) #define GITS_IIDR_RAW(impl, prod, var, rev) \ ((prod) << GITS_IIDR_PRODUCT_SHIFT | \ (var) << GITS_IIDR_VARIANT_SHIFT | \ (rev) << GITS_IIDR_REVISION_SHIFT | \ (impl) << GITS_IIDR_IMPLEMENTOR_SHIFT) #define GITS_IIDR_IMPL_ARM (0x43B) #define GITS_IIDR_PROD_GIC500 (0x0) #define GITS_IIDR_IMPL_CAVIUM (0x34c) #define GITS_IIDR_PROD_THUNDER (0xa1) #define GITS_IIDR_VAR_THUNDER_1 (0x0) #define GITS_CBASER (0x0080) #define GITS_CBASER_VALID (1UL << 63) /* * Cacheability * 0x0 - Device-nGnRnE * 0x1 - Normal Inner Non-cacheable * 0x2 - Normal Inner Read-allocate, Write-through * 0x3 - Normal Inner Read-allocate, Write-back * 0x4 - Normal Inner Write-allocate, Write-through * 0x5 - Normal Inner Write-allocate, Write-back * 0x6 - Normal Inner Read-allocate, Write-allocate, Write-through * 0x7 - Normal Inner Read-allocate, Write-allocate, Write-back */ #define GITS_CBASER_CACHE_SHIFT 59 #define GITS_CBASER_CACHE_DnGnRnE 0x0UL #define GITS_CBASER_CACHE_NIN 0x1UL #define GITS_CBASER_CACHE_NIRAWT 0x2UL #define GITS_CBASER_CACHE_NIRAWB 0x3UL #define GITS_CBASER_CACHE_NIWAWT 0x4UL #define GITS_CBASER_CACHE_NIWAWB 0x5UL #define GITS_CBASER_CACHE_NIRAWAWT 0x6UL #define GITS_CBASER_CACHE_NIRAWAWB 0x7UL #define GITS_CBASER_CACHE_MASK (0x7UL << GITS_CBASER_CACHE_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GITS_CBASER_SHARE_SHIFT 10 #define GITS_CBASER_SHARE_NS 0x0UL #define GITS_CBASER_SHARE_IS 0x1UL #define GITS_CBASER_SHARE_OS 0x2UL #define GITS_CBASER_SHARE_RES 0x3UL #define GITS_CBASER_SHARE_MASK \ (0x3UL << GITS_CBASER_SHARE_SHIFT) #define GITS_CBASER_PA_SHIFT 12 #define GITS_CBASER_PA_MASK (0xFFFFFFFFFUL << GITS_CBASER_PA_SHIFT) #define GITS_CBASER_SIZE_SHIFT 0 #define GITS_CBASER_SIZE_MASK (0xFF << GITS_CBASER_SIZE_SHIFT) #define GITS_CBASER_SIZE(x) \ (4096 * (((x) & GITS_CBASER_SIZE_MASK) + 1)) #define GITS_CWRITER (0x0088) #define GITS_CREADR (0x0090) #define GITS_CREADR_STALL 1ul #define GITS_CMD_OFFSET(reg) ((reg) & 0xfffe0ul) #define GITS_CMD_SIZE 0x20 #define GITS_BASER_BASE (0x0100) #define GITS_BASER(x) (GITS_BASER_BASE + (x) * 8) #define GITS_BASER_VALID (1UL << 63) #define GITS_BASER_TYPE_SHIFT 56 #define GITS_BASER_TYPE(x) \ (((x) & GITS_BASER_TYPE_MASK) >> GITS_BASER_TYPE_SHIFT) #define GITS_BASER_TYPE_UNIMPL 0x0UL /* Unimplemented */ #define GITS_BASER_TYPE_DEV 0x1UL /* Devices */ #define GITS_BASER_TYPE_VP 0x2UL /* Virtual Processors */ #define GITS_BASER_TYPE_PP 0x3UL /* Physical Processors */ #define GITS_BASER_TYPE_IC 0x4UL /* Interrupt Collections */ #define GITS_BASER_TYPE_RES5 0x5UL /* Reserved */ #define GITS_BASER_TYPE_RES6 0x6UL /* Reserved */ #define GITS_BASER_TYPE_RES7 0x7UL /* Reserved */ #define GITS_BASER_TYPE_MASK (0x7UL << GITS_BASER_TYPE_SHIFT) /* * Cacheability * 0x0 - Non-cacheable, non-bufferable * 0x1 - Non-cacheable * 0x2 - Read-allocate, Write-through * 0x3 - Read-allocate, Write-back * 0x4 - Write-allocate, Write-through * 0x5 - Write-allocate, Write-back * 0x6 - Read-allocate, Write-allocate, Write-through * 0x7 - Read-allocate, Write-allocate, Write-back */ #define GITS_BASER_CACHE_SHIFT 59 #define GITS_BASER_CACHE_NCNB 0x0UL #define GITS_BASER_CACHE_NC 0x1UL #define GITS_BASER_CACHE_RAWT 0x2UL #define GITS_BASER_CACHE_RAWB 0x3UL #define GITS_BASER_CACHE_WAWT 0x4UL #define GITS_BASER_CACHE_WAWB 0x5UL #define GITS_BASER_CACHE_RAWAWT 0x6UL #define GITS_BASER_CACHE_RAWAWB 0x7UL #define GITS_BASER_CACHE_MASK (0x7UL << GITS_BASER_CACHE_SHIFT) #define GITS_BASER_ESIZE_SHIFT 48 #define GITS_BASER_ESIZE_MASK (0x1FUL << GITS_BASER_ESIZE_SHIFT) #define GITS_BASER_ESIZE(x) \ ((((x) & GITS_BASER_ESIZE_MASK) >> GITS_BASER_ESIZE_SHIFT) + 1) #define GITS_BASER_PA_SHIFT 12 #define GITS_BASER_PA_MASK (0xFFFFFFFFFUL << GITS_BASER_PA_SHIFT) /* * Shareability * 0x0 - Non-shareable * 0x1 - Inner-shareable * 0x2 - Outer-shareable * 0x3 - Reserved. Threated as 0x0 */ #define GITS_BASER_SHARE_SHIFT 10 #define GITS_BASER_SHARE_NS 0x0UL #define GITS_BASER_SHARE_IS 0x1UL #define GITS_BASER_SHARE_OS 0x2UL #define GITS_BASER_SHARE_RES 0x3UL #define GITS_BASER_SHARE_MASK (0x3UL << GITS_BASER_SHARE_SHIFT) #define GITS_BASER_PSZ_SHIFT 8 #define GITS_BASER_PSZ_4K 0x0UL #define GITS_BASER_PSZ_16K 0x1UL #define GITS_BASER_PSZ_64K 0x2UL #define GITS_BASER_PSZ_MASK (0x3UL << GITS_BASER_PSZ_SHIFT) #define GITS_BASER_SIZE_MASK 0xFFUL #define GITS_BASER_NUM 8 #define GITS_TYPER (0x0008) #define GITS_TYPER_HCC_SHIFT 24 #define GITS_TYPER_PTA (1UL << 19) #define GITS_TYPER_DEVB_SHIFT 13 #define GITS_TYPER_DEVB_MASK (0x1FUL << GITS_TYPER_DEVB_SHIFT) /* Number of device identifiers implemented */ #define GITS_TYPER_DEVB(x) \ ((((x) & GITS_TYPER_DEVB_MASK) >> GITS_TYPER_DEVB_SHIFT) + 1) #define GITS_TYPER_ITTES_SHIFT 4 #define GITS_TYPER_ITTES_MASK (0xFUL << GITS_TYPER_ITTES_SHIFT) /* Number of bytes per ITT Entry */ #define GITS_TYPER_ITTES(x) \ ((((x) & GITS_TYPER_ITTES_MASK) >> GITS_TYPER_ITTES_SHIFT) + 1) #define GITS_TRANSLATER (0x10040) /* * LPI related */ #define LPI_CONF_PRIO_MASK (0xFC) #define LPI_CONF_GROUP1 (1 << 1) #define LPI_CONF_ENABLE (1 << 0) /* * GIC 500 ITS tracking facility */ #define GITS_TRKCTLR 0xC000 #define GITS_TRKR 0xC004 #define GITS_TRKDIDR 0xC008 #define GITS_TRKPIDR 0xC00C #define GITS_TRKVIDR 0xC010 #define GITS_TRKTGTR 0xC014 #define GITS_TRKICR 0xC018 #define GITS_TRKLCR 0xC018 /* * CPU interface */ /* * Registers list (ICC_xyz_EL1): * * PMR - Priority Mask Register * * interrupts of priority higher than specified * in this mask will be signalled to the CPU. * (0xff - lowest possible prio., 0x00 - highest prio.) * * CTLR - Control Register * * controls behavior of the CPU interface and displays * implemented features. * * IGRPEN1 - Interrupt Group 1 Enable Register * * IAR1 - Interrupt Acknowledge Register Group 1 * * contains number of the highest priority pending * interrupt from the Group 1. * * EOIR1 - End of Interrupt Register Group 1 * * Writes inform CPU interface about completed Group 1 * interrupts processing. */ #define gic_icc_write(reg, val) \ do { \ WRITE_SPECIALREG(icc_ ##reg ##_el1, val); \ isb(); \ } while (0) #define gic_icc_read(reg) \ ({ \ uint64_t val; \ \ val = READ_SPECIALREG(icc_ ##reg ##_el1); \ (val); \ }) #define gic_icc_set(reg, mask) \ do { \ uint64_t val; \ val = gic_icc_read(reg); \ val |= (mask); \ gic_icc_write(reg, val); \ } while (0) #define gic_icc_clear(reg, mask) \ do { \ uint64_t val; \ val = gic_icc_read(reg); \ val &= ~(mask); \ gic_icc_write(reg, val); \ } while (0) #endif /* _GIC_V3_REG_H_ */ diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h index 1645c417fd8d..7722a48cb456 100644 --- a/sys/arm64/arm64/gic_v3_var.h +++ b/sys/arm64/arm64/gic_v3_var.h @@ -1,150 +1,149 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_V3_VAR_H_ #define _GIC_V3_VAR_H_ #include #define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" DECLARE_CLASS(gic_v3_driver); struct gic_v3_irqsrc; struct redist_pcpu { struct resource res; /* mem resource for redist */ vm_offset_t pend_base; bool lpi_enabled; /* redist LPI configured? */ }; struct gic_redists { /* * Re-Distributor region description. * We will have few of those depending * on the #redistributor-regions property in FDT. */ struct resource ** regions; /* Number of Re-Distributor regions */ u_int nregions; /* Per-CPU Re-Distributor data */ struct redist_pcpu *pcpu[MAXCPU]; }; struct gic_v3_softc { device_t dev; struct resource ** gic_res; struct mtx gic_mtx; /* Distributor */ struct resource * gic_dist; /* Re-Distributors */ struct gic_redists gic_redists; /* Message Based Interrupts */ u_int gic_mbi_start; u_int gic_mbi_end; struct mtx gic_mbi_mtx; uint32_t gic_pidr2; u_int gic_bus; u_int gic_nirqs; u_int gic_idbits; boolean_t gic_registered; int gic_nchildren; device_t *gic_children; struct intr_pic *gic_pic; struct gic_v3_irqsrc *gic_irqs; }; struct gic_v3_devinfo { int gic_domain; int msi_xref; }; #define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) MALLOC_DECLARE(M_GIC_V3); /* ivars */ #define GICV3_IVAR_NIRQS 1000 /* 1001 was GICV3_IVAR_REDIST_VADDR */ #define GICV3_IVAR_REDIST 1002 __BUS_ACCESSOR(gicv3, nirqs, GICV3, NIRQS, u_int); __BUS_ACCESSOR(gicv3, redist, GICV3, REDIST, void *); /* Device methods */ int gic_v3_attach(device_t dev); int gic_v3_detach(device_t dev); int arm_gic_v3_intr(void *); uint32_t gic_r_read_4(device_t, bus_size_t); uint64_t gic_r_read_8(device_t, bus_size_t); void gic_r_write_4(device_t, bus_size_t, uint32_t var); void gic_r_write_8(device_t, bus_size_t, uint64_t var); /* * GIC Distributor accessors. * Notice that only GIC sofc can be passed. */ #define gic_d_read(sc, len, reg) \ ({ \ bus_read_##len(sc->gic_dist, reg); \ }) #define gic_d_write(sc, len, reg, val) \ ({ \ bus_write_##len(sc->gic_dist, reg, val);\ }) /* GIC Re-Distributor accessors (per-CPU) */ #define gic_r_read(sc, len, reg) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_read_##len( \ &sc->gic_redists.pcpu[cpu]->res, \ reg); \ }) #define gic_r_write(sc, len, reg, val) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_write_##len( \ &sc->gic_redists.pcpu[cpu]->res, \ reg, val); \ }) #endif /* _GIC_V3_VAR_H_ */ diff --git a/sys/arm64/arm64/gicv3_its.c b/sys/arm64/arm64/gicv3_its.c index 8ba7aa3b2047..af587247626b 100644 --- a/sys/arm64/arm64/gicv3_its.c +++ b/sys/arm64/arm64/gicv3_its.c @@ -1,2032 +1,2031 @@ /*- * Copyright (c) 2015-2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * the sponsorship of the FreeBSD Foundation. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include "opt_platform.h" #include "opt_iommu.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include #include #ifdef IOMMU #include #include #endif #include "pcib_if.h" #include "pic_if.h" #include "msi_if.h" MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", "ARM GICv3 Interrupt Translation Service"); #define LPI_NIRQS (64 * 1024) /* The size and alignment of the command circular buffer */ #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ #define ITS_CMDQ_ALIGN (64 * 1024) #define LPI_CONFTAB_SIZE LPI_NIRQS #define LPI_CONFTAB_ALIGN (64 * 1024) #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) #define LPI_PENDTAB_ALIGN (64 * 1024) #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ #define LPI_INT_TRANS_TAB_ALIGN 256 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) /* ITS commands encoding */ #define ITS_CMD_MOVI (0x01) #define ITS_CMD_SYNC (0x05) #define ITS_CMD_MAPD (0x08) #define ITS_CMD_MAPC (0x09) #define ITS_CMD_MAPTI (0x0a) #define ITS_CMD_MAPI (0x0b) #define ITS_CMD_INV (0x0c) #define ITS_CMD_INVALL (0x0d) /* Command */ #define CMD_COMMAND_MASK (0xFFUL) /* PCI device ID */ #define CMD_DEVID_SHIFT (32) #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) /* Size of IRQ ID bitfield */ #define CMD_SIZE_MASK (0xFFUL) /* Virtual LPI ID */ #define CMD_ID_MASK (0xFFFFFFFFUL) /* Physical LPI ID */ #define CMD_PID_SHIFT (32) #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) /* Collection */ #define CMD_COL_MASK (0xFFFFUL) /* Target (CPU or Re-Distributor) */ #define CMD_TARGET_SHIFT (16) #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) /* Interrupt Translation Table address */ #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) /* Valid command bit */ #define CMD_VALID_SHIFT (63) #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) #define ITS_TARGET_NONE 0xFBADBEEF /* LPI chunk owned by ITS device */ struct lpi_chunk { u_int lpi_base; u_int lpi_free; /* First free LPI in set */ u_int lpi_num; /* Total number of LPIs in chunk */ u_int lpi_busy; /* Number of busy LPIs in chink */ }; /* ITS device */ struct its_dev { TAILQ_ENTRY(its_dev) entry; /* PCI device */ device_t pci_dev; /* Device ID (i.e. PCI device ID) */ uint32_t devid; /* List of assigned LPIs */ struct lpi_chunk lpis; /* Virtual address of ITT */ vm_offset_t itt; size_t itt_size; }; /* * ITS command descriptor. * Idea for command description passing taken from Linux. */ struct its_cmd_desc { uint8_t cmd_type; union { struct { struct its_dev *its_dev; struct its_col *col; uint32_t id; } cmd_desc_movi; struct { struct its_col *col; } cmd_desc_sync; struct { struct its_col *col; uint8_t valid; } cmd_desc_mapc; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; uint32_t id; } cmd_desc_mapvi; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_mapi; struct { struct its_dev *its_dev; uint8_t valid; } cmd_desc_mapd; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_inv; struct { struct its_col *col; } cmd_desc_invall; }; }; /* ITS command. Each command is 32 bytes long */ struct its_cmd { uint64_t cmd_dword[4]; /* ITS command double word */ }; /* An ITS private table */ struct its_ptable { vm_offset_t ptab_vaddr; unsigned long ptab_size; }; /* ITS collection description. */ struct its_col { uint64_t col_target; /* Target Re-Distributor */ uint64_t col_id; /* Collection ID */ }; struct gicv3_its_irqsrc { struct intr_irqsrc gi_isrc; u_int gi_id; u_int gi_lpi; struct its_dev *gi_its_dev; TAILQ_ENTRY(gicv3_its_irqsrc) gi_link; }; struct gicv3_its_softc { device_t dev; struct intr_pic *sc_pic; struct resource *sc_its_res; cpuset_t sc_cpus; struct domainset *sc_ds; u_int gic_irq_cpu; struct its_ptable sc_its_ptab[GITS_BASER_NUM]; struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ /* * TODO: We should get these from the parent as we only want a * single copy of each across the interrupt controller. */ uint8_t *sc_conf_base; vm_offset_t sc_pend_base[MAXCPU]; /* Command handling */ struct mtx sc_its_cmd_lock; struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ size_t sc_its_cmd_next_idx; vmem_t *sc_irq_alloc; struct gicv3_its_irqsrc **sc_irqs; u_int sc_irq_base; u_int sc_irq_length; u_int sc_irq_count; struct mtx sc_its_dev_lock; TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 u_int sc_its_flags; bool trace_enable; vm_page_t ma; /* fake msi page */ }; static void *conf_base; typedef void (its_quirk_func_t)(device_t); static its_quirk_func_t its_quirk_cavium_22375; static const struct { const char *desc; uint32_t iidr; uint32_t iidr_mask; its_quirk_func_t *func; } its_quirks[] = { { /* Cavium ThunderX Pass 1.x */ .desc = "Cavium ThunderX errata: 22375, 24313", .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), .iidr_mask = ~GITS_IIDR_REVISION_MASK, .func = its_quirk_cavium_22375, }, }; #define gic_its_read_4(sc, reg) \ bus_read_4((sc)->sc_its_res, (reg)) #define gic_its_read_8(sc, reg) \ bus_read_8((sc)->sc_its_res, (reg)) #define gic_its_write_4(sc, reg, val) \ bus_write_4((sc)->sc_its_res, (reg), (val)) #define gic_its_write_8(sc, reg, val) \ bus_write_8((sc)->sc_its_res, (reg), (val)) static device_attach_t gicv3_its_attach; static device_detach_t gicv3_its_detach; static pic_disable_intr_t gicv3_its_disable_intr; static pic_enable_intr_t gicv3_its_enable_intr; static pic_map_intr_t gicv3_its_map_intr; static pic_setup_intr_t gicv3_its_setup_intr; static pic_post_filter_t gicv3_its_post_filter; static pic_post_ithread_t gicv3_its_post_ithread; static pic_pre_ithread_t gicv3_its_pre_ithread; static pic_bind_intr_t gicv3_its_bind_intr; #ifdef SMP static pic_init_secondary_t gicv3_its_init_secondary; #endif static msi_alloc_msi_t gicv3_its_alloc_msi; static msi_release_msi_t gicv3_its_release_msi; static msi_alloc_msix_t gicv3_its_alloc_msix; static msi_release_msix_t gicv3_its_release_msix; static msi_map_msi_t gicv3_its_map_msi; #ifdef IOMMU static msi_iommu_init_t gicv3_iommu_init; static msi_iommu_deinit_t gicv3_iommu_deinit; #endif static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); static void its_cmd_mapc(device_t, struct its_col *, uint8_t); static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); static void its_cmd_invall(device_t, struct its_col *); static device_method_t gicv3_its_methods[] = { /* Device interface */ DEVMETHOD(device_detach, gicv3_its_detach), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), DEVMETHOD(pic_map_intr, gicv3_its_map_intr), DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), DEVMETHOD(pic_post_filter, gicv3_its_post_filter), DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), #ifdef SMP DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), #endif /* MSI/MSI-X */ DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), DEVMETHOD(msi_release_msi, gicv3_its_release_msi), DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), DEVMETHOD(msi_release_msix, gicv3_its_release_msix), DEVMETHOD(msi_map_msi, gicv3_its_map_msi), #ifdef IOMMU DEVMETHOD(msi_iommu_init, gicv3_iommu_init), DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit), #endif /* End */ DEVMETHOD_END }; static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, sizeof(struct gicv3_its_softc)); static void gicv3_its_cmdq_init(struct gicv3_its_softc *sc) { vm_paddr_t cmd_paddr; uint64_t reg, tmp; /* Set up the command circular buffer */ sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); sc->sc_its_cmd_next_idx = 0; cmd_paddr = vtophys(sc->sc_its_cmd_base); /* Set the base of the command buffer */ reg = GITS_CBASER_VALID | (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | (ITS_CMDQ_SIZE / 4096 - 1); gic_its_write_8(sc, GITS_CBASER, reg); /* Read back to check for fixed value fields */ tmp = gic_its_read_8(sc, GITS_CBASER); if ((tmp & GITS_CBASER_SHARE_MASK) != (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { /* Check if the hardware reported non-shareable */ if ((tmp & GITS_CBASER_SHARE_MASK) == (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { /* If so remove the cache attribute */ reg &= ~GITS_CBASER_CACHE_MASK; reg &= ~GITS_CBASER_SHARE_MASK; /* Set to Non-cacheable, Non-shareable */ reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; gic_its_write_8(sc, GITS_CBASER, reg); } /* The command queue has to be flushed after each command */ sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; } /* Get the next command from the start of the buffer */ gic_its_write_8(sc, GITS_CWRITER, 0x0); } static int gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) { vm_offset_t table; vm_paddr_t paddr; uint64_t cache, reg, share, tmp, type; size_t esize, its_tbl_size, nidents, nitspages, npages; int i, page_size; int devbits; if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { /* * GITS_TYPER[17:13] of ThunderX reports that device IDs * are to be 21 bits in length. The entry size of the ITS * table can be read from GITS_BASERn[52:48] and on ThunderX * is supposed to be 8 bytes in length (for device table). * Finally the page size that is to be used by ITS to access * this table will be set to 64KB. * * This gives 0x200000 entries of size 0x8 bytes covered by * 256 pages each of which 64KB in size. The number of pages * (minus 1) should then be written to GITS_BASERn[7:0]. In * that case this value would be 0xFF but on ThunderX the * maximum value that HW accepts is 0xFD. * * Set an arbitrary number of device ID bits to 20 in order * to limit the number of entries in ITS device table to * 0x100000 and the table size to 8MB. */ devbits = 20; cache = 0; } else { devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); cache = GITS_BASER_CACHE_WAWB; } share = GITS_BASER_SHARE_IS; page_size = PAGE_SIZE_64K; for (i = 0; i < GITS_BASER_NUM; i++) { reg = gic_its_read_8(sc, GITS_BASER(i)); /* The type of table */ type = GITS_BASER_TYPE(reg); /* The table entry size */ esize = GITS_BASER_ESIZE(reg); switch(type) { case GITS_BASER_TYPE_DEV: nidents = (1 << devbits); its_tbl_size = esize * nidents; its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); break; case GITS_BASER_TYPE_VP: case GITS_BASER_TYPE_PP: /* Undocumented? */ case GITS_BASER_TYPE_IC: its_tbl_size = page_size; break; default: continue; } npages = howmany(its_tbl_size, PAGE_SIZE); /* Allocate the table */ table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE, M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, PAGE_SIZE_64K, 0); sc->sc_its_ptab[i].ptab_vaddr = table; sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; paddr = vtophys(table); while (1) { nitspages = howmany(its_tbl_size, page_size); /* Clear the fields we will be setting */ reg &= ~(GITS_BASER_VALID | GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | GITS_BASER_SIZE_MASK); /* Set the new values */ reg |= GITS_BASER_VALID | (cache << GITS_BASER_CACHE_SHIFT) | (type << GITS_BASER_TYPE_SHIFT) | ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | paddr | (share << GITS_BASER_SHARE_SHIFT) | (nitspages - 1); switch (page_size) { case PAGE_SIZE_4K: /* 4KB */ reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; break; case PAGE_SIZE_16K: /* 16KB */ reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; break; case PAGE_SIZE_64K: /* 64KB */ reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; break; } gic_its_write_8(sc, GITS_BASER(i), reg); /* Read back to check */ tmp = gic_its_read_8(sc, GITS_BASER(i)); /* Do the shareability masks line up? */ if ((tmp & GITS_BASER_SHARE_MASK) != (reg & GITS_BASER_SHARE_MASK)) { share = (tmp & GITS_BASER_SHARE_MASK) >> GITS_BASER_SHARE_SHIFT; continue; } if ((tmp & GITS_BASER_PSZ_MASK) != (reg & GITS_BASER_PSZ_MASK)) { switch (page_size) { case PAGE_SIZE_16K: page_size = PAGE_SIZE_4K; continue; case PAGE_SIZE_64K: page_size = PAGE_SIZE_16K; continue; } } if (tmp != reg) { device_printf(dev, "GITS_BASER%d: " "unable to be updated: %lx != %lx\n", i, reg, tmp); return (ENXIO); } /* We should have made all needed changes */ break; } } return (0); } static void gicv3_its_conftable_init(struct gicv3_its_softc *sc) { void *conf_table; conf_table = atomic_load_ptr(&conf_base); if (conf_table == NULL) { conf_table = contigmalloc(LPI_CONFTAB_SIZE, M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, 0); if (atomic_cmpset_ptr((uintptr_t *)&conf_base, (uintptr_t)NULL, (uintptr_t)conf_table) == 0) { contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS); conf_table = atomic_load_ptr(&conf_base); } } sc->sc_conf_base = conf_table; /* Set the default configuration */ memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, LPI_CONFTAB_SIZE); /* Flush the table to memory */ cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE); } static void gicv3_its_pendtables_init(struct gicv3_its_softc *sc) { int i; for (i = 0; i <= mp_maxid; i++) { if (CPU_ISSET(i, &sc->sc_cpus) == 0) continue; sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); /* Flush so the ITS can see the memory */ cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], LPI_PENDTAB_SIZE); } } static void its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) { device_t gicv3; uint64_t xbaser, tmp; uint32_t ctlr; u_int cpuid; gicv3 = device_get_parent(dev); cpuid = PCPU_GET(cpuid); /* Disable LPIs */ ctlr = gic_r_read_4(gicv3, GICR_CTLR); ctlr &= ~GICR_CTLR_LPI_ENABLE; gic_r_write_4(gicv3, GICR_CTLR, ctlr); /* Make sure changes are observable my the GIC */ dsb(sy); /* * Set the redistributor base */ xbaser = vtophys(sc->sc_conf_base) | (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); /* Check the cache attributes we set */ tmp = gic_r_read_8(gicv3, GICR_PROPBASER); if ((tmp & GICR_PROPBASER_SHARE_MASK) != (xbaser & GICR_PROPBASER_SHARE_MASK)) { if ((tmp & GICR_PROPBASER_SHARE_MASK) == (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { /* We need to mark as non-cacheable */ xbaser &= ~(GICR_PROPBASER_SHARE_MASK | GICR_PROPBASER_CACHE_MASK); /* Non-cacheable */ xbaser |= GICR_PROPBASER_CACHE_NIN << GICR_PROPBASER_CACHE_SHIFT; /* Non-sareable */ xbaser |= GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT; gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); } sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; } /* * Set the LPI pending table base */ xbaser = vtophys(sc->sc_pend_base[cpuid]) | (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); tmp = gic_r_read_8(gicv3, GICR_PENDBASER); if ((tmp & GICR_PENDBASER_SHARE_MASK) == (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { /* Clear the cahce and shareability bits */ xbaser &= ~(GICR_PENDBASER_CACHE_MASK | GICR_PENDBASER_SHARE_MASK); /* Mark as non-shareable */ xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; /* And non-cacheable */ xbaser |= GICR_PENDBASER_CACHE_NIN << GICR_PENDBASER_CACHE_SHIFT; } /* Enable LPIs */ ctlr = gic_r_read_4(gicv3, GICR_CTLR); ctlr |= GICR_CTLR_LPI_ENABLE; gic_r_write_4(gicv3, GICR_CTLR, ctlr); /* Make sure the GIC has seen everything */ dsb(sy); } static int its_init_cpu(device_t dev, struct gicv3_its_softc *sc) { device_t gicv3; vm_paddr_t target; u_int cpuid; struct redist_pcpu *rpcpu; gicv3 = device_get_parent(dev); cpuid = PCPU_GET(cpuid); if (!CPU_ISSET(cpuid, &sc->sc_cpus)) return (0); /* Check if the ITS is enabled on this CPU */ if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) return (ENXIO); rpcpu = gicv3_get_redist(dev); /* Do per-cpu LPI init once */ if (!rpcpu->lpi_enabled) { its_init_cpu_lpi(dev, sc); rpcpu->lpi_enabled = true; } if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { /* This ITS wants the redistributor physical address */ target = vtophys(rman_get_virtual(&rpcpu->res)); } else { /* This ITS wants the unique processor number */ target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) << CMD_TARGET_SHIFT; } sc->sc_its_cols[cpuid]->col_target = target; sc->sc_its_cols[cpuid]->col_id = cpuid; its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); its_cmd_invall(dev, sc->sc_its_cols[cpuid]); return (0); } static int gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS) { struct gicv3_its_softc *sc; int rv; sc = arg1; rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req); if (rv != 0 || req->newptr == NULL) return (rv); if (sc->trace_enable) gic_its_write_8(sc, GITS_TRKCTLR, 3); else gic_its_write_8(sc, GITS_TRKCTLR, 0); return (0); } static int gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS) { struct gicv3_its_softc *sc; struct sbuf *sb; int err; sc = arg1; sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (sb == NULL) { device_printf(sc->dev, "Could not allocate sbuf for output.\n"); return (ENOMEM); } sbuf_cat(sb, "\n"); sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKCTLR)); sbuf_printf(sb, "GITS_TRKR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKR)); sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKDIDR)); sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKPIDR)); sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKVIDR)); sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n", gic_its_read_4(sc, GITS_TRKTGTR)); err = sbuf_finish(sb); if (err) device_printf(sc->dev, "Error finishing sbuf: %d\n", err); sbuf_delete(sb); return(err); } static int gicv3_its_init_sysctl(struct gicv3_its_softc *sc) { struct sysctl_oid *oid, *child; struct sysctl_ctx_list *ctx_list; ctx_list = device_get_sysctl_ctx(sc->dev); child = device_get_sysctl_tree(sc->dev); oid = SYSCTL_ADD_NODE(ctx_list, SYSCTL_CHILDREN(child), OID_AUTO, "tracing", CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing"); if (oid == NULL) return (ENXIO); /* Add registers */ SYSCTL_ADD_PROC(ctx_list, SYSCTL_CHILDREN(oid), OID_AUTO, "enable", CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, gicv3_its_sysctl_trace_enable, "CU", "Enable tracing"); SYSCTL_ADD_PROC(ctx_list, SYSCTL_CHILDREN(oid), OID_AUTO, "capture", CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, gicv3_its_sysctl_trace_regs, "", "Captured tracing registers."); return (0); } static int gicv3_its_attach(device_t dev) { struct gicv3_its_softc *sc; int domain, err, i, rid; uint64_t phys; uint32_t iidr; sc = device_get_softc(dev); sc->sc_irq_length = gicv3_get_nirqs(dev); sc->sc_irq_base = GIC_FIRST_LPI; sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; rid = 0; sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_its_res == NULL) { device_printf(dev, "Could not allocate memory\n"); return (ENXIO); } phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER, PAGE_SIZE); sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO); vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT); CPU_COPY(&all_cpus, &sc->sc_cpus); iidr = gic_its_read_4(sc, GITS_IIDR); for (i = 0; i < nitems(its_quirks); i++) { if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { if (bootverbose) { device_printf(dev, "Applying %s\n", its_quirks[i].desc); } its_quirks[i].func(dev); break; } } if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) { sc->sc_ds = DOMAINSET_PREF(domain); } else { sc->sc_ds = DOMAINSET_RR(); } /* Allocate the private tables */ err = gicv3_its_table_init(dev, sc); if (err != 0) return (err); /* Protects access to the device list */ mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); /* Protects access to the ITS command circular buffer. */ mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); /* Allocate the command circular buffer */ gicv3_its_cmdq_init(sc); /* Allocate the per-CPU collections */ for (int cpu = 0; cpu <= mp_maxid; cpu++) if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) sc->sc_its_cols[cpu] = malloc_domainset( sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), M_WAITOK | M_ZERO); else sc->sc_its_cols[cpu] = NULL; /* Enable the ITS */ gic_its_write_4(sc, GITS_CTLR, gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); /* Create the LPI configuration table */ gicv3_its_conftable_init(sc); /* And the pending tebles */ gicv3_its_pendtables_init(sc); /* Enable LPIs on this CPU */ its_init_cpu(dev, sc); TAILQ_INIT(&sc->sc_its_dev_list); TAILQ_INIT(&sc->sc_free_irqs); /* * Create the vmem object to allocate INTRNG IRQs from. We try to * use all IRQs not already used by the GICv3. * XXX: This assumes there are no other interrupt controllers in the * system. */ sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0, gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK); sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, M_GICV3_ITS, M_WAITOK | M_ZERO); /* For GIC-500 install tracking sysctls. */ if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) == GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0)) gicv3_its_init_sysctl(sc); return (0); } static int gicv3_its_detach(device_t dev) { return (ENXIO); } static void its_quirk_cavium_22375(device_t dev) { struct gicv3_its_softc *sc; int domain; sc = device_get_softc(dev); sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; /* * We need to limit which CPUs we send these interrupts to on * the original dual socket ThunderX as it is unable to * forward them between the two sockets. */ if (bus_get_domain(dev, &domain) == 0) { if (domain < MAXMEMDOM) { CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); } else { CPU_ZERO(&sc->sc_cpus); } } } static void gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; uint8_t *conf; sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; conf = sc->sc_conf_base; conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE; if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { /* Clean D-cache under command. */ cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); } else { /* DSB inner shareable, store */ dsb(ishst); } its_cmd_inv(dev, girq->gi_its_dev, girq); } static void gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; uint8_t *conf; sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; conf = sc->sc_conf_base; conf[girq->gi_lpi] |= LPI_CONF_ENABLE; if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { /* Clean D-cache under command. */ cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); } else { /* DSB inner shareable, store */ dsb(ishst); } its_cmd_inv(dev, girq->gi_its_dev, girq); } static int gicv3_its_intr(void *arg, uintptr_t irq) { struct gicv3_its_softc *sc = arg; struct gicv3_its_irqsrc *girq; struct trapframe *tf; irq -= sc->sc_irq_base; girq = sc->sc_irqs[irq]; if (girq == NULL) panic("gicv3_its_intr: Invalid interrupt %ld", irq + sc->sc_irq_base); tf = curthread->td_intr_frame; intr_isrc_dispatch(&girq->gi_isrc, tf); return (FILTER_HANDLED); } static void gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_irqsrc *girq; struct gicv3_its_softc *sc; sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); } static void gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) { } static void gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_irqsrc *girq; struct gicv3_its_softc *sc; sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); } static int gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_softc *sc; sc = device_get_softc(dev); if (CPU_EMPTY(&isrc->isrc_cpu)) { sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, &sc->sc_cpus); CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); } return (0); } static int gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct gicv3_its_irqsrc *girq; gicv3_its_select_cpu(dev, isrc); girq = (struct gicv3_its_irqsrc *)isrc; its_cmd_movi(dev, girq); return (0); } static int gicv3_its_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { /* * This should never happen, we only call this function to map * interrupts found before the controller driver is ready. */ panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); } static int gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { /* Bind the interrupt to a CPU */ gicv3_its_bind_intr(dev, isrc); return (0); } #ifdef SMP static void gicv3_its_init_secondary(device_t dev) { struct gicv3_its_softc *sc; sc = device_get_softc(dev); /* * This is fatal as otherwise we may bind interrupts to this CPU. * We need a way to tell the interrupt framework to only bind to a * subset of given CPUs when it performs the shuffle. */ if (its_init_cpu(dev, sc) != 0) panic("gicv3_its_init_secondary: No usable ITS on CPU%d", PCPU_GET(cpuid)); } #endif static uint32_t its_get_devid(device_t pci_dev) { uintptr_t id; if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) panic("its_get_devid: Unable to get the MSI DeviceID"); return (id); } static struct its_dev * its_device_find(device_t dev, device_t child) { struct gicv3_its_softc *sc; struct its_dev *its_dev = NULL; sc = device_get_softc(dev); mtx_lock_spin(&sc->sc_its_dev_lock); TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { if (its_dev->pci_dev == child) break; } mtx_unlock_spin(&sc->sc_its_dev_lock); return (its_dev); } static struct its_dev * its_device_get(device_t dev, device_t child, u_int nvecs) { struct gicv3_its_softc *sc; struct its_dev *its_dev; vmem_addr_t irq_base; size_t esize; sc = device_get_softc(dev); its_dev = its_device_find(dev, child); if (its_dev != NULL) return (its_dev); its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); if (its_dev == NULL) return (NULL); its_dev->pci_dev = child; its_dev->devid = its_get_devid(child); its_dev->lpis.lpi_busy = 0; its_dev->lpis.lpi_num = nvecs; its_dev->lpis.lpi_free = nvecs; if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, &irq_base) != 0) { free(its_dev, M_GICV3_ITS); return (NULL); } its_dev->lpis.lpi_base = irq_base; /* Get ITT entry size */ esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); /* * Allocate ITT for this device. * PA has to be 256 B aligned. At least two entries for device. */ its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size, M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); if (its_dev->itt == 0) { vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); free(its_dev, M_GICV3_ITS); return (NULL); } /* Make sure device sees zeroed ITT. */ if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size); mtx_lock_spin(&sc->sc_its_dev_lock); TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); mtx_unlock_spin(&sc->sc_its_dev_lock); /* Map device to its ITT */ its_cmd_mapd(dev, its_dev, 1); return (its_dev); } static void its_device_release(device_t dev, struct its_dev *its_dev) { struct gicv3_its_softc *sc; KASSERT(its_dev->lpis.lpi_busy == 0, ("its_device_release: Trying to release an inuse ITS device")); /* Unmap device in ITS */ its_cmd_mapd(dev, its_dev, 0); sc = device_get_softc(dev); /* Remove the device from the list of devices */ mtx_lock_spin(&sc->sc_its_dev_lock); TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); mtx_unlock_spin(&sc->sc_its_dev_lock); /* Free ITT */ KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); /* Free the IRQ allocation */ vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, its_dev->lpis.lpi_num); free(its_dev, M_GICV3_ITS); } static struct gicv3_its_irqsrc * gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq) { struct gicv3_its_irqsrc *girq = NULL; KASSERT(sc->sc_irqs[irq] == NULL, ("%s: Interrupt %u already allocated", __func__, irq)); mtx_lock_spin(&sc->sc_its_dev_lock); if (!TAILQ_EMPTY(&sc->sc_free_irqs)) { girq = TAILQ_FIRST(&sc->sc_free_irqs); TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link); } mtx_unlock_spin(&sc->sc_its_dev_lock); if (girq == NULL) { girq = malloc(sizeof(*girq), M_GICV3_ITS, M_NOWAIT | M_ZERO); if (girq == NULL) return (NULL); girq->gi_id = -1; if (intr_isrc_register(&girq->gi_isrc, dev, 0, "%s,%u", device_get_nameunit(dev), irq) != 0) { free(girq, M_GICV3_ITS); return (NULL); } } girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI; sc->sc_irqs[irq] = girq; return (girq); } static void gicv3_its_release_irqsrc(struct gicv3_its_softc *sc, struct gicv3_its_irqsrc *girq) { u_int irq; mtx_assert(&sc->sc_its_dev_lock, MA_OWNED); irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base; sc->sc_irqs[irq] = NULL; girq->gi_id = -1; girq->gi_its_dev = NULL; TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link); } static int gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, device_t *pic, struct intr_irqsrc **srcs) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; struct its_dev *its_dev; u_int irq; int i; its_dev = its_device_get(dev, child, count); if (its_dev == NULL) return (ENXIO); KASSERT(its_dev->lpis.lpi_free >= count, ("gicv3_its_alloc_msi: No free LPIs")); sc = device_get_softc(dev); irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - its_dev->lpis.lpi_free; /* Allocate the irqsrc for each MSI */ for (i = 0; i < count; i++, irq++) { its_dev->lpis.lpi_free--; srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev, sc, irq); if (srcs[i] == NULL) break; } /* The allocation failed, release them */ if (i != count) { mtx_lock_spin(&sc->sc_its_dev_lock); for (i = 0; i < count; i++) { girq = (struct gicv3_its_irqsrc *)srcs[i]; if (girq == NULL) break; gicv3_its_release_irqsrc(sc, girq); srcs[i] = NULL; } mtx_unlock_spin(&sc->sc_its_dev_lock); return (ENXIO); } /* Finish the allocation now we have all MSI irqsrcs */ for (i = 0; i < count; i++) { girq = (struct gicv3_its_irqsrc *)srcs[i]; girq->gi_id = i; girq->gi_its_dev = its_dev; /* Map the message to the given IRQ */ gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); its_cmd_mapti(dev, girq); } its_dev->lpis.lpi_busy += count; *pic = dev; return (0); } static int gicv3_its_release_msi(device_t dev, device_t child, int count, struct intr_irqsrc **isrc) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; struct its_dev *its_dev; int i; its_dev = its_device_find(dev, child); KASSERT(its_dev != NULL, ("gicv3_its_release_msi: Releasing a MSI interrupt with " "no ITS device")); KASSERT(its_dev->lpis.lpi_busy >= count, ("gicv3_its_release_msi: Releasing more interrupts than " "were allocated: releasing %d, allocated %d", count, its_dev->lpis.lpi_busy)); sc = device_get_softc(dev); mtx_lock_spin(&sc->sc_its_dev_lock); for (i = 0; i < count; i++) { girq = (struct gicv3_its_irqsrc *)isrc[i]; gicv3_its_release_irqsrc(sc, girq); } mtx_unlock_spin(&sc->sc_its_dev_lock); its_dev->lpis.lpi_busy -= count; if (its_dev->lpis.lpi_busy == 0) its_device_release(dev, its_dev); return (0); } static int gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, struct intr_irqsrc **isrcp) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; struct its_dev *its_dev; u_int nvecs, irq; nvecs = pci_msix_count(child); its_dev = its_device_get(dev, child, nvecs); if (its_dev == NULL) return (ENXIO); KASSERT(its_dev->lpis.lpi_free > 0, ("gicv3_its_alloc_msix: No free LPIs")); sc = device_get_softc(dev); irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - its_dev->lpis.lpi_free; girq = gicv3_its_alloc_irqsrc(dev, sc, irq); if (girq == NULL) return (ENXIO); girq->gi_id = its_dev->lpis.lpi_busy; girq->gi_its_dev = its_dev; its_dev->lpis.lpi_free--; its_dev->lpis.lpi_busy++; /* Map the message to the given IRQ */ gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); its_cmd_mapti(dev, girq); *pic = dev; *isrcp = (struct intr_irqsrc *)girq; return (0); } static int gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; struct its_dev *its_dev; its_dev = its_device_find(dev, child); KASSERT(its_dev != NULL, ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " "no ITS device")); KASSERT(its_dev->lpis.lpi_busy > 0, ("gicv3_its_release_msix: Releasing more interrupts than " "were allocated: allocated %d", its_dev->lpis.lpi_busy)); sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; mtx_lock_spin(&sc->sc_its_dev_lock); gicv3_its_release_irqsrc(sc, girq); mtx_unlock_spin(&sc->sc_its_dev_lock); its_dev->lpis.lpi_busy--; if (its_dev->lpis.lpi_busy == 0) its_device_release(dev, its_dev); return (0); } static int gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, uint64_t *addr, uint32_t *data) { struct gicv3_its_softc *sc; struct gicv3_its_irqsrc *girq; sc = device_get_softc(dev); girq = (struct gicv3_its_irqsrc *)isrc; *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; *data = girq->gi_id; return (0); } #ifdef IOMMU static int gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain) { struct gicv3_its_softc *sc; struct iommu_ctx *ctx; int error; sc = device_get_softc(dev); ctx = iommu_get_dev_ctx(child); error = iommu_map_msi(ctx, PAGE_SIZE, GITS_TRANSLATER, IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma); *domain = iommu_get_ctx_domain(ctx); return (error); } static void gicv3_iommu_deinit(device_t dev, device_t child) { struct iommu_ctx *ctx; ctx = iommu_get_dev_ctx(child); iommu_unmap_msi(ctx); } #endif /* * Commands handling. */ static __inline void cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) { /* Command field: DW0 [7:0] */ cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); cmd->cmd_dword[0] |= htole64(cmd_type); } static __inline void cmd_format_devid(struct its_cmd *cmd, uint32_t devid) { /* Device ID field: DW0 [63:32] */ cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); } static __inline void cmd_format_size(struct its_cmd *cmd, uint16_t size) { /* Size field: DW1 [4:0] */ cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); } static __inline void cmd_format_id(struct its_cmd *cmd, uint32_t id) { /* ID field: DW1 [31:0] */ cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); cmd->cmd_dword[1] |= htole64(id); } static __inline void cmd_format_pid(struct its_cmd *cmd, uint32_t pid) { /* Physical ID field: DW1 [63:32] */ cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); } static __inline void cmd_format_col(struct its_cmd *cmd, uint16_t col_id) { /* Collection field: DW2 [16:0] */ cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); cmd->cmd_dword[2] |= htole64(col_id); } static __inline void cmd_format_target(struct its_cmd *cmd, uint64_t target) { /* Target Address field: DW2 [47:16] */ cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); } static __inline void cmd_format_itt(struct its_cmd *cmd, uint64_t itt) { /* ITT Address field: DW2 [47:8] */ cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); } static __inline void cmd_format_valid(struct its_cmd *cmd, uint8_t valid) { /* Valid field: DW2 [63] */ cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); } static inline bool its_cmd_queue_full(struct gicv3_its_softc *sc) { size_t read_idx, next_write_idx; /* Get the index of the next command */ next_write_idx = (sc->sc_its_cmd_next_idx + 1) % (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); /* And the index of the current command being read */ read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); /* * The queue is full when the write offset points * at the command before the current read offset. */ return (next_write_idx == read_idx); } static inline void its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) { if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { /* Clean D-cache under command. */ cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); } else { /* DSB inner shareable, store */ dsb(ishst); } } static inline uint64_t its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) { uint64_t off; off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); return (off); } static void its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, struct its_cmd *cmd_last) { struct gicv3_its_softc *sc; uint64_t first, last, read; size_t us_left; sc = device_get_softc(dev); /* * XXX ARM64TODO: This is obviously a significant delay. * The reason for that is that currently the time frames for * the command to complete are not known. */ us_left = 1000000; first = its_cmd_cwriter_offset(sc, cmd_first); last = its_cmd_cwriter_offset(sc, cmd_last); for (;;) { read = gic_its_read_8(sc, GITS_CREADR); if (first < last) { if (read < first || read >= last) break; } else if (read < first && read >= last) break; if (us_left-- == 0) { /* This means timeout */ device_printf(dev, "Timeout while waiting for CMD completion.\n"); return; } DELAY(1); } } static struct its_cmd * its_cmd_alloc_locked(device_t dev) { struct gicv3_its_softc *sc; struct its_cmd *cmd; size_t us_left; sc = device_get_softc(dev); /* * XXX ARM64TODO: This is obviously a significant delay. * The reason for that is that currently the time frames for * the command to complete (and therefore free the descriptor) * are not known. */ us_left = 1000000; mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); while (its_cmd_queue_full(sc)) { if (us_left-- == 0) { /* Timeout while waiting for free command */ device_printf(dev, "Timeout while waiting for free command\n"); return (NULL); } DELAY(1); } cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; sc->sc_its_cmd_next_idx++; sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); return (cmd); } static uint64_t its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) { uint64_t target; uint8_t cmd_type; u_int size; cmd_type = desc->cmd_type; target = ITS_TARGET_NONE; switch (cmd_type) { case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ target = desc->cmd_desc_movi.col->col_target; cmd_format_command(cmd, ITS_CMD_MOVI); cmd_format_id(cmd, desc->cmd_desc_movi.id); cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); break; case ITS_CMD_SYNC: /* Wait for previous commands completion */ target = desc->cmd_desc_sync.col->col_target; cmd_format_command(cmd, ITS_CMD_SYNC); cmd_format_target(cmd, target); break; case ITS_CMD_MAPD: /* Assign ITT to device */ cmd_format_command(cmd, ITS_CMD_MAPD); cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); /* * Size describes number of bits to encode interrupt IDs * supported by the device minus one. * When V (valid) bit is zero, this field should be written * as zero. */ if (desc->cmd_desc_mapd.valid != 0) { size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); size = MAX(1, size) - 1; } else size = 0; cmd_format_size(cmd, size); cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); break; case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ target = desc->cmd_desc_mapc.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPC); cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); cmd_format_target(cmd, target); break; case ITS_CMD_MAPTI: target = desc->cmd_desc_mapvi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPTI); cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_mapvi.id); cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); break; case ITS_CMD_MAPI: target = desc->cmd_desc_mapi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPI); cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_mapi.pid); cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); break; case ITS_CMD_INV: target = desc->cmd_desc_inv.col->col_target; cmd_format_command(cmd, ITS_CMD_INV); cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_inv.pid); break; case ITS_CMD_INVALL: cmd_format_command(cmd, ITS_CMD_INVALL); cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); break; default: panic("its_cmd_prepare: Invalid command: %x", cmd_type); } return (target); } static int its_cmd_send(device_t dev, struct its_cmd_desc *desc) { struct gicv3_its_softc *sc; struct its_cmd *cmd, *cmd_sync, *cmd_write; struct its_col col_sync; struct its_cmd_desc desc_sync; uint64_t target, cwriter; sc = device_get_softc(dev); mtx_lock_spin(&sc->sc_its_cmd_lock); cmd = its_cmd_alloc_locked(dev); if (cmd == NULL) { device_printf(dev, "could not allocate ITS command\n"); mtx_unlock_spin(&sc->sc_its_cmd_lock); return (EBUSY); } target = its_cmd_prepare(cmd, desc); its_cmd_sync(sc, cmd); if (target != ITS_TARGET_NONE) { cmd_sync = its_cmd_alloc_locked(dev); if (cmd_sync != NULL) { desc_sync.cmd_type = ITS_CMD_SYNC; col_sync.col_target = target; desc_sync.cmd_desc_sync.col = &col_sync; its_cmd_prepare(cmd_sync, &desc_sync); its_cmd_sync(sc, cmd_sync); } } /* Update GITS_CWRITER */ cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); gic_its_write_8(sc, GITS_CWRITER, cwriter); cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; mtx_unlock_spin(&sc->sc_its_cmd_lock); its_cmd_wait_completion(dev, cmd, cmd_write); return (0); } /* Handlers to send commands */ static void its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) { struct gicv3_its_softc *sc; struct its_cmd_desc desc; struct its_col *col; sc = device_get_softc(dev); col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; desc.cmd_type = ITS_CMD_MOVI; desc.cmd_desc_movi.its_dev = girq->gi_its_dev; desc.cmd_desc_movi.col = col; desc.cmd_desc_movi.id = girq->gi_id; its_cmd_send(dev, &desc); } static void its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_MAPC; desc.cmd_desc_mapc.col = col; /* * Valid bit set - map the collection. * Valid bit cleared - unmap the collection. */ desc.cmd_desc_mapc.valid = valid; its_cmd_send(dev, &desc); } static void its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) { struct gicv3_its_softc *sc; struct its_cmd_desc desc; struct its_col *col; u_int col_id; sc = device_get_softc(dev); col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; col = sc->sc_its_cols[col_id]; desc.cmd_type = ITS_CMD_MAPTI; desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; desc.cmd_desc_mapvi.col = col; /* The EventID sent to the device */ desc.cmd_desc_mapvi.id = girq->gi_id; /* The physical interrupt presented to softeware */ desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI; its_cmd_send(dev, &desc); } static void its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_MAPD; desc.cmd_desc_mapd.its_dev = its_dev; desc.cmd_desc_mapd.valid = valid; its_cmd_send(dev, &desc); } static void its_cmd_inv(device_t dev, struct its_dev *its_dev, struct gicv3_its_irqsrc *girq) { struct gicv3_its_softc *sc; struct its_cmd_desc desc; struct its_col *col; sc = device_get_softc(dev); col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; desc.cmd_type = ITS_CMD_INV; /* The EventID sent to the device */ desc.cmd_desc_inv.pid = girq->gi_id; desc.cmd_desc_inv.its_dev = its_dev; desc.cmd_desc_inv.col = col; its_cmd_send(dev, &desc); } static void its_cmd_invall(device_t dev, struct its_col *col) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_INVALL; desc.cmd_desc_invall.col = col; its_cmd_send(dev, &desc); } #ifdef FDT static device_probe_t gicv3_its_fdt_probe; static device_attach_t gicv3_its_fdt_attach; static device_method_t gicv3_its_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gicv3_its_fdt_probe), DEVMETHOD(device_attach, gicv3_its_fdt_attach), /* End */ DEVMETHOD_END }; #define its_baseclasses its_fdt_baseclasses DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, sizeof(struct gicv3_its_softc), gicv3_its_driver); #undef its_baseclasses static devclass_t gicv3_its_fdt_devclass; EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); static int gicv3_its_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) return (ENXIO); device_set_desc(dev, "ARM GIC Interrupt Translation Service"); return (BUS_PROBE_DEFAULT); } static int gicv3_its_fdt_attach(device_t dev) { struct gicv3_its_softc *sc; phandle_t xref; int err; sc = device_get_softc(dev); sc->dev = dev; err = gicv3_its_attach(dev); if (err != 0) return (err); /* Register this device as a interrupt controller */ xref = OF_xref_from_node(ofw_bus_get_node(dev)); sc->sc_pic = intr_pic_register(dev, xref); intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); /* Register this device to handle MSI interrupts */ intr_msi_register(dev, xref); return (0); } #endif #ifdef DEV_ACPI static device_probe_t gicv3_its_acpi_probe; static device_attach_t gicv3_its_acpi_attach; static device_method_t gicv3_its_acpi_methods[] = { /* Device interface */ DEVMETHOD(device_probe, gicv3_its_acpi_probe), DEVMETHOD(device_attach, gicv3_its_acpi_attach), /* End */ DEVMETHOD_END }; #define its_baseclasses its_acpi_baseclasses DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, sizeof(struct gicv3_its_softc), gicv3_its_driver); #undef its_baseclasses static devclass_t gicv3_its_acpi_devclass; EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); static int gicv3_its_acpi_probe(device_t dev) { if (gic_get_bus(dev) != GIC_BUS_ACPI) return (EINVAL); if (gic_get_hw_rev(dev) < 3) return (EINVAL); device_set_desc(dev, "ARM GIC Interrupt Translation Service"); return (BUS_PROBE_DEFAULT); } static int gicv3_its_acpi_attach(device_t dev) { struct gicv3_its_softc *sc; struct gic_v3_devinfo *di; int err; sc = device_get_softc(dev); sc->dev = dev; err = gicv3_its_attach(dev); if (err != 0) return (err); di = device_get_ivars(dev); sc->sc_pic = intr_pic_register(dev, di->msi_xref); intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); /* Register this device to handle MSI interrupts */ intr_msi_register(dev, di->msi_xref); return (0); } #endif diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c index 3b1a8cc89cab..63715c64641c 100644 --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -1,932 +1,931 @@ /*- * Copyright (c) 2015-2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_acpi.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #ifdef DEV_ACPI #include #include #endif #ifdef FDT #include #include #include #include #endif #include #include "pic_if.h" #define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */ /* don't panic if one fails to start */ static uint32_t mp_quirks; #ifdef FDT static struct { const char *compat; uint32_t quirks; } fdt_quirks[] = { { "arm,foundation-aarch64", MP_QUIRK_CPULIST }, { "arm,fvp-base", MP_QUIRK_CPULIST }, /* This is incorrect in some DTS files */ { "arm,vfp-base", MP_QUIRK_CPULIST }, { NULL, 0 }, }; #endif typedef void intr_ipi_send_t(void *, cpuset_t, u_int); typedef void intr_ipi_handler_t(void *); #define INTR_IPI_NAMELEN (MAXCOMLEN + 1) struct intr_ipi { intr_ipi_handler_t * ii_handler; void * ii_handler_arg; intr_ipi_send_t * ii_send; void * ii_send_arg; char ii_name[INTR_IPI_NAMELEN]; u_long * ii_count; }; static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; static struct intr_ipi *intr_ipi_lookup(u_int); static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *); static void ipi_ast(void *); static void ipi_hardclock(void *); static void ipi_preempt(void *); static void ipi_rendezvous(void *); static void ipi_stop(void *); struct pcb stoppcbs[MAXCPU]; #ifdef FDT static u_int fdt_cpuid; #endif void mpentry(unsigned long cpuid); void init_secondary(uint64_t); /* Synchronize AP startup. */ static struct mtx ap_boot_mtx; /* Stacks for AP initialization, discarded once idle threads are started. */ void *bootstack; static void *bootstacks[MAXCPU]; /* Count of started APs, used to synchronize access to bootstack. */ static volatile int aps_started; /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready; /* Temporary variables for init_secondary() */ void *dpcpu[MAXCPU - 1]; static bool is_boot_cpu(uint64_t target_cpu) { return (__pcpu[0].pc_mpidr == (target_cpu & CPU_AFF_MASK)); } static void release_aps(void *dummy __unused) { int i, started; /* Only release CPUs if they exist */ if (mp_ncpus == 1) return; intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); /* Wake up the other CPUs */ __asm __volatile( "dsb ishst \n" "sev \n" ::: "memory"); printf("Release APs..."); started = 0; for (i = 0; i < 2000; i++) { if (smp_started) { printf("done\n"); return; } /* * Don't time out while we are making progress. Some large * systems can take a while to start all CPUs. */ if (smp_cpus > started) { i = 0; started = smp_cpus; } DELAY(1000); } printf("APs not started\n"); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); void init_secondary(uint64_t cpu) { struct pcpu *pcpup; pmap_t pmap0; u_int mpidr; /* * Verify that the value passed in 'cpu' argument (aka context_id) is * valid. Some older U-Boot based PSCI implementations are buggy, * they can pass random value in it. */ mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; if (cpu >= MAXCPU || __pcpu[cpu].pc_mpidr != mpidr) { for (cpu = 0; cpu < mp_maxid; cpu++) if (__pcpu[cpu].pc_mpidr == mpidr) break; if ( cpu >= MAXCPU) panic("MPIDR for this CPU is not in pcpu table"); } pcpup = &__pcpu[cpu]; /* * Set the pcpu pointer with a backup in tpidr_el1 to be * loaded when entering the kernel from userland. */ __asm __volatile( "mov x18, %0 \n" "msr tpidr_el1, %0" :: "r"(pcpup)); /* * Identify current CPU. This is necessary to setup * affinity registers and to provide support for * runtime chip identification. * * We need this before signalling the CPU is ready to * let the boot CPU use the results. */ identify_cpu(cpu); /* Ensure the stores in identify_cpu have completed */ atomic_thread_fence_acq_rel(); /* Signal the BSP and spin until it has released all APs. */ atomic_add_int(&aps_started, 1); while (!atomic_load_int(&aps_ready)) __asm __volatile("wfe"); pcpup->pc_midr = get_midr(); /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pcpup->pc_curthread = pcpup->pc_idlethread; /* Initialize curpmap to match TTBR0's current setting. */ pmap0 = vmspace_pmap(&vmspace0); KASSERT(pmap_to_ttbr0(pmap0) == READ_SPECIALREG(ttbr0_el1), ("pmap0 doesn't match cpu %ld's ttbr0", cpu)); pcpup->pc_curpmap = pmap0; install_cpu_errata(); intr_pic_init_secondary(); /* Start per-CPU event timers. */ cpu_initclocks_ap(); #ifdef VFP vfp_init(); #endif dbg_init(); pan_enable(); mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); } mtx_unlock_spin(&ap_boot_mtx); kcsan_cpu_init(cpu); /* * Assert that smp_after_idle_runnable condition is reasonable. */ MPASS(PCPU_GET(curpcb) == NULL); /* Enter the scheduler */ sched_throw(NULL); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ } static void smp_after_idle_runnable(void *arg __unused) { struct pcpu *pc; int cpu; for (cpu = 1; cpu < mp_ncpus; cpu++) { if (bootstacks[cpu] != NULL) { pc = pcpu_find(cpu); while (atomic_load_ptr(&pc->pc_curpcb) == NULL) cpu_spinwait(); kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE); } } } SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); /* * Send IPI thru interrupt controller. */ static void pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi) { KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); /* * Ensure that this CPU's stores will be visible to IPI * recipients before starting to send the interrupts. */ dsb(ishst); PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi); } /* * Setup IPI handler on interrupt controller. * * Not SMP coherent. */ static void intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, void *arg) { struct intr_irqsrc *isrc; struct intr_ipi *ii; int error; KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); if (error != 0) return; isrc->isrc_handlers++; ii = intr_ipi_lookup(ipi); KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); ii->ii_handler = hand; ii->ii_handler_arg = arg; ii->ii_send = pic_ipi_send; ii->ii_send_arg = isrc; strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); ii->ii_count = intr_ipi_setup_counters(name); PIC_ENABLE_INTR(intr_irq_root_dev, isrc); } static void intr_ipi_send(cpuset_t cpus, u_int ipi) { struct intr_ipi *ii; ii = intr_ipi_lookup(ipi); if (ii->ii_count == NULL) panic("%s: not setup IPI %u", __func__, ipi); ii->ii_send(ii->ii_send_arg, cpus, ipi); } static void ipi_ast(void *dummy __unused) { CTR0(KTR_SMP, "IPI_AST"); } static void ipi_hardclock(void *dummy __unused) { CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); } static void ipi_preempt(void *dummy __unused) { CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(curthread); } static void ipi_rendezvous(void *dummy __unused) { CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); } static void ipi_stop(void *dummy __unused) { u_int cpu; CTR0(KTR_SMP, "IPI_STOP"); cpu = PCPU_GET(cpuid); savectx(&stoppcbs[cpu]); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); #ifdef DDB dbg_register_sync(NULL); #endif CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); CTR0(KTR_SMP, "IPI_STOP (restart)"); } struct cpu_group * cpu_topo(void) { struct cpu_group *dom, *root; int i; root = smp_topo_alloc(1); dom = smp_topo_alloc(vm_ndomains); root->cg_parent = NULL; root->cg_child = dom; CPU_COPY(&all_cpus, &root->cg_mask); root->cg_count = mp_ncpus; root->cg_children = vm_ndomains; root->cg_level = CG_SHARE_NONE; root->cg_flags = 0; /* * Redundant layers will be collapsed by the caller so we don't need a * special case for a single domain. */ for (i = 0; i < vm_ndomains; i++, dom++) { dom->cg_parent = root; dom->cg_child = NULL; CPU_COPY(&cpuset_domain[i], &dom->cg_mask); dom->cg_count = CPU_COUNT(&dom->cg_mask); dom->cg_children = 0; dom->cg_level = CG_SHARE_L3; dom->cg_flags = 0; } return (root); } /* Determine if we running MP machine */ int cpu_mp_probe(void) { /* ARM64TODO: Read the u bit of mpidr_el1 to determine this */ return (1); } /* * Starts a given CPU. If the CPU is already running, i.e. it is the boot CPU, * do nothing. Returns true if the CPU is present and running. */ static bool start_cpu(u_int cpuid, uint64_t target_cpu) { struct pcpu *pcpup; vm_paddr_t pa; int err, naps; /* Check we are able to start this cpu */ if (cpuid > mp_maxid) return (false); /* Skip boot CPU */ if (is_boot_cpu(target_cpu)) return (true); KASSERT(cpuid < MAXCPU, ("Too many CPUs")); pcpup = &__pcpu[cpuid]; pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); pcpup->pc_mpidr = target_cpu & CPU_AFF_MASK; dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); dpcpu_init(dpcpu[cpuid - 1], cpuid); bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); naps = atomic_load_int(&aps_started); bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE; printf("Starting CPU %u (%lx)\n", cpuid, target_cpu); pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry); err = psci_cpu_on(target_cpu, pa, cpuid); if (err != PSCI_RETVAL_SUCCESS) { /* * Panic here if INVARIANTS are enabled and PSCI failed to * start the requested CPU. psci_cpu_on() returns PSCI_MISSING * to indicate we are unable to use it to start the given CPU. */ KASSERT(err == PSCI_MISSING || (mp_quirks & MP_QUIRK_CPULIST) == MP_QUIRK_CPULIST, ("Failed to start CPU %u (%lx), error %d\n", cpuid, target_cpu, err)); pcpu_destroy(pcpup); kmem_free((vm_offset_t)dpcpu[cpuid - 1], DPCPU_SIZE); dpcpu[cpuid - 1] = NULL; kmem_free((vm_offset_t)bootstacks[cpuid], PAGE_SIZE); bootstacks[cpuid] = NULL; mp_ncpus--; return (false); } /* Wait for the AP to switch to its boot stack. */ while (atomic_load_int(&aps_started) < naps + 1) cpu_spinwait(); CPU_SET(cpuid, &all_cpus); return (true); } #ifdef DEV_ACPI static void madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_INTERRUPT *intr; u_int *cpuid; u_int id; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_INTERRUPT: intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; cpuid = arg; if (is_boot_cpu(intr->ArmMpidr)) id = 0; else id = *cpuid; if (start_cpu(id, intr->ArmMpidr)) { __pcpu[id].pc_acpi_id = intr->Uid; /* * Don't increment for the boot CPU, its CPU ID is * reserved. */ if (!is_boot_cpu(intr->ArmMpidr)) (*cpuid)++; } break; default: break; } } static void cpu_init_acpi(void) { ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; u_int cpuid; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return; madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { printf("Unable to map the MADT, not starting APs\n"); return; } /* Boot CPU is always 0 */ cpuid = 1; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, madt_handler, &cpuid); acpi_unmap_table(madt); #if MAXMEMDOM > 1 acpi_pxm_set_cpu_locality(); #endif } #endif #ifdef FDT static boolean_t start_cpu_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) { uint64_t target_cpu; int domain; int cpuid; target_cpu = reg[0]; if (addr_size == 2) { target_cpu <<= 32; target_cpu |= reg[1]; } if (is_boot_cpu(target_cpu)) cpuid = 0; else cpuid = fdt_cpuid; if (!start_cpu(cpuid, target_cpu)) return (FALSE); /* * Don't increment for the boot CPU, its CPU ID is reserved. */ if (!is_boot_cpu(target_cpu)) fdt_cpuid++; /* Try to read the numa node of this cpu */ if (vm_ndomains == 1 || OF_getencprop(node, "numa-node-id", &domain, sizeof(domain)) <= 0) domain = 0; __pcpu[cpuid].pc_domain = domain; if (domain < MAXMEMDOM) CPU_SET(cpuid, &cpuset_domain[domain]); return (TRUE); } static void cpu_init_fdt(void) { phandle_t node; int i; node = OF_peer(0); for (i = 0; fdt_quirks[i].compat != NULL; i++) { if (ofw_bus_node_is_compatible(node, fdt_quirks[i].compat) != 0) { mp_quirks = fdt_quirks[i].quirks; } } fdt_cpuid = 1; ofw_cpu_early_foreach(start_cpu_fdt, true); } #endif /* Initialize and fire up non-boot processors */ void cpu_mp_start(void) { mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* CPU 0 is always boot CPU. */ CPU_SET(0, &all_cpus); __pcpu[0].pc_mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; switch(arm64_bus_method) { #ifdef DEV_ACPI case ARM64_BUS_ACPI: mp_quirks = MP_QUIRK_CPULIST; cpu_init_acpi(); break; #endif #ifdef FDT case ARM64_BUS_FDT: cpu_init_fdt(); break; #endif default: break; } } /* Introduce rest of cores to the world */ void cpu_mp_announce(void) { } #ifdef DEV_ACPI static void cpu_count_acpi_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) { ACPI_MADT_GENERIC_INTERRUPT *intr; u_int *cores = arg; switch(entry->Type) { case ACPI_MADT_TYPE_GENERIC_INTERRUPT: intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; (*cores)++; break; default: break; } } static u_int cpu_count_acpi(void) { ACPI_TABLE_MADT *madt; vm_paddr_t physaddr; u_int cores; physaddr = acpi_find_table(ACPI_SIG_MADT); if (physaddr == 0) return (0); madt = acpi_map_table(physaddr, ACPI_SIG_MADT); if (madt == NULL) { printf("Unable to map the MADT, not starting APs\n"); return (0); } cores = 0; acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, cpu_count_acpi_handler, &cores); acpi_unmap_table(madt); return (cores); } #endif void cpu_mp_setmaxid(void) { int cores; mp_ncpus = 1; mp_maxid = 0; switch(arm64_bus_method) { #ifdef DEV_ACPI case ARM64_BUS_ACPI: cores = cpu_count_acpi(); if (cores > 0) { cores = MIN(cores, MAXCPU); if (bootverbose) printf("Found %d CPUs in the ACPI tables\n", cores); mp_ncpus = cores; mp_maxid = cores - 1; } break; #endif #ifdef FDT case ARM64_BUS_FDT: cores = ofw_cpu_early_foreach(NULL, false); if (cores > 0) { cores = MIN(cores, MAXCPU); if (bootverbose) printf("Found %d CPUs in the device tree\n", cores); mp_ncpus = cores; mp_maxid = cores - 1; } break; #endif default: if (bootverbose) printf("No CPU data, limiting to 1 core\n"); break; } if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { if (cores > 0 && cores < mp_ncpus) { mp_ncpus = cores; mp_maxid = cores - 1; } } } /* * Lookup IPI source. */ static struct intr_ipi * intr_ipi_lookup(u_int ipi) { if (ipi >= INTR_IPI_COUNT) panic("%s: no such IPI %u", __func__, ipi); return (&ipi_sources[ipi]); } /* * interrupt controller dispatch function for IPIs. It should * be called straight from the interrupt controller, when associated * interrupt source is learned. Or from anybody who has an interrupt * source mapped. */ void intr_ipi_dispatch(u_int ipi, struct trapframe *tf) { void *arg; struct intr_ipi *ii; ii = intr_ipi_lookup(ipi); if (ii->ii_count == NULL) panic("%s: not setup IPI %u", __func__, ipi); intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); /* * Supply ipi filter with trapframe argument * if none is registered. */ arg = ii->ii_handler_arg != NULL ? ii->ii_handler_arg : tf; ii->ii_handler(arg); } #ifdef notyet /* * Map IPI into interrupt controller. * * Not SMP coherent. */ static int ipi_map(struct intr_irqsrc *isrc, u_int ipi) { boolean_t is_percpu; int error; if (ipi >= INTR_IPI_COUNT) panic("%s: no such IPI %u", __func__, ipi); KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); isrc->isrc_type = INTR_ISRCT_NAMESPACE; isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; isrc->isrc_nspc_num = ipi_next_num; error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu); if (error == 0) { isrc->isrc_dev = intr_irq_root_dev; ipi_next_num++; } return (error); } /* * Setup IPI handler to interrupt source. * * Note that there could be more ways how to send and receive IPIs * on a platform like fast interrupts for example. In that case, * one can call this function with ASIF_NOALLOC flag set and then * call intr_ipi_dispatch() when appropriate. * * Not SMP coherent. */ int intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, void *arg, u_int flags) { struct intr_irqsrc *isrc; int error; if (filter == NULL) return(EINVAL); isrc = intr_ipi_lookup(ipi); if (isrc->isrc_ipifilter != NULL) return (EEXIST); if ((flags & AISHF_NOALLOC) == 0) { error = ipi_map(isrc, ipi); if (error != 0) return (error); } isrc->isrc_ipifilter = filter; isrc->isrc_arg = arg; isrc->isrc_handlers = 1; isrc->isrc_count = intr_ipi_setup_counters(name); isrc->isrc_index = 0; /* it should not be used in IPI case */ if (isrc->isrc_dev != NULL) { PIC_ENABLE_INTR(isrc->isrc_dev, isrc); PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); } return (0); } #endif /* Sending IPI */ void ipi_all_but_self(u_int ipi) { cpuset_t cpus; cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); intr_ipi_send(cpus, ipi); } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); intr_ipi_send(cpus, ipi); } void ipi_selected(cpuset_t cpus, u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); intr_ipi_send(cpus, ipi); } diff --git a/sys/arm64/arm64/stack_machdep.c b/sys/arm64/arm64/stack_machdep.c index 9486e4815294..001164e7b6e3 100644 --- a/sys/arm64/arm64/stack_machdep.c +++ b/sys/arm64/arm64/stack_machdep.c @@ -1,88 +1,87 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include static void stack_capture(struct thread *td, struct stack *st, struct unwind_state *frame) { stack_zero(st); while (1) { if (!unwind_frame(td, frame)) break; if (!INKERNEL((vm_offset_t)frame->pc)) break; if (stack_put(st, frame->pc) == -1) break; } } int stack_save_td(struct stack *st, struct thread *td) { struct unwind_state frame; THREAD_LOCK_ASSERT(td, MA_OWNED); KASSERT(!TD_IS_SWAPPED(td), ("stack_save_td: thread %p is swapped", td)); if (TD_IS_RUNNING(td)) return (EOPNOTSUPP); frame.fp = td->td_pcb->pcb_x[29]; frame.pc = td->td_pcb->pcb_lr; stack_capture(td, st, &frame); return (0); } void stack_save(struct stack *st) { struct unwind_state frame; frame.fp = (uintptr_t)__builtin_frame_address(0); frame.pc = (uintptr_t)stack_save; stack_capture(curthread, st, &frame); } diff --git a/sys/arm64/arm64/sys_machdep.c b/sys/arm64/arm64/sys_machdep.c index dfb2c4ad85b8..937a9f761a7c 100644 --- a/sys/arm64/arm64/sys_machdep.c +++ b/sys/arm64/arm64/sys_machdep.c @@ -1,45 +1,44 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include int sysarch(struct thread *td, struct sysarch_args *uap) { return (ENOTSUP); } diff --git a/sys/arm64/arm64/unwind.c b/sys/arm64/arm64/unwind.c index cd994fffb953..770158295fb5 100644 --- a/sys/arm64/arm64/unwind.c +++ b/sys/arm64/arm64/unwind.c @@ -1,54 +1,53 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include bool unwind_frame(struct thread *td, struct unwind_state *frame) { uintptr_t fp; fp = frame->fp; if (!kstack_contains(td, fp, sizeof(uintptr_t) * 2)) return (false); /* FP to previous frame (X29) */ frame->fp = ((uintptr_t *)fp)[0]; /* LR (X30) */ frame->pc = ((uintptr_t *)fp)[1] - 4; return (true); } diff --git a/sys/arm64/arm64/vfp.c b/sys/arm64/arm64/vfp.c index 9de27349fb8d..a481b5ebb3ba 100644 --- a/sys/arm64/arm64/vfp.c +++ b/sys/arm64/arm64/vfp.c @@ -1,411 +1,410 @@ /*- * Copyright (c) 2015-2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef VFP #include #include #include #include #include #include #include #include #include #include #include /* Sanity check we can store all the VFP registers */ CTASSERT(sizeof(((struct pcb *)0)->pcb_fpustate.vfp_regs) == 16 * 32); static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", "Kernel contexts for VFP state"); struct fpu_kern_ctx { struct vfpstate *prev; #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */ #define FPU_KERN_CTX_INUSE 0x02 uint32_t flags; struct vfpstate state; }; static void vfp_enable(void) { uint32_t cpacr; cpacr = READ_SPECIALREG(cpacr_el1); cpacr = (cpacr & ~CPACR_FPEN_MASK) | CPACR_FPEN_TRAP_NONE; WRITE_SPECIALREG(cpacr_el1, cpacr); isb(); } static void vfp_disable(void) { uint32_t cpacr; cpacr = READ_SPECIALREG(cpacr_el1); cpacr = (cpacr & ~CPACR_FPEN_MASK) | CPACR_FPEN_TRAP_ALL1; WRITE_SPECIALREG(cpacr_el1, cpacr); isb(); } /* * Called when the thread is dying or when discarding the kernel VFP state. * If the thread was the last to use the VFP unit mark it as unused to tell * the kernel the fp state is unowned. Ensure the VFP unit is off so we get * an exception on the next access. */ void vfp_discard(struct thread *td) { #ifdef INVARIANTS if (td != NULL) CRITICAL_ASSERT(td); #endif if (PCPU_GET(fpcurthread) == td) PCPU_SET(fpcurthread, NULL); vfp_disable(); } static void vfp_store(struct vfpstate *state) { __uint128_t *vfp_state; uint64_t fpcr, fpsr; vfp_state = state->vfp_regs; __asm __volatile( "mrs %0, fpcr \n" "mrs %1, fpsr \n" "stp q0, q1, [%2, #16 * 0]\n" "stp q2, q3, [%2, #16 * 2]\n" "stp q4, q5, [%2, #16 * 4]\n" "stp q6, q7, [%2, #16 * 6]\n" "stp q8, q9, [%2, #16 * 8]\n" "stp q10, q11, [%2, #16 * 10]\n" "stp q12, q13, [%2, #16 * 12]\n" "stp q14, q15, [%2, #16 * 14]\n" "stp q16, q17, [%2, #16 * 16]\n" "stp q18, q19, [%2, #16 * 18]\n" "stp q20, q21, [%2, #16 * 20]\n" "stp q22, q23, [%2, #16 * 22]\n" "stp q24, q25, [%2, #16 * 24]\n" "stp q26, q27, [%2, #16 * 26]\n" "stp q28, q29, [%2, #16 * 28]\n" "stp q30, q31, [%2, #16 * 30]\n" : "=&r"(fpcr), "=&r"(fpsr) : "r"(vfp_state)); state->vfp_fpcr = fpcr; state->vfp_fpsr = fpsr; } static void vfp_restore(struct vfpstate *state) { __uint128_t *vfp_state; uint64_t fpcr, fpsr; vfp_state = state->vfp_regs; fpcr = state->vfp_fpcr; fpsr = state->vfp_fpsr; __asm __volatile( "ldp q0, q1, [%2, #16 * 0]\n" "ldp q2, q3, [%2, #16 * 2]\n" "ldp q4, q5, [%2, #16 * 4]\n" "ldp q6, q7, [%2, #16 * 6]\n" "ldp q8, q9, [%2, #16 * 8]\n" "ldp q10, q11, [%2, #16 * 10]\n" "ldp q12, q13, [%2, #16 * 12]\n" "ldp q14, q15, [%2, #16 * 14]\n" "ldp q16, q17, [%2, #16 * 16]\n" "ldp q18, q19, [%2, #16 * 18]\n" "ldp q20, q21, [%2, #16 * 20]\n" "ldp q22, q23, [%2, #16 * 22]\n" "ldp q24, q25, [%2, #16 * 24]\n" "ldp q26, q27, [%2, #16 * 26]\n" "ldp q28, q29, [%2, #16 * 28]\n" "ldp q30, q31, [%2, #16 * 30]\n" "msr fpcr, %0 \n" "msr fpsr, %1 \n" : : "r"(fpcr), "r"(fpsr), "r"(vfp_state)); } void vfp_save_state(struct thread *td, struct pcb *pcb) { uint32_t cpacr; KASSERT(pcb != NULL, ("NULL vfp pcb")); KASSERT(td == NULL || td->td_pcb == pcb, ("Invalid vfp pcb")); /* * savectx() will be called on panic with dumppcb as an argument, * dumppcb doesn't have pcb_fpusaved set, so set it to save * the VFP registers. */ if (pcb->pcb_fpusaved == NULL) pcb->pcb_fpusaved = &pcb->pcb_fpustate; if (td == NULL) td = curthread; critical_enter(); /* * Only store the registers if the VFP is enabled, * i.e. return if we are trapping on FP access. */ cpacr = READ_SPECIALREG(cpacr_el1); if ((cpacr & CPACR_FPEN_MASK) == CPACR_FPEN_TRAP_NONE) { KASSERT(PCPU_GET(fpcurthread) == td, ("Storing an invalid VFP state")); vfp_store(pcb->pcb_fpusaved); dsb(ish); vfp_disable(); } critical_exit(); } /* * Reset the FP state to avoid leaking state from the parent process across * execve() (and to ensure that we get a consistent floating point environment * in every new process). */ void vfp_reset_state(struct thread *td, struct pcb *pcb) { /* Discard the threads VFP state before resetting it */ critical_enter(); vfp_discard(td); critical_exit(); /* * Clear the thread state. The VFP is disabled and is not the current * VFP thread so we won't change any of these on context switch. */ bzero(&pcb->pcb_fpustate.vfp_regs, sizeof(pcb->pcb_fpustate.vfp_regs)); KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("pcb_fpusaved should point to pcb_fpustate.")); pcb->pcb_fpustate.vfp_fpcr = VFPCR_INIT; pcb->pcb_fpustate.vfp_fpsr = 0; pcb->pcb_vfpcpu = UINT_MAX; pcb->pcb_fpflags = 0; } void vfp_restore_state(void) { struct pcb *curpcb; u_int cpu; critical_enter(); cpu = PCPU_GET(cpuid); curpcb = curthread->td_pcb; curpcb->pcb_fpflags |= PCB_FP_STARTED; vfp_enable(); /* * If the previous thread on this cpu to use the VFP was not the * current thread, or the current thread last used it on a different * cpu we need to restore the old state. */ if (PCPU_GET(fpcurthread) != curthread || cpu != curpcb->pcb_vfpcpu) { vfp_restore(curthread->td_pcb->pcb_fpusaved); PCPU_SET(fpcurthread, curthread); curpcb->pcb_vfpcpu = cpu; } critical_exit(); } void vfp_init(void) { uint64_t pfr; /* Check if there is a vfp unit present */ pfr = READ_SPECIALREG(id_aa64pfr0_el1); if ((pfr & ID_AA64PFR0_FP_MASK) == ID_AA64PFR0_FP_NONE) return; /* Disable to be enabled when it's used */ vfp_disable(); if (PCPU_GET(cpuid) == 0) thread0.td_pcb->pcb_fpusaved->vfp_fpcr = VFPCR_INIT; } SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL); struct fpu_kern_ctx * fpu_kern_alloc_ctx(u_int flags) { struct fpu_kern_ctx *res; size_t sz; sz = sizeof(struct fpu_kern_ctx); res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? M_NOWAIT : M_WAITOK) | M_ZERO); return (res); } void fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) { KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx")); /* XXXAndrew clear the memory ? */ free(ctx, M_FPUKERN_CTX); } void fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) { struct pcb *pcb; pcb = td->td_pcb; KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL, ("ctx is required when !FPU_KERN_NOCTX")); KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("using inuse ctx")); KASSERT((pcb->pcb_fpflags & PCB_FP_NOSAVE) == 0, ("recursive fpu_kern_enter while in PCB_FP_NOSAVE state")); if ((flags & FPU_KERN_NOCTX) != 0) { critical_enter(); if (curthread == PCPU_GET(fpcurthread)) { vfp_save_state(curthread, pcb); } PCPU_SET(fpcurthread, NULL); vfp_enable(); pcb->pcb_fpflags |= PCB_FP_KERN | PCB_FP_NOSAVE | PCB_FP_STARTED; return; } if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; return; } /* * Check either we are already using the VFP in the kernel, or * the the saved state points to the default user space. */ KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0 || pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Mangled pcb_fpusaved %x %p %p", pcb->pcb_fpflags, pcb->pcb_fpusaved, &pcb->pcb_fpustate)); ctx->flags = FPU_KERN_CTX_INUSE; vfp_save_state(curthread, pcb); ctx->prev = pcb->pcb_fpusaved; pcb->pcb_fpusaved = &ctx->state; pcb->pcb_fpflags |= PCB_FP_KERN; pcb->pcb_fpflags &= ~PCB_FP_STARTED; return; } int fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) { struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_NOSAVE) != 0) { KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX")); KASSERT(PCPU_GET(fpcurthread) == NULL, ("non-NULL fpcurthread for PCB_FP_NOSAVE")); CRITICAL_ASSERT(td); vfp_disable(); pcb->pcb_fpflags &= ~(PCB_FP_NOSAVE | PCB_FP_STARTED); critical_exit(); } else { KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, ("FPU context not inuse")); ctx->flags &= ~FPU_KERN_CTX_INUSE; if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) return (0); KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); critical_enter(); vfp_discard(td); critical_exit(); pcb->pcb_fpflags &= ~PCB_FP_STARTED; pcb->pcb_fpusaved = ctx->prev; } if (pcb->pcb_fpusaved == &pcb->pcb_fpustate) { pcb->pcb_fpflags &= ~PCB_FP_KERN; } else { KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0, ("unpaired fpu_kern_leave")); } return (0); } int fpu_kern_thread(u_int flags __unused) { struct pcb *pcb = curthread->td_pcb; KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, ("Only kthread may use fpu_kern_thread")); KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, ("Mangled pcb_fpusaved")); KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) == 0, ("Thread already setup for the VFP")); pcb->pcb_fpflags |= PCB_FP_KERN; return (0); } int is_fpu_kern_thread(u_int flags __unused) { struct pcb *curpcb; if ((curthread->td_pflags & TDP_KTHREAD) == 0) return (0); curpcb = curthread->td_pcb; return ((curpcb->pcb_fpflags & PCB_FP_KERN) != 0); } #endif diff --git a/sys/arm64/cavium/thunder_pcie_common.c b/sys/arm64/cavium/thunder_pcie_common.c index 8f1d4834b1a5..a47b3ba18908 100644 --- a/sys/arm64/cavium/thunder_pcie_common.c +++ b/sys/arm64/cavium/thunder_pcie_common.c @@ -1,209 +1,208 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Common PCIe functions for Cavium Thunder SOC */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #include #endif #include #include #include #include #include #include #ifdef FDT #include #endif #include "thunder_pcie_common.h" MALLOC_DEFINE(M_THUNDER_PCIE, "Thunder PCIe driver", "Thunder PCIe driver memory"); #define THUNDER_CFG_BASE_TO_ECAM(x) ((((x) >> 36UL) & 0x3) | (((x) >> 42UL) & 0x4)) uint32_t range_addr_is_pci(struct pcie_range *ranges, uint64_t addr, uint64_t size) { struct pcie_range *r; int tuple; for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { r = &ranges[tuple]; if (addr >= r->pci_base && addr < (r->pci_base + r->size) && size < r->size) { /* Address is within PCI range */ return (1); } } /* Address is outside PCI range */ return (0); } uint32_t range_addr_is_phys(struct pcie_range *ranges, uint64_t addr, uint64_t size) { struct pcie_range *r; int tuple; for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { r = &ranges[tuple]; if (addr >= r->phys_base && addr < (r->phys_base + r->size) && size < r->size) { /* Address is within Physical range */ return (1); } } /* Address is outside Physical range */ return (0); } uint64_t range_addr_phys_to_pci(struct pcie_range *ranges, uint64_t phys_addr) { struct pcie_range *r; uint64_t offset; int tuple; /* Find physical address corresponding to given bus address */ for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { r = &ranges[tuple]; if (phys_addr >= r->phys_base && phys_addr < (r->phys_base + r->size)) { /* Given phys addr is in this range. * Translate phys addr to bus addr. */ offset = phys_addr - r->phys_base; return (r->pci_base + offset); } } return (0); } uint64_t range_addr_pci_to_phys(struct pcie_range *ranges, uint64_t pci_addr) { struct pcie_range *r; uint64_t offset; int tuple; /* Find physical address corresponding to given bus address */ for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { r = &ranges[tuple]; if (pci_addr >= r->pci_base && pci_addr < (r->pci_base + r->size)) { /* Given pci addr is in this range. * Translate bus addr to phys addr. */ offset = pci_addr - r->pci_base; return (r->phys_base + offset); } } return (0); } int thunder_pcie_identify_ecam(device_t dev, int *ecam) { rman_res_t start; /* Check if we're running on Cavium ThunderX */ if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_CAVIUM, CPU_PART_THUNDERX, 0, 0)) return (EINVAL); start = bus_get_resource_start(dev, SYS_RES_MEMORY, 0); *ecam = THUNDER_CFG_BASE_TO_ECAM(start); device_printf(dev, "ThunderX quirk, setting ECAM to %d\n", *ecam); return (0); } #ifdef THUNDERX_PASS_1_1_ERRATA struct resource * thunder_pcie_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { pci_addr_t map, testval; /* * If Enhanced Allocation is not used, we can't allocate any random * range. All internal devices have hardcoded place where they can * be located within PCI address space. Fortunately, we can read * this value from BAR. */ if (((type == SYS_RES_IOPORT) || (type == SYS_RES_MEMORY)) && RMAN_IS_DEFAULT_RANGE(start, end)) { /* Read BAR manually to get resource address and size */ pci_read_bar(child, *rid, &map, &testval, NULL); /* Mask the information bits */ if (PCI_BAR_MEM(map)) map &= PCIM_BAR_MEM_BASE; else map &= PCIM_BAR_IO_BASE; if (PCI_BAR_MEM(testval)) testval &= PCIM_BAR_MEM_BASE; else testval &= PCIM_BAR_IO_BASE; start = map; end = start + count - 1; } return (pci_host_generic_core_alloc_resource(dev, child, type, rid, start, end, count, flags)); } #endif diff --git a/sys/arm64/cavium/thunder_pcie_common.h b/sys/arm64/cavium/thunder_pcie_common.h index 057dff5d3b4b..d9a34148a513 100644 --- a/sys/arm64/cavium/thunder_pcie_common.h +++ b/sys/arm64/cavium/thunder_pcie_common.h @@ -1,48 +1,47 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _CAVIUM_THUNDER_PCIE_COMMON_H_ #define _CAVIUM_THUNDER_PCIE_COMMON_H_ DECLARE_CLASS(thunder_pcie_driver); DECLARE_CLASS(thunder_pem_driver); MALLOC_DECLARE(M_THUNDER_PCIE); uint32_t range_addr_is_pci(struct pcie_range *, uint64_t, uint64_t); uint32_t range_addr_is_phys(struct pcie_range *, uint64_t, uint64_t); uint64_t range_addr_phys_to_pci(struct pcie_range *, uint64_t); uint64_t range_addr_pci_to_phys(struct pcie_range *, uint64_t); int thunder_pcie_identify_ecam(device_t, int *); #ifdef THUNDERX_PASS_1_1_ERRATA struct resource *thunder_pcie_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); #endif #endif /* _CAVIUM_THUNDER_PCIE_COMMON_H_ */ diff --git a/sys/arm64/cavium/thunder_pcie_pem.c b/sys/arm64/cavium/thunder_pcie_pem.c index 9296280ad95f..45d9880571b0 100644 --- a/sys/arm64/cavium/thunder_pcie_pem.c +++ b/sys/arm64/cavium/thunder_pcie_pem.c @@ -1,921 +1,920 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* PCIe external MAC root complex driver (PEM) for Cavium Thunder SOC */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #include #endif #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" #define THUNDER_PEM_DEVICE_ID 0xa020 #define THUNDER_PEM_VENDOR_ID 0x177d /* ThunderX specific defines */ #define THUNDER_PEMn_REG_BASE(unit) (0x87e0c0000000UL | ((unit) << 24)) #define PCIERC_CFG002 0x08 #define PCIERC_CFG006 0x18 #define PCIERC_CFG032 0x80 #define PCIERC_CFG006_SEC_BUS(reg) (((reg) >> 8) & 0xFF) #define PEM_CFG_RD_REG_ALIGN(reg) ((reg) & ~0x3) #define PEM_CFG_RD_REG_DATA(val) (((val) >> 32) & 0xFFFFFFFF) #define PEM_CFG_RD 0x30 #define PEM_CFG_LINK_MASK 0x3 #define PEM_CFG_LINK_RDY 0x3 #define PEM_CFG_SLIX_TO_REG(slix) ((slix) << 4) #define SBNUM_OFFSET 0x8 #define SBNUM_MASK 0xFF #define PEM_ON_REG 0x420 #define PEM_CTL_STATUS 0x0 #define PEM_LINK_ENABLE (1 << 4) #define PEM_LINK_DLLA (1 << 29) #define PEM_LINK_LT (1 << 27) #define PEM_BUS_SHIFT (24) #define PEM_SLOT_SHIFT (19) #define PEM_FUNC_SHIFT (16) #define SLIX_S2M_REGX_ACC 0x874001000000UL #define SLIX_S2M_REGX_ACC_SIZE 0x1000 #define SLIX_S2M_REGX_ACC_SPACING 0x001000000000UL #define SLI_BASE 0x880000000000UL #define SLI_WINDOW_SPACING 0x004000000000UL #define SLI_PCI_OFFSET 0x001000000000UL #define SLI_NODE_SHIFT (44) #define SLI_NODE_MASK (3) #define SLI_GROUP_SHIFT (40) #define SLI_ID_SHIFT (24) #define SLI_ID_MASK (7) #define SLI_PEMS_PER_GROUP (3) #define SLI_GROUPS_PER_NODE (2) #define SLI_PEMS_PER_NODE (SLI_PEMS_PER_GROUP * SLI_GROUPS_PER_NODE) #define SLI_ACC_REG_CNT (256) /* * Each PEM device creates its own bus with * own address translation, so we can adjust bus addresses * as we want. To support 32-bit cards let's assume * PCI window assignment looks as following: * * 0x00000000 - 0x000FFFFF IO * 0x00100000 - 0xFFFFFFFF Memory */ #define PCI_IO_BASE 0x00000000UL #define PCI_IO_SIZE 0x00100000UL #define PCI_MEMORY_BASE PCI_IO_SIZE #define PCI_MEMORY_SIZE 0xFFF00000UL #define RID_PEM_SPACE 1 static int thunder_pem_activate_resource(device_t, device_t, int, int, struct resource *); static int thunder_pem_adjust_resource(device_t, device_t, int, struct resource *, rman_res_t, rman_res_t); static struct resource * thunder_pem_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static int thunder_pem_alloc_msi(device_t, device_t, int, int, int *); static int thunder_pem_release_msi(device_t, device_t, int, int *); static int thunder_pem_alloc_msix(device_t, device_t, int *); static int thunder_pem_release_msix(device_t, device_t, int); static int thunder_pem_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); static int thunder_pem_get_id(device_t, device_t, enum pci_id_type, uintptr_t *); static int thunder_pem_attach(device_t); static int thunder_pem_deactivate_resource(device_t, device_t, int, int, struct resource *); static bus_dma_tag_t thunder_pem_get_dma_tag(device_t, device_t); static int thunder_pem_detach(device_t); static uint64_t thunder_pem_config_reg_read(struct thunder_pem_softc *, int); static int thunder_pem_link_init(struct thunder_pem_softc *); static int thunder_pem_maxslots(device_t); static int thunder_pem_probe(device_t); static uint32_t thunder_pem_read_config(device_t, u_int, u_int, u_int, u_int, int); static int thunder_pem_read_ivar(device_t, device_t, int, uintptr_t *); static void thunder_pem_release_all(device_t); static int thunder_pem_release_resource(device_t, device_t, int, int, struct resource *); static struct rman * thunder_pem_rman(struct thunder_pem_softc *, int); static void thunder_pem_slix_s2m_regx_acc_modify(struct thunder_pem_softc *, int, int); static void thunder_pem_write_config(device_t, u_int, u_int, u_int, u_int, uint32_t, int); static int thunder_pem_write_ivar(device_t, device_t, int, uintptr_t); /* Global handlers for SLI interface */ static bus_space_handle_t sli0_s2m_regx_base = 0; static bus_space_handle_t sli1_s2m_regx_base = 0; static device_method_t thunder_pem_methods[] = { /* Device interface */ DEVMETHOD(device_probe, thunder_pem_probe), DEVMETHOD(device_attach, thunder_pem_attach), DEVMETHOD(device_detach, thunder_pem_detach), /* Bus interface */ DEVMETHOD(bus_read_ivar, thunder_pem_read_ivar), DEVMETHOD(bus_write_ivar, thunder_pem_write_ivar), DEVMETHOD(bus_alloc_resource, thunder_pem_alloc_resource), DEVMETHOD(bus_release_resource, thunder_pem_release_resource), DEVMETHOD(bus_adjust_resource, thunder_pem_adjust_resource), DEVMETHOD(bus_activate_resource, thunder_pem_activate_resource), DEVMETHOD(bus_deactivate_resource, thunder_pem_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_get_dma_tag, thunder_pem_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, thunder_pem_maxslots), DEVMETHOD(pcib_read_config, thunder_pem_read_config), DEVMETHOD(pcib_write_config, thunder_pem_write_config), DEVMETHOD(pcib_alloc_msix, thunder_pem_alloc_msix), DEVMETHOD(pcib_release_msix, thunder_pem_release_msix), DEVMETHOD(pcib_alloc_msi, thunder_pem_alloc_msi), DEVMETHOD(pcib_release_msi, thunder_pem_release_msi), DEVMETHOD(pcib_map_msi, thunder_pem_map_msi), DEVMETHOD(pcib_get_id, thunder_pem_get_id), DEVMETHOD_END }; DEFINE_CLASS_0(pcib, thunder_pem_driver, thunder_pem_methods, sizeof(struct thunder_pem_softc)); static devclass_t thunder_pem_devclass; extern struct bus_space memmap_bus; DRIVER_MODULE(thunder_pem, pci, thunder_pem_driver, thunder_pem_devclass, 0, 0); MODULE_DEPEND(thunder_pem, pci, 1, 1, 1); static int thunder_pem_maxslots(device_t dev) { #if 0 /* max slots per bus acc. to standard */ return (PCI_SLOTMAX); #else /* * ARM64TODO Workaround - otherwise an em(4) interface appears to be * present on every PCI function on the bus to which it is connected */ return (0); #endif } static int thunder_pem_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct thunder_pem_softc *sc; int secondary_bus = 0; sc = device_get_softc(dev); if (index == PCIB_IVAR_BUS) { secondary_bus = thunder_pem_config_reg_read(sc, PCIERC_CFG006); *result = PCIERC_CFG006_SEC_BUS(secondary_bus); return (0); } if (index == PCIB_IVAR_DOMAIN) { *result = sc->id; return (0); } return (ENOENT); } static int thunder_pem_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { return (ENOENT); } static int thunder_pem_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { int err; bus_addr_t paddr; bus_size_t psize; bus_space_handle_t vaddr; struct thunder_pem_softc *sc; if ((err = rman_activate_resource(r)) != 0) return (err); sc = device_get_softc(dev); /* * If this is a memory resource, map it into the kernel. */ if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) { paddr = (bus_addr_t)rman_get_start(r); psize = (bus_size_t)rman_get_size(r); paddr = range_addr_pci_to_phys(sc->ranges, paddr); err = bus_space_map(&memmap_bus, paddr, psize, 0, &vaddr); if (err != 0) { rman_deactivate_resource(r); return (err); } rman_set_bustag(r, &memmap_bus); rman_set_virtual(r, (void *)vaddr); rman_set_bushandle(r, vaddr); } return (0); } /* * This function is an exact copy of nexus_deactivate_resource() * Keep it up-to-date with all changes in nexus. To be removed * once bus-mapping interface is developed. */ static int thunder_pem_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { bus_size_t psize; bus_space_handle_t vaddr; psize = (bus_size_t)rman_get_size(r); vaddr = rman_get_bushandle(r); if (vaddr != 0) { bus_space_unmap(&memmap_bus, vaddr, psize); rman_set_virtual(r, NULL); rman_set_bushandle(r, 0); } return (rman_deactivate_resource(r)); } static int thunder_pem_adjust_resource(device_t dev, device_t child, int type, struct resource *res, rman_res_t start, rman_res_t end) { struct thunder_pem_softc *sc; struct rman *rm; sc = device_get_softc(dev); #if defined(NEW_PCIB) && defined(PCI_RES_BUS) if (type == PCI_RES_BUS) return (pci_domain_adjust_bus(sc->id, child, res, start, end)); #endif rm = thunder_pem_rman(sc, type); if (rm == NULL) return (bus_generic_adjust_resource(dev, child, type, res, start, end)); if (!rman_is_region_manager(res, rm)) /* * This means a child device has a memory or I/O * resource not from you which shouldn't happen. */ return (EINVAL); return (rman_adjust_resource(res, start, end)); } static bus_dma_tag_t thunder_pem_get_dma_tag(device_t dev, device_t child) { struct thunder_pem_softc *sc; sc = device_get_softc(dev); return (sc->dmat); } static int thunder_pem_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { device_t bus; bus = device_get_parent(pci); return (PCIB_ALLOC_MSI(device_get_parent(bus), child, count, maxcount, irqs)); } static int thunder_pem_release_msi(device_t pci, device_t child, int count, int *irqs) { device_t bus; bus = device_get_parent(pci); return (PCIB_RELEASE_MSI(device_get_parent(bus), child, count, irqs)); } static int thunder_pem_alloc_msix(device_t pci, device_t child, int *irq) { device_t bus; bus = device_get_parent(pci); return (PCIB_ALLOC_MSIX(device_get_parent(bus), child, irq)); } static int thunder_pem_release_msix(device_t pci, device_t child, int irq) { device_t bus; bus = device_get_parent(pci); return (PCIB_RELEASE_MSIX(device_get_parent(bus), child, irq)); } static int thunder_pem_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { device_t bus; bus = device_get_parent(pci); return (PCIB_MAP_MSI(device_get_parent(bus), child, irq, addr, data)); } static int thunder_pem_get_id(device_t pci, device_t child, enum pci_id_type type, uintptr_t *id) { int bsf; int pem; if (type != PCI_ID_MSI) return (pcib_get_id(pci, child, type, id)); bsf = pci_get_rid(child); /* PEM (PCIe MAC/root complex) number is equal to domain */ pem = pci_get_domain(child); /* * Set appropriate device ID (passed by the HW along with * the transaction to memory) for different root complex * numbers using hard-coded domain portion for each group. */ if (pem < 3) *id = (0x1 << PCI_RID_DOMAIN_SHIFT) | bsf; else if (pem < 6) *id = (0x3 << PCI_RID_DOMAIN_SHIFT) | bsf; else if (pem < 9) *id = (0x9 << PCI_RID_DOMAIN_SHIFT) | bsf; else if (pem < 12) *id = (0xB << PCI_RID_DOMAIN_SHIFT) | bsf; else return (ENXIO); return (0); } static int thunder_pem_identify(device_t dev) { struct thunder_pem_softc *sc; rman_res_t start; sc = device_get_softc(dev); start = rman_get_start(sc->reg); /* Calculate PEM designations from its address */ sc->node = (start >> SLI_NODE_SHIFT) & SLI_NODE_MASK; sc->id = ((start >> SLI_ID_SHIFT) & SLI_ID_MASK) + (SLI_PEMS_PER_NODE * sc->node); sc->sli = sc->id % SLI_PEMS_PER_GROUP; sc->sli_group = (sc->id / SLI_PEMS_PER_GROUP) % SLI_GROUPS_PER_NODE; sc->sli_window_base = SLI_BASE | (((uint64_t)sc->node) << SLI_NODE_SHIFT) | ((uint64_t)sc->sli_group << SLI_GROUP_SHIFT); sc->sli_window_base += SLI_WINDOW_SPACING * sc->sli; return (0); } static void thunder_pem_slix_s2m_regx_acc_modify(struct thunder_pem_softc *sc, int sli_group, int slix) { uint64_t regval; bus_space_handle_t handle = 0; KASSERT(slix >= 0 && slix <= SLI_ACC_REG_CNT, ("Invalid SLI index")); if (sli_group == 0) handle = sli0_s2m_regx_base; else if (sli_group == 1) handle = sli1_s2m_regx_base; else device_printf(sc->dev, "SLI group is not correct\n"); if (handle) { /* Clear lower 32-bits of the SLIx register */ regval = bus_space_read_8(sc->reg_bst, handle, PEM_CFG_SLIX_TO_REG(slix)); regval &= ~(0xFFFFFFFFUL); bus_space_write_8(sc->reg_bst, handle, PEM_CFG_SLIX_TO_REG(slix), regval); } } static int thunder_pem_link_init(struct thunder_pem_softc *sc) { uint64_t regval; /* check whether PEM is safe to access. */ regval = bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_ON_REG); if ((regval & PEM_CFG_LINK_MASK) != PEM_CFG_LINK_RDY) { device_printf(sc->dev, "PEM%d is not ON\n", sc->id); return (ENXIO); } regval = bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_CTL_STATUS); regval |= PEM_LINK_ENABLE; bus_space_write_8(sc->reg_bst, sc->reg_bsh, PEM_CTL_STATUS, regval); /* Wait 1ms as per Cavium specification */ DELAY(1000); regval = thunder_pem_config_reg_read(sc, PCIERC_CFG032); if (((regval & PEM_LINK_DLLA) == 0) || ((regval & PEM_LINK_LT) != 0)) { device_printf(sc->dev, "PCIe RC: Port %d Link Timeout\n", sc->id); return (ENXIO); } return (0); } static int thunder_pem_init(struct thunder_pem_softc *sc) { int i, retval = 0; retval = thunder_pem_link_init(sc); if (retval) { device_printf(sc->dev, "%s failed\n", __func__); return retval; } /* To support 32-bit PCIe devices, set S2M_REGx_ACC[BA]=0x0 */ for (i = 0; i < SLI_ACC_REG_CNT; i++) { thunder_pem_slix_s2m_regx_acc_modify(sc, sc->sli_group, i); } return (retval); } static uint64_t thunder_pem_config_reg_read(struct thunder_pem_softc *sc, int reg) { uint64_t data; /* Write to ADDR register */ bus_space_write_8(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD, PEM_CFG_RD_REG_ALIGN(reg)); bus_space_barrier(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD, 8, BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); /* Read from DATA register */ data = PEM_CFG_RD_REG_DATA(bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD)); return (data); } static uint32_t thunder_pem_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes) { uint64_t offset; uint32_t data; struct thunder_pem_softc *sc; bus_space_tag_t t; bus_space_handle_t h; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return (~0U); sc = device_get_softc(dev); /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; bus_space_map(sc->reg_bst, sc->sli_window_base + offset, PCIE_REGMAX, 0, &h); switch (bytes) { case 1: data = bus_space_read_1(t, h, reg); break; case 2: data = le16toh(bus_space_read_2(t, h, reg)); break; case 4: data = le32toh(bus_space_read_4(t, h, reg)); break; default: data = ~0U; break; } bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); return (data); } static void thunder_pem_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes) { uint64_t offset; struct thunder_pem_softc *sc; bus_space_tag_t t; bus_space_handle_t h; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return; sc = device_get_softc(dev); /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; bus_space_map(sc->reg_bst, sc->sli_window_base + offset, PCIE_REGMAX, 0, &h); switch (bytes) { case 1: bus_space_write_1(t, h, reg, val); break; case 2: bus_space_write_2(t, h, reg, htole16(val)); break; case 4: bus_space_write_4(t, h, reg, htole32(val)); break; default: break; } bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); } static struct resource * thunder_pem_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct thunder_pem_softc *sc = device_get_softc(dev); struct rman *rm = NULL; struct resource *res; device_t parent_dev; #if defined(NEW_PCIB) && defined(PCI_RES_BUS) if (type == PCI_RES_BUS) return (pci_domain_alloc_bus(sc->id, child, rid, start, end, count, flags)); #endif rm = thunder_pem_rman(sc, type); if (rm == NULL) { /* Find parent device. On ThunderX we know an exact path. */ parent_dev = device_get_parent(device_get_parent(dev)); return (BUS_ALLOC_RESOURCE(parent_dev, dev, type, rid, start, end, count, flags)); } if (!RMAN_IS_DEFAULT_RANGE(start, end)) { /* * We might get PHYS addresses here inherited from EFI. * Convert to PCI if necessary. */ if (range_addr_is_phys(sc->ranges, start, count)) { start = range_addr_phys_to_pci(sc->ranges, start); end = start + count - 1; } } if (bootverbose) { device_printf(dev, "thunder_pem_alloc_resource: start=%#lx, end=%#lx, count=%#lx\n", start, end, count); } res = rman_reserve_resource(rm, start, end, count, flags, child); if (res == NULL) goto fail; rman_set_rid(res, *rid); if (flags & RF_ACTIVE) if (bus_activate_resource(child, type, *rid, res)) { rman_release_resource(res); goto fail; } return (res); fail: if (bootverbose) { device_printf(dev, "%s FAIL: type=%d, rid=%d, " "start=%016lx, end=%016lx, count=%016lx, flags=%x\n", __func__, type, *rid, start, end, count, flags); } return (NULL); } static int thunder_pem_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { device_t parent_dev; #if defined(NEW_PCIB) && defined(PCI_RES_BUS) struct thunder_pem_softc *sc = device_get_softc(dev); if (type == PCI_RES_BUS) return (pci_domain_release_bus(sc->id, child, rid, res)); #endif /* Find parent device. On ThunderX we know an exact path. */ parent_dev = device_get_parent(device_get_parent(dev)); if ((type != SYS_RES_MEMORY) && (type != SYS_RES_IOPORT)) return (BUS_RELEASE_RESOURCE(parent_dev, child, type, rid, res)); return (rman_release_resource(res)); } static struct rman * thunder_pem_rman(struct thunder_pem_softc *sc, int type) { switch (type) { case SYS_RES_IOPORT: return (&sc->io_rman); case SYS_RES_MEMORY: return (&sc->mem_rman); default: break; } return (NULL); } static int thunder_pem_probe(device_t dev) { uint16_t pci_vendor_id; uint16_t pci_device_id; pci_vendor_id = pci_get_vendor(dev); pci_device_id = pci_get_device(dev); if ((pci_vendor_id == THUNDER_PEM_VENDOR_ID) && (pci_device_id == THUNDER_PEM_DEVICE_ID)) { device_set_desc_copy(dev, THUNDER_PEM_DESC); return (0); } return (ENXIO); } static int thunder_pem_attach(device_t dev) { devclass_t pci_class; device_t parent; struct thunder_pem_softc *sc; int error; int rid; int tuple; uint64_t base, size; struct rman *rman; sc = device_get_softc(dev); sc->dev = dev; /* Allocate memory for resource */ pci_class = devclass_find("pci"); parent = device_get_parent(dev); if (device_get_devclass(parent) == pci_class) rid = PCIR_BAR(0); else rid = RID_PEM_SPACE; sc->reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->reg == NULL) { device_printf(dev, "Failed to allocate resource\n"); return (ENXIO); } sc->reg_bst = rman_get_bustag(sc->reg); sc->reg_bsh = rman_get_bushandle(sc->reg); /* Create the parent DMA tag to pass down the coherent flag */ error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE, /* maxsize */ BUS_SPACE_UNRESTRICTED, /* nsegments */ BUS_SPACE_MAXSIZE, /* maxsegsize */ BUS_DMA_COHERENT, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->dmat); if (error != 0) return (error); /* Map SLI, do it only once */ if (!sli0_s2m_regx_base) { bus_space_map(sc->reg_bst, SLIX_S2M_REGX_ACC, SLIX_S2M_REGX_ACC_SIZE, 0, &sli0_s2m_regx_base); } if (!sli1_s2m_regx_base) { bus_space_map(sc->reg_bst, SLIX_S2M_REGX_ACC + SLIX_S2M_REGX_ACC_SPACING, SLIX_S2M_REGX_ACC_SIZE, 0, &sli1_s2m_regx_base); } if ((sli0_s2m_regx_base == 0) || (sli1_s2m_regx_base == 0)) { device_printf(dev, "bus_space_map failed to map slix_s2m_regx_base\n"); goto fail; } /* Identify PEM */ if (thunder_pem_identify(dev) != 0) goto fail; /* Initialize rman and allocate regions */ sc->mem_rman.rm_type = RMAN_ARRAY; sc->mem_rman.rm_descr = "PEM PCIe Memory"; error = rman_init(&sc->mem_rman); if (error != 0) { device_printf(dev, "memory rman_init() failed. error = %d\n", error); goto fail; } sc->io_rman.rm_type = RMAN_ARRAY; sc->io_rman.rm_descr = "PEM PCIe IO"; error = rman_init(&sc->io_rman); if (error != 0) { device_printf(dev, "IO rman_init() failed. error = %d\n", error); goto fail_mem; } /* * We ignore the values that may have been provided in FDT * and configure ranges according to the below formula * for all types of devices. This is because some DTBs provided * by EFI do not have proper ranges property or don't have them * at all. */ /* Fill memory window */ sc->ranges[0].pci_base = PCI_MEMORY_BASE; sc->ranges[0].size = PCI_MEMORY_SIZE; sc->ranges[0].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[0].pci_base; sc->ranges[0].flags = SYS_RES_MEMORY; /* Fill IO window */ sc->ranges[1].pci_base = PCI_IO_BASE; sc->ranges[1].size = PCI_IO_SIZE; sc->ranges[1].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[1].pci_base; sc->ranges[1].flags = SYS_RES_IOPORT; for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { base = sc->ranges[tuple].pci_base; size = sc->ranges[tuple].size; if (size == 0) continue; /* empty range element */ rman = thunder_pem_rman(sc, sc->ranges[tuple].flags); if (rman != NULL) error = rman_manage_region(rman, base, base + size - 1); else error = EINVAL; if (error) { device_printf(dev, "rman_manage_region() failed. error = %d\n", error); rman_fini(&sc->mem_rman); return (error); } if (bootverbose) { device_printf(dev, "\tPCI addr: 0x%jx, CPU addr: 0x%jx, Size: 0x%jx, Flags:0x%jx\n", sc->ranges[tuple].pci_base, sc->ranges[tuple].phys_base, sc->ranges[tuple].size, sc->ranges[tuple].flags); } } if (thunder_pem_init(sc)) { device_printf(dev, "Failure during PEM init\n"); goto fail_io; } device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); fail_io: rman_fini(&sc->io_rman); fail_mem: rman_fini(&sc->mem_rman); fail: bus_free_resource(dev, SYS_RES_MEMORY, sc->reg); return (ENXIO); } static void thunder_pem_release_all(device_t dev) { struct thunder_pem_softc *sc; sc = device_get_softc(dev); rman_fini(&sc->io_rman); rman_fini(&sc->mem_rman); if (sc->reg != NULL) bus_free_resource(dev, SYS_RES_MEMORY, sc->reg); } static int thunder_pem_detach(device_t dev) { thunder_pem_release_all(dev); return (0); } diff --git a/sys/arm64/include/bus_dma_impl.h b/sys/arm64/include/bus_dma_impl.h index 736376c13864..1488bd3a55da 100644 --- a/sys/arm64/include/bus_dma_impl.h +++ b/sys/arm64/include/bus_dma_impl.h @@ -1,99 +1,98 @@ /*- * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_BUS_DMA_IMPL_H_ #define _MACHINE_BUS_DMA_IMPL_H_ struct bus_dma_tag_common { struct bus_dma_impl *impl; struct bus_dma_tag_common *parent; bus_size_t alignment; bus_addr_t boundary; bus_addr_t lowaddr; bus_addr_t highaddr; bus_dma_filter_t *filter; void *filterarg; bus_size_t maxsize; u_int nsegments; bus_size_t maxsegsz; int flags; bus_dma_lock_t *lockfunc; void *lockfuncarg; int ref_count; int domain; }; struct bus_dma_impl { int (*tag_create)(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); int (*tag_destroy)(bus_dma_tag_t dmat); int (*tag_set_domain)(bus_dma_tag_t); bool (*id_mapped)(bus_dma_tag_t, vm_paddr_t, bus_size_t); int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp); void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map); int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp); int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp); int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, int *segp); void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg); bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error); void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); int bus_dma_run_filter(struct bus_dma_tag_common *dmat, bus_addr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat); extern struct bus_dma_impl bus_dma_bounce_impl; #endif diff --git a/sys/arm64/include/debug_monitor.h b/sys/arm64/include/debug_monitor.h index 4ca1a4c9248e..befc51e88304 100644 --- a/sys/arm64/include/debug_monitor.h +++ b/sys/arm64/include/debug_monitor.h @@ -1,70 +1,69 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_DEBUG_MONITOR_H_ #define _MACHINE_DEBUG_MONITOR_H_ #define DBG_BRP_MAX 16 #define DBG_WRP_MAX 16 struct debug_monitor_state { uint32_t dbg_enable_count; uint32_t dbg_flags; #define DBGMON_ENABLED (1 << 0) #define DBGMON_KERNEL (1 << 1) uint64_t dbg_bcr[DBG_BRP_MAX]; uint64_t dbg_bvr[DBG_BRP_MAX]; uint64_t dbg_wcr[DBG_WRP_MAX]; uint64_t dbg_wvr[DBG_WRP_MAX]; }; #ifdef _KERNEL enum dbg_access_t { HW_BREAKPOINT_X = 0, HW_BREAKPOINT_R = 1, HW_BREAKPOINT_W = 2, HW_BREAKPOINT_RW = HW_BREAKPOINT_R | HW_BREAKPOINT_W, }; void dbg_monitor_init(void); void dbg_register_sync(struct debug_monitor_state *); int dbg_setup_watchpoint(struct debug_monitor_state *, vm_offset_t, vm_size_t, enum dbg_access_t); int dbg_remove_watchpoint(struct debug_monitor_state *, vm_offset_t, vm_size_t); #ifdef DDB void dbg_show_watchpoint(void); #endif #endif /* _KERNEL */ #endif /* _MACHINE_DEBUG_MONITOR_H_ */ diff --git a/sys/arm64/include/ifunc.h b/sys/arm64/include/ifunc.h index cf89af7c7142..da4d856f7f92 100644 --- a/sys/arm64/include/ifunc.h +++ b/sys/arm64/include/ifunc.h @@ -1,51 +1,50 @@ /*- * Copyright (c) 2015-2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __ARM64_IFUNC_H #define __ARM64_IFUNC_H #define DEFINE_IFUNC(qual, ret_type, name, args) \ static ret_type (*name##_resolver(void))args __used; \ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ static ret_type (*name##_resolver(void))args #define DEFINE_UIFUNC(qual, ret_type, name, args) \ static ret_type (*name##_resolver(uint64_t, uint64_t, \ uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, \ uint64_t))args __used; \ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ static ret_type (*name##_resolver(uint64_t _arg1 __unused, \ uint64_t _arg2 __unused, uint64_t _arg3 __unused, \ uint64_t _arg4 __unused, uint64_t _arg5 __unused, \ uint64_t _arg6 __unused, uint64_t _arg7 __unused, \ uint64_t _arg8 __unused))args #endif diff --git a/sys/arm64/include/iodev.h b/sys/arm64/include/iodev.h index 5521ff71bc5f..8752c4cb7f6b 100644 --- a/sys/arm64/include/iodev.h +++ b/sys/arm64/include/iodev.h @@ -1,65 +1,64 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_IODEV_H_ #define _MACHINE_IODEV_H_ #define iodev_read_1(a) \ ({ \ uint8_t val; \ __asm __volatile("ldrb %w0, [%1]" : "=&r" (val) : "r"(a)); \ val; \ }) #define iodev_read_2(a) \ ({ \ uint16_t val; \ __asm __volatile("ldrh %w0, [%1]" : "=&r" (val) : "r"(a)); \ val; \ }) #define iodev_read_4(a) \ ({ \ uint32_t val; \ __asm __volatile("ldr %w0, [%1]" : "=&r" (val) : "r"(a)); \ val; \ }) #define iodev_write_1(a, v) \ __asm __volatile("strb %w0, [%1]" :: "r" (v), "r"(a)) #define iodev_write_2(a, v) \ __asm __volatile("strh %w0, [%1]" :: "r" (v), "r"(a)) #define iodev_write_4(a, v) \ __asm __volatile("str %w0, [%1]" :: "r" (v), "r"(a)) #endif /* _MACHINE_IODEV_H_ */ diff --git a/sys/arm64/include/ofw_machdep.h b/sys/arm64/include/ofw_machdep.h index 511fc8d71c19..b2ba37cb441c 100644 --- a/sys/arm64/include/ofw_machdep.h +++ b/sys/arm64/include/ofw_machdep.h @@ -1,44 +1,43 @@ /*- * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_OFW_MACHDEP_H_ #define _MACHINE_OFW_MACHDEP_H_ #include typedef uint32_t cell_t; struct mem_region { vm_offset_t mr_start; vm_size_t mr_size; }; #endif /* _MACHINE_OFW_MACHDEP_H_ */ diff --git a/sys/arm64/include/pci_cfgreg.h b/sys/arm64/include/pci_cfgreg.h index 68b38aaeb5bb..fa7e93f1a7e7 100644 --- a/sys/arm64/include/pci_cfgreg.h +++ b/sys/arm64/include/pci_cfgreg.h @@ -1,36 +1,35 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PCI_CFGREG_H #define _MACHINE_PCI_CFGREG_H int pci_cfgregopen(void); uint32_t pci_cfgregread(int, int, int, int, int); void pci_cfgregwrite(int, int, int, int, u_int32_t, int); #endif /* !_MACHINE_PCI_CFGREG_H */ diff --git a/sys/arm64/include/vfp.h b/sys/arm64/include/vfp.h index 6689be1b15d7..629b1b7fadf4 100644 --- a/sys/arm64/include/vfp.h +++ b/sys/arm64/include/vfp.h @@ -1,102 +1,101 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_VFP_H_ #define _MACHINE_VFP_H_ /* VFPCR */ #define VFPCR_AHP (0x04000000) /* alt. half-precision: */ #define VFPCR_DN (0x02000000) /* default NaN enable */ #define VFPCR_FZ (0x01000000) /* flush to zero enabled */ #define VFPCR_INIT VFPCR_DN /* Default fpcr after exec */ #define VFPCR_RMODE_OFF 22 /* rounding mode offset */ #define VFPCR_RMODE_MASK (0x00c00000) /* rounding mode mask */ #define VFPCR_RMODE_RN (0x00000000) /* round nearest */ #define VFPCR_RMODE_RPI (0x00400000) /* round to plus infinity */ #define VFPCR_RMODE_RNI (0x00800000) /* round to neg infinity */ #define VFPCR_RMODE_RM (0x00c00000) /* round to zero */ #define VFPCR_STRIDE_OFF 20 /* vector stride -1 */ #define VFPCR_STRIDE_MASK (0x00300000) #define VFPCR_LEN_OFF 16 /* vector length -1 */ #define VFPCR_LEN_MASK (0x00070000) #define VFPCR_IDE (0x00008000) /* input subnormal exc enable */ #define VFPCR_IXE (0x00001000) /* inexact exception enable */ #define VFPCR_UFE (0x00000800) /* underflow exception enable */ #define VFPCR_OFE (0x00000400) /* overflow exception enable */ #define VFPCR_DZE (0x00000200) /* div by zero exception en */ #define VFPCR_IOE (0x00000100) /* invalid op exec enable */ #ifndef LOCORE struct vfpstate { __uint128_t vfp_regs[32]; uint32_t vfp_fpcr; uint32_t vfp_fpsr; }; #ifdef _KERNEL struct pcb; struct thread; void vfp_init(void); void vfp_discard(struct thread *); void vfp_reset_state(struct thread *, struct pcb *); void vfp_restore_state(void); void vfp_save_state(struct thread *, struct pcb *); struct fpu_kern_ctx; /* * Flags for fpu_kern_alloc_ctx(), fpu_kern_enter() and fpu_kern_thread(). */ #define FPU_KERN_NORMAL 0x0000 #define FPU_KERN_NOWAIT 0x0001 #define FPU_KERN_KTHR 0x0002 #define FPU_KERN_NOCTX 0x0004 struct fpu_kern_ctx *fpu_kern_alloc_ctx(u_int); void fpu_kern_free_ctx(struct fpu_kern_ctx *); void fpu_kern_enter(struct thread *, struct fpu_kern_ctx *, u_int); int fpu_kern_leave(struct thread *, struct fpu_kern_ctx *); int fpu_kern_thread(u_int); int is_fpu_kern_thread(u_int); /* Convert to and from Aarch32 FPSCR to Aarch64 FPCR/FPSR */ #define VFP_FPSCR_FROM_SRCR(vpsr, vpcr) ((vpsr) | ((vpcr) & 0x7c00000)) #define VFP_FPSR_FROM_FPSCR(vpscr) ((vpscr) &~ 0x7c00000) #define VFP_FPCR_FROM_FPSCR(vpsrc) ((vpsrc) & 0x7c00000) #endif #endif #endif /* !_MACHINE_VFP_H_ */ diff --git a/sys/compat/freebsd32/freebsd32_capability.c b/sys/compat/freebsd32/freebsd32_capability.c index 9a39b4f4cd71..2796d2e9384f 100644 --- a/sys/compat/freebsd32/freebsd32_capability.c +++ b/sys/compat/freebsd32/freebsd32_capability.c @@ -1,157 +1,156 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_capsicum.h" #include #include #include #include #include #include #include #include #include #include #ifdef CAPABILITIES MALLOC_DECLARE(M_FILECAPS); int freebsd32_cap_ioctls_limit(struct thread *td, struct freebsd32_cap_ioctls_limit_args *uap) { u_long *cmds; uint32_t *cmds32; size_t ncmds; u_int i; int error; ncmds = uap->ncmds; if (ncmds > 256) /* XXX: Is 256 sane? */ return (EINVAL); if (ncmds == 0) { cmds = NULL; } else { cmds32 = malloc(sizeof(cmds32[0]) * ncmds, M_FILECAPS, M_WAITOK); error = copyin(uap->cmds, cmds32, sizeof(cmds32[0]) * ncmds); if (error != 0) { free(cmds32, M_FILECAPS); return (error); } cmds = malloc(sizeof(cmds[0]) * ncmds, M_FILECAPS, M_WAITOK); for (i = 0; i < ncmds; i++) cmds[i] = cmds32[i]; free(cmds32, M_FILECAPS); } return (kern_cap_ioctls_limit(td, uap->fd, cmds, ncmds)); } int freebsd32_cap_ioctls_get(struct thread *td, struct freebsd32_cap_ioctls_get_args *uap) { struct filedesc *fdp; struct filedescent *fdep; uint32_t *cmds32; u_long *cmds; size_t maxcmds; int error, fd; u_int i; fd = uap->fd; cmds32 = uap->cmds; maxcmds = uap->maxcmds; AUDIT_ARG_FD(fd); fdp = td->td_proc->p_fd; FILEDESC_SLOCK(fdp); if (fget_locked(fdp, fd) == NULL) { error = EBADF; goto out; } /* * If all ioctls are allowed (fde_nioctls == -1 && fde_ioctls == NULL) * the only sane thing we can do is to not populate the given array and * return CAP_IOCTLS_ALL (actually, INT_MAX). */ fdep = &fdp->fd_ofiles[fd]; cmds = fdep->fde_ioctls; if (cmds32 != NULL && cmds != NULL) { for (i = 0; i < MIN(fdep->fde_nioctls, maxcmds); i++) { error = suword32(&cmds32[i], cmds[i]); if (error != 0) goto out; } } if (fdep->fde_nioctls == -1) td->td_retval[0] = INT_MAX; else td->td_retval[0] = fdep->fde_nioctls; error = 0; out: FILEDESC_SUNLOCK(fdp); return (error); } #else /* !CAPABILITIES */ int freebsd32_cap_ioctls_limit(struct thread *td, struct freebsd32_cap_ioctls_limit_args *uap) { return (ENOSYS); } int freebsd32_cap_ioctls_get(struct thread *td, struct freebsd32_cap_ioctls_get_args *uap) { return (ENOSYS); } #endif /* CAPABILITIES */ diff --git a/sys/compat/freebsd32/freebsd32_misc.h b/sys/compat/freebsd32/freebsd32_misc.h index 9e0f55ae8478..f6f85adc8e72 100644 --- a/sys/compat/freebsd32/freebsd32_misc.h +++ b/sys/compat/freebsd32/freebsd32_misc.h @@ -1,49 +1,48 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _COMPAT_FREEBSD32_MISC_H_ #define _COMPAT_FREEBSD32_MISC_H_ #include #if BYTE_ORDER == BIG_ENDIAN #define PAIR32TO64(type, name) ((name ## 2) | ((type)(name ## 1) << 32)) #define RETVAL_HI 0 #define RETVAL_LO 1 #else #define PAIR32TO64(type, name) ((name ## 1) | ((type)(name ## 2) << 32)) #define RETVAL_HI 1 #define RETVAL_LO 0 #endif #endif /* !_COMPAT_FREEBSD32_MISC_H_ */ diff --git a/sys/contrib/libnv/dnvlist.c b/sys/contrib/libnv/dnvlist.c index b118d379406f..3c6898d51256 100644 --- a/sys/contrib/libnv/dnvlist.c +++ b/sys/contrib/libnv/dnvlist.c @@ -1,130 +1,129 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef _KERNEL #include #include #include #include #include #include #else #include #include #include #include #endif #include #include #include "nv_impl.h" #define DNVLIST_GET(ftype, type) \ ftype \ dnvlist_get_##type(const nvlist_t *nvl, const char *name, ftype defval) \ { \ \ if (nvlist_exists_##type(nvl, name)) \ return (nvlist_get_##type(nvl, name)); \ else \ return (defval); \ } DNVLIST_GET(bool, bool) DNVLIST_GET(uint64_t, number) DNVLIST_GET(const char *, string) DNVLIST_GET(const nvlist_t *, nvlist) #ifndef _KERNEL DNVLIST_GET(int, descriptor) #endif #undef DNVLIST_GET const void * dnvlist_get_binary(const nvlist_t *nvl, const char *name, size_t *sizep, const void *defval, size_t defsize) { const void *value; if (nvlist_exists_binary(nvl, name)) value = nvlist_get_binary(nvl, name, sizep); else { if (sizep != NULL) *sizep = defsize; value = defval; } return (value); } #define DNVLIST_TAKE(ftype, type) \ ftype \ dnvlist_take_##type(nvlist_t *nvl, const char *name, ftype defval) \ { \ \ if (nvlist_exists_##type(nvl, name)) \ return (nvlist_take_##type(nvl, name)); \ else \ return (defval); \ } DNVLIST_TAKE(bool, bool) DNVLIST_TAKE(uint64_t, number) DNVLIST_TAKE(char *, string) DNVLIST_TAKE(nvlist_t *, nvlist) #ifndef _KERNEL DNVLIST_TAKE(int, descriptor) #endif #undef DNVLIST_TAKE void * dnvlist_take_binary(nvlist_t *nvl, const char *name, size_t *sizep, void *defval, size_t defsize) { void *value; if (nvlist_exists_binary(nvl, name)) value = nvlist_take_binary(nvl, name, sizep); else { if (sizep != NULL) *sizep = defsize; value = defval; } return (value); } diff --git a/sys/crypto/armv8/armv8_crypto.h b/sys/crypto/armv8/armv8_crypto.h index 0e4cd91e7062..855aabd8bac3 100644 --- a/sys/crypto/armv8/armv8_crypto.h +++ b/sys/crypto/armv8/armv8_crypto.h @@ -1,87 +1,86 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARMV8_CRYPTO_H_ #define _ARMV8_CRYPTO_H_ #define AES256_ROUNDS 14 #define AES_SCHED_LEN ((AES256_ROUNDS + 1) * AES_BLOCK_LEN) typedef struct { uint32_t aes_key[AES_SCHED_LEN/4]; int aes_rounds; } AES_key_t; typedef union { uint64_t u[2]; uint32_t d[4]; uint8_t c[16]; size_t t[16 / sizeof(size_t)]; } __uint128_val_t; struct armv8_crypto_session { AES_key_t enc_schedule; AES_key_t dec_schedule; AES_key_t xts_schedule; __uint128_val_t Htable[16]; }; /* Prototypes for aesv8-armx.S */ void aes_v8_encrypt(uint8_t *in, uint8_t *out, const AES_key_t *key); int aes_v8_set_encrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key); int aes_v8_set_decrypt_key(const unsigned char *userKey, const int bits, const AES_key_t *key); /* Prototypes for ghashv8-armx.S */ void gcm_init_v8(__uint128_val_t Htable[16], const uint64_t Xi[2]); void gcm_gmult_v8(uint64_t Xi[2], const __uint128_val_t Htable[16]); void gcm_ghash_v8(uint64_t Xi[2], const __uint128_val_t Htable[16], const uint8_t *inp, size_t len); void armv8_aes_encrypt_cbc(const AES_key_t *, size_t, const uint8_t *, uint8_t *, const uint8_t[static AES_BLOCK_LEN]); void armv8_aes_decrypt_cbc(const AES_key_t *, size_t, uint8_t *, const uint8_t[static AES_BLOCK_LEN]); void armv8_aes_encrypt_gcm(AES_key_t *, size_t, const uint8_t *, uint8_t *, size_t, const uint8_t*, uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t[static AES_BLOCK_LEN], const __uint128_val_t *); int armv8_aes_decrypt_gcm(AES_key_t *, size_t, const uint8_t *, uint8_t *, size_t, const uint8_t*, const uint8_t tag[static GMAC_DIGEST_LEN], const uint8_t[static AES_BLOCK_LEN], const __uint128_val_t *); void armv8_aes_encrypt_xts(AES_key_t *, const void *, size_t, const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); void armv8_aes_decrypt_xts(AES_key_t *, const void *, size_t, const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]); #endif /* _ARMV8_CRYPTO_H_ */ diff --git a/sys/dev/acpica/acpi_bus_if.m b/sys/dev/acpica/acpi_bus_if.m index df3f930af086..9142a89453c5 100644 --- a/sys/dev/acpica/acpi_bus_if.m +++ b/sys/dev/acpica/acpi_bus_if.m @@ -1,67 +1,66 @@ #- # Copyright (c) 2016 The FreeBSD Foundation -# All rights reserved. # # This software was developed by Andrew Turner under # sponsorship from the FreeBSD Foundation. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ #include #include #include INTERFACE acpi_bus; CODE { static acpi_bus_map_intr_t acpi_bus_default_map_intr; int acpi_bus_default_map_intr(device_t bus, device_t dev, u_int irq, int trig, int pol) { device_t parent; /* Pass up the hierarchy */ parent = device_get_parent(bus); if (parent != NULL) return (ACPI_BUS_MAP_INTR(parent, dev, irq, trig, pol)); panic("Unable to map interrupt %u", irq); } }; # Map an interrupt from ACPI space to the FreeBSD IRQ space. Note that # both of these may be different than the pysical interrupt space as this # may be local to each interrupt controller. # # This method also associates interrupt metadata with the interrupt, # removing the need for a post hoc configure step. METHOD int map_intr { device_t bus; device_t dev; u_int irq; int trig; int pol; } DEFAULT acpi_bus_default_map_intr; diff --git a/sys/dev/agp/agp_i810.h b/sys/dev/agp/agp_i810.h index 211675c4a67d..d3633150fed5 100644 --- a/sys/dev/agp/agp_i810.h +++ b/sys/dev/agp/agp_i810.h @@ -1,104 +1,103 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef AGP_AGP_I810_H #define AGP_AGP_I810_H #include #include #include #include #include /* Special gtt memory types */ #define AGP_DCACHE_MEMORY 1 #define AGP_PHYS_MEMORY 2 /* New caching attributes for gen6/sandybridge */ #define AGP_USER_CACHED_MEMORY_LLC_MLC (AGP_USER_TYPES + 2) #define AGP_USER_UNCACHED_MEMORY (AGP_USER_TYPES + 4) /* flag for GFDT type */ #define AGP_USER_CACHED_MEMORY_GFDT (1 << 3) struct intel_gtt { /* Size of memory reserved for graphics by the BIOS */ unsigned int stolen_size; /* Total number of gtt entries. */ unsigned int gtt_total_entries; /* Part of the gtt that is mappable by the cpu, for those chips where * this is not the full gtt. */ unsigned int gtt_mappable_entries; /* Whether i915 needs to use the dmar apis or not. */ unsigned int needs_dmar : 1; /* Whether we idle the gpu before mapping/unmapping */ unsigned int do_idle_maps : 1; /* Share the scratch page dma with ppgtts. */ vm_paddr_t scratch_page_dma; vm_page_t scratch_page; /* for ppgtt PDE access */ uint32_t *gtt; /* needed for ioremap in drm/i915 */ bus_addr_t gma_bus_addr; }; struct intel_gtt agp_intel_gtt_get(device_t dev); int agp_intel_gtt_chipset_flush(device_t dev); void agp_intel_gtt_unmap_memory(device_t dev, struct sglist *sg_list); void agp_intel_gtt_clear_range(device_t dev, u_int first_entry, u_int num_entries); int agp_intel_gtt_map_memory(device_t dev, vm_page_t *pages, u_int num_entries, struct sglist **sg_list); void agp_intel_gtt_insert_sg_entries(device_t dev, struct sglist *sg_list, u_int pg_start, u_int flags); void agp_intel_gtt_insert_pages(device_t dev, u_int first_entry, u_int num_entries, vm_page_t *pages, u_int flags); struct intel_gtt *intel_gtt_get(void); int intel_gtt_chipset_flush(void); void intel_gtt_unmap_memory(struct sglist *sg_list); void intel_gtt_clear_range(u_int first_entry, u_int num_entries); void intel_gtt_install_pte(u_int index, vm_paddr_t addr, u_int flags); int intel_gtt_map_memory(vm_page_t *pages, u_int num_entries, struct sglist **sg_list); void intel_gtt_insert_sg_entries(struct sglist *sg_list, u_int pg_start, u_int flags); void intel_gtt_insert_pages(u_int first_entry, u_int num_entries, vm_page_t *pages, u_int flags); vm_paddr_t intel_gtt_read_pte_paddr(u_int entry); u_int32_t intel_gtt_read_pte(u_int entry); device_t intel_gtt_get_bridge_device(void); void intel_gtt_write(u_int entry, uint32_t val); #endif diff --git a/sys/dev/ata/chipsets/ata-fsl.c b/sys/dev/ata/chipsets/ata-fsl.c index 32fe2a04bd19..25187390611a 100644 --- a/sys/dev/ata/chipsets/ata-fsl.c +++ b/sys/dev/ata/chipsets/ata-fsl.c @@ -1,242 +1,241 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* local prototypes */ static int imx_ata_ch_attach(device_t dev); static int imx_ata_setmode(device_t dev, int target, int mode); static int imx_ata_probe(device_t dev) { struct ata_pci_controller *ctrl; if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,imx51-ata") && !ofw_bus_is_compatible(dev, "fsl,imx53-ata")) return (ENXIO); ctrl = device_get_softc(dev); device_set_desc(dev, "Freescale Integrated PATA Controller"); return (BUS_PROBE_LOW_PRIORITY); } static void imx_ata_intr(void *data) { struct ata_pci_controller *ctrl = data; bus_write_2(ctrl->r_res1, 0x28, bus_read_2(ctrl->r_res1, 0x28)); ctrl->interrupt[0].function(ctrl->interrupt[0].argument); } static int imx_ata_attach(device_t dev) { struct ata_pci_controller *ctrl; device_t child; int unit; ctrl = device_get_softc(dev); /* do chipset specific setups only needed once */ ctrl->legacy = ata_legacy(dev); ctrl->channels = 1; ctrl->ichannels = -1; ctrl->ch_attach = ata_pci_ch_attach; ctrl->ch_detach = ata_pci_ch_detach; ctrl->dev = dev; ctrl->r_type1 = SYS_RES_MEMORY; ctrl->r_rid1 = 0; ctrl->r_res1 = bus_alloc_resource_any(dev, ctrl->r_type1, &ctrl->r_rid1, RF_ACTIVE); if (ata_setup_interrupt(dev, imx_ata_intr)) { device_printf(dev, "failed to setup interrupt\n"); return ENXIO; } ctrl->channels = 1; ctrl->ch_attach = imx_ata_ch_attach; ctrl->setmode = imx_ata_setmode; /* attach all channels on this controller */ unit = 0; child = device_add_child(dev, "ata", ((unit == 0) && ctrl->legacy) ? unit : devclass_find_free_unit(ata_devclass, 2)); if (child == NULL) device_printf(dev, "failed to add ata child device\n"); else device_set_ivars(child, (void *)(intptr_t)unit); bus_generic_attach(dev); return 0; } static int imx_ata_ch_attach(device_t dev) { struct ata_pci_controller *ctrl; struct ata_channel *ch; int i; ctrl = device_get_softc(device_get_parent(dev)); ch = device_get_softc(dev); for (i = ATA_DATA; i < ATA_MAX_RES; i++) ch->r_io[i].res = ctrl->r_res1; bus_write_2(ctrl->r_res1, 0x24, 0x80); DELAY(100); bus_write_2(ctrl->r_res1, 0x24, 0xc0); DELAY(100); /* Write TIME_OFF/ON/1/2W */ bus_write_1(ctrl->r_res1, 0x00, 3); bus_write_1(ctrl->r_res1, 0x01, 3); bus_write_1(ctrl->r_res1, 0x02, (25 + 15) / 15); bus_write_1(ctrl->r_res1, 0x03, (70 + 15) / 15); /* Write TIME_2R/AX/RDX/4 */ bus_write_1(ctrl->r_res1, 0x04, (70 + 15) / 15); bus_write_1(ctrl->r_res1, 0x05, (50 + 15) / 15 + 2); bus_write_1(ctrl->r_res1, 0x06, 1); bus_write_1(ctrl->r_res1, 0x07, (10 + 15) / 15); /* Write TIME_9 ; the rest of timing registers is irrelevant for PIO */ bus_write_1(ctrl->r_res1, 0x08, (10 + 15) / 15); bus_write_2(ctrl->r_res1, 0x24, 0xc1); DELAY(30000); /* setup ATA registers */ ch->r_io[ATA_DATA ].offset = 0xa0; ch->r_io[ATA_FEATURE].offset = 0xa4; ch->r_io[ATA_ERROR ].offset = 0xa4; ch->r_io[ATA_COUNT ].offset = 0xa8; ch->r_io[ATA_SECTOR ].offset = 0xac; ch->r_io[ATA_CYL_LSB].offset = 0xb0; ch->r_io[ATA_CYL_MSB].offset = 0xb4; ch->r_io[ATA_DRIVE ].offset = 0xb8; ch->r_io[ATA_COMMAND].offset = 0xbc; ch->r_io[ATA_STATUS ].offset = 0xbc; ch->r_io[ATA_ALTSTAT].offset = 0xd8; ch->r_io[ATA_CONTROL].offset = 0xd8; ata_pci_hw(dev); ch->flags |= ATA_NO_SLAVE; ch->flags |= ATA_USE_16BIT; ch->flags |= ATA_CHECKS_CABLE; ch->flags |= ATA_KNOWN_PRESENCE; /* Clear pending interrupts. */ bus_write_2(ctrl->r_res1, 0x28, 0xf8); /* Enable all, but Idle interrupts. */ bus_write_2(ctrl->r_res1, 0x2c, 0x88); return 0; } static int imx_ata_setmode(device_t dev, int target, int mode) { return (min(mode, ATA_PIO4)); } static device_method_t imx_ata_methods[] = { DEVMETHOD(device_probe, imx_ata_probe), DEVMETHOD(device_attach, imx_ata_attach), DEVMETHOD(device_detach, ata_pci_detach), DEVMETHOD(device_suspend, ata_pci_suspend), DEVMETHOD(device_resume, ata_pci_resume), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(bus_read_ivar, ata_pci_read_ivar), DEVMETHOD(bus_write_ivar, ata_pci_write_ivar), DEVMETHOD(bus_alloc_resource, ata_pci_alloc_resource), DEVMETHOD(bus_release_resource, ata_pci_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, ata_pci_setup_intr), DEVMETHOD(bus_teardown_intr, ata_pci_teardown_intr), DEVMETHOD(pci_read_config, ata_pci_read_config), DEVMETHOD(pci_write_config, ata_pci_write_config), DEVMETHOD(bus_print_child, ata_pci_print_child), DEVMETHOD(bus_child_location, ata_pci_child_location), DEVMETHOD_END }; static driver_t imx_ata_driver = { "atapci", imx_ata_methods, sizeof(struct ata_pci_controller) }; DRIVER_MODULE(imx_ata, simplebus, imx_ata_driver, ata_pci_devclass, NULL, NULL); MODULE_VERSION(imx_ata, 1); MODULE_DEPEND(imx_ata, ata, 1, 1, 1); MODULE_DEPEND(imx_ata, atapci, 1, 1, 1); diff --git a/sys/dev/bhnd/bhnd_eromvar.h b/sys/dev/bhnd/bhnd_eromvar.h index a7e6e2cf98f2..dbfed9f520ed 100644 --- a/sys/dev/bhnd/bhnd_eromvar.h +++ b/sys/dev/bhnd/bhnd_eromvar.h @@ -1,89 +1,88 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ #ifndef _BHND_EROM_BHND_EROMVAR_H_ #define _BHND_EROM_BHND_EROMVAR_H_ #include #include "bhnd_erom.h" /* forward declarations */ struct bhnd_erom_io; struct bhnd_erom_iobus; /** @see bhnd_erom_io_map() */ typedef int (bhnd_erom_io_map_t)(struct bhnd_erom_io *eio, bhnd_addr_t addr, bhnd_size_t size); /** @see bhnd_erom_io_tell() */ typedef int (bhnd_erom_io_tell_t)(struct bhnd_erom_io *eio, bhnd_addr_t *addr, bhnd_size_t *size); /** @see bhnd_erom_io_read() */ typedef uint32_t (bhnd_erom_io_read_t)(struct bhnd_erom_io *eio, bhnd_size_t offset, u_int width); /** @see bhnd_erom_io_fini() */ typedef void (bhnd_erom_io_fini_t)(struct bhnd_erom_io *eio); int bhnd_erom_read_chipid(struct bhnd_erom_io *eio, struct bhnd_chipid *cid); /** * Abstract EROM bus I/O support. */ struct bhnd_erom_io { bhnd_erom_io_map_t *map; /**< @see bhnd_erom_io_map() */ bhnd_erom_io_tell_t *tell; /**< @see bhnd_erom_io_tell() */ bhnd_erom_io_read_t *read; /**< @see bhnd_erom_io_read() */ bhnd_erom_io_fini_t *fini; /**< @see bhnd_erom_io_fini(). May be NULL */ }; /** * EROM bus handle/tag I/O instance state. */ struct bhnd_erom_iobus { struct bhnd_erom_io eio; bhnd_addr_t addr; /**< the address of @p bsh */ bhnd_size_t size; /**< the size of @p bsh */ bus_space_tag_t bst; /**< bus space tag */ bus_space_handle_t bsh; /**< bus space handle mapping the full enumeration space */ bool mapped; /**< if a mapping is active */ bus_size_t offset; /**< the current mapped offset within bsh */ bus_size_t limit; /**< the current mapped size relative to offset */ }; #endif /* _BHND_EROM_BHND_EROMVAR_H_ */ diff --git a/sys/dev/bhnd/bhnd_private.h b/sys/dev/bhnd/bhnd_private.h index aae25349e90e..224ea67a0b76 100644 --- a/sys/dev/bhnd/bhnd_private.h +++ b/sys/dev/bhnd/bhnd_private.h @@ -1,104 +1,103 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BHND_BHND_PRIVATE_H_ #define _BHND_BHND_PRIVATE_H_ #include #include #include "bhnd_types.h" /* * Private bhnd(4) driver definitions. */ /** * A bhnd(4) service registry entry. */ struct bhnd_service_entry { device_t provider; /**< service provider */ bhnd_service_t service; /**< service implemented */ uint32_t flags; /**< entry flags (see BHND_SPF_*) */ volatile u_int refs; /**< reference count; updated atomically with only a shared lock held */ STAILQ_ENTRY(bhnd_service_entry) link; }; /** * bhnd(4) per-core PMU clkctl quirks. */ enum { /** On BCM4328-derived chipsets, the CLK_CTL_ST register CCS_HTAVAIL * and CCS_ALPAVAIL bits are swapped in the ChipCommon and PCMCIA * cores; the BHND_CCS0_* constants should be used. */ BHND_CLKCTL_QUIRK_CCS0 = 1 }; /** * Per-core bhnd(4) PMU clkctl registers. */ struct bhnd_core_clkctl { device_t cc_dev; /**< core device */ device_t cc_pmu_dev; /**< pmu device */ uint32_t cc_quirks; /**< core-specific clkctl quirks */ struct bhnd_resource *cc_res; /**< resource mapping core's clkctl register */ bus_size_t cc_res_offset; /**< offset to clkctl register */ u_int cc_max_latency; /**< maximum PMU transition latency, in microseconds */ struct mtx cc_mtx; /**< register read/modify/write lock */ }; #define BHND_ASSERT_CLKCTL_AVAIL(_clkctl) \ KASSERT(!bhnd_is_hw_suspended((_clkctl)->cc_dev), \ ("reading clkctl on suspended core will trigger system livelock")) #define BHND_CLKCTL_LOCK_INIT(_clkctl) mtx_init(&(_clkctl)->cc_mtx, \ device_get_nameunit((_clkctl)->cc_dev), NULL, MTX_DEF) #define BHND_CLKCTL_LOCK(_clkctl) mtx_lock(&(_clkctl)->cc_mtx) #define BHND_CLKCTL_UNLOCK(_clkctl) mtx_unlock(&(_clkctl)->cc_mtx) #define BHND_CLKCTL_LOCK_ASSERT(_clkctl, what) \ mtx_assert(&(_clkctl)->cc_mtx, what) #define BHND_CLKCTL_LOCK_DESTROY(_clkctl) mtx_destroy(&(_clkctl->cc_mtx)) #define BHND_CLKCTL_READ_4(_clkctl) \ bhnd_bus_read_4((_clkctl)->cc_res, (_clkctl)->cc_res_offset) #define BHND_CLKCTL_WRITE_4(_clkctl, _val) \ bhnd_bus_write_4((_clkctl)->cc_res, (_clkctl)->cc_res_offset, (_val)) #define BHND_CLKCTL_SET_4(_clkctl, _val, _mask) \ BHND_CLKCTL_WRITE_4((_clkctl), \ ((_val) & (_mask)) | (BHND_CLKCTL_READ_4(_clkctl) & ~(_mask))) #endif /* _BHND_BHND_PRIVATE_H_ */ diff --git a/sys/dev/bhnd/cores/chipc/chipc_gpio.c b/sys/dev/bhnd/cores/chipc/chipc_gpio.c index 9123531d3ab6..7c75259c226f 100644 --- a/sys/dev/bhnd/cores/chipc/chipc_gpio.c +++ b/sys/dev/bhnd/cores/chipc/chipc_gpio.c @@ -1,846 +1,845 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "bhnd_nvram_map.h" #include "chipcreg.h" #include "chipc_gpiovar.h" /* * ChipCommon GPIO driver */ static int chipc_gpio_check_flags( struct chipc_gpio_softc *sc, uint32_t pin_num, uint32_t flags, chipc_gpio_pin_mode *mode); static int chipc_gpio_pin_update( struct chipc_gpio_softc *sc, struct chipc_gpio_update *update, uint32_t pin_num, uint32_t flags); static int chipc_gpio_commit_update( struct chipc_gpio_softc *sc, struct chipc_gpio_update *update); static chipc_gpio_pin_mode chipc_gpio_pin_get_mode( struct chipc_gpio_softc *sc, uint32_t pin_num); /* Debugging flags */ static u_long chipc_gpio_debug = 0; TUNABLE_ULONG("hw.bhnd_chipc.gpio_debug", &chipc_gpio_debug); enum { /** Allow userspace GPIO access on bridged network (e.g. wi-fi) * adapters */ CC_GPIO_DEBUG_ADAPTER_GPIOC = 1 << 0, }; #define CC_GPIO_DEBUG(_type) (CC_GPIO_DEBUG_ ## _type & chipc_gpio_debug) static struct bhnd_device_quirk chipc_gpio_quirks[]; /* Supported parent core device identifiers */ static const struct bhnd_device chipc_gpio_devices[] = { BHND_DEVICE(BCM, CC, "Broadcom ChipCommon GPIO", chipc_gpio_quirks), BHND_DEVICE_END }; /* Device quirks table */ static struct bhnd_device_quirk chipc_gpio_quirks[] = { BHND_CORE_QUIRK (HWREV_LTE(10), CC_GPIO_QUIRK_NO_EVENTS), BHND_CORE_QUIRK (HWREV_LTE(15), CC_GPIO_QUIRK_NO_DCTIMER), BHND_CORE_QUIRK (HWREV_LTE(19), CC_GPIO_QUIRK_NO_PULLUPDOWN), BHND_DEVICE_QUIRK_END }; static int chipc_gpio_probe(device_t dev) { const struct bhnd_device *id; device_t chipc; /* Look for compatible chipc parent */ chipc = device_get_parent(dev); id = bhnd_device_lookup(chipc, chipc_gpio_devices, sizeof(chipc_gpio_devices[0])); if (id == NULL) return (ENXIO); device_set_desc(dev, id->desc); return (BUS_PROBE_NOWILDCARD); } static int chipc_gpio_attach(device_t dev) { struct chipc_gpio_softc *sc; device_t chipc; int error; chipc = device_get_parent(dev); sc = device_get_softc(dev); sc->dev = dev; sc->quirks = bhnd_device_quirks(chipc, chipc_gpio_devices, sizeof(chipc_gpio_devices[0])); /* If this is a bridged wi-fi adapter, we don't want to support * userspace requests via gpioc(4) */ if (bhnd_get_attach_type(chipc) == BHND_ATTACH_ADAPTER) { if (!CC_GPIO_DEBUG(ADAPTER_GPIOC)) sc->quirks |= CC_GPIO_QUIRK_NO_GPIOC; } CC_GPIO_LOCK_INIT(sc); sc->mem_rid = 0; sc->mem_res = bhnd_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, RF_ACTIVE|RF_SHAREABLE); if (sc->mem_res == NULL) { device_printf(dev, "failed to allocate chipcommon registers\n"); error = ENXIO; goto failed; } /* * If hardware 'pulsate' support is available, set the timer duty-cycle * to either the NVRAM 'leddc' value if available, or the default duty * cycle. */ if (!CC_GPIO_QUIRK(sc, NO_DCTIMER)) { uint32_t dctimerval; error = bhnd_nvram_getvar_uint32(chipc, BHND_NVAR_LEDDC, &dctimerval); if (error == ENOENT) { /* Fall back on default duty cycle */ dctimerval = CHIPC_GPIOTIMERVAL_DEFAULT; } else if (error) { device_printf(dev, "error reading %s from NVRAM: %d\n", BHND_NVAR_LEDDC, error); goto failed; } CC_GPIO_WR4(sc, CHIPC_GPIOTIMERVAL, dctimerval); } /* Attach gpioc/gpiobus */ if (CC_GPIO_QUIRK(sc, NO_GPIOC)) { sc->gpiobus = NULL; } else { if ((sc->gpiobus = gpiobus_attach_bus(dev)) == NULL) { device_printf(dev, "failed to attach gpiobus\n"); error = ENXIO; goto failed; } } /* Register as the bus GPIO provider */ if ((error = bhnd_register_provider(dev, BHND_SERVICE_GPIO))) { device_printf(dev, "failed to register gpio with bus: %d\n", error); goto failed; } return (0); failed: device_delete_children(dev); if (sc->mem_res != NULL) { bhnd_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); } CC_GPIO_LOCK_DESTROY(sc); return (error); } static int chipc_gpio_detach(device_t dev) { struct chipc_gpio_softc *sc; int error; sc = device_get_softc(dev); if ((error = bus_generic_detach(dev))) return (error); if ((error = bhnd_deregister_provider(dev, BHND_SERVICE_ANY))) return (error); bhnd_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem_res); CC_GPIO_LOCK_DESTROY(sc); return (0); } static device_t chipc_gpio_get_bus(device_t dev) { struct chipc_gpio_softc *sc = device_get_softc(dev); return (sc->gpiobus); } static int chipc_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = CC_GPIO_NPINS-1; return (0); } static int chipc_gpio_pin_set(device_t dev, uint32_t pin_num, uint32_t pin_value) { struct chipc_gpio_softc *sc; bool pin_high; int error; sc = device_get_softc(dev); error = 0; if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); switch (pin_value) { case GPIO_PIN_HIGH: pin_high = true; break; case GPIO_PIN_LOW: pin_high = false; break; default: return (EINVAL); } CC_GPIO_LOCK(sc); switch (chipc_gpio_pin_get_mode(sc, pin_num)) { case CC_GPIO_PIN_INPUT: case CC_GPIO_PIN_TRISTATE: error = ENODEV; break; case CC_GPIO_PIN_OUTPUT: CC_GPIO_WRFLAG(sc, pin_num, GPIOOUT, pin_high); break; } CC_GPIO_UNLOCK(sc); return (error); } static int chipc_gpio_pin_get(device_t dev, uint32_t pin_num, uint32_t *pin_value) { struct chipc_gpio_softc *sc; bool pin_high; if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); sc = device_get_softc(dev); pin_high = false; CC_GPIO_LOCK(sc); switch (chipc_gpio_pin_get_mode(sc, pin_num)) { case CC_GPIO_PIN_INPUT: pin_high = CC_GPIO_RDFLAG(sc, pin_num, GPIOIN); break; case CC_GPIO_PIN_OUTPUT: pin_high = CC_GPIO_RDFLAG(sc, pin_num, GPIOOUT); break; case CC_GPIO_PIN_TRISTATE: pin_high = false; break; } CC_GPIO_UNLOCK(sc); *pin_value = pin_high ? GPIO_PIN_HIGH : GPIO_PIN_LOW; return (0); } static int chipc_gpio_pin_toggle(device_t dev, uint32_t pin_num) { struct chipc_gpio_softc *sc; bool pin_high; int error; if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); sc = device_get_softc(dev); error = 0; CC_GPIO_LOCK(sc); switch (chipc_gpio_pin_get_mode(sc, pin_num)) { case CC_GPIO_PIN_INPUT: case CC_GPIO_PIN_TRISTATE: error = ENODEV; break; case CC_GPIO_PIN_OUTPUT: pin_high = CC_GPIO_RDFLAG(sc, pin_num, GPIOOUT); CC_GPIO_WRFLAG(sc, pin_num, GPIOOUT, !pin_high); break; } CC_GPIO_UNLOCK(sc); return (error); } static int chipc_gpio_pin_getcaps(device_t dev, uint32_t pin_num, uint32_t *caps) { struct chipc_gpio_softc *sc = device_get_softc(dev); if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); *caps = (GPIO_PIN_INPUT | GPIO_PIN_OUTPUT | GPIO_PIN_TRISTATE); if (!CC_GPIO_QUIRK(sc, NO_PULLUPDOWN)) *caps |= (GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN); if (!CC_GPIO_QUIRK(sc, NO_DCTIMER)) *caps |= GPIO_PIN_PULSATE; return (0); } static int chipc_gpio_pin_getflags(device_t dev, uint32_t pin_num, uint32_t *flags) { struct chipc_gpio_softc *sc = device_get_softc(dev); if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); CC_GPIO_LOCK(sc); switch (chipc_gpio_pin_get_mode(sc, pin_num)) { case CC_GPIO_PIN_INPUT: *flags = GPIO_PIN_INPUT; if (!CC_GPIO_QUIRK(sc, NO_PULLUPDOWN)) { if (CC_GPIO_RDFLAG(sc, pin_num, GPIOPU)) { *flags |= GPIO_PIN_PULLUP; } else if (CC_GPIO_RDFLAG(sc, pin_num, GPIOPD)) { *flags |= GPIO_PIN_PULLDOWN; } } break; case CC_GPIO_PIN_OUTPUT: *flags = GPIO_PIN_OUTPUT; if (!CC_GPIO_QUIRK(sc, NO_DCTIMER)) { if (CC_GPIO_RDFLAG(sc, pin_num, GPIOTIMEROUTMASK)) *flags |= GPIO_PIN_PULSATE; } break; case CC_GPIO_PIN_TRISTATE: *flags = GPIO_PIN_TRISTATE|GPIO_PIN_OUTPUT; break; } CC_GPIO_UNLOCK(sc); return (0); } static int chipc_gpio_pin_getname(device_t dev, uint32_t pin_num, char *name) { int ret; if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); ret = snprintf(name, GPIOMAXNAME, "bhnd_gpio%02" PRIu32, pin_num); if (ret < 0) return (ENXIO); if (ret >= GPIOMAXNAME) return (ENOMEM); return (0); } static int chipc_gpio_pin_setflags(device_t dev, uint32_t pin_num, uint32_t flags) { struct chipc_gpio_softc *sc; struct chipc_gpio_update upd; int error; sc = device_get_softc(dev); if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); /* Produce an update descriptor */ memset(&upd, 0, sizeof(upd)); if ((error = chipc_gpio_pin_update(sc, &upd, pin_num, flags))) return (error); /* Commit the update */ CC_GPIO_LOCK(sc); error = chipc_gpio_commit_update(sc, &upd); CC_GPIO_UNLOCK(sc); return (error); } static int chipc_gpio_pin_access_32(device_t dev, uint32_t first_pin, uint32_t clear_pins, uint32_t change_pins, uint32_t *orig_pins) { struct chipc_gpio_softc *sc; struct chipc_gpio_update upd; uint32_t out, outen, ctrl; uint32_t num_pins; int error; sc = device_get_softc(dev); if (first_pin >= CC_GPIO_NPINS) return (EINVAL); /* Determine the actual number of referenced pins */ if (clear_pins == 0 && change_pins == 0) { num_pins = CC_GPIO_NPINS - first_pin; } else { int num_clear_pins, num_change_pins; num_clear_pins = flsl((u_long)clear_pins); num_change_pins = flsl((u_long)change_pins); num_pins = MAX(num_clear_pins, num_change_pins); } /* Validate the full pin range */ if (!CC_GPIO_VALID_PINS(first_pin, num_pins)) return (EINVAL); /* Produce an update descriptor for all pins, relative to the current * pin state */ CC_GPIO_LOCK(sc); memset(&upd, 0, sizeof(upd)); out = CC_GPIO_RD4(sc, CHIPC_GPIOOUT); outen = CC_GPIO_RD4(sc, CHIPC_GPIOOUTEN); ctrl = CC_GPIO_RD4(sc, CHIPC_GPIOCTRL); for (uint32_t i = 0; i < num_pins; i++) { uint32_t pin; bool pin_high; pin = first_pin + i; /* The pin must be configured for output */ if ((outen & (1 << pin)) == 0) { CC_GPIO_UNLOCK(sc); return (EINVAL); } /* The pin must not tristated */ if ((ctrl & (1 << pin)) != 0) { CC_GPIO_UNLOCK(sc); return (EINVAL); } /* Fetch current state */ if (out & (1 << pin)) { pin_high = true; } else { pin_high = false; } /* Apply clear/toggle request */ if (clear_pins & (1 << pin)) pin_high = false; if (change_pins & (1 << pin)) pin_high = !pin_high; /* Add to our update descriptor */ CC_GPIO_UPDATE(&upd, pin, out, pin_high); } /* Commit the update */ error = chipc_gpio_commit_update(sc, &upd); CC_GPIO_UNLOCK(sc); return (error); } static int chipc_gpio_pin_config_32(device_t dev, uint32_t first_pin, uint32_t num_pins, uint32_t *pin_flags) { struct chipc_gpio_softc *sc; struct chipc_gpio_update upd; int error; sc = device_get_softc(dev); if (!CC_GPIO_VALID_PINS(first_pin, num_pins)) return (EINVAL); /* Produce an update descriptor */ memset(&upd, 0, sizeof(upd)); for (uint32_t i = 0; i < num_pins; i++) { uint32_t pin, flags; pin = first_pin + i; flags = pin_flags[i]; /* As per the gpio_config_32 API documentation, any pins for * which neither GPIO_PIN_OUTPUT or GPIO_PIN_INPUT are set * should be ignored and left unmodified */ if ((flags & (GPIO_PIN_OUTPUT|GPIO_PIN_INPUT)) == 0) continue; if ((error = chipc_gpio_pin_update(sc, &upd, pin, flags))) return (error); } /* Commit the update */ CC_GPIO_LOCK(sc); error = chipc_gpio_commit_update(sc, &upd); CC_GPIO_UNLOCK(sc); return (error); } /** * Commit a single @p reg register update. */ static void chipc_gpio_commit_reg(struct chipc_gpio_softc *sc, bus_size_t offset, struct chipc_gpio_reg *reg) { uint32_t value; CC_GPIO_LOCK_ASSERT(sc, MA_OWNED); if (reg->mask == 0) return; value = bhnd_bus_read_4(sc->mem_res, offset); value &= ~reg->mask; value |= reg->value; bhnd_bus_write_4(sc->mem_res, offset, value); } /** * Commit the set of GPIO register updates described by @p update. */ static int chipc_gpio_commit_update(struct chipc_gpio_softc *sc, struct chipc_gpio_update *update) { CC_GPIO_LOCK_ASSERT(sc, MA_OWNED); /* Commit pulldown/pullup before potentially disabling an output pin */ chipc_gpio_commit_reg(sc, CHIPC_GPIOPD, &update->pulldown); chipc_gpio_commit_reg(sc, CHIPC_GPIOPU, &update->pullup); /* Commit output settings before potentially enabling an output pin */ chipc_gpio_commit_reg(sc, CHIPC_GPIOTIMEROUTMASK, &update->timeroutmask); chipc_gpio_commit_reg(sc, CHIPC_GPIOOUT, &update->out); /* Commit input/output/tristate modes */ chipc_gpio_commit_reg(sc, CHIPC_GPIOOUTEN, &update->outen); chipc_gpio_commit_reg(sc, CHIPC_GPIOCTRL, &update->ctrl); return (0); } /** * Apply the changes described by @p flags for @p pin_num to the given @p update * descriptor. */ static int chipc_gpio_pin_update(struct chipc_gpio_softc *sc, struct chipc_gpio_update *update, uint32_t pin_num, uint32_t flags) { chipc_gpio_pin_mode mode; int error; if (!CC_GPIO_VALID_PIN(pin_num)) return (EINVAL); /* Verify flag compatibility and determine the pin mode */ if ((error = chipc_gpio_check_flags(sc, pin_num, flags, &mode))) return (error); /* Apply the mode-specific changes */ switch (mode) { case CC_GPIO_PIN_INPUT: CC_GPIO_UPDATE(update, pin_num, pullup, false); CC_GPIO_UPDATE(update, pin_num, pulldown, false); CC_GPIO_UPDATE(update, pin_num, out, false); CC_GPIO_UPDATE(update, pin_num, outen, false); CC_GPIO_UPDATE(update, pin_num, timeroutmask, false); CC_GPIO_UPDATE(update, pin_num, ctrl, false); if (flags & GPIO_PIN_PULLUP) { CC_GPIO_UPDATE(update, pin_num, pullup, true); } else if (flags & GPIO_PIN_PULLDOWN) { CC_GPIO_UPDATE(update, pin_num, pulldown, true); } return (0); case CC_GPIO_PIN_OUTPUT: CC_GPIO_UPDATE(update, pin_num, pullup, false); CC_GPIO_UPDATE(update, pin_num, pulldown, false); CC_GPIO_UPDATE(update, pin_num, outen, true); CC_GPIO_UPDATE(update, pin_num, timeroutmask, false); CC_GPIO_UPDATE(update, pin_num, ctrl, false); if (flags & GPIO_PIN_PRESET_HIGH) { CC_GPIO_UPDATE(update, pin_num, out, true); } else if (flags & GPIO_PIN_PRESET_LOW) { CC_GPIO_UPDATE(update, pin_num, out, false); } if (flags & GPIO_PIN_PULSATE) CC_GPIO_UPDATE(update, pin_num, timeroutmask, true); return (0); case CC_GPIO_PIN_TRISTATE: CC_GPIO_UPDATE(update, pin_num, pullup, false); CC_GPIO_UPDATE(update, pin_num, pulldown, false); CC_GPIO_UPDATE(update, pin_num, out, false); CC_GPIO_UPDATE(update, pin_num, outen, false); CC_GPIO_UPDATE(update, pin_num, timeroutmask, false); CC_GPIO_UPDATE(update, pin_num, ctrl, true); if (flags & GPIO_PIN_OUTPUT) CC_GPIO_UPDATE(update, pin_num, outen, true); return (0); } device_printf(sc->dev, "unknown pin mode %d\n", mode); return (EINVAL); } /** * Verify that @p flags are valid for use with @p pin_num, and on success, * return the pin mode described by @p flags in @p mode. * * @param sc GPIO driver instance state. * @param pin_num The pin number to configure. * @param flags The pin flags to be validated. * @param[out] mode On success, will be populated with the GPIO pin mode * defined by @p flags. * * @retval 0 success * @retval EINVAL if @p flags are invalid. */ static int chipc_gpio_check_flags(struct chipc_gpio_softc *sc, uint32_t pin_num, uint32_t flags, chipc_gpio_pin_mode *mode) { uint32_t mode_flag, input_flag, output_flag; CC_GPIO_ASSERT_VALID_PIN(sc, pin_num); mode_flag = flags & (GPIO_PIN_OUTPUT | GPIO_PIN_INPUT | GPIO_PIN_TRISTATE); output_flag = flags & (GPIO_PIN_PRESET_HIGH | GPIO_PIN_PRESET_LOW | GPIO_PIN_PULSATE); input_flag = flags & (GPIO_PIN_PULLUP | GPIO_PIN_PULLDOWN); switch (mode_flag) { case GPIO_PIN_OUTPUT: /* No input flag(s) should be set */ if (input_flag != 0) return (EINVAL); /* Validate our output flag(s) */ switch (output_flag) { case GPIO_PIN_PRESET_HIGH: case GPIO_PIN_PRESET_LOW: case (GPIO_PIN_PRESET_HIGH|GPIO_PIN_PULSATE): case (GPIO_PIN_PRESET_LOW|GPIO_PIN_PULSATE): case 0: /* Check for unhandled flags */ if ((flags & ~(mode_flag | output_flag)) != 0) return (EINVAL); *mode = CC_GPIO_PIN_OUTPUT; return (0); default: /* Incompatible output flags */ return (EINVAL); } case GPIO_PIN_INPUT: /* No output flag(s) should be set */ if (output_flag != 0) return (EINVAL); /* Validate our input flag(s) */ switch (input_flag) { case GPIO_PIN_PULLUP: case GPIO_PIN_PULLDOWN: case 0: /* Check for unhandled flags */ if ((flags & ~(mode_flag | input_flag)) != 0) return (EINVAL); *mode = CC_GPIO_PIN_INPUT; return (0); default: /* Incompatible input flags */ return (EINVAL); } break; case (GPIO_PIN_TRISTATE|GPIO_PIN_OUTPUT): case GPIO_PIN_TRISTATE: /* No input or output flag(s) should be set */ if (input_flag != 0 || output_flag != 0) return (EINVAL); /* Check for unhandled flags */ if ((flags & ~mode_flag) != 0) return (EINVAL); *mode = CC_GPIO_PIN_TRISTATE; return (0); default: /* Incompatible mode flags */ return (EINVAL); } } /** * Return the current pin mode for @p pin_num. * * @param sc GPIO driver instance state. * @param pin_num The pin number to query. */ static chipc_gpio_pin_mode chipc_gpio_pin_get_mode(struct chipc_gpio_softc *sc, uint32_t pin_num) { CC_GPIO_LOCK_ASSERT(sc, MA_OWNED); CC_GPIO_ASSERT_VALID_PIN(sc, pin_num); if (CC_GPIO_RDFLAG(sc, pin_num, GPIOCTRL)) { return (CC_GPIO_PIN_TRISTATE); } else if (CC_GPIO_RDFLAG(sc, pin_num, GPIOOUTEN)) { return (CC_GPIO_PIN_OUTPUT); } else { return (CC_GPIO_PIN_INPUT); } } static device_method_t chipc_gpio_methods[] = { /* Device interface */ DEVMETHOD(device_probe, chipc_gpio_probe), DEVMETHOD(device_attach, chipc_gpio_attach), DEVMETHOD(device_detach, chipc_gpio_detach), /* GPIO interface */ DEVMETHOD(gpio_get_bus, chipc_gpio_get_bus), DEVMETHOD(gpio_pin_max, chipc_gpio_pin_max), DEVMETHOD(gpio_pin_getname, chipc_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, chipc_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, chipc_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, chipc_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, chipc_gpio_pin_get), DEVMETHOD(gpio_pin_set, chipc_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, chipc_gpio_pin_toggle), DEVMETHOD(gpio_pin_access_32, chipc_gpio_pin_access_32), DEVMETHOD(gpio_pin_config_32, chipc_gpio_pin_config_32), DEVMETHOD_END }; static devclass_t gpio_devclass; DEFINE_CLASS_0(gpio, chipc_gpio_driver, chipc_gpio_methods, sizeof(struct chipc_gpio_softc)); EARLY_DRIVER_MODULE(chipc_gpio, bhnd_chipc, chipc_gpio_driver, gpio_devclass, NULL, NULL, BUS_PASS_RESOURCE + BUS_PASS_ORDER_MIDDLE); MODULE_DEPEND(chipc_gpio, bhnd, 1, 1, 1); MODULE_DEPEND(chipc_gpio, gpiobus, 1, 1, 1); MODULE_VERSION(chipc_gpio, 1); diff --git a/sys/dev/bhnd/cores/chipc/chipc_gpiovar.h b/sys/dev/bhnd/cores/chipc/chipc_gpiovar.h index 640fbc1560d4..c4bbcdf47688 100644 --- a/sys/dev/bhnd/cores/chipc/chipc_gpiovar.h +++ b/sys/dev/bhnd/cores/chipc/chipc_gpiovar.h @@ -1,163 +1,162 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BHND_CORES_CHIPC_CHIPC_GPIOVAR_H_ #define _BHND_CORES_CHIPC_CHIPC_GPIOVAR_H_ #include #include #include #include #include /** * ChipCommon GPIO device quirks. */ enum { /** * No GPIO event support. * * The CHIPC_GPIOEVENT, CHIPC_GPIOEVENT_INTM, and * CHIPC_GPIOEVENT_INTPOLARITY registers are not available. */ CC_GPIO_QUIRK_NO_EVENTS = (1<<0), /** * No GPIO duty-cycle timer support. * * The CHIPC_GPIOTIMERVAL and CHIPC_GPIOTIMEROUTMASK registers are not * available. */ CC_GPIO_QUIRK_NO_DCTIMER = (1<<1), /** * No GPIO pull-up/pull-down configuration support. * * The CHIPC_GPIOPU and CHIPC_GPIOPD registers are not available. */ CC_GPIO_QUIRK_NO_PULLUPDOWN = (1<<2), /** * Do not attach a child gpioc(4) device. * * This is primarily intended for use on bridged Wi-Fi adapters, where * userspace modification of GPIO pin configuration could introduce * significant undesirable behavior. */ CC_GPIO_QUIRK_NO_GPIOC = (1<<3), }; /** ChipCommon GPIO pin modes */ typedef enum { CC_GPIO_PIN_INPUT, CC_GPIO_PIN_OUTPUT, CC_GPIO_PIN_TRISTATE } chipc_gpio_pin_mode; /** * A single GPIO update register. */ struct chipc_gpio_reg { uint32_t value; /**< register update value */ uint32_t mask; /**< register update mask */ }; /** * A GPIO register update descriptor. */ struct chipc_gpio_update { struct chipc_gpio_reg pullup; /**< CHIPC_GPIOPU changes */ struct chipc_gpio_reg pulldown; /**< CHIPC_GPIOPD changes */ struct chipc_gpio_reg out; /**< CHIPC_GPIOOUT changes */ struct chipc_gpio_reg outen; /**< CHIPC_GPIOOUTEN changes */ struct chipc_gpio_reg timeroutmask; /**< CHIPC_GPIOTIMEROUTMASK changes */ struct chipc_gpio_reg ctrl; /**< CHIPC_GPIOCTRL changes */ }; #define CC_GPIO_UPDATE(_upd, _pin, _reg, _val) do { \ (_upd)->_reg.mask |= (1 << (_pin)); \ if (_val) \ (_upd)->_reg.value |= (1 << (_pin)); \ else \ (_upd)->_reg.value &= ~(1 << (_pin)); \ } while(0) /** * ChipCommon GPIO driver instance state. */ struct chipc_gpio_softc { device_t dev; device_t gpiobus; /**< attached gpiobus child */ struct bhnd_resource *mem_res; /**< chipcommon register block */ int mem_rid; /**< resource ID of mem_res */ uint32_t quirks; /**< device quirks (see CC_GPIO_QUIRK_*) */ struct mtx mtx; /**< lock protecting RMW register access */ }; #define CC_GPIO_LOCK_INIT(sc) mtx_init(&(sc)->mtx, \ device_get_nameunit((sc)->dev), NULL, MTX_DEF) #define CC_GPIO_LOCK(sc) mtx_lock(&(sc)->mtx) #define CC_GPIO_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define CC_GPIO_LOCK_ASSERT(sc, what) mtx_assert(&(sc)->mtx, what) #define CC_GPIO_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx) #define CC_GPIO_WR4(sc, off, val) \ bhnd_bus_write_4((sc)->mem_res, (off), (val)) #define CC_GPIO_WRFLAG(sc, pin_num, flag, val) \ CC_GPIO_WR4(sc, CHIPC_ ## flag, \ (CC_GPIO_RD4(sc, CHIPC_ ## flag) & ~(1 << pin_num)) | \ (val ? (1 << pin_num) : 0)) #define CC_GPIO_RD4(sc, off) \ bhnd_bus_read_4((sc)->mem_res, (off)) #define CC_GPIO_RDFLAG(sc, pin_num, flag) \ ((CC_GPIO_RD4(sc, CHIPC_ ## flag) & (1 << pin_num)) != 0) #define CC_GPIO_NPINS 32 #define CC_GPIO_VALID_PIN(_pin) \ ((_pin) >= 0 && (_pin) < CC_GPIO_NPINS) #define CC_GPIO_VALID_PINS(_first, _num) \ ((_num) <= CC_GPIO_NPINS && CC_GPIO_NPINS - (_num) >= _first) #define CC_GPIO_ASSERT_VALID_PIN(sc, pin_num) \ KASSERT(CC_GPIO_VALID_PIN(pin_num), ("invalid pin# %" PRIu32, pin_num)); #define CC_GPIO_QUIRK(_sc, _name) \ ((_sc)->quirks & CC_GPIO_QUIRK_ ## _name) #define CC_GPIO_ASSERT_QUIRK(_sc, name) \ KASSERT(CC_GPIO_QUIRK((_sc), name), ("quirk " __STRING(_name) " not set")) #endif /* _BHND_PWRCTL_BHND_PWRCTLVAR_H_ */ diff --git a/sys/dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl.h b/sys/dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl.h index 3d05709d5b38..01d3e76298dd 100644 --- a/sys/dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl.h +++ b/sys/dev/bhnd/cores/chipc/pwrctl/bhnd_pwrctl.h @@ -1,94 +1,93 @@ /*- * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BHND_PWRCTL_BHND_PWRCTL_H_ #define _BHND_PWRCTL_BHND_PWRCTL_H_ #include #include #include "bhnd_pwrctl_if.h" /** * Request that @p clock (or a faster clock) be enabled on behalf of * @p child. * * @param dev PWRCTL device. * @param child The requesting bhnd(4) device. * @param clock Clock requested. * * @retval 0 success * @retval ENODEV If an unsupported clock was requested. */ static inline int bhnd_pwrctl_request_clock(device_t dev, device_t child, bhnd_clock clock) { return (BHND_PWRCTL_REQUEST_CLOCK(dev, child, clock)); } /** * Return the transition latency required for @p clock in microseconds, if * known. * * The BHND_CLOCK_HT latency value is suitable for use as the D11 core's * 'fastpwrup_dly' value. * * @param dev PWRCTL device. * @param clock The clock to be queried for transition latency. * @param[out] latency On success, the transition latency of @p clock in * microseconds. * * @retval 0 success * @retval ENODEV If the transition latency for @p clock is not available. */ static inline int bhnd_pwrctl_get_clock_latency(device_t dev, bhnd_clock clock, u_int *latency) { return (BHND_PWRCTL_GET_CLOCK_LATENCY(dev, clock, latency)); } /** * Return the frequency for @p clock in Hz, if known. * * @param dev PWRCTL device. * @param clock The clock to be queried. * @param[out] freq On success, the frequency of @p clock in Hz. * * @retval 0 success * @retval ENODEV If the frequency for @p clock is not available. */ static inline int bhnd_pwrctl_get_clock_freq(device_t dev, bhnd_clock clock, u_int *freq) { return (BHND_PWRCTL_GET_CLOCK_FREQ(dev, clock, freq)); } #endif /* _BHND_PWRCTL_BHND_PWRCTL_H_ */ diff --git a/sys/dev/bhnd/cores/pmu/bhnd_pmu_types.h b/sys/dev/bhnd/cores/pmu/bhnd_pmu_types.h index 700ff3287b16..48f54f235d80 100644 --- a/sys/dev/bhnd/cores/pmu/bhnd_pmu_types.h +++ b/sys/dev/bhnd/cores/pmu/bhnd_pmu_types.h @@ -1,55 +1,54 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BHND_CORES_PMU_BHND_PMU_TYPES_H_ #define _BHND_CORES_PMU_BHND_PMU_TYPES_H_ #include /** * bhnd_pmu(4) regulators. */ typedef enum bhnd_pmu_regulator { BHND_REGULATOR_PAREF_LDO = 0, /**< PA reference LDO */ } bhnd_pmu_regulator; /** * bhnd_pmu(4) spurious signal avoidance modes. */ typedef enum bhnd_pmu_spuravoid { BHND_PMU_SPURAVOID_NONE = 0, /**< spur avoidance disabled */ BHND_PMU_SPURAVOID_M1 = 1, /**< chipset-specific mode 1 */ BHND_PMU_SPURAVOID_M2 = 2, /**< chipset-specific mode 2 */ } bhnd_pmu_spuravoid; #endif /* _BHND_CORES_PMU_BHND_PMU_TYPES_H_ */ diff --git a/sys/dev/drm2/drm_gem.c b/sys/dev/drm2/drm_gem.c index 47d363a811ad..2692eb2499ae 100644 --- a/sys/dev/drm2/drm_gem.c +++ b/sys/dev/drm2/drm_gem.c @@ -1,498 +1,497 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include /* * We make up offsets for buffer objects so we can recognize them at * mmap time. */ /* pgoff in mmap is an unsigned long, so we need to make sure that * the faked up offset will fit */ #if BITS_PER_LONG == 64 #define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1) #define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 16) #else #define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1) #define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16) #endif /** * Initialize the GEM device fields */ int drm_gem_init(struct drm_device *dev) { struct drm_gem_mm *mm; drm_gem_names_init(&dev->object_names); mm = malloc(sizeof(*mm), DRM_MEM_DRIVER, M_NOWAIT); if (!mm) { DRM_ERROR("out of memory\n"); return -ENOMEM; } dev->mm_private = mm; if (drm_ht_create(&mm->offset_hash, 19)) { free(mm, DRM_MEM_DRIVER); return -ENOMEM; } mm->idxunr = new_unrhdr(0, DRM_GEM_MAX_IDX, NULL); return 0; } void drm_gem_destroy(struct drm_device *dev) { struct drm_gem_mm *mm = dev->mm_private; dev->mm_private = NULL; drm_ht_remove(&mm->offset_hash); delete_unrhdr(mm->idxunr); free(mm, DRM_MEM_DRIVER); drm_gem_names_fini(&dev->object_names); } int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj, size_t size) { KASSERT((size & (PAGE_SIZE - 1)) == 0, ("Bad size %ju", (uintmax_t)size)); obj->dev = dev; obj->vm_obj = vm_pager_allocate(OBJT_DEFAULT, NULL, size, VM_PROT_READ | VM_PROT_WRITE, 0, curthread->td_ucred); obj->refcount = 1; obj->handle_count = 0; obj->size = size; return 0; } EXPORT_SYMBOL(drm_gem_object_init); /** * Initialize an already allocated GEM object of the specified size with * no GEM provided backing store. Instead the caller is responsible for * backing the object and handling it. */ int drm_gem_private_object_init(struct drm_device *dev, struct drm_gem_object *obj, size_t size) { MPASS((size & (PAGE_SIZE - 1)) == 0); obj->dev = dev; obj->vm_obj = NULL; obj->refcount = 1; atomic_store_rel_int(&obj->handle_count, 0); obj->size = size; return 0; } EXPORT_SYMBOL(drm_gem_private_object_init); struct drm_gem_object * drm_gem_object_alloc(struct drm_device *dev, size_t size) { struct drm_gem_object *obj; obj = malloc(sizeof(*obj), DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); if (!obj) goto free; if (drm_gem_object_init(dev, obj, size) != 0) goto free; if (dev->driver->gem_init_object != NULL && dev->driver->gem_init_object(obj) != 0) { goto dealloc; } return obj; dealloc: vm_object_deallocate(obj->vm_obj); free: free(obj, DRM_MEM_DRIVER); return NULL; } EXPORT_SYMBOL(drm_gem_object_alloc); #if defined(FREEBSD_NOTYET) static void drm_gem_remove_prime_handles(struct drm_gem_object *obj, struct drm_file *filp) { if (obj->import_attach) { drm_prime_remove_buf_handle(&filp->prime, obj->import_attach->dmabuf); } if (obj->export_dma_buf) { drm_prime_remove_buf_handle(&filp->prime, obj->export_dma_buf); } } #endif /** * Removes the mapping from handle to filp for this object. */ int drm_gem_handle_delete(struct drm_file *filp, u32 handle) { struct drm_device *dev; struct drm_gem_object *obj; obj = drm_gem_names_remove(&filp->object_names, handle); if (obj == NULL) { return -EINVAL; } dev = obj->dev; #if defined(FREEBSD_NOTYET) drm_gem_remove_prime_handles(obj, filp); #endif if (dev->driver->gem_close_object) dev->driver->gem_close_object(obj, filp); drm_gem_object_handle_unreference_unlocked(obj); return 0; } EXPORT_SYMBOL(drm_gem_handle_delete); /** * Create a handle for this object. This adds a handle reference * to the object, which includes a regular reference count. Callers * will likely want to dereference the object afterwards. */ int drm_gem_handle_create(struct drm_file *file_priv, struct drm_gem_object *obj, u32 *handlep) { struct drm_device *dev = obj->dev; int ret; *handlep = 0; ret = drm_gem_name_create(&file_priv->object_names, obj, handlep); if (ret != 0) return ret; drm_gem_object_handle_reference(obj); if (dev->driver->gem_open_object) { ret = dev->driver->gem_open_object(obj, file_priv); if (ret) { drm_gem_handle_delete(file_priv, *handlep); return ret; } } return 0; } EXPORT_SYMBOL(drm_gem_handle_create); void drm_gem_free_mmap_offset(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; struct drm_gem_mm *mm = dev->mm_private; struct drm_hash_item *list = &obj->map_list; if (!obj->on_map) return; drm_ht_remove_item(&mm->offset_hash, list); free_unr(mm->idxunr, list->key); obj->on_map = false; } EXPORT_SYMBOL(drm_gem_free_mmap_offset); int drm_gem_create_mmap_offset(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; struct drm_gem_mm *mm = dev->mm_private; int ret; if (obj->on_map) return 0; obj->map_list.key = alloc_unr(mm->idxunr); ret = drm_ht_insert_item(&mm->offset_hash, &obj->map_list); if (ret) { DRM_ERROR("failed to add to map hash\n"); free_unr(mm->idxunr, obj->map_list.key); return ret; } obj->on_map = true; return 0; } EXPORT_SYMBOL(drm_gem_create_mmap_offset); /** Returns a reference to the object named by the handle. */ struct drm_gem_object * drm_gem_object_lookup(struct drm_device *dev, struct drm_file *filp, u32 handle) { struct drm_gem_object *obj; obj = drm_gem_name_ref(&filp->object_names, handle, (void (*)(void *))drm_gem_object_reference); return obj; } EXPORT_SYMBOL(drm_gem_object_lookup); int drm_gem_close_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_gem_close *args = data; int ret; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; ret = drm_gem_handle_delete(file_priv, args->handle); return ret; } int drm_gem_flink_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_gem_flink *args = data; struct drm_gem_object *obj; int ret; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; obj = drm_gem_object_lookup(dev, file_priv, args->handle); if (obj == NULL) return -ENOENT; ret = drm_gem_name_create(&dev->object_names, obj, &obj->name); if (ret != 0) { if (ret == -EALREADY) ret = 0; drm_gem_object_unreference_unlocked(obj); } if (ret == 0) args->name = obj->name; return ret; } int drm_gem_open_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_gem_open *args = data; struct drm_gem_object *obj; int ret; u32 handle; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; obj = drm_gem_name_ref(&dev->object_names, args->name, (void (*)(void *))drm_gem_object_reference); if (!obj) return -ENOENT; ret = drm_gem_handle_create(file_priv, obj, &handle); drm_gem_object_unreference_unlocked(obj); if (ret) return ret; args->handle = handle; args->size = obj->size; return 0; } void drm_gem_open(struct drm_device *dev, struct drm_file *file_private) { drm_gem_names_init(&file_private->object_names); } static int drm_gem_object_release_handle(uint32_t name, void *ptr, void *data) { struct drm_file *file_priv = data; struct drm_gem_object *obj = ptr; struct drm_device *dev = obj->dev; #if defined(FREEBSD_NOTYET) drm_gem_remove_prime_handles(obj, file_priv); #endif if (dev->driver->gem_close_object) dev->driver->gem_close_object(obj, file_priv); drm_gem_object_handle_unreference_unlocked(obj); return 0; } void drm_gem_release(struct drm_device *dev, struct drm_file *file_private) { drm_gem_names_foreach(&file_private->object_names, drm_gem_object_release_handle, file_private); drm_gem_names_fini(&file_private->object_names); } void drm_gem_object_release(struct drm_gem_object *obj) { /* * obj->vm_obj can be NULL for private gem objects. */ vm_object_deallocate(obj->vm_obj); } EXPORT_SYMBOL(drm_gem_object_release); void drm_gem_object_free(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; DRM_LOCK_ASSERT(dev); if (dev->driver->gem_free_object != NULL) dev->driver->gem_free_object(obj); } EXPORT_SYMBOL(drm_gem_object_free); void drm_gem_object_handle_free(struct drm_gem_object *obj) { struct drm_device *dev = obj->dev; struct drm_gem_object *obj1; if (obj->name) { obj1 = drm_gem_names_remove(&dev->object_names, obj->name); obj->name = 0; drm_gem_object_unreference(obj1); } } static struct drm_gem_object * drm_gem_object_from_offset(struct drm_device *dev, vm_ooffset_t offset) { struct drm_gem_object *obj; struct drm_gem_mm *mm; struct drm_hash_item *map_list; if ((offset & DRM_GEM_MAPPING_MASK) != DRM_GEM_MAPPING_KEY) return (NULL); offset &= ~DRM_GEM_MAPPING_KEY; mm = dev->mm_private; if (drm_ht_find_item(&mm->offset_hash, DRM_GEM_MAPPING_IDX(offset), &map_list) != 0) { DRM_DEBUG("drm_gem_object_from_offset: offset 0x%jx obj not found\n", (uintmax_t)offset); return (NULL); } obj = __containerof(map_list, struct drm_gem_object, map_list); return (obj); } int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot) { struct drm_gem_object *gem_obj; struct vm_object *vm_obj; DRM_LOCK(dev); gem_obj = drm_gem_object_from_offset(dev, *offset); if (gem_obj == NULL) { DRM_UNLOCK(dev); return (-ENODEV); } drm_gem_object_reference(gem_obj); DRM_UNLOCK(dev); vm_obj = cdev_pager_allocate(gem_obj, OBJT_MGTDEVICE, dev->driver->gem_pager_ops, size, nprot, DRM_GEM_MAPPING_MAPOFF(*offset), curthread->td_ucred); if (vm_obj == NULL) { drm_gem_object_unreference_unlocked(gem_obj); return (-EINVAL); } *offset = DRM_GEM_MAPPING_MAPOFF(*offset); *obj_res = vm_obj; return (0); } void drm_gem_pager_dtr(void *handle) { struct drm_gem_object *obj; struct drm_device *dev; obj = handle; dev = obj->dev; DRM_LOCK(dev); drm_gem_free_mmap_offset(obj); drm_gem_object_unreference(obj); DRM_UNLOCK(dev); } diff --git a/sys/dev/drm2/drm_gem_names.c b/sys/dev/drm2/drm_gem_names.c index 6b0046fc042e..58dbc5a26c8f 100644 --- a/sys/dev/drm2/drm_gem_names.c +++ b/sys/dev/drm2/drm_gem_names.c @@ -1,231 +1,230 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include MALLOC_DEFINE(M_GEM_NAMES, "gem_name", "Hash headers for the gem names"); static void drm_gem_names_delete_name(struct drm_gem_names *names, struct drm_gem_name *np); void drm_gem_names_init(struct drm_gem_names *names) { names->unr = new_unrhdr(1, INT_MAX, NULL); /* XXXKIB */ names->names_hash = hashinit(1000 /* XXXKIB */, M_GEM_NAMES, &names->hash_mask); mtx_init(&names->lock, "drmnames", NULL, MTX_DEF); } void drm_gem_names_fini(struct drm_gem_names *names) { struct drm_gem_name *np; int i; mtx_lock(&names->lock); for (i = 0; i <= names->hash_mask; i++) { while ((np = LIST_FIRST(&names->names_hash[i])) != NULL) { drm_gem_names_delete_name(names, np); mtx_lock(&names->lock); } } mtx_unlock(&names->lock); mtx_destroy(&names->lock); hashdestroy(names->names_hash, M_GEM_NAMES, names->hash_mask); delete_unrhdr(names->unr); } static struct drm_gem_names_head * gem_name_hash_index(struct drm_gem_names *names, int name) { return (&names->names_hash[name & names->hash_mask]); } void * drm_gem_name_ref(struct drm_gem_names *names, uint32_t name, void (*ref)(void *)) { struct drm_gem_name *n; mtx_lock(&names->lock); LIST_FOREACH(n, gem_name_hash_index(names, name), link) { if (n->name == name) { if (ref != NULL) ref(n->ptr); mtx_unlock(&names->lock); return (n->ptr); } } mtx_unlock(&names->lock); return (NULL); } struct drm_gem_ptr_match_arg { uint32_t res; void *ptr; }; static int drm_gem_ptr_match(uint32_t name, void *ptr, void *arg) { struct drm_gem_ptr_match_arg *a; a = arg; if (ptr == a->ptr) { a->res = name; return (1); } else return (0); } uint32_t drm_gem_find_name(struct drm_gem_names *names, void *ptr) { struct drm_gem_ptr_match_arg arg; arg.res = 0; arg.ptr = ptr; drm_gem_names_foreach(names, drm_gem_ptr_match, &arg); return (arg.res); } void * drm_gem_find_ptr(struct drm_gem_names *names, uint32_t name) { struct drm_gem_name *n; void *res; mtx_lock(&names->lock); LIST_FOREACH(n, gem_name_hash_index(names, name), link) { if (n->name == name) { res = n->ptr; mtx_unlock(&names->lock); return (res); } } mtx_unlock(&names->lock); return (NULL); } int drm_gem_name_create(struct drm_gem_names *names, void *p, uint32_t *name) { struct drm_gem_name *np; if (*name != 0) { return (-EALREADY); } np = malloc(sizeof(struct drm_gem_name), M_GEM_NAMES, M_WAITOK); mtx_lock(&names->lock); np->name = alloc_unr(names->unr); if (np->name == -1) { mtx_unlock(&names->lock); free(np, M_GEM_NAMES); return (-ENOMEM); } *name = np->name; np->ptr = p; LIST_INSERT_HEAD(gem_name_hash_index(names, np->name), np, link); mtx_unlock(&names->lock); return (0); } static void drm_gem_names_delete_name(struct drm_gem_names *names, struct drm_gem_name *np) { mtx_assert(&names->lock, MA_OWNED); LIST_REMOVE(np, link); mtx_unlock(&names->lock); free_unr(names->unr, np->name); free(np, M_GEM_NAMES); } void * drm_gem_names_remove(struct drm_gem_names *names, uint32_t name) { struct drm_gem_name *n; void *res; mtx_lock(&names->lock); LIST_FOREACH(n, gem_name_hash_index(names, name), link) { if (n->name == name) { res = n->ptr; drm_gem_names_delete_name(names, n); return (res); } } mtx_unlock(&names->lock); return (NULL); } void drm_gem_names_foreach(struct drm_gem_names *names, int (*f)(uint32_t, void *, void *), void *arg) { struct drm_gem_name *np; struct drm_gem_name marker; int i, fres; bzero(&marker, sizeof(marker)); marker.name = -1; mtx_lock(&names->lock); for (i = 0; i <= names->hash_mask; i++) { for (np = LIST_FIRST(&names->names_hash[i]); np != NULL; ) { if (np->name == -1) { np = LIST_NEXT(np, link); continue; } LIST_INSERT_AFTER(np, &marker, link); mtx_unlock(&names->lock); fres = f(np->name, np->ptr, arg); mtx_lock(&names->lock); np = LIST_NEXT(&marker, link); LIST_REMOVE(&marker, link); if (fres) break; } } mtx_unlock(&names->lock); } diff --git a/sys/dev/drm2/drm_gem_names.h b/sys/dev/drm2/drm_gem_names.h index bb97c0797f74..dad66f6ef9a6 100644 --- a/sys/dev/drm2/drm_gem_names.h +++ b/sys/dev/drm2/drm_gem_names.h @@ -1,67 +1,66 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef DRM_GEM_NAMES_H #define DRM_GEM_NAMES_H #include #include #include #include struct drm_gem_name { uint32_t name; void *ptr; LIST_ENTRY(drm_gem_name) link; }; struct drm_gem_names { struct mtx lock; LIST_HEAD(drm_gem_names_head, drm_gem_name) *names_hash; u_long hash_mask; struct unrhdr *unr; }; void drm_gem_names_init(struct drm_gem_names *names); void drm_gem_names_fini(struct drm_gem_names *names); uint32_t drm_gem_find_name(struct drm_gem_names *names, void *ptr); void *drm_gem_find_ptr(struct drm_gem_names *names, uint32_t name); void *drm_gem_name_ref(struct drm_gem_names *names, uint32_t name, void (*ref)(void *)); int drm_gem_name_create(struct drm_gem_names *names, void *obj, uint32_t *name); void drm_gem_names_foreach(struct drm_gem_names *names, int (*f)(uint32_t, void *, void *), void *arg); void *drm_gem_names_remove(struct drm_gem_names *names, uint32_t name); #endif diff --git a/sys/dev/drm2/drm_linux_list_sort.c b/sys/dev/drm2/drm_linux_list_sort.c index bd7085e2a86d..f9a64154c796 100644 --- a/sys/dev/drm2/drm_linux_list_sort.c +++ b/sys/dev/drm2/drm_linux_list_sort.c @@ -1,77 +1,76 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); struct drm_list_sort_thunk { int (*cmp)(void *, struct list_head *, struct list_head *); void *priv; }; static int drm_le_cmp(void *priv, const void *d1, const void *d2) { struct list_head *le1, *le2; struct drm_list_sort_thunk *thunk; thunk = priv; le1 = *(__DECONST(struct list_head **, d1)); le2 = *(__DECONST(struct list_head **, d2)); return ((thunk->cmp)(thunk->priv, le1, le2)); } /* * Punt and use array sort. */ void drm_list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, struct list_head *a, struct list_head *b)) { struct drm_list_sort_thunk thunk; struct list_head **ar, *le; int count, i; count = 0; list_for_each(le, head) count++; ar = malloc(sizeof(struct list_head *) * count, M_TEMP, M_WAITOK); i = 0; list_for_each(le, head) ar[i++] = le; thunk.cmp = cmp; thunk.priv = priv; qsort_r(ar, count, sizeof(struct list_head *), &thunk, drm_le_cmp); INIT_LIST_HEAD(head); for (i = 0; i < count; i++) list_add_tail(ar[i], head); free(ar, M_TEMP); } diff --git a/sys/dev/fb/fbd.c b/sys/dev/fb/fbd.c index 36439af9256c..e577768eafdb 100644 --- a/sys/dev/fb/fbd.c +++ b/sys/dev/fb/fbd.c @@ -1,372 +1,371 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* Generic framebuffer */ /* TODO unlink from VT(9) */ /* TODO done normal /dev/fb methods */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fb_if.h" LIST_HEAD(fb_list_head_t, fb_list_entry) fb_list_head = LIST_HEAD_INITIALIZER(fb_list_head); struct fb_list_entry { struct fb_info *fb_info; struct cdev *fb_si; LIST_ENTRY(fb_list_entry) fb_list; }; struct fbd_softc { device_t sc_dev; struct fb_info *sc_info; }; static void fbd_evh_init(void *); /* SI_ORDER_SECOND, just after EVENTHANDLERs initialized. */ SYSINIT(fbd_evh_init, SI_SUB_CONFIGURE, SI_ORDER_SECOND, fbd_evh_init, NULL); static d_open_t fb_open; static d_close_t fb_close; static d_read_t fb_read; static d_write_t fb_write; static d_ioctl_t fb_ioctl; static d_mmap_t fb_mmap; static struct cdevsw fb_cdevsw = { .d_version = D_VERSION, .d_flags = D_NEEDGIANT, .d_open = fb_open, .d_close = fb_close, .d_read = fb_read, .d_write = fb_write, .d_ioctl = fb_ioctl, .d_mmap = fb_mmap, .d_name = "fb", }; static int framebuffer_dev_unit = 0; static int fb_open(struct cdev *dev, int oflags, int devtype, struct thread *td) { return (0); } static int fb_close(struct cdev *dev, int fflag, int devtype, struct thread *td) { return (0); } static int fb_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { struct fb_info *info; int error; error = 0; info = dev->si_drv1; switch (cmd) { case FBIOGTYPE: bcopy(info, (struct fbtype *)data, sizeof(struct fbtype)); break; case FBIO_GETWINORG: /* get frame buffer window origin */ *(u_int *)data = 0; break; case FBIO_GETDISPSTART: /* get display start address */ ((video_display_start_t *)data)->x = 0; ((video_display_start_t *)data)->y = 0; break; case FBIO_GETLINEWIDTH: /* get scan line width in bytes */ *(u_int *)data = info->fb_stride; break; case FBIO_BLANK: /* blank display */ if (info->setblankmode != NULL) error = info->setblankmode(info->fb_priv, *(int *)data); break; default: error = ENOIOCTL; break; } return (error); } static int fb_read(struct cdev *dev, struct uio *uio, int ioflag) { return (0); /* XXX nothing to read, yet */ } static int fb_write(struct cdev *dev, struct uio *uio, int ioflag) { return (0); /* XXX nothing written */ } static int fb_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr) { struct fb_info *info; info = dev->si_drv1; if (info->fb_flags & FB_FLAG_NOMMAP) return (ENODEV); if (offset < info->fb_size) { if (info->fb_pbase == 0) *paddr = vtophys((uint8_t *)info->fb_vbase + offset); else *paddr = info->fb_pbase + offset; if (info->fb_flags & FB_FLAG_MEMATTR) *memattr = info->fb_memattr; return (0); } return (EINVAL); } static int fb_init(struct fb_list_entry *entry, int unit) { struct fb_info *info; info = entry->fb_info; entry->fb_si = make_dev(&fb_cdevsw, unit, UID_ROOT, GID_WHEEL, 0600, "fb%d", unit); entry->fb_si->si_drv1 = info; info->fb_cdev = entry->fb_si; return (0); } int fbd_list() { struct fb_list_entry *entry; if (LIST_EMPTY(&fb_list_head)) return (ENOENT); LIST_FOREACH(entry, &fb_list_head, fb_list) { printf("FB %s @%p\n", entry->fb_info->fb_name, (void *)entry->fb_info->fb_pbase); } return (0); } static struct fb_list_entry * fbd_find(struct fb_info* info) { struct fb_list_entry *entry, *tmp; LIST_FOREACH_SAFE(entry, &fb_list_head, fb_list, tmp) { if (entry->fb_info == info) { return (entry); } } return (NULL); } int fbd_register(struct fb_info* info) { struct fb_list_entry *entry; int err, first; first = 0; if (LIST_EMPTY(&fb_list_head)) first++; entry = fbd_find(info); if (entry != NULL) { /* XXX Update framebuffer params */ return (0); } entry = malloc(sizeof(struct fb_list_entry), M_DEVBUF, M_WAITOK|M_ZERO); entry->fb_info = info; LIST_INSERT_HEAD(&fb_list_head, entry, fb_list); err = fb_init(entry, framebuffer_dev_unit++); if (err) return (err); if (first) { err = vt_fb_attach(info); if (err) return (err); } return (0); } int fbd_unregister(struct fb_info* info) { struct fb_list_entry *entry, *tmp; LIST_FOREACH_SAFE(entry, &fb_list_head, fb_list, tmp) { if (entry->fb_info == info) { LIST_REMOVE(entry, fb_list); if (LIST_EMPTY(&fb_list_head)) vt_fb_detach(info); free(entry, M_DEVBUF); return (0); } } return (ENOENT); } static void register_fb_wrap(void *arg, void *ptr) { fbd_register((struct fb_info *)ptr); } static void unregister_fb_wrap(void *arg, void *ptr) { fbd_unregister((struct fb_info *)ptr); } static void fbd_evh_init(void *ctx) { EVENTHANDLER_REGISTER(register_framebuffer, register_fb_wrap, NULL, EVENTHANDLER_PRI_ANY); EVENTHANDLER_REGISTER(unregister_framebuffer, unregister_fb_wrap, NULL, EVENTHANDLER_PRI_ANY); } /* Newbus methods. */ static int fbd_probe(device_t dev) { return (BUS_PROBE_NOWILDCARD); } static int fbd_attach(device_t dev) { struct fbd_softc *sc; int err; sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_info = FB_GETINFO(device_get_parent(dev)); if (sc->sc_info == NULL) return (ENXIO); err = fbd_register(sc->sc_info); return (err); } static int fbd_detach(device_t dev) { struct fbd_softc *sc; int err; sc = device_get_softc(dev); err = fbd_unregister(sc->sc_info); return (err); } static device_method_t fbd_methods[] = { /* Device interface */ DEVMETHOD(device_probe, fbd_probe), DEVMETHOD(device_attach, fbd_attach), DEVMETHOD(device_detach, fbd_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), { 0, 0 } }; driver_t fbd_driver = { "fbd", fbd_methods, sizeof(struct fbd_softc) }; devclass_t fbd_devclass; DRIVER_MODULE(fbd, fb, fbd_driver, fbd_devclass, 0, 0); DRIVER_MODULE(fbd, drmn, fbd_driver, fbd_devclass, 0, 0); DRIVER_MODULE(fbd, udl, fbd_driver, fbd_devclass, 0, 0); MODULE_VERSION(fbd, 1); diff --git a/sys/dev/fdt/fdt_common.c b/sys/dev/fdt/fdt_common.c index 107c91e11425..18ac3957f7f9 100644 --- a/sys/dev/fdt/fdt_common.c +++ b/sys/dev/fdt/fdt_common.c @@ -1,623 +1,622 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "ofw_bus_if.h" #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #define FDT_COMPAT_LEN 255 #define FDT_REG_CELLS 4 #define FDT_RANGES_SIZE 48 SYSCTL_NODE(_hw, OID_AUTO, fdt, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Flattened Device Tree"); vm_paddr_t fdt_immr_pa; vm_offset_t fdt_immr_va; vm_offset_t fdt_immr_size; struct fdt_ic_list fdt_ic_list_head = SLIST_HEAD_INITIALIZER(fdt_ic_list_head); static int fdt_get_range_by_busaddr(phandle_t node, u_long addr, u_long *base, u_long *size) { pcell_t ranges[32], *rangesptr; pcell_t addr_cells, size_cells, par_addr_cells; u_long bus_addr, par_bus_addr, pbase, psize; int err, i, len, tuple_size, tuples; if (node == 0) { *base = 0; *size = ULONG_MAX; return (0); } if ((fdt_addrsize_cells(node, &addr_cells, &size_cells)) != 0) return (ENXIO); /* * Process 'ranges' property. */ par_addr_cells = fdt_parent_addr_cells(node); if (par_addr_cells > 2) { return (ERANGE); } len = OF_getproplen(node, "ranges"); if (len < 0) return (-1); if (len > sizeof(ranges)) return (ENOMEM); if (len == 0) { return (fdt_get_range_by_busaddr(OF_parent(node), addr, base, size)); } if (OF_getprop(node, "ranges", ranges, sizeof(ranges)) <= 0) return (EINVAL); tuple_size = addr_cells + par_addr_cells + size_cells; tuples = len / (tuple_size * sizeof(cell_t)); if (par_addr_cells > 2 || addr_cells > 2 || size_cells > 2) return (ERANGE); *base = 0; *size = 0; for (i = 0; i < tuples; i++) { rangesptr = &ranges[i * tuple_size]; bus_addr = fdt_data_get((void *)rangesptr, addr_cells); if (bus_addr != addr) continue; rangesptr += addr_cells; par_bus_addr = fdt_data_get((void *)rangesptr, par_addr_cells); rangesptr += par_addr_cells; err = fdt_get_range_by_busaddr(OF_parent(node), par_bus_addr, &pbase, &psize); if (err > 0) return (err); if (err == 0) *base = pbase; else *base = par_bus_addr; *size = fdt_data_get((void *)rangesptr, size_cells); return (0); } return (EINVAL); } int fdt_get_range(phandle_t node, int range_id, u_long *base, u_long *size) { pcell_t ranges[FDT_RANGES_SIZE], *rangesptr; pcell_t addr_cells, size_cells, par_addr_cells; u_long par_bus_addr, pbase, psize; int err, len; if ((fdt_addrsize_cells(node, &addr_cells, &size_cells)) != 0) return (ENXIO); /* * Process 'ranges' property. */ par_addr_cells = fdt_parent_addr_cells(node); if (par_addr_cells > 2) return (ERANGE); len = OF_getproplen(node, "ranges"); if (len > sizeof(ranges)) return (ENOMEM); if (len == 0) { *base = 0; *size = ULONG_MAX; return (0); } if (!(range_id < len)) return (ERANGE); if (OF_getprop(node, "ranges", ranges, sizeof(ranges)) <= 0) return (EINVAL); if (par_addr_cells > 2 || addr_cells > 2 || size_cells > 2) return (ERANGE); *base = 0; *size = 0; rangesptr = &ranges[range_id]; *base = fdt_data_get((void *)rangesptr, addr_cells); rangesptr += addr_cells; par_bus_addr = fdt_data_get((void *)rangesptr, par_addr_cells); rangesptr += par_addr_cells; err = fdt_get_range_by_busaddr(OF_parent(node), par_bus_addr, &pbase, &psize); if (err == 0) *base += pbase; else *base += par_bus_addr; *size = fdt_data_get((void *)rangesptr, size_cells); return (0); } int fdt_immr_addr(vm_offset_t immr_va) { phandle_t node; u_long base, size; int r; /* * Try to access the SOC node directly i.e. through /aliases/. */ if ((node = OF_finddevice("soc")) != -1) if (ofw_bus_node_is_compatible(node, "simple-bus")) goto moveon; /* * Find the node the long way. */ if ((node = OF_finddevice("/")) == -1) return (ENXIO); if ((node = fdt_find_compatible(node, "simple-bus", 0)) == 0) return (ENXIO); moveon: if ((r = fdt_get_range(node, 0, &base, &size)) == 0) { fdt_immr_pa = base; fdt_immr_va = immr_va; fdt_immr_size = size; } return (r); } int fdt_is_compatible_strict(phandle_t node, const char *compatible) { char compat[FDT_COMPAT_LEN]; if (OF_getproplen(node, "compatible") <= 0) return (0); if (OF_getprop(node, "compatible", compat, FDT_COMPAT_LEN) < 0) return (0); if (strncasecmp(compat, compatible, FDT_COMPAT_LEN) == 0) /* This fits. */ return (1); return (0); } phandle_t fdt_find_compatible(phandle_t start, const char *compat, int strict) { phandle_t child; /* * Traverse all children of 'start' node, and find first with * matching 'compatible' property. */ for (child = OF_child(start); child != 0; child = OF_peer(child)) if (ofw_bus_node_is_compatible(child, compat)) { if (strict) if (!fdt_is_compatible_strict(child, compat)) continue; return (child); } return (0); } phandle_t fdt_depth_search_compatible(phandle_t start, const char *compat, int strict) { phandle_t child, node; /* * Depth-search all descendants of 'start' node, and find first with * matching 'compatible' property. */ for (node = OF_child(start); node != 0; node = OF_peer(node)) { if (ofw_bus_node_is_compatible(node, compat) && (strict == 0 || fdt_is_compatible_strict(node, compat))) { return (node); } child = fdt_depth_search_compatible(node, compat, strict); if (child != 0) return (child); } return (0); } int fdt_parent_addr_cells(phandle_t node) { pcell_t addr_cells; /* Find out #address-cells of the superior bus. */ if (OF_searchprop(OF_parent(node), "#address-cells", &addr_cells, sizeof(addr_cells)) <= 0) return (2); return ((int)fdt32_to_cpu(addr_cells)); } u_long fdt_data_get(void *data, int cells) { if (cells == 1) return (fdt32_to_cpu(*((uint32_t *)data))); return (fdt64_to_cpu(*((uint64_t *)data))); } int fdt_addrsize_cells(phandle_t node, int *addr_cells, int *size_cells) { pcell_t cell; int cell_size; /* * Retrieve #{address,size}-cells. */ cell_size = sizeof(cell); if (OF_getencprop(node, "#address-cells", &cell, cell_size) < cell_size) cell = 2; *addr_cells = (int)cell; if (OF_getencprop(node, "#size-cells", &cell, cell_size) < cell_size) cell = 1; *size_cells = (int)cell; if (*addr_cells > 3 || *size_cells > 2) return (ERANGE); return (0); } int fdt_data_to_res(pcell_t *data, int addr_cells, int size_cells, u_long *start, u_long *count) { /* Address portion. */ if (addr_cells > 2) return (ERANGE); *start = fdt_data_get((void *)data, addr_cells); data += addr_cells; /* Size portion. */ if (size_cells > 2) return (ERANGE); *count = fdt_data_get((void *)data, size_cells); return (0); } int fdt_regsize(phandle_t node, u_long *base, u_long *size) { pcell_t reg[4]; int addr_cells, len, size_cells; if (fdt_addrsize_cells(OF_parent(node), &addr_cells, &size_cells)) return (ENXIO); if ((sizeof(pcell_t) * (addr_cells + size_cells)) > sizeof(reg)) return (ENOMEM); len = OF_getprop(node, "reg", ®, sizeof(reg)); if (len <= 0) return (EINVAL); *base = fdt_data_get(®[0], addr_cells); *size = fdt_data_get(®[addr_cells], size_cells); return (0); } int fdt_get_phyaddr(phandle_t node, device_t dev, int *phy_addr, void **phy_sc) { phandle_t phy_node; pcell_t phy_handle, phy_reg; uint32_t i; device_t parent, child; if (OF_getencprop(node, "phy-handle", (void *)&phy_handle, sizeof(phy_handle)) <= 0) return (ENXIO); phy_node = OF_node_from_xref(phy_handle); if (OF_getencprop(phy_node, "reg", (void *)&phy_reg, sizeof(phy_reg)) <= 0) return (ENXIO); *phy_addr = phy_reg; if (phy_sc == NULL) return (0); /* * Search for softc used to communicate with phy. */ /* * Step 1: Search for ancestor of the phy-node with a "phy-handle" * property set. */ phy_node = OF_parent(phy_node); while (phy_node != 0) { if (OF_getprop(phy_node, "phy-handle", (void *)&phy_handle, sizeof(phy_handle)) > 0) break; phy_node = OF_parent(phy_node); } if (phy_node == 0) return (ENXIO); /* * Step 2: For each device with the same parent and name as ours * compare its node with the one found in step 1, ancestor of phy * node (stored in phy_node). */ parent = device_get_parent(dev); i = 0; child = device_find_child(parent, device_get_name(dev), i); while (child != NULL) { if (ofw_bus_get_node(child) == phy_node) break; i++; child = device_find_child(parent, device_get_name(dev), i); } if (child == NULL) return (ENXIO); /* * Use softc of the device found. */ *phy_sc = (void *)device_get_softc(child); return (0); } int fdt_get_reserved_regions(struct mem_region *mr, int *mrcnt) { pcell_t reserve[FDT_REG_CELLS * FDT_MEM_REGIONS]; pcell_t *reservep; phandle_t memory, root; int addr_cells, size_cells; int i, res_len, rv, tuple_size, tuples; root = OF_finddevice("/"); memory = OF_finddevice("/memory"); if (memory == -1) { rv = ENXIO; goto out; } if ((rv = fdt_addrsize_cells(OF_parent(memory), &addr_cells, &size_cells)) != 0) goto out; if (addr_cells > 2) { rv = ERANGE; goto out; } tuple_size = sizeof(pcell_t) * (addr_cells + size_cells); res_len = OF_getproplen(root, "memreserve"); if (res_len <= 0 || res_len > sizeof(reserve)) { rv = ERANGE; goto out; } if (OF_getprop(root, "memreserve", reserve, res_len) <= 0) { rv = ENXIO; goto out; } tuples = res_len / tuple_size; reservep = (pcell_t *)&reserve; for (i = 0; i < tuples; i++) { rv = fdt_data_to_res(reservep, addr_cells, size_cells, (u_long *)&mr[i].mr_start, (u_long *)&mr[i].mr_size); if (rv != 0) goto out; reservep += addr_cells + size_cells; } *mrcnt = i; rv = 0; out: return (rv); } int fdt_get_reserved_mem(struct mem_region *reserved, int *mreserved) { pcell_t reg[FDT_REG_CELLS]; phandle_t child, root; int addr_cells, size_cells; int i, rv; root = OF_finddevice("/reserved-memory"); if (root == -1) { return (ENXIO); } if ((rv = fdt_addrsize_cells(root, &addr_cells, &size_cells)) != 0) return (rv); if (addr_cells + size_cells > FDT_REG_CELLS) panic("Too many address and size cells %d %d", addr_cells, size_cells); i = 0; for (child = OF_child(root); child != 0; child = OF_peer(child)) { if (!OF_hasprop(child, "no-map")) continue; rv = OF_getprop(child, "reg", reg, sizeof(reg)); if (rv <= 0) /* XXX: Does a no-map of a dynamic range make sense? */ continue; fdt_data_to_res(reg, addr_cells, size_cells, (u_long *)&reserved[i].mr_start, (u_long *)&reserved[i].mr_size); i++; } *mreserved = i; return (0); } int fdt_get_mem_regions(struct mem_region *mr, int *mrcnt, uint64_t *memsize) { pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS]; pcell_t *regp; phandle_t memory; uint64_t memory_size; int addr_cells, size_cells; int i, reg_len, rv, tuple_size, tuples; memory = OF_finddevice("/memory"); if (memory == -1) { rv = ENXIO; goto out; } if ((rv = fdt_addrsize_cells(OF_parent(memory), &addr_cells, &size_cells)) != 0) goto out; if (addr_cells > 2) { rv = ERANGE; goto out; } tuple_size = sizeof(pcell_t) * (addr_cells + size_cells); reg_len = OF_getproplen(memory, "reg"); if (reg_len <= 0 || reg_len > sizeof(reg)) { rv = ERANGE; goto out; } if (OF_getprop(memory, "reg", reg, reg_len) <= 0) { rv = ENXIO; goto out; } memory_size = 0; tuples = reg_len / tuple_size; regp = (pcell_t *)® for (i = 0; i < tuples; i++) { rv = fdt_data_to_res(regp, addr_cells, size_cells, (u_long *)&mr[i].mr_start, (u_long *)&mr[i].mr_size); if (rv != 0) goto out; regp += addr_cells + size_cells; memory_size += mr[i].mr_size; } if (memory_size == 0) { rv = ERANGE; goto out; } *mrcnt = i; if (memsize != NULL) *memsize = memory_size; rv = 0; out: return (rv); } int fdt_get_chosen_bootargs(char *bootargs, size_t max_size) { phandle_t chosen; chosen = OF_finddevice("/chosen"); if (chosen == -1) return (ENXIO); if (OF_getprop(chosen, "bootargs", bootargs, max_size) == -1) return (ENXIO); return (0); } diff --git a/sys/dev/fdt/fdt_common.h b/sys/dev/fdt/fdt_common.h index 3bf4df41973c..fc2990cdae29 100644 --- a/sys/dev/fdt/fdt_common.h +++ b/sys/dev/fdt/fdt_common.h @@ -1,98 +1,97 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _FDT_COMMON_H_ #define _FDT_COMMON_H_ #include #include #include #include #define FDT_MEM_REGIONS 16 #define DI_MAX_INTR_NUM 32 struct fdt_sense_level { enum intr_trigger trig; enum intr_polarity pol; }; #if defined(__arm__) && !defined(INTRNG) typedef int (*fdt_pic_decode_t)(phandle_t, pcell_t *, int *, int *, int *); extern fdt_pic_decode_t fdt_pic_table[]; #endif #if defined(__arm__) || defined(__powerpc__) typedef void (*fdt_fixup_t)(phandle_t); struct fdt_fixup_entry { char *model; fdt_fixup_t handler; }; extern struct fdt_fixup_entry fdt_fixup_table[]; #endif extern SLIST_HEAD(fdt_ic_list, fdt_ic) fdt_ic_list_head; struct fdt_ic { SLIST_ENTRY(fdt_ic) fdt_ics; ihandle_t iph; device_t dev; }; extern vm_paddr_t fdt_immr_pa; extern vm_offset_t fdt_immr_va; extern vm_offset_t fdt_immr_size; #if defined(FDT_DTB_STATIC) extern u_char fdt_static_dtb; #endif SYSCTL_DECL(_hw_fdt); int fdt_addrsize_cells(phandle_t, int *, int *); u_long fdt_data_get(void *, int); int fdt_data_to_res(pcell_t *, int, int, u_long *, u_long *); phandle_t fdt_find_compatible(phandle_t, const char *, int); phandle_t fdt_depth_search_compatible(phandle_t, const char *, int); int fdt_get_mem_regions(struct mem_region *, int *, uint64_t *); int fdt_get_reserved_mem(struct mem_region *, int *); int fdt_get_reserved_regions(struct mem_region *, int *); int fdt_get_phyaddr(phandle_t, device_t, int *, void **); int fdt_get_range(phandle_t, int, u_long *, u_long *); int fdt_immr_addr(vm_offset_t); int fdt_regsize(phandle_t, u_long *, u_long *); int fdt_is_compatible_strict(phandle_t, const char *); int fdt_parent_addr_cells(phandle_t); int fdt_get_chosen_bootargs(char *bootargs, size_t max_size); #endif /* _FDT_COMMON_H_ */ diff --git a/sys/dev/fdt/fdt_static_dtb.S b/sys/dev/fdt/fdt_static_dtb.S index ae1fc03e5b4a..1686bc4b0099 100644 --- a/sys/dev/fdt/fdt_static_dtb.S +++ b/sys/dev/fdt/fdt_static_dtb.S @@ -1,47 +1,46 @@ /*- * Copyright (c) 2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include "fdt_static_dtb.h" .data .ascii "Device Tree Blob STARTS here" .global fdt_static_dtb /* * The device tree blob must be aligned at 8-bytes boundary. Use * gas-specific 'balign' extension to ensure the same alignment behaviour on * all archs (the .align directive meaning can vary across gas arch * variations). */ .balign 8 fdt_static_dtb: .incbin FDT_DTB_FILE .ascii "Device Tree Blob STOPS here" diff --git a/sys/dev/iommu/busdma_iommu.c b/sys/dev/iommu/busdma_iommu.c index 9cafd49807ff..07a16e87eeda 100644 --- a/sys/dev/iommu/busdma_iommu.c +++ b/sys/dev/iommu/busdma_iommu.c @@ -1,1116 +1,1115 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * busdma_iommu.c, the implementation of the busdma(9) interface using * IOMMU units from Intel VT-d. */ static bool iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func) { char str[128], *env; int default_bounce; bool ret; static const char bounce_str[] = "bounce"; static const char iommu_str[] = "iommu"; static const char dmar_str[] = "dmar"; /* compatibility */ default_bounce = 0; env = kern_getenv("hw.busdma.default"); if (env != NULL) { if (strcmp(env, bounce_str) == 0) default_bounce = 1; else if (strcmp(env, iommu_str) == 0 || strcmp(env, dmar_str) == 0) default_bounce = 0; freeenv(env); } snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d", domain, bus, slot, func); env = kern_getenv(str); if (env == NULL) return (default_bounce != 0); if (strcmp(env, bounce_str) == 0) ret = true; else if (strcmp(env, iommu_str) == 0 || strcmp(env, dmar_str) == 0) ret = false; else ret = default_bounce != 0; freeenv(env); return (ret); } /* * Given original device, find the requester ID that will be seen by * the IOMMU unit and used for page table lookup. PCI bridges may take * ownership of transactions from downstream devices, so it may not be * the same as the BSF of the target device. In those cases, all * devices downstream of the bridge must share a single mapping * domain, and must collectively be assigned to use either IOMMU or * bounce mapping. */ device_t iommu_get_requester(device_t dev, uint16_t *rid) { devclass_t pci_class; device_t l, pci, pcib, pcip, pcibp, requester; int cap_offset; uint16_t pcie_flags; bool bridge_is_pcie; pci_class = devclass_find("pci"); l = requester = dev; *rid = pci_get_rid(dev); /* * Walk the bridge hierarchy from the target device to the * host port to find the translating bridge nearest the IOMMU * unit. */ for (;;) { pci = device_get_parent(l); KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent " "for %s", device_get_name(dev), device_get_name(l))); KASSERT(device_get_devclass(pci) == pci_class, ("iommu_get_requester(%s): non-pci parent %s for %s", device_get_name(dev), device_get_name(pci), device_get_name(l))); pcib = device_get_parent(pci); KASSERT(pcib != NULL, ("iommu_get_requester(%s): NULL bridge " "for %s", device_get_name(dev), device_get_name(pci))); /* * The parent of our "bridge" isn't another PCI bus, * so pcib isn't a PCI->PCI bridge but rather a host * port, and the requester ID won't be translated * further. */ pcip = device_get_parent(pcib); if (device_get_devclass(pcip) != pci_class) break; pcibp = device_get_parent(pcip); if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) { /* * Do not stop the loop even if the target * device is PCIe, because it is possible (but * unlikely) to have a PCI->PCIe bridge * somewhere in the hierarchy. */ l = pcib; } else { /* * Device is not PCIe, it cannot be seen as a * requester by IOMMU unit. Check whether the * bridge is PCIe. */ bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS, &cap_offset) == 0; requester = pcib; /* * Check for a buggy PCIe/PCI bridge that * doesn't report the express capability. If * the bridge above it is express but isn't a * PCI bridge, then we know pcib is actually a * PCIe/PCI bridge. */ if (!bridge_is_pcie && pci_find_cap(pcibp, PCIY_EXPRESS, &cap_offset) == 0) { pcie_flags = pci_read_config(pcibp, cap_offset + PCIER_FLAGS, 2); if ((pcie_flags & PCIEM_FLAGS_TYPE) != PCIEM_TYPE_PCI_BRIDGE) bridge_is_pcie = true; } if (bridge_is_pcie) { /* * The current device is not PCIe, but * the bridge above it is. This is a * PCIe->PCI bridge. Assume that the * requester ID will be the secondary * bus number with slot and function * set to zero. * * XXX: Doesn't handle the case where * the bridge is PCIe->PCI-X, and the * bridge will only take ownership of * requests in some cases. We should * provide context entries with the * same page tables for taken and * non-taken transactions. */ *rid = PCI_RID(pci_get_bus(l), 0, 0); l = pcibp; } else { /* * Neither the device nor the bridge * above it are PCIe. This is a * conventional PCI->PCI bridge, which * will use the bridge's BSF as the * requester ID. */ *rid = pci_get_rid(pcib); l = pcib; } } } return (requester); } struct iommu_ctx * iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr) { device_t requester; struct iommu_ctx *ctx; bool disabled; uint16_t rid; requester = iommu_get_requester(dev, &rid); /* * If the user requested the IOMMU disabled for the device, we * cannot disable the IOMMU unit, due to possibility of other * devices on the same IOMMU unit still requiring translation. * Instead provide the identity mapping for the device * context. */ disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester), pci_get_bus(requester), pci_get_slot(requester), pci_get_function(requester)); ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr); if (ctx == NULL) return (NULL); if (disabled) { /* * Keep the first reference on context, release the * later refs. */ IOMMU_LOCK(unit); if ((ctx->flags & IOMMU_CTX_DISABLED) == 0) { ctx->flags |= IOMMU_CTX_DISABLED; IOMMU_UNLOCK(unit); } else { iommu_free_ctx_locked(unit, ctx); } ctx = NULL; } return (ctx); } struct iommu_ctx * iommu_get_dev_ctx(device_t dev) { struct iommu_unit *unit; unit = iommu_find(dev, bootverbose); /* Not in scope of any IOMMU ? */ if (unit == NULL) return (NULL); if (!unit->dma_enabled) return (NULL); #if defined(__amd64__) || defined(__i386__) dmar_quirks_pre_use(unit); dmar_instantiate_rmrr_ctxs(unit); #endif return (iommu_instantiate_ctx(unit, dev, false)); } bus_dma_tag_t iommu_get_dma_tag(device_t dev, device_t child) { struct iommu_ctx *ctx; bus_dma_tag_t res; ctx = iommu_get_dev_ctx(child); if (ctx == NULL) return (NULL); res = (bus_dma_tag_t)ctx->tag; return (res); } bool bus_dma_iommu_set_buswide(device_t dev) { struct iommu_unit *unit; device_t parent; u_int busno, slot, func; parent = device_get_parent(dev); if (device_get_devclass(parent) != devclass_find("pci")) return (false); unit = iommu_find(dev, bootverbose); if (unit == NULL) return (false); busno = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); if (slot != 0 || func != 0) { if (bootverbose) { device_printf(dev, "iommu%d pci%d:%d:%d requested buswide busdma\n", unit->unit, busno, slot, func); } return (false); } iommu_set_buswide_ctx(unit, busno); return (true); } void iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno) { MPASS(busno <= PCI_BUSMAX); IOMMU_LOCK(unit); unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |= 1 << (busno % (NBBY * sizeof(uint32_t))); IOMMU_UNLOCK(unit); } bool iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno) { MPASS(busno <= PCI_BUSMAX); return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] & (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0); } static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map"); static void iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map); static int iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) { struct bus_dma_tag_iommu *newtag, *oldtag; int error; *dmat = NULL; error = common_bus_dma_tag_create(parent != NULL ? &((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg, maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, sizeof(struct bus_dma_tag_iommu), (void **)&newtag); if (error != 0) goto out; oldtag = (struct bus_dma_tag_iommu *)parent; newtag->common.impl = &bus_dma_iommu_impl; newtag->ctx = oldtag->ctx; newtag->owner = oldtag->owner; *dmat = (bus_dma_tag_t)newtag; out: CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), error); return (error); } static int iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat) { return (0); } static int iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1) { struct bus_dma_tag_iommu *dmat, *dmat_copy, *parent; int error; error = 0; dmat_copy = dmat = (struct bus_dma_tag_iommu *)dmat1; if (dmat != NULL) { if (dmat->map_count != 0) { error = EBUSY; goto out; } while (dmat != NULL) { parent = (struct bus_dma_tag_iommu *)dmat->common.parent; if (atomic_fetchadd_int(&dmat->common.ref_count, -1) == 1) { if (dmat == dmat->ctx->tag) iommu_free_ctx(dmat->ctx); free(dmat->segments, M_IOMMU_DMAMAP); free(dmat, M_DEVBUF); dmat = parent; } else dmat = NULL; } } out: CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); return (error); } static bool iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) { return (false); } static int iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; tag = (struct bus_dma_tag_iommu *)dmat; map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP, DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO); if (map == NULL) { *mapp = NULL; return (ENOMEM); } if (tag->segments == NULL) { tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) * tag->common.nsegments, M_IOMMU_DMAMAP, DOMAINSET_PREF(tag->common.domain), M_NOWAIT); if (tag->segments == NULL) { free(map, M_IOMMU_DMAMAP); *mapp = NULL; return (ENOMEM); } } TAILQ_INIT(&map->map_entries); map->tag = tag; map->locked = true; map->cansleep = false; tag->map_count++; *mapp = (bus_dmamap_t)map; return (0); } static int iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; struct iommu_domain *domain; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; if (map != NULL) { domain = tag->ctx->domain; IOMMU_DOMAIN_LOCK(domain); if (!TAILQ_EMPTY(&map->map_entries)) { IOMMU_DOMAIN_UNLOCK(domain); return (EBUSY); } IOMMU_DOMAIN_UNLOCK(domain); free(map, M_IOMMU_DMAMAP); } tag->map_count--; return (0); } static int iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; int error, mflags; vm_memattr_t attr; error = iommu_bus_dmamap_create(dmat, flags, mapp); if (error != 0) return (error); mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK; mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0; attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE : VM_MEMATTR_DEFAULT; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)*mapp; if (tag->common.maxsize < PAGE_SIZE && tag->common.alignment <= tag->common.maxsize && attr == VM_MEMATTR_DEFAULT) { *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF, DOMAINSET_PREF(tag->common.domain), mflags); map->flags |= BUS_DMAMAP_IOMMU_MALLOC; } else { *vaddr = (void *)kmem_alloc_attr_domainset( DOMAINSET_PREF(tag->common.domain), tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR, attr); map->flags |= BUS_DMAMAP_IOMMU_KMEM_ALLOC; } if (*vaddr == NULL) { iommu_bus_dmamap_destroy(dmat, *mapp); *mapp = NULL; return (ENOMEM); } return (0); } static void iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) { free(vaddr, M_DEVBUF); map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC; } else { KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0, ("iommu_bus_dmamem_free for non alloced map %p", map)); kmem_free((vm_offset_t)vaddr, tag->common.maxsize); map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC; } iommu_bus_dmamap_destroy(dmat, map1); } static int iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag, struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp, struct iommu_map_entries_tailq *unroll_list) { struct iommu_ctx *ctx; struct iommu_domain *domain; struct iommu_map_entry *entry; iommu_gaddr_t size; bus_size_t buflen1; int error, idx, gas_flags, seg; KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset)); if (segs == NULL) segs = tag->segments; ctx = tag->ctx; domain = ctx->domain; seg = *segp; error = 0; idx = 0; while (buflen > 0) { seg++; if (seg >= tag->common.nsegments) { error = EFBIG; break; } buflen1 = buflen > tag->common.maxsegsz ? tag->common.maxsegsz : buflen; size = round_page(offset + buflen1); /* * (Too) optimistically allow split if there are more * then one segments left. */ gas_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0; if (seg + 1 < tag->common.nsegments) gas_flags |= IOMMU_MF_CANSPLIT; error = iommu_map(domain, &tag->common, size, offset, IOMMU_MAP_ENTRY_READ | ((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE : 0), gas_flags, ma + idx, &entry); if (error != 0) break; if ((gas_flags & IOMMU_MF_CANSPLIT) != 0) { KASSERT(size >= entry->end - entry->start, ("split increased entry size %jx %jx %jx", (uintmax_t)size, (uintmax_t)entry->start, (uintmax_t)entry->end)); size = entry->end - entry->start; if (buflen1 > size) buflen1 = size; } else { KASSERT(entry->end - entry->start == size, ("no split allowed %jx %jx %jx", (uintmax_t)size, (uintmax_t)entry->start, (uintmax_t)entry->end)); } if (offset + buflen1 > size) buflen1 = size - offset; if (buflen1 > tag->common.maxsegsz) buflen1 = tag->common.maxsegsz; KASSERT(((entry->start + offset) & (tag->common.alignment - 1)) == 0, ("alignment failed: ctx %p start 0x%jx offset %x " "align 0x%jx", ctx, (uintmax_t)entry->start, offset, (uintmax_t)tag->common.alignment)); KASSERT(entry->end <= tag->common.lowaddr || entry->start >= tag->common.highaddr, ("entry placement failed: ctx %p start 0x%jx end 0x%jx " "lowaddr 0x%jx highaddr 0x%jx", ctx, (uintmax_t)entry->start, (uintmax_t)entry->end, (uintmax_t)tag->common.lowaddr, (uintmax_t)tag->common.highaddr)); KASSERT(iommu_test_boundary(entry->start + offset, buflen1, tag->common.boundary), ("boundary failed: ctx %p start 0x%jx end 0x%jx " "boundary 0x%jx", ctx, (uintmax_t)entry->start, (uintmax_t)entry->end, (uintmax_t)tag->common.boundary)); KASSERT(buflen1 <= tag->common.maxsegsz, ("segment too large: ctx %p start 0x%jx end 0x%jx " "buflen1 0x%jx maxsegsz 0x%jx", ctx, (uintmax_t)entry->start, (uintmax_t)entry->end, (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz)); IOMMU_DOMAIN_LOCK(domain); TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); entry->flags |= IOMMU_MAP_ENTRY_MAP; IOMMU_DOMAIN_UNLOCK(domain); TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link); segs[seg].ds_addr = entry->start + offset; segs[seg].ds_len = buflen1; idx += OFF_TO_IDX(trunc_page(offset + buflen1)); offset += buflen1; offset &= IOMMU_PAGE_MASK; buflen -= buflen1; } if (error == 0) *segp = seg; return (error); } static int iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag, struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { struct iommu_ctx *ctx; struct iommu_domain *domain; struct iommu_map_entry *entry, *entry1; struct iommu_map_entries_tailq unroll_list; int error; ctx = tag->ctx; domain = ctx->domain; atomic_add_long(&ctx->loads, 1); TAILQ_INIT(&unroll_list); error = iommu_bus_dmamap_load_something1(tag, map, ma, offset, buflen, flags, segs, segp, &unroll_list); if (error != 0) { /* * The busdma interface does not allow us to report * partial buffer load, so unfortunately we have to * revert all work done. */ IOMMU_DOMAIN_LOCK(domain); TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link, entry1) { /* * No entries other than what we have created * during the failed run might have been * inserted there in between, since we own ctx * pglock. */ TAILQ_REMOVE(&map->map_entries, entry, dmamap_link); TAILQ_REMOVE(&unroll_list, entry, unroll_link); TAILQ_INSERT_TAIL(&domain->unload_entries, entry, dmamap_link); } IOMMU_DOMAIN_UNLOCK(domain); taskqueue_enqueue(domain->iommu->delayed_taskqueue, &domain->unload_task); } if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 && !map->cansleep) error = EINPROGRESS; if (error == EINPROGRESS) iommu_bus_schedule_dmamap(domain->iommu, map); return (error); } static int iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen, flags, segs, segp)); } static int iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; vm_page_t *ma, fma; vm_paddr_t pstart, pend, paddr; int error, i, ma_cnt, mflags, offset; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; pstart = trunc_page(buf); pend = round_page(buf + buflen); offset = buf & PAGE_MASK; ma_cnt = OFF_TO_IDX(pend - pstart); mflags = map->cansleep ? M_WAITOK : M_NOWAIT; ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); if (ma == NULL) return (ENOMEM); fma = NULL; for (i = 0; i < ma_cnt; i++) { paddr = pstart + ptoa(i); ma[i] = PHYS_TO_VM_PAGE(paddr); if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { /* * If PHYS_TO_VM_PAGE() returned NULL or the * vm_page was not initialized we'll use a * fake page. */ if (fma == NULL) { fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF, M_ZERO | mflags); if (fma == NULL) { free(ma, M_DEVBUF); return (ENOMEM); } } vm_page_initfake(&fma[i], pstart + ptoa(i), VM_MEMATTR_DEFAULT); ma[i] = &fma[i]; } } error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen, flags, segs, segp); free(fma, M_DEVBUF); free(ma, M_DEVBUF); return (error); } static int iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf, bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, int *segp) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; vm_page_t *ma, fma; vm_paddr_t pstart, pend, paddr; int error, i, ma_cnt, mflags, offset; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; pstart = trunc_page((vm_offset_t)buf); pend = round_page((vm_offset_t)buf + buflen); offset = (vm_offset_t)buf & PAGE_MASK; ma_cnt = OFF_TO_IDX(pend - pstart); mflags = map->cansleep ? M_WAITOK : M_NOWAIT; ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags); if (ma == NULL) return (ENOMEM); fma = NULL; for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) { if (pmap == kernel_pmap) paddr = pmap_kextract(pstart); else paddr = pmap_extract(pmap, pstart); ma[i] = PHYS_TO_VM_PAGE(paddr); if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) { /* * If PHYS_TO_VM_PAGE() returned NULL or the * vm_page was not initialized we'll use a * fake page. */ if (fma == NULL) { fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF, M_ZERO | mflags); if (fma == NULL) { free(ma, M_DEVBUF); return (ENOMEM); } } vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT); ma[i] = &fma[i]; } } error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen, flags, segs, segp); free(ma, M_DEVBUF); free(fma, M_DEVBUF); return (error); } static void iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) { struct bus_dmamap_iommu *map; if (map1 == NULL) return; map = (struct bus_dmamap_iommu *)map1; map->mem = *mem; map->tag = (struct bus_dma_tag_iommu *)dmat; map->callback = callback; map->callback_arg = callback_arg; } static bus_dma_segment_t * iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1, bus_dma_segment_t *segs, int nsegs, int error) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; if (!map->locked) { KASSERT(map->cansleep, ("map not locked and not sleepable context %p", map)); /* * We are called from the delayed context. Relock the * driver. */ (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK); map->locked = true; } if (segs == NULL) segs = tag->segments; return (segs); } /* * The limitations of busdma KPI forces the iommu to perform the actual * unload, consisting of the unmapping of the map entries page tables, * from the delayed context on i386, since page table page mapping * might require a sleep to be successfull. The unfortunate * consequence is that the DMA requests can be served some time after * the bus_dmamap_unload() call returned. * * On amd64, we assume that sf allocation cannot fail. */ static void iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; struct iommu_ctx *ctx; struct iommu_domain *domain; #ifndef IOMMU_DOMAIN_UNLOAD_SLEEP struct iommu_map_entries_tailq entries; #endif tag = (struct bus_dma_tag_iommu *)dmat; map = (struct bus_dmamap_iommu *)map1; ctx = tag->ctx; domain = ctx->domain; atomic_add_long(&ctx->unloads, 1); #if defined(IOMMU_DOMAIN_UNLOAD_SLEEP) IOMMU_DOMAIN_LOCK(domain); TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link); IOMMU_DOMAIN_UNLOCK(domain); taskqueue_enqueue(domain->iommu->delayed_taskqueue, &domain->unload_task); #else TAILQ_INIT(&entries); IOMMU_DOMAIN_LOCK(domain); TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link); IOMMU_DOMAIN_UNLOCK(domain); THREAD_NO_SLEEPING(); iommu_domain_unload(domain, &entries, false); THREAD_SLEEPING_OK(); KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_ctx_unload %p", ctx)); #endif } static void iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { } struct bus_dma_impl bus_dma_iommu_impl = { .tag_create = iommu_bus_dma_tag_create, .tag_destroy = iommu_bus_dma_tag_destroy, .tag_set_domain = iommu_bus_dma_tag_set_domain, .id_mapped = iommu_bus_dma_id_mapped, .map_create = iommu_bus_dmamap_create, .map_destroy = iommu_bus_dmamap_destroy, .mem_alloc = iommu_bus_dmamem_alloc, .mem_free = iommu_bus_dmamem_free, .load_phys = iommu_bus_dmamap_load_phys, .load_buffer = iommu_bus_dmamap_load_buffer, .load_ma = iommu_bus_dmamap_load_ma, .map_waitok = iommu_bus_dmamap_waitok, .map_complete = iommu_bus_dmamap_complete, .map_unload = iommu_bus_dmamap_unload, .map_sync = iommu_bus_dmamap_sync, }; static void iommu_bus_task_dmamap(void *arg, int pending) { struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; struct iommu_unit *unit; unit = arg; IOMMU_LOCK(unit); while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) { TAILQ_REMOVE(&unit->delayed_maps, map, delay_link); IOMMU_UNLOCK(unit); tag = map->tag; map->cansleep = true; map->locked = false; bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map, &map->mem, map->callback, map->callback_arg, BUS_DMA_WAITOK); map->cansleep = false; if (map->locked) { (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_UNLOCK); } else map->locked = true; map->cansleep = false; IOMMU_LOCK(unit); } IOMMU_UNLOCK(unit); } static void iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map) { map->locked = false; IOMMU_LOCK(unit); TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link); IOMMU_UNLOCK(unit); taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task); } int iommu_init_busdma(struct iommu_unit *unit) { int error; unit->dma_enabled = 1; error = TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled); if (error == 0) /* compatibility */ TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled); TAILQ_INIT(&unit->delayed_maps); TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit); unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK, taskqueue_thread_enqueue, &unit->delayed_taskqueue); taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK, "iommu%d busdma taskq", unit->unit); return (0); } void iommu_fini_busdma(struct iommu_unit *unit) { if (unit->delayed_taskqueue == NULL) return; taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task); taskqueue_free(unit->delayed_taskqueue); unit->delayed_taskqueue = NULL; } int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1, vm_paddr_t start, vm_size_t length, int flags) { struct bus_dma_tag_common *tc; struct bus_dma_tag_iommu *tag; struct bus_dmamap_iommu *map; struct iommu_ctx *ctx; struct iommu_domain *domain; struct iommu_map_entry *entry; vm_page_t *ma; vm_size_t i; int error; bool waitok; MPASS((start & PAGE_MASK) == 0); MPASS((length & PAGE_MASK) == 0); MPASS(length > 0); MPASS(start + length >= start); MPASS((flags & ~(BUS_DMA_NOWAIT | BUS_DMA_NOWRITE)) == 0); tc = (struct bus_dma_tag_common *)dmat; if (tc->impl != &bus_dma_iommu_impl) return (0); tag = (struct bus_dma_tag_iommu *)dmat; ctx = tag->ctx; domain = ctx->domain; map = (struct bus_dmamap_iommu *)map1; waitok = (flags & BUS_DMA_NOWAIT) != 0; entry = iommu_map_alloc_entry(domain, waitok ? 0 : IOMMU_PGF_WAITOK); if (entry == NULL) return (ENOMEM); entry->start = start; entry->end = start + length; ma = malloc(sizeof(vm_page_t) * atop(length), M_TEMP, waitok ? M_WAITOK : M_NOWAIT); if (ma == NULL) { iommu_map_free_entry(domain, entry); return (ENOMEM); } for (i = 0; i < atop(length); i++) { ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, VM_MEMATTR_DEFAULT); } error = iommu_map_region(domain, entry, IOMMU_MAP_ENTRY_READ | ((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE), waitok ? IOMMU_MF_CANWAIT : 0, ma); if (error == 0) { IOMMU_DOMAIN_LOCK(domain); TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link); entry->flags |= IOMMU_MAP_ENTRY_MAP; IOMMU_DOMAIN_UNLOCK(domain); } else { iommu_domain_unload_entry(entry, true); } for (i = 0; i < atop(length); i++) vm_page_putfake(ma[i]); free(ma, M_TEMP); return (error); } static void iommu_domain_unload_task(void *arg, int pending) { struct iommu_domain *domain; struct iommu_map_entries_tailq entries; domain = arg; TAILQ_INIT(&entries); for (;;) { IOMMU_DOMAIN_LOCK(domain); TAILQ_SWAP(&domain->unload_entries, &entries, iommu_map_entry, dmamap_link); IOMMU_DOMAIN_UNLOCK(domain); if (TAILQ_EMPTY(&entries)) break; iommu_domain_unload(domain, &entries, true); } } void iommu_domain_init(struct iommu_unit *unit, struct iommu_domain *domain, const struct iommu_domain_map_ops *ops) { domain->ops = ops; domain->iommu = unit; TASK_INIT(&domain->unload_task, 0, iommu_domain_unload_task, domain); RB_INIT(&domain->rb_root); TAILQ_INIT(&domain->unload_entries); mtx_init(&domain->lock, "iodom", NULL, MTX_DEF); } void iommu_domain_fini(struct iommu_domain *domain) { mtx_destroy(&domain->lock); } diff --git a/sys/dev/iommu/busdma_iommu.h b/sys/dev/iommu/busdma_iommu.h index 857d3670fc6f..f911f26fa0f7 100644 --- a/sys/dev/iommu/busdma_iommu.h +++ b/sys/dev/iommu/busdma_iommu.h @@ -1,65 +1,64 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_IOMMU_BUSDMA_DMAR_H #define __X86_IOMMU_BUSDMA_DMAR_H #include #include struct bus_dma_tag_iommu { struct bus_dma_tag_common common; struct iommu_ctx *ctx; device_t owner; int map_count; bus_dma_segment_t *segments; }; struct bus_dmamap_iommu { struct bus_dma_tag_iommu *tag; struct memdesc mem; bus_dmamap_callback_t *callback; void *callback_arg; struct iommu_map_entries_tailq map_entries; TAILQ_ENTRY(bus_dmamap_iommu) delay_link; bool locked; bool cansleep; int flags; }; #define BUS_DMAMAP_IOMMU_MALLOC 0x0001 #define BUS_DMAMAP_IOMMU_KMEM_ALLOC 0x0002 extern struct bus_dma_impl bus_dma_iommu_impl; #endif diff --git a/sys/dev/iommu/iommu.h b/sys/dev/iommu/iommu.h index ba592ea08ff1..dd803e84c2ee 100644 --- a/sys/dev/iommu/iommu.h +++ b/sys/dev/iommu/iommu.h @@ -1,221 +1,220 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_IOMMU_IOMMU_H_ #define _DEV_IOMMU_IOMMU_H_ #include struct bus_dma_tag_common; struct iommu_map_entry; TAILQ_HEAD(iommu_map_entries_tailq, iommu_map_entry); RB_HEAD(iommu_gas_entries_tree, iommu_map_entry); RB_PROTOTYPE(iommu_gas_entries_tree, iommu_map_entry, rb_entry, iommu_gas_cmp_entries); struct iommu_qi_genseq { u_int gen; uint32_t seq; }; struct iommu_map_entry { iommu_gaddr_t start; iommu_gaddr_t end; iommu_gaddr_t first; /* Least start in subtree */ iommu_gaddr_t last; /* Greatest end in subtree */ iommu_gaddr_t free_down; /* Max free space below the current R/B tree node */ u_int flags; TAILQ_ENTRY(iommu_map_entry) dmamap_link; /* Link for dmamap entries */ RB_ENTRY(iommu_map_entry) rb_entry; /* Links for domain entries */ TAILQ_ENTRY(iommu_map_entry) unroll_link; /* Link for unroll after dmamap_load failure */ struct iommu_domain *domain; struct iommu_qi_genseq gseq; }; struct iommu_unit { struct mtx lock; device_t dev; int unit; int dma_enabled; /* Busdma delayed map load */ struct task dmamap_load_task; TAILQ_HEAD(, bus_dmamap_iommu) delayed_maps; struct taskqueue *delayed_taskqueue; /* * Bitmap of buses for which context must ignore slot:func, * duplicating the page table pointer into all context table * entries. This is a client-controlled quirk to support some * NTBs. */ uint32_t buswide_ctxs[(PCI_BUSMAX + 1) / NBBY / sizeof(uint32_t)]; }; struct iommu_domain_map_ops { int (*map)(struct iommu_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags); int (*unmap)(struct iommu_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, int flags); }; /* * Locking annotations: * (u) - Protected by iommu unit lock * (d) - Protected by domain lock * (c) - Immutable after initialization */ struct iommu_domain { struct iommu_unit *iommu; /* (c) */ const struct iommu_domain_map_ops *ops; struct mtx lock; /* (c) */ struct task unload_task; /* (c) */ u_int entries_cnt; /* (d) */ struct iommu_map_entries_tailq unload_entries; /* (d) Entries to unload */ struct iommu_gas_entries_tree rb_root; /* (d) */ iommu_gaddr_t end; /* (c) Highest address + 1 in the guest AS */ struct iommu_map_entry *first_place, *last_place; /* (d) */ struct iommu_map_entry *msi_entry; /* (d) Arch-specific */ iommu_gaddr_t msi_base; /* (d) Arch-specific */ vm_paddr_t msi_phys; /* (d) Arch-specific */ u_int flags; /* (u) */ }; struct iommu_ctx { struct iommu_domain *domain; /* (c) */ struct bus_dma_tag_iommu *tag; /* (c) Root tag */ u_long loads; /* atomic updates, for stat only */ u_long unloads; /* same */ u_int flags; /* (u) */ uint16_t rid; /* (c) pci RID */ }; /* struct iommu_ctx flags */ #define IOMMU_CTX_FAULTED 0x0001 /* Fault was reported, last_fault_rec is valid */ #define IOMMU_CTX_DISABLED 0x0002 /* Device is disabled, the ephemeral reference is kept to prevent context destruction */ #define IOMMU_DOMAIN_GAS_INITED 0x0001 #define IOMMU_DOMAIN_PGTBL_INITED 0x0002 #define IOMMU_DOMAIN_IDMAP 0x0010 /* Domain uses identity page table */ #define IOMMU_DOMAIN_RMRR 0x0020 /* Domain contains RMRR entry, cannot be turned off */ #define IOMMU_LOCK(unit) mtx_lock(&(unit)->lock) #define IOMMU_UNLOCK(unit) mtx_unlock(&(unit)->lock) #define IOMMU_ASSERT_LOCKED(unit) mtx_assert(&(unit)->lock, MA_OWNED) #define IOMMU_DOMAIN_LOCK(dom) mtx_lock(&(dom)->lock) #define IOMMU_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->lock) #define IOMMU_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->lock, MA_OWNED) static inline bool iommu_test_boundary(iommu_gaddr_t start, iommu_gaddr_t size, iommu_gaddr_t boundary) { if (boundary == 0) return (true); return (start + size <= ((start + boundary) & ~(boundary - 1))); } void iommu_free_ctx(struct iommu_ctx *ctx); void iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ctx); struct iommu_ctx *iommu_get_ctx(struct iommu_unit *, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init); struct iommu_unit *iommu_find(device_t dev, bool verbose); void iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free); void iommu_domain_unload(struct iommu_domain *domain, struct iommu_map_entries_tailq *entries, bool cansleep); struct iommu_ctx *iommu_instantiate_ctx(struct iommu_unit *iommu, device_t dev, bool rmrr); device_t iommu_get_requester(device_t dev, uint16_t *rid); int iommu_init_busdma(struct iommu_unit *unit); void iommu_fini_busdma(struct iommu_unit *unit); struct iommu_map_entry *iommu_map_alloc_entry(struct iommu_domain *iodom, u_int flags); void iommu_map_free_entry(struct iommu_domain *, struct iommu_map_entry *); int iommu_map(struct iommu_domain *iodom, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res); int iommu_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma); void iommu_gas_init_domain(struct iommu_domain *domain); void iommu_gas_fini_domain(struct iommu_domain *domain); struct iommu_map_entry *iommu_gas_alloc_entry(struct iommu_domain *domain, u_int flags); void iommu_gas_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry); void iommu_gas_free_space(struct iommu_domain *domain, struct iommu_map_entry *entry); int iommu_gas_map(struct iommu_domain *domain, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res); void iommu_gas_free_region(struct iommu_domain *domain, struct iommu_map_entry *entry); int iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma); int iommu_gas_reserve_region(struct iommu_domain *domain, iommu_gaddr_t start, iommu_gaddr_t end, struct iommu_map_entry **entry0); int iommu_gas_reserve_region_extend(struct iommu_domain *domain, iommu_gaddr_t start, iommu_gaddr_t end); void iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno); bool iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno); void iommu_domain_init(struct iommu_unit *unit, struct iommu_domain *domain, const struct iommu_domain_map_ops *ops); void iommu_domain_fini(struct iommu_domain *domain); bool bus_dma_iommu_set_buswide(device_t dev); int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t start, vm_size_t length, int flags); bus_dma_tag_t iommu_get_dma_tag(device_t dev, device_t child); struct iommu_ctx *iommu_get_dev_ctx(device_t dev); struct iommu_domain *iommu_get_ctx_domain(struct iommu_ctx *ctx); SYSCTL_DECL(_hw_iommu); #endif /* !_DEV_IOMMU_IOMMU_H_ */ diff --git a/sys/dev/iommu/iommu_gas.c b/sys/dev/iommu/iommu_gas.c index 431c0c195c35..ae54f41953c8 100644 --- a/sys/dev/iommu/iommu_gas.c +++ b/sys/dev/iommu/iommu_gas.c @@ -1,899 +1,898 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #define RB_AUGMENT(entry) iommu_gas_augment_entry(entry) #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Guest Address Space management. */ static uma_zone_t iommu_map_entry_zone; #ifdef INVARIANTS static int iommu_check_free; #endif static void intel_gas_init(void) { iommu_map_entry_zone = uma_zcreate("IOMMU_MAP_ENTRY", sizeof(struct iommu_map_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); } SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); struct iommu_map_entry * iommu_gas_alloc_entry(struct iommu_domain *domain, u_int flags) { struct iommu_map_entry *res; KASSERT((flags & ~(IOMMU_PGF_WAITOK)) == 0, ("unsupported flags %x", flags)); res = uma_zalloc(iommu_map_entry_zone, ((flags & IOMMU_PGF_WAITOK) != 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); if (res != NULL) { res->domain = domain; atomic_add_int(&domain->entries_cnt, 1); } return (res); } void iommu_gas_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) { KASSERT(domain == entry->domain, ("mismatched free domain %p entry %p entry->domain %p", domain, entry, entry->domain)); atomic_subtract_int(&domain->entries_cnt, 1); uma_zfree(iommu_map_entry_zone, entry); } static int iommu_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b) { /* Last entry have zero size, so <= */ KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", a, (uintmax_t)a->start, (uintmax_t)a->end)); KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", b, (uintmax_t)b->start, (uintmax_t)b->end)); KASSERT(a->end <= b->start || b->end <= a->start || a->end == a->start || b->end == b->start, ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", a, (uintmax_t)a->start, (uintmax_t)a->end, b, (uintmax_t)b->start, (uintmax_t)b->end)); if (a->end < b->end) return (-1); else if (b->end < a->end) return (1); return (0); } static void iommu_gas_augment_entry(struct iommu_map_entry *entry) { struct iommu_map_entry *child; iommu_gaddr_t free_down; free_down = 0; if ((child = RB_LEFT(entry, rb_entry)) != NULL) { free_down = MAX(free_down, child->free_down); free_down = MAX(free_down, entry->start - child->last); entry->first = child->first; } else entry->first = entry->start; if ((child = RB_RIGHT(entry, rb_entry)) != NULL) { free_down = MAX(free_down, child->free_down); free_down = MAX(free_down, child->first - entry->end); entry->last = child->last; } else entry->last = entry->end; entry->free_down = free_down; } RB_GENERATE(iommu_gas_entries_tree, iommu_map_entry, rb_entry, iommu_gas_cmp_entries); #ifdef INVARIANTS static void iommu_gas_check_free(struct iommu_domain *domain) { struct iommu_map_entry *entry, *l, *r; iommu_gaddr_t v; RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { KASSERT(domain == entry->domain, ("mismatched free domain %p entry %p entry->domain %p", domain, entry, entry->domain)); l = RB_LEFT(entry, rb_entry); r = RB_RIGHT(entry, rb_entry); v = 0; if (l != NULL) { v = MAX(v, l->free_down); v = MAX(v, entry->start - l->last); } if (r != NULL) { v = MAX(v, r->free_down); v = MAX(v, r->first - entry->end); } MPASS(entry->free_down == v); } } #endif static bool iommu_gas_rb_insert(struct iommu_domain *domain, struct iommu_map_entry *entry) { struct iommu_map_entry *found; found = RB_INSERT(iommu_gas_entries_tree, &domain->rb_root, entry); return (found == NULL); } static void iommu_gas_rb_remove(struct iommu_domain *domain, struct iommu_map_entry *entry) { RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); } struct iommu_domain * iommu_get_ctx_domain(struct iommu_ctx *ctx) { return (ctx->domain); } void iommu_gas_init_domain(struct iommu_domain *domain) { struct iommu_map_entry *begin, *end; begin = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); end = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); IOMMU_DOMAIN_LOCK(domain); KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); KASSERT(RB_EMPTY(&domain->rb_root), ("non-empty entries %p", domain)); begin->start = 0; begin->end = IOMMU_PAGE_SIZE; begin->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; iommu_gas_rb_insert(domain, begin); end->start = domain->end; end->end = domain->end; end->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; iommu_gas_rb_insert(domain, end); domain->first_place = begin; domain->last_place = end; domain->flags |= IOMMU_DOMAIN_GAS_INITED; IOMMU_DOMAIN_UNLOCK(domain); } void iommu_gas_fini_domain(struct iommu_domain *domain) { struct iommu_map_entry *entry, *entry1; IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT(domain->entries_cnt == 2, ("domain still in use %p", domain)); entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root); KASSERT(entry->start == 0, ("start entry start %p", domain)); KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain)); KASSERT(entry->flags == (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), ("start entry flags %p", domain)); RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); iommu_gas_free_entry(domain, entry); entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root); KASSERT(entry->start == domain->end, ("end entry start %p", domain)); KASSERT(entry->end == domain->end, ("end entry end %p", domain)); KASSERT(entry->flags == (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), ("end entry flags %p", domain)); RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); iommu_gas_free_entry(domain, entry); RB_FOREACH_SAFE(entry, iommu_gas_entries_tree, &domain->rb_root, entry1) { KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0, ("non-RMRR entry left %p", domain)); RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); iommu_gas_free_entry(domain, entry); } } struct iommu_gas_match_args { struct iommu_domain *domain; iommu_gaddr_t size; int offset; const struct bus_dma_tag_common *common; u_int gas_flags; struct iommu_map_entry *entry; }; /* * The interval [beg, end) is a free interval between two iommu_map_entries. * maxaddr is an upper bound on addresses that can be allocated. Try to * allocate space in the free interval, subject to the conditions expressed * by a, and return 'true' if and only if the allocation attempt succeeds. */ static bool iommu_gas_match_one(struct iommu_gas_match_args *a, iommu_gaddr_t beg, iommu_gaddr_t end, iommu_gaddr_t maxaddr) { iommu_gaddr_t bs, start; a->entry->start = roundup2(beg + IOMMU_PAGE_SIZE, a->common->alignment); if (a->entry->start + a->size > maxaddr) return (false); /* IOMMU_PAGE_SIZE to create gap after new entry. */ if (a->entry->start < beg + IOMMU_PAGE_SIZE || a->entry->start + a->size + a->offset + IOMMU_PAGE_SIZE > end) return (false); /* No boundary crossing. */ if (iommu_test_boundary(a->entry->start + a->offset, a->size, a->common->boundary)) return (true); /* * The start + offset to start + offset + size region crosses * the boundary. Check if there is enough space after the * next boundary after the beg. */ bs = rounddown2(a->entry->start + a->offset + a->common->boundary, a->common->boundary); start = roundup2(bs, a->common->alignment); /* IOMMU_PAGE_SIZE to create gap after new entry. */ if (start + a->offset + a->size + IOMMU_PAGE_SIZE <= end && start + a->offset + a->size <= maxaddr && iommu_test_boundary(start + a->offset, a->size, a->common->boundary)) { a->entry->start = start; return (true); } /* * Not enough space to align at the requested boundary, or * boundary is smaller than the size, but allowed to split. * We already checked that start + size does not overlap maxaddr. * * XXXKIB. It is possible that bs is exactly at the start of * the next entry, then we do not have gap. Ignore for now. */ if ((a->gas_flags & IOMMU_MF_CANSPLIT) != 0) { a->size = bs - a->entry->start; return (true); } return (false); } static void iommu_gas_match_insert(struct iommu_gas_match_args *a) { bool found; /* * The prev->end is always aligned on the page size, which * causes page alignment for the entry->start too. The size * is checked to be multiple of the page size. * * The page sized gap is created between consequent * allocations to ensure that out-of-bounds accesses fault. */ a->entry->end = a->entry->start + a->size; found = iommu_gas_rb_insert(a->domain, a->entry); KASSERT(found, ("found dup %p start %jx size %jx", a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); a->entry->flags = IOMMU_MAP_ENTRY_MAP; } static int iommu_gas_lowermatch(struct iommu_gas_match_args *a, struct iommu_map_entry *entry) { struct iommu_map_entry *child; child = RB_RIGHT(entry, rb_entry); if (child != NULL && entry->end < a->common->lowaddr && iommu_gas_match_one(a, entry->end, child->first, a->common->lowaddr)) { iommu_gas_match_insert(a); return (0); } if (entry->free_down < a->size + a->offset + IOMMU_PAGE_SIZE) return (ENOMEM); if (entry->first >= a->common->lowaddr) return (ENOMEM); child = RB_LEFT(entry, rb_entry); if (child != NULL && 0 == iommu_gas_lowermatch(a, child)) return (0); if (child != NULL && child->last < a->common->lowaddr && iommu_gas_match_one(a, child->last, entry->start, a->common->lowaddr)) { iommu_gas_match_insert(a); return (0); } child = RB_RIGHT(entry, rb_entry); if (child != NULL && 0 == iommu_gas_lowermatch(a, child)) return (0); return (ENOMEM); } static int iommu_gas_uppermatch(struct iommu_gas_match_args *a, struct iommu_map_entry *entry) { struct iommu_map_entry *child; if (entry->free_down < a->size + a->offset + IOMMU_PAGE_SIZE) return (ENOMEM); if (entry->last < a->common->highaddr) return (ENOMEM); child = RB_LEFT(entry, rb_entry); if (child != NULL && 0 == iommu_gas_uppermatch(a, child)) return (0); if (child != NULL && child->last >= a->common->highaddr && iommu_gas_match_one(a, child->last, entry->start, a->domain->end)) { iommu_gas_match_insert(a); return (0); } child = RB_RIGHT(entry, rb_entry); if (child != NULL && entry->end >= a->common->highaddr && iommu_gas_match_one(a, entry->end, child->first, a->domain->end)) { iommu_gas_match_insert(a); return (0); } if (child != NULL && 0 == iommu_gas_uppermatch(a, child)) return (0); return (ENOMEM); } static int iommu_gas_find_space(struct iommu_domain *domain, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, u_int flags, struct iommu_map_entry *entry) { struct iommu_gas_match_args a; int error; IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); KASSERT((size & IOMMU_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); a.domain = domain; a.size = size; a.offset = offset; a.common = common; a.gas_flags = flags; a.entry = entry; /* Handle lower region. */ if (common->lowaddr > 0) { error = iommu_gas_lowermatch(&a, RB_ROOT(&domain->rb_root)); if (error == 0) return (0); KASSERT(error == ENOMEM, ("error %d from iommu_gas_lowermatch", error)); } /* Handle upper region. */ if (common->highaddr >= domain->end) return (ENOMEM); error = iommu_gas_uppermatch(&a, RB_ROOT(&domain->rb_root)); KASSERT(error == ENOMEM, ("error %d from iommu_gas_uppermatch", error)); return (error); } static int iommu_gas_alloc_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int flags) { struct iommu_map_entry *next, *prev; bool found; IOMMU_DOMAIN_ASSERT_LOCKED(domain); if ((entry->start & IOMMU_PAGE_MASK) != 0 || (entry->end & IOMMU_PAGE_MASK) != 0) return (EINVAL); if (entry->start >= entry->end) return (EINVAL); if (entry->end >= domain->end) return (EINVAL); next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, entry); KASSERT(next != NULL, ("next must be non-null %p %jx", domain, (uintmax_t)entry->start)); prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); /* prev could be NULL */ /* * Adapt to broken BIOSes which specify overlapping RMRR * entries. * * XXXKIB: this does not handle a case when prev or next * entries are completely covered by the current one, which * extends both ways. */ if (prev != NULL && prev->end > entry->start && (prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { if ((flags & IOMMU_MF_RMRR) == 0 || (prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0) return (EBUSY); entry->start = prev->end; } if (next->start < entry->end && (next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { if ((flags & IOMMU_MF_RMRR) == 0 || (next->flags & IOMMU_MAP_ENTRY_RMRR) == 0) return (EBUSY); entry->end = next->start; } if (entry->end == entry->start) return (0); if (prev != NULL && prev->end > entry->start) { /* This assumes that prev is the placeholder entry. */ iommu_gas_rb_remove(domain, prev); prev = NULL; } if (next->start < entry->end) { iommu_gas_rb_remove(domain, next); next = NULL; } found = iommu_gas_rb_insert(domain, entry); KASSERT(found, ("found RMRR dup %p start %jx end %jx", domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); if ((flags & IOMMU_MF_RMRR) != 0) entry->flags = IOMMU_MAP_ENTRY_RMRR; #ifdef INVARIANTS struct iommu_map_entry *ip, *in; ip = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); in = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); KASSERT(prev == NULL || ip == prev, ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", entry, entry->start, entry->end, prev, prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); KASSERT(next == NULL || in == next, ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", entry, entry->start, entry->end, next, next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); #endif return (0); } void iommu_gas_free_space(struct iommu_domain *domain, struct iommu_map_entry *entry) { IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP, ("permanent entry %p %p", domain, entry)); iommu_gas_rb_remove(domain, entry); entry->flags &= ~IOMMU_MAP_ENTRY_MAP; #ifdef INVARIANTS if (iommu_check_free) iommu_gas_check_free(domain); #endif } void iommu_gas_free_region(struct iommu_domain *domain, struct iommu_map_entry *entry) { struct iommu_map_entry *next, *prev; IOMMU_DOMAIN_ASSERT_LOCKED(domain); KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR, ("non-RMRR entry %p %p", domain, entry)); prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); next = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); iommu_gas_rb_remove(domain, entry); entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; if (prev == NULL) iommu_gas_rb_insert(domain, domain->first_place); if (next == NULL) iommu_gas_rb_insert(domain, domain->last_place); } int iommu_gas_map(struct iommu_domain *domain, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) { struct iommu_map_entry *entry; int error; KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_CANSPLIT)) == 0, ("invalid flags 0x%x", flags)); entry = iommu_gas_alloc_entry(domain, (flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0); if (entry == NULL) return (ENOMEM); IOMMU_DOMAIN_LOCK(domain); error = iommu_gas_find_space(domain, common, size, offset, flags, entry); if (error == ENOMEM) { IOMMU_DOMAIN_UNLOCK(domain); iommu_gas_free_entry(domain, entry); return (error); } #ifdef INVARIANTS if (iommu_check_free) iommu_gas_check_free(domain); #endif KASSERT(error == 0, ("unexpected error %d from iommu_gas_find_entry", error)); KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", (uintmax_t)entry->end, (uintmax_t)domain->end)); entry->flags |= eflags; IOMMU_DOMAIN_UNLOCK(domain); error = domain->ops->map(domain, entry->start, entry->end - entry->start, ma, eflags, ((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0)); if (error == ENOMEM) { iommu_domain_unload_entry(entry, true); return (error); } KASSERT(error == 0, ("unexpected error %d from domain_map_buf", error)); *res = entry; return (0); } int iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma) { iommu_gaddr_t start; int error; KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, entry, entry->flags)); KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0, ("invalid flags 0x%x", flags)); start = entry->start; IOMMU_DOMAIN_LOCK(domain); error = iommu_gas_alloc_region(domain, entry, flags); if (error != 0) { IOMMU_DOMAIN_UNLOCK(domain); return (error); } entry->flags |= eflags; IOMMU_DOMAIN_UNLOCK(domain); if (entry->end == entry->start) return (0); error = domain->ops->map(domain, entry->start, entry->end - entry->start, ma + OFF_TO_IDX(start - entry->start), eflags, ((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0)); if (error == ENOMEM) { iommu_domain_unload_entry(entry, false); return (error); } KASSERT(error == 0, ("unexpected error %d from domain_map_buf", error)); return (0); } static int iommu_gas_reserve_region_locked(struct iommu_domain *domain, iommu_gaddr_t start, iommu_gaddr_t end, struct iommu_map_entry *entry) { int error; IOMMU_DOMAIN_ASSERT_LOCKED(domain); entry->start = start; entry->end = end; error = iommu_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT); if (error == 0) entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; return (error); } int iommu_gas_reserve_region(struct iommu_domain *domain, iommu_gaddr_t start, iommu_gaddr_t end, struct iommu_map_entry **entry0) { struct iommu_map_entry *entry; int error; entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); IOMMU_DOMAIN_LOCK(domain); error = iommu_gas_reserve_region_locked(domain, start, end, entry); IOMMU_DOMAIN_UNLOCK(domain); if (error != 0) iommu_gas_free_entry(domain, entry); else if (entry0 != NULL) *entry0 = entry; return (error); } /* * As in iommu_gas_reserve_region, reserve [start, end), but allow for existing * entries. */ int iommu_gas_reserve_region_extend(struct iommu_domain *domain, iommu_gaddr_t start, iommu_gaddr_t end) { struct iommu_map_entry *entry, *next, *prev, key = {}; iommu_gaddr_t entry_start, entry_end; int error; error = 0; entry = NULL; end = ummin(end, domain->end); while (start < end) { /* Preallocate an entry. */ if (entry == NULL) entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); /* Calculate the free region from here to the next entry. */ key.start = key.end = start; IOMMU_DOMAIN_LOCK(domain); next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, &key); KASSERT(next != NULL, ("domain %p with end %#jx has no entry " "after %#jx", domain, (uintmax_t)domain->end, (uintmax_t)start)); entry_end = ummin(end, next->start); prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); if (prev != NULL) entry_start = ummax(start, prev->end); else entry_start = start; start = next->end; /* Reserve the region if non-empty. */ if (entry_start != entry_end) { error = iommu_gas_reserve_region_locked(domain, entry_start, entry_end, entry); if (error != 0) break; entry = NULL; } IOMMU_DOMAIN_UNLOCK(domain); } /* Release a preallocated entry if it was not used. */ if (entry != NULL) iommu_gas_free_entry(domain, entry); return (error); } struct iommu_map_entry * iommu_map_alloc_entry(struct iommu_domain *domain, u_int flags) { struct iommu_map_entry *res; res = iommu_gas_alloc_entry(domain, flags); return (res); } void iommu_map_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) { iommu_gas_free_entry(domain, entry); } int iommu_map(struct iommu_domain *domain, const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) { int error; error = iommu_gas_map(domain, common, size, offset, eflags, flags, ma, res); return (error); } void iommu_unmap_msi(struct iommu_ctx *ctx) { struct iommu_map_entry *entry; struct iommu_domain *domain; domain = ctx->domain; entry = domain->msi_entry; if (entry == NULL) return; domain->ops->unmap(domain, entry->start, entry->end - entry->start, IOMMU_PGF_WAITOK); IOMMU_DOMAIN_LOCK(domain); iommu_gas_free_space(domain, entry); IOMMU_DOMAIN_UNLOCK(domain); iommu_gas_free_entry(domain, entry); domain->msi_entry = NULL; domain->msi_base = 0; domain->msi_phys = 0; } int iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, u_int eflags, u_int flags, vm_page_t *ma) { struct iommu_domain *domain; struct iommu_map_entry *entry; int error; error = 0; domain = ctx->domain; /* Check if there is already an MSI page allocated */ IOMMU_DOMAIN_LOCK(domain); entry = domain->msi_entry; IOMMU_DOMAIN_UNLOCK(domain); if (entry == NULL) { error = iommu_gas_map(domain, &ctx->tag->common, size, offset, eflags, flags, ma, &entry); IOMMU_DOMAIN_LOCK(domain); if (error == 0) { if (domain->msi_entry == NULL) { MPASS(domain->msi_base == 0); MPASS(domain->msi_phys == 0); domain->msi_entry = entry; domain->msi_base = entry->start; domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]); } else { /* * We lost the race and already have an * MSI page allocated. Free the unneeded entry. */ iommu_gas_free_entry(domain, entry); } } else if (domain->msi_entry != NULL) { /* * The allocation failed, but another succeeded. * Return success as there is a valid MSI page. */ error = 0; } IOMMU_DOMAIN_UNLOCK(domain); } return (error); } void iommu_translate_msi(struct iommu_domain *domain, uint64_t *addr) { *addr = (*addr - domain->msi_phys) + domain->msi_base; KASSERT(*addr >= domain->msi_entry->start, ("%s: Address is below the MSI entry start address (%jx < %jx)", __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->start)); KASSERT(*addr + sizeof(*addr) <= domain->msi_entry->end, ("%s: Address is above the MSI entry end address (%jx < %jx)", __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->end)); } int iommu_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, u_int eflags, u_int flags, vm_page_t *ma) { int error; error = iommu_gas_map_region(domain, entry, eflags, flags, ma); return (error); } SYSCTL_NODE(_hw, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, ""); #ifdef INVARIANTS SYSCTL_INT(_hw_iommu, OID_AUTO, check_free, CTLFLAG_RWTUN, &iommu_check_free, 0, "Check the GPA RBtree for free_down and free_after validity"); #endif diff --git a/sys/dev/iommu/iommu_gas.h b/sys/dev/iommu/iommu_gas.h index ed64f9763721..c32a098538b0 100644 --- a/sys/dev/iommu/iommu_gas.h +++ b/sys/dev/iommu/iommu_gas.h @@ -1,60 +1,59 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_IOMMU_IOMMU_GAS_H_ #define _DEV_IOMMU_IOMMU_GAS_H_ /* Map flags */ #define IOMMU_MF_CANWAIT 0x0001 #define IOMMU_MF_CANSPLIT 0x0002 #define IOMMU_MF_RMRR 0x0004 #define IOMMU_PGF_WAITOK 0x0001 #define IOMMU_PGF_ZERO 0x0002 #define IOMMU_PGF_ALLOC 0x0004 #define IOMMU_PGF_NOALLOC 0x0008 #define IOMMU_PGF_OBJL 0x0010 #define IOMMU_MAP_ENTRY_PLACE 0x0001 /* Fake entry */ #define IOMMU_MAP_ENTRY_RMRR 0x0002 /* Permanent, not linked by dmamap_link */ #define IOMMU_MAP_ENTRY_MAP 0x0004 /* Busdma created, linked by dmamap_link */ #define IOMMU_MAP_ENTRY_UNMAPPED 0x0010 /* No backing pages */ #define IOMMU_MAP_ENTRY_QI_NF 0x0020 /* qi task, do not free entry */ #define IOMMU_MAP_ENTRY_READ 0x1000 /* Read permitted */ #define IOMMU_MAP_ENTRY_WRITE 0x2000 /* Write permitted */ #define IOMMU_MAP_ENTRY_SNOOP 0x4000 /* Snoop */ #define IOMMU_MAP_ENTRY_TM 0x8000 /* Transient */ #endif /* !_DEV_IOMMU_IOMMU_GAS_H_ */ diff --git a/sys/dev/iommu/iommu_types.h b/sys/dev/iommu/iommu_types.h index cdc47dfc9df4..78f3000d0f65 100644 --- a/sys/dev/iommu/iommu_types.h +++ b/sys/dev/iommu/iommu_types.h @@ -1,42 +1,41 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_IOMMU_IOMMU_TYPES_H_ #define _DEV_IOMMU_IOMMU_TYPES_H_ /* Host or physical memory address, after translation. */ typedef uint64_t iommu_haddr_t; /* Guest or bus address, before translation. */ typedef uint64_t iommu_gaddr_t; #endif /* !_DEV_IOMMU_IOMMU_TYPES_H_ */ diff --git a/sys/dev/ofw/ofw_cpu.h b/sys/dev/ofw/ofw_cpu.h index 9ac91713e612..d96991946237 100644 --- a/sys/dev/ofw/ofw_cpu.h +++ b/sys/dev/ofw/ofw_cpu.h @@ -1,38 +1,37 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_OFW_OFW_CPU_H_ #define _DEV_OFW_OFW_CPU_H_ typedef boolean_t (*ofw_cpu_foreach_cb)(u_int, phandle_t, u_int, pcell_t *); int ofw_cpu_early_foreach(ofw_cpu_foreach_cb, boolean_t); #endif /* _DEV_OFW_OFW_CPU_H_ */ diff --git a/sys/dev/ofw/ofw_fdt.c b/sys/dev/ofw/ofw_fdt.c index 304ffb6b1436..a83f7cbdcb60 100644 --- a/sys/dev/ofw/ofw_fdt.c +++ b/sys/dev/ofw/ofw_fdt.c @@ -1,516 +1,515 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "ofw_if.h" #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #if defined(__arm__) #if defined(SOC_MV_ARMADAXP) || defined(SOC_MV_ARMADA38X) || \ defined(SOC_MV_DISCOVERY) || defined(SOC_MV_DOVE) || \ defined(SOC_MV_FREY) || defined(SOC_MV_KIRKWOOD) || \ defined(SOC_MV_LOKIPLUS) || defined(SOC_MV_ORION) #define FDT_MARVELL #endif #endif static int ofw_fdt_init(ofw_t, void *); static phandle_t ofw_fdt_peer(ofw_t, phandle_t); static phandle_t ofw_fdt_child(ofw_t, phandle_t); static phandle_t ofw_fdt_parent(ofw_t, phandle_t); static phandle_t ofw_fdt_instance_to_package(ofw_t, ihandle_t); static ssize_t ofw_fdt_getproplen(ofw_t, phandle_t, const char *); static ssize_t ofw_fdt_getprop(ofw_t, phandle_t, const char *, void *, size_t); static int ofw_fdt_nextprop(ofw_t, phandle_t, const char *, char *, size_t); static int ofw_fdt_setprop(ofw_t, phandle_t, const char *, const void *, size_t); static ssize_t ofw_fdt_canon(ofw_t, const char *, char *, size_t); static phandle_t ofw_fdt_finddevice(ofw_t, const char *); static ssize_t ofw_fdt_instance_to_path(ofw_t, ihandle_t, char *, size_t); static ssize_t ofw_fdt_package_to_path(ofw_t, phandle_t, char *, size_t); static int ofw_fdt_interpret(ofw_t, const char *, int, cell_t *); static ofw_method_t ofw_fdt_methods[] = { OFWMETHOD(ofw_init, ofw_fdt_init), OFWMETHOD(ofw_peer, ofw_fdt_peer), OFWMETHOD(ofw_child, ofw_fdt_child), OFWMETHOD(ofw_parent, ofw_fdt_parent), OFWMETHOD(ofw_instance_to_package, ofw_fdt_instance_to_package), OFWMETHOD(ofw_getproplen, ofw_fdt_getproplen), OFWMETHOD(ofw_getprop, ofw_fdt_getprop), OFWMETHOD(ofw_nextprop, ofw_fdt_nextprop), OFWMETHOD(ofw_setprop, ofw_fdt_setprop), OFWMETHOD(ofw_canon, ofw_fdt_canon), OFWMETHOD(ofw_finddevice, ofw_fdt_finddevice), OFWMETHOD(ofw_instance_to_path, ofw_fdt_instance_to_path), OFWMETHOD(ofw_package_to_path, ofw_fdt_package_to_path), OFWMETHOD(ofw_interpret, ofw_fdt_interpret), { 0, 0 } }; static ofw_def_t ofw_fdt = { OFW_FDT, ofw_fdt_methods, 0 }; OFW_DEF(ofw_fdt); #define FDT_FBSDVER_LEN 16 #define FDT_MODEL_LEN 80 #define FDT_COMPAT_LEN 255 static void *fdtp = NULL; static char fdt_model[FDT_MODEL_LEN]; static char fdt_compatible[FDT_COMPAT_LEN]; static char fdt_fbsd_version[FDT_FBSDVER_LEN]; static int sysctl_handle_dtb(SYSCTL_HANDLER_ARGS) { return (sysctl_handle_opaque(oidp, fdtp, fdt_totalsize(fdtp), req)); } static void sysctl_register_fdt_oid(void *arg) { /* If there is no FDT registered, skip adding the sysctl */ if (fdtp == NULL) return; SYSCTL_ADD_PROC(NULL, SYSCTL_STATIC_CHILDREN(_hw_fdt), OID_AUTO, "dtb", CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_handle_dtb, "", "Device Tree Blob"); if (fdt_model[0] != '\0') SYSCTL_ADD_STRING(NULL, SYSCTL_STATIC_CHILDREN(_hw_fdt), OID_AUTO, "model", CTLFLAG_RD, fdt_model, FDT_MODEL_LEN, "System board model"); if (fdt_compatible[0] != '\0') SYSCTL_ADD_STRING(NULL, SYSCTL_STATIC_CHILDREN(_hw_fdt), OID_AUTO, "compatible", CTLFLAG_RD, fdt_compatible, FDT_COMPAT_LEN, "Compatible platforms"); if (fdt_fbsd_version[0] != '\0') SYSCTL_ADD_STRING(NULL, SYSCTL_STATIC_CHILDREN(_hw_fdt), OID_AUTO, "freebsd-version", CTLFLAG_RD, fdt_fbsd_version, FDT_FBSDVER_LEN, "FreeBSD DTS branding version"); } SYSINIT(dtb_oid, SI_SUB_KMEM, SI_ORDER_ANY, sysctl_register_fdt_oid, NULL); static int ofw_fdt_init(ofw_t ofw, void *data) { phandle_t root; ssize_t len; int err; int i; /* Check FDT blob integrity */ if ((err = fdt_check_header(data)) != 0) return (err); fdtp = data; root = ofw_fdt_finddevice(NULL, "/"); len = ofw_fdt_getproplen(NULL, root, "model"); bzero(fdt_model, FDT_MODEL_LEN); ofw_fdt_getprop(NULL, root, "model", fdt_model, FDT_MODEL_LEN); len = ofw_fdt_getproplen(NULL, root, "compatible"); bzero(fdt_compatible, FDT_COMPAT_LEN); ofw_fdt_getprop(NULL, root, "compatible", fdt_compatible, FDT_COMPAT_LEN); /* Replace the middle '\0' with ' ' */ for (i = 0; i < len - 1; i++) if (fdt_compatible[i] == '\0') fdt_compatible[i] = ' '; if ((len = ofw_fdt_getproplen(NULL, root, "freebsd,dts-version")) > 0) { bzero(fdt_fbsd_version, FDT_FBSDVER_LEN); ofw_fdt_getprop(NULL, root, "freebsd,dts-version", fdt_fbsd_version, FDT_FBSDVER_LEN); } return (0); } /* * Device tree functions. * * We use the offset from fdtp to the node as the 'phandle' in OF interface. * * phandle is a u32 value, therefore we cannot use the pointer to node as * phandle in 64 bit. We also do not use the usual fdt offset as phandle, * as it can be 0, and the OF interface has special meaning for phandle 0. */ static phandle_t fdt_offset_phandle(int offset) { if (offset < 0) return (0); return ((phandle_t)offset + fdt_off_dt_struct(fdtp)); } static int fdt_phandle_offset(phandle_t p) { int pint = (int)p; int dtoff = fdt_off_dt_struct(fdtp); if (pint < dtoff) return (-1); return (pint - dtoff); } /* Return the next sibling of this node or 0. */ static phandle_t ofw_fdt_peer(ofw_t ofw, phandle_t node) { int offset; if (node == 0) { /* Find root node */ offset = fdt_path_offset(fdtp, "/"); return (fdt_offset_phandle(offset)); } offset = fdt_phandle_offset(node); if (offset < 0) return (0); offset = fdt_next_subnode(fdtp, offset); return (fdt_offset_phandle(offset)); } /* Return the first child of this node or 0. */ static phandle_t ofw_fdt_child(ofw_t ofw, phandle_t node) { int offset; offset = fdt_phandle_offset(node); if (offset < 0) return (0); offset = fdt_first_subnode(fdtp, offset); return (fdt_offset_phandle(offset)); } /* Return the parent of this node or 0. */ static phandle_t ofw_fdt_parent(ofw_t ofw, phandle_t node) { int offset, paroffset; offset = fdt_phandle_offset(node); if (offset < 0) return (0); paroffset = fdt_parent_offset(fdtp, offset); return (fdt_offset_phandle(paroffset)); } /* Return the package handle that corresponds to an instance handle. */ static phandle_t ofw_fdt_instance_to_package(ofw_t ofw, ihandle_t instance) { /* Where real OF uses ihandles in the tree, FDT uses xref phandles */ return (OF_node_from_xref(instance)); } /* Get the length of a property of a package. */ static ssize_t ofw_fdt_getproplen(ofw_t ofw, phandle_t package, const char *propname) { const void *prop; int offset, len; offset = fdt_phandle_offset(package); if (offset < 0) return (-1); len = -1; prop = fdt_getprop(fdtp, offset, propname, &len); if (prop == NULL && strcmp(propname, "name") == 0) { /* Emulate the 'name' property */ fdt_get_name(fdtp, offset, &len); return (len + 1); } if (prop == NULL && offset == fdt_path_offset(fdtp, "/chosen")) { if (strcmp(propname, "fdtbootcpu") == 0) return (sizeof(cell_t)); if (strcmp(propname, "fdtmemreserv") == 0) return (sizeof(uint64_t)*2*fdt_num_mem_rsv(fdtp)); } if (prop == NULL) return (-1); return (len); } /* Get the value of a property of a package. */ static ssize_t ofw_fdt_getprop(ofw_t ofw, phandle_t package, const char *propname, void *buf, size_t buflen) { const void *prop; const char *name; int len, offset; uint32_t cpuid; offset = fdt_phandle_offset(package); if (offset < 0) return (-1); prop = fdt_getprop(fdtp, offset, propname, &len); if (prop == NULL && strcmp(propname, "name") == 0) { /* Emulate the 'name' property */ name = fdt_get_name(fdtp, offset, &len); strncpy(buf, name, buflen); return (len + 1); } if (prop == NULL && offset == fdt_path_offset(fdtp, "/chosen")) { if (strcmp(propname, "fdtbootcpu") == 0) { cpuid = cpu_to_fdt32(fdt_boot_cpuid_phys(fdtp)); len = sizeof(cpuid); prop = &cpuid; } if (strcmp(propname, "fdtmemreserv") == 0) { prop = (char *)fdtp + fdt_off_mem_rsvmap(fdtp); len = sizeof(uint64_t)*2*fdt_num_mem_rsv(fdtp); } } if (prop == NULL) return (-1); bcopy(prop, buf, min(len, buflen)); return (len); } /* * Get the next property of a package. Return values: * -1: package or previous property does not exist * 0: no more properties * 1: success */ static int ofw_fdt_nextprop(ofw_t ofw, phandle_t package, const char *previous, char *buf, size_t size) { const void *prop; const char *name; int offset; offset = fdt_phandle_offset(package); if (offset < 0) return (-1); if (previous == NULL) /* Find the first prop in the node */ offset = fdt_first_property_offset(fdtp, offset); else { fdt_for_each_property_offset(offset, fdtp, offset) { prop = fdt_getprop_by_offset(fdtp, offset, &name, NULL); if (prop == NULL) return (-1); /* Internal error */ /* Skip until we find 'previous', then bail out */ if (strcmp(name, previous) != 0) continue; offset = fdt_next_property_offset(fdtp, offset); break; } } if (offset < 0) return (0); /* No properties */ prop = fdt_getprop_by_offset(fdtp, offset, &name, &offset); if (prop == NULL) return (-1); /* Internal error */ strncpy(buf, name, size); return (1); } /* Set the value of a property of a package. */ static int ofw_fdt_setprop(ofw_t ofw, phandle_t package, const char *propname, const void *buf, size_t len) { int offset; offset = fdt_phandle_offset(package); if (offset < 0) return (-1); if (fdt_setprop_inplace(fdtp, offset, propname, buf, len) != 0) /* Try to add property, when setting value inplace failed */ return (fdt_setprop(fdtp, offset, propname, buf, len)); return (0); } /* Convert a device specifier to a fully qualified pathname. */ static ssize_t ofw_fdt_canon(ofw_t ofw, const char *device, char *buf, size_t len) { return (-1); } /* Return a package handle for the specified device. */ static phandle_t ofw_fdt_finddevice(ofw_t ofw, const char *device) { int offset; offset = fdt_path_offset(fdtp, device); if (offset < 0) return (-1); return (fdt_offset_phandle(offset)); } /* Return the fully qualified pathname corresponding to an instance. */ static ssize_t ofw_fdt_instance_to_path(ofw_t ofw, ihandle_t instance, char *buf, size_t len) { phandle_t phandle; phandle = OF_instance_to_package(instance); if (phandle == -1) return (-1); return (OF_package_to_path(phandle, buf, len)); } /* Return the fully qualified pathname corresponding to a package. */ static ssize_t ofw_fdt_package_to_path(ofw_t ofw, phandle_t package, char *buf, size_t len) { return (-1); } #if defined(FDT_MARVELL) static int ofw_fdt_fixup(ofw_t ofw) { char model[FDT_MODEL_LEN]; phandle_t root; ssize_t len; int i; if ((root = ofw_fdt_finddevice(ofw, "/")) == -1) return (ENODEV); if ((len = ofw_fdt_getproplen(ofw, root, "model")) <= 0) return (0); bzero(model, FDT_MODEL_LEN); if (ofw_fdt_getprop(ofw, root, "model", model, FDT_MODEL_LEN) <= 0) return (0); /* * Search fixup table and call handler if appropriate. */ for (i = 0; fdt_fixup_table[i].model != NULL; i++) { if (strncmp(model, fdt_fixup_table[i].model, FDT_MODEL_LEN) != 0) /* * Sometimes it's convenient to provide one * fixup entry that refers to many boards. * To handle this case, simply check if model * is compatible parameter */ if(!ofw_bus_node_is_compatible(root, fdt_fixup_table[i].model)) continue; if (fdt_fixup_table[i].handler != NULL) (*fdt_fixup_table[i].handler)(root); } return (0); } #endif static int ofw_fdt_interpret(ofw_t ofw, const char *cmd, int nret, cell_t *retvals) { #if defined(FDT_MARVELL) int rv; /* * Note: FDT does not have the possibility to 'interpret' commands, * but we abuse the interface a bit to use it for doing non-standard * operations on the device tree blob. * * Currently the only supported 'command' is to trigger performing * fixups. */ if (strncmp("perform-fixup", cmd, 13) != 0) return (0); rv = ofw_fdt_fixup(ofw); if (nret > 0) retvals[0] = rv; return (rv); #else return (0); #endif } diff --git a/sys/dev/uart/uart_bus_fdt.c b/sys/dev/uart/uart_bus_fdt.c index 5f09c9520c33..985d58c850f0 100644 --- a/sys/dev/uart/uart_bus_fdt.c +++ b/sys/dev/uart/uart_bus_fdt.c @@ -1,298 +1,297 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include static int uart_fdt_probe(device_t); static device_method_t uart_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, uart_fdt_probe), DEVMETHOD(device_attach, uart_bus_attach), DEVMETHOD(device_detach, uart_bus_detach), { 0, 0 } }; static driver_t uart_fdt_driver = { uart_driver_name, uart_fdt_methods, sizeof(struct uart_softc), }; int uart_fdt_get_clock(phandle_t node, pcell_t *cell) { /* clock-frequency is a FreeBSD-only extension. */ if ((OF_getencprop(node, "clock-frequency", cell, sizeof(*cell))) <= 0) { /* Try to retrieve parent 'bus-frequency' */ /* XXX this should go to simple-bus fixup or so */ if ((OF_getencprop(OF_parent(node), "bus-frequency", cell, sizeof(*cell))) <= 0) *cell = 0; } return (0); } int uart_fdt_get_shift(phandle_t node, pcell_t *cell) { if ((OF_getencprop(node, "reg-shift", cell, sizeof(*cell))) <= 0) return (-1); return (0); } int uart_fdt_get_io_width(phandle_t node, pcell_t *cell) { if ((OF_getencprop(node, "reg-io-width", cell, sizeof(*cell))) <= 0) return (-1); return (0); } static uintptr_t uart_fdt_find_device(device_t dev) { struct ofw_compat_data **cd; const struct ofw_compat_data *ocd; SET_FOREACH(cd, uart_fdt_class_and_device_set) { ocd = ofw_bus_search_compatible(dev, *cd); if (ocd->ocd_data != 0) return (ocd->ocd_data); } return (0); } static int phandle_chosen_propdev(phandle_t chosen, const char *name, phandle_t *node) { char buf[64]; char *sep; if (OF_getprop(chosen, name, buf, sizeof(buf)) <= 0) return (ENXIO); /* * stdout-path may have a ':' to separate the device from the * connection settings. Split the string so we just pass the former * to OF_finddevice. */ sep = strchr(buf, ':'); if (sep != NULL) *sep = '\0'; if ((*node = OF_finddevice(buf)) == -1) return (ENXIO); return (0); } static const struct ofw_compat_data * uart_fdt_find_compatible(phandle_t node, const struct ofw_compat_data *cd) { const struct ofw_compat_data *ocd; for (ocd = cd; ocd->ocd_str != NULL; ocd++) { if (ofw_bus_node_is_compatible(node, ocd->ocd_str)) return (ocd); } return (NULL); } static uintptr_t uart_fdt_find_by_node(phandle_t node, int class_list) { struct ofw_compat_data **cd; const struct ofw_compat_data *ocd; if (class_list) { SET_FOREACH(cd, uart_fdt_class_set) { ocd = uart_fdt_find_compatible(node, *cd); if ((ocd != NULL) && (ocd->ocd_data != 0)) return (ocd->ocd_data); } } else { SET_FOREACH(cd, uart_fdt_class_and_device_set) { ocd = uart_fdt_find_compatible(node, *cd); if ((ocd != NULL) && (ocd->ocd_data != 0)) return (ocd->ocd_data); } } return (0); } int uart_cpu_fdt_probe(struct uart_class **classp, bus_space_tag_t *bst, bus_space_handle_t *bsh, int *baud, u_int *rclk, u_int *shiftp, u_int *iowidthp, const int devtype) { const char *propnames[] = {"stdout-path", "linux,stdout-path", "stdout", "stdin-path", "stdin", NULL}; const char *propnames_dbgport[] = {"freebsd,debug-path", NULL}; const char **name; struct uart_class *class; phandle_t node, chosen; pcell_t br, clk, shift, iowidth; char *cp = NULL; int err; /* Has the user forced a specific device node? */ switch (devtype) { case UART_DEV_DBGPORT: cp = kern_getenv("hw.fdt.dbgport"); name = propnames_dbgport; break; case UART_DEV_CONSOLE: cp = kern_getenv("hw.fdt.console"); name = propnames; break; default: return (ENXIO); } if (cp == NULL) { /* * Retrieve a node from /chosen. */ node = -1; if ((chosen = OF_finddevice("/chosen")) != -1) { for (; *name != NULL; name++) { if (phandle_chosen_propdev(chosen, *name, &node) == 0) break; } } if (chosen == -1 || *name == NULL) node = OF_finddevice("serial0"); /* Last ditch */ } else { node = OF_finddevice(cp); } if (node == -1) return (ENXIO); /* * Check old style of UART definition first. Unfortunately, the common * FDT processing is not possible if we have clock, power domains and * pinmux stuff. */ class = (struct uart_class *)uart_fdt_find_by_node(node, 0); if (class != NULL) { if ((err = uart_fdt_get_clock(node, &clk)) != 0) return (err); } else { /* Check class only linker set */ class = (struct uart_class *)uart_fdt_find_by_node(node, 1); if (class == NULL) return (ENXIO); clk = 0; } /* * Retrieve serial attributes. */ if (uart_fdt_get_shift(node, &shift) != 0) shift = uart_getregshift(class); if (uart_fdt_get_io_width(node, &iowidth) != 0) iowidth = uart_getregiowidth(class); if (OF_getencprop(node, "current-speed", &br, sizeof(br)) <= 0) br = 0; err = OF_decode_addr(node, 0, bst, bsh, NULL); if (err != 0) return (err); *classp = class; *baud = br; *rclk = clk; *shiftp = shift; *iowidthp = iowidth; return (0); } static int uart_fdt_probe(device_t dev) { struct uart_softc *sc; phandle_t node; pcell_t clock, shift, iowidth; int err; sc = device_get_softc(dev); if (!ofw_bus_status_okay(dev)) return (ENXIO); sc->sc_class = (struct uart_class *)uart_fdt_find_device(dev); if (sc->sc_class == NULL) return (ENXIO); node = ofw_bus_get_node(dev); if ((err = uart_fdt_get_clock(node, &clock)) != 0) return (err); if (uart_fdt_get_shift(node, &shift) != 0) shift = uart_getregshift(sc->sc_class); if (uart_fdt_get_io_width(node, &iowidth) != 0) iowidth = uart_getregiowidth(sc->sc_class); return (uart_bus_probe(dev, (int)shift, (int)iowidth, (int)clock, 0, 0, 0)); } DRIVER_MODULE(uart, simplebus, uart_fdt_driver, uart_devclass, 0, 0); DRIVER_MODULE(uart, ofwbus, uart_fdt_driver, uart_devclass, 0, 0); diff --git a/sys/dev/uart/uart_cpu_arm64.c b/sys/dev/uart/uart_cpu_arm64.c index 1d88a1e882a0..148bf749f0bb 100644 --- a/sys/dev/uart/uart_cpu_arm64.c +++ b/sys/dev/uart/uart_cpu_arm64.c @@ -1,131 +1,130 @@ /*- * Copyright (c) 2016 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_ACPI #include #include #include #include #endif #ifdef FDT #include #include #include #include #endif /* * UART console routines. */ extern struct bus_space memmap_bus; bus_space_tag_t uart_bus_space_io; bus_space_tag_t uart_bus_space_mem = &memmap_bus; int uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2) { if (pmap_kextract(b1->bsh) == 0) return (0); if (pmap_kextract(b2->bsh) == 0) return (0); return ((pmap_kextract(b1->bsh) == pmap_kextract(b2->bsh)) ? 1 : 0); } int uart_cpu_getdev(int devtype, struct uart_devinfo *di) { struct uart_class *class; bus_space_handle_t bsh; bus_space_tag_t bst; u_int rclk, shift, iowidth; int br, err; /* Allow overriding the FDT using the environment. */ class = &uart_ns8250_class; err = uart_getenv(devtype, di, class); if (err == 0) return (0); #ifdef DEV_ACPI /* Check if SPCR can tell us what console to use. */ if (uart_cpu_acpi_spcr(devtype, di) == 0) return (0); #endif err = ENXIO; #ifdef FDT if (err != 0) { err = uart_cpu_fdt_probe(&class, &bst, &bsh, &br, &rclk, &shift, &iowidth, devtype); } #endif if (err != 0) return (err); /* * Finalize configuration. */ di->bas.chan = 0; di->bas.regshft = shift; di->bas.regiowidth = iowidth; di->baudrate = br; di->bas.rclk = rclk; di->ops = uart_getops(class); di->databits = 8; di->stopbits = 1; di->parity = UART_PARITY_NONE; di->bas.bst = bst; di->bas.bsh = bsh; uart_bus_space_mem = di->bas.bst; uart_bus_space_io = NULL; return (0); } diff --git a/sys/dev/uart/uart_cpu_fdt.c b/sys/dev/uart/uart_cpu_fdt.c index 467e29515d00..c3b7b13d2bf7 100644 --- a/sys/dev/uart/uart_cpu_fdt.c +++ b/sys/dev/uart/uart_cpu_fdt.c @@ -1,114 +1,113 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009-2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * UART console routines. */ bus_space_tag_t uart_bus_space_io; bus_space_tag_t uart_bus_space_mem; int uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2) { if (b1->bst != b2->bst) return (0); if (pmap_kextract(b1->bsh) == 0) return (0); if (pmap_kextract(b2->bsh) == 0) return (0); return ((pmap_kextract(b1->bsh) == pmap_kextract(b2->bsh)) ? 1 : 0); } int uart_cpu_getdev(int devtype, struct uart_devinfo *di) { struct uart_class *class; bus_space_tag_t bst; bus_space_handle_t bsh; u_int shift, iowidth, rclk; int br, err; /* Allow overriding the FDT using the environment. */ class = &uart_ns8250_class; err = uart_getenv(devtype, di, class); if (!err) return (0); err = uart_cpu_fdt_probe(&class, &bst, &bsh, &br, &rclk, &shift, &iowidth, devtype); if (err != 0) return (err); /* * Finalize configuration. */ di->bas.chan = 0; di->bas.regshft = shift; di->bas.regiowidth = iowidth; di->baudrate = br; di->bas.rclk = rclk; di->ops = uart_getops(class); di->databits = 8; di->stopbits = 1; di->parity = UART_PARITY_NONE; di->bas.bst = bst; di->bas.bsh = bsh; uart_bus_space_mem = di->bas.bst; uart_bus_space_io = NULL; return (err); } diff --git a/sys/dev/uart/uart_dev_imx.c b/sys/dev/uart/uart_dev_imx.c index 14dba82dd9ee..24b2e4c65f75 100644 --- a/sys/dev/uart/uart_dev_imx.c +++ b/sys/dev/uart/uart_dev_imx.c @@ -1,673 +1,672 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #if defined(EXT_RESOURCES) && defined(__aarch64__) #define IMX_ENABLE_CLOCKS #endif #ifdef IMX_ENABLE_CLOCKS #include #endif #include "uart_if.h" #include /* * The hardare FIFOs are 32 bytes. We want an interrupt when there are 24 bytes * available to read or space for 24 more bytes to write. While 8 bytes of * slack before over/underrun might seem excessive, the hardware can run at * 5mbps, which means 2uS per char, so at full speed 8 bytes provides only 16uS * to get into the interrupt handler and service the fifo. */ #define IMX_FIFOSZ 32 #define IMX_RXFIFO_LEVEL 24 #define IMX_TXFIFO_LEVEL 24 /* * Low-level UART interface. */ static int imx_uart_probe(struct uart_bas *bas); static void imx_uart_init(struct uart_bas *bas, int, int, int, int); static void imx_uart_term(struct uart_bas *bas); static void imx_uart_putc(struct uart_bas *bas, int); static int imx_uart_rxready(struct uart_bas *bas); static int imx_uart_getc(struct uart_bas *bas, struct mtx *); static struct uart_ops uart_imx_uart_ops = { .probe = imx_uart_probe, .init = imx_uart_init, .term = imx_uart_term, .putc = imx_uart_putc, .rxready = imx_uart_rxready, .getc = imx_uart_getc, }; #if 0 /* Handy when debugging. */ static void dumpregs(struct uart_bas *bas, const char * msg) { if (!bootverbose) return; printf("%s bsh 0x%08lx UCR1 0x%08x UCR2 0x%08x " "UCR3 0x%08x UCR4 0x%08x USR1 0x%08x USR2 0x%08x\n", msg, bas->bsh, GETREG(bas, REG(UCR1)), GETREG(bas, REG(UCR2)), GETREG(bas, REG(UCR3)), GETREG(bas, REG(UCR4)), GETREG(bas, REG(USR1)), GETREG(bas, REG(USR2))); } #endif static int imx_uart_probe(struct uart_bas *bas) { return (0); } static u_int imx_uart_getbaud(struct uart_bas *bas) { uint32_t rate, ubir, ubmr; u_int baud, blo, bhi, i; static const u_int predivs[] = {6, 5, 4, 3, 2, 1, 7, 1}; static const u_int std_rates[] = { 9600, 14400, 19200, 38400, 57600, 115200, 230400, 460800, 921600 }; /* * Get the baud rate the hardware is programmed for, then search the * table of standard baud rates for a number that's within 3% of the * actual rate the hardware is programmed for. It's more comforting to * see that your console is running at 115200 than 114942. Note that * here we cannot make a simplifying assumption that the predivider and * numerator are 1 (like we do when setting the baud rate), because we * don't know what u-boot might have set up. */ i = (GETREG(bas, REG(UFCR)) & IMXUART_UFCR_RFDIV_MASK) >> IMXUART_UFCR_RFDIV_SHIFT; rate = bas->rclk / predivs[i]; ubir = GETREG(bas, REG(UBIR)) + 1; ubmr = GETREG(bas, REG(UBMR)) + 1; baud = ((rate / 16 ) * ubir) / ubmr; blo = (baud * 100) / 103; bhi = (baud * 100) / 97; for (i = 0; i < nitems(std_rates); i++) { rate = std_rates[i]; if (rate >= blo && rate <= bhi) { baud = rate; break; } } return (baud); } static void imx_uart_init(struct uart_bas *bas, int baudrate, int databits, int stopbits, int parity) { uint32_t baseclk, reg; /* Enable the device and the RX/TX channels. */ SET(bas, REG(UCR1), FLD(UCR1, UARTEN)); SET(bas, REG(UCR2), FLD(UCR2, RXEN) | FLD(UCR2, TXEN)); if (databits == 7) DIS(bas, UCR2, WS); else ENA(bas, UCR2, WS); if (stopbits == 2) ENA(bas, UCR2, STPB); else DIS(bas, UCR2, STPB); switch (parity) { case UART_PARITY_ODD: DIS(bas, UCR2, PROE); ENA(bas, UCR2, PREN); break; case UART_PARITY_EVEN: ENA(bas, UCR2, PROE); ENA(bas, UCR2, PREN); break; case UART_PARITY_MARK: case UART_PARITY_SPACE: /* FALLTHROUGH: Hardware doesn't support mark/space. */ case UART_PARITY_NONE: default: DIS(bas, UCR2, PREN); break; } /* * The hardware has an extremely flexible baud clock: it allows setting * both the numerator and denominator of the divider, as well as a * separate pre-divider. We simplify the problem of coming up with a * workable pair of numbers by assuming a pre-divider and numerator of * one because our base clock is so fast we can reach virtually any * reasonable speed with a simple divisor. The numerator value actually * includes the 16x over-sampling (so a value of 16 means divide by 1); * the register value is the numerator-1, so we have a hard-coded 15. * Note that a quirk of the hardware requires that both UBIR and UBMR be * set back to back in order for the change to take effect. */ if ((baudrate > 0) && (bas->rclk != 0)) { baseclk = bas->rclk; reg = GETREG(bas, REG(UFCR)); reg = (reg & ~IMXUART_UFCR_RFDIV_MASK) | IMXUART_UFCR_RFDIV_DIV1; SETREG(bas, REG(UFCR), reg); SETREG(bas, REG(UBIR), 15); SETREG(bas, REG(UBMR), (baseclk / baudrate) - 1); } /* * Program the tx lowater and rx hiwater levels at which fifo-service * interrupts are signaled. The tx value is interpetted as "when there * are only this many bytes remaining" (not "this many free"). */ reg = GETREG(bas, REG(UFCR)); reg &= ~(IMXUART_UFCR_TXTL_MASK | IMXUART_UFCR_RXTL_MASK); reg |= (IMX_FIFOSZ - IMX_TXFIFO_LEVEL) << IMXUART_UFCR_TXTL_SHIFT; reg |= IMX_RXFIFO_LEVEL << IMXUART_UFCR_RXTL_SHIFT; SETREG(bas, REG(UFCR), reg); } static void imx_uart_term(struct uart_bas *bas) { } static void imx_uart_putc(struct uart_bas *bas, int c) { while (!(IS(bas, USR1, TRDY))) ; SETREG(bas, REG(UTXD), c); } static int imx_uart_rxready(struct uart_bas *bas) { return ((IS(bas, USR2, RDR)) ? 1 : 0); } static int imx_uart_getc(struct uart_bas *bas, struct mtx *hwmtx) { int c; uart_lock(hwmtx); while (!(IS(bas, USR2, RDR))) ; c = GETREG(bas, REG(URXD)); uart_unlock(hwmtx); #if defined(KDB) if (c & FLD(URXD, BRK)) { if (kdb_break()) return (0); } #endif return (c & 0xff); } /* * High-level UART interface. */ struct imx_uart_softc { struct uart_softc base; }; static int imx_uart_bus_attach(struct uart_softc *); static int imx_uart_bus_detach(struct uart_softc *); static int imx_uart_bus_flush(struct uart_softc *, int); static int imx_uart_bus_getsig(struct uart_softc *); static int imx_uart_bus_ioctl(struct uart_softc *, int, intptr_t); static int imx_uart_bus_ipend(struct uart_softc *); static int imx_uart_bus_param(struct uart_softc *, int, int, int, int); static int imx_uart_bus_probe(struct uart_softc *); static int imx_uart_bus_receive(struct uart_softc *); static int imx_uart_bus_setsig(struct uart_softc *, int); static int imx_uart_bus_transmit(struct uart_softc *); static void imx_uart_bus_grab(struct uart_softc *); static void imx_uart_bus_ungrab(struct uart_softc *); static kobj_method_t imx_uart_methods[] = { KOBJMETHOD(uart_attach, imx_uart_bus_attach), KOBJMETHOD(uart_detach, imx_uart_bus_detach), KOBJMETHOD(uart_flush, imx_uart_bus_flush), KOBJMETHOD(uart_getsig, imx_uart_bus_getsig), KOBJMETHOD(uart_ioctl, imx_uart_bus_ioctl), KOBJMETHOD(uart_ipend, imx_uart_bus_ipend), KOBJMETHOD(uart_param, imx_uart_bus_param), KOBJMETHOD(uart_probe, imx_uart_bus_probe), KOBJMETHOD(uart_receive, imx_uart_bus_receive), KOBJMETHOD(uart_setsig, imx_uart_bus_setsig), KOBJMETHOD(uart_transmit, imx_uart_bus_transmit), KOBJMETHOD(uart_grab, imx_uart_bus_grab), KOBJMETHOD(uart_ungrab, imx_uart_bus_ungrab), { 0, 0 } }; static struct uart_class uart_imx_class = { "imx", imx_uart_methods, sizeof(struct imx_uart_softc), .uc_ops = &uart_imx_uart_ops, .uc_range = 0x100, .uc_rclk = 24000000, /* TODO: get value from CCM */ .uc_rshift = 0 }; static struct ofw_compat_data compat_data[] = { {"fsl,imx6q-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx53-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx51-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx31-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx27-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx25-uart", (uintptr_t)&uart_imx_class}, {"fsl,imx21-uart", (uintptr_t)&uart_imx_class}, {NULL, (uintptr_t)NULL}, }; UART_FDT_CLASS_AND_DEVICE(compat_data); #define SIGCHG(c, i, s, d) \ if (c) { \ i |= (i & s) ? s : s | d; \ } else { \ i = (i & s) ? (i & ~s) | d : i; \ } #ifdef IMX_ENABLE_CLOCKS static int imx_uart_setup_clocks(struct uart_softc *sc) { struct uart_bas *bas; clk_t ipgclk, perclk; uint64_t freq; int error; bas = &sc->sc_bas; if (clk_get_by_ofw_name(sc->sc_dev, 0, "ipg", &ipgclk) != 0) return (ENOENT); if (clk_get_by_ofw_name(sc->sc_dev, 0, "per", &perclk) != 0) { return (ENOENT); } error = clk_enable(ipgclk); if (error != 0) { device_printf(sc->sc_dev, "cannot enable ipg clock\n"); return (error); } error = clk_get_freq(perclk, &freq); if (error != 0) { device_printf(sc->sc_dev, "cannot get frequency\n"); return (error); } bas->rclk = (uint32_t)freq; return (0); } #endif static int imx_uart_bus_attach(struct uart_softc *sc) { struct uart_bas *bas; struct uart_devinfo *di; bas = &sc->sc_bas; #ifdef IMX_ENABLE_CLOCKS int error = imx_uart_setup_clocks(sc); if (error) return (error); #else bas->rclk = imx_ccm_uart_hz(); #endif if (sc->sc_sysdev != NULL) { di = sc->sc_sysdev; imx_uart_init(bas, di->baudrate, di->databits, di->stopbits, di->parity); } else { imx_uart_init(bas, 115200, 8, 1, 0); } (void)imx_uart_bus_getsig(sc); /* Clear all pending interrupts. */ SETREG(bas, REG(USR1), 0xffff); SETREG(bas, REG(USR2), 0xffff); DIS(bas, UCR4, DREN); ENA(bas, UCR1, RRDYEN); DIS(bas, UCR1, IDEN); DIS(bas, UCR3, RXDSEN); ENA(bas, UCR2, ATEN); DIS(bas, UCR1, TXMPTYEN); DIS(bas, UCR1, TRDYEN); DIS(bas, UCR4, TCEN); DIS(bas, UCR4, OREN); ENA(bas, UCR4, BKEN); DIS(bas, UCR4, WKEN); DIS(bas, UCR1, ADEN); DIS(bas, UCR3, ACIEN); DIS(bas, UCR2, ESCI); DIS(bas, UCR4, ENIRI); DIS(bas, UCR3, AIRINTEN); DIS(bas, UCR3, AWAKEN); DIS(bas, UCR3, FRAERREN); DIS(bas, UCR3, PARERREN); DIS(bas, UCR1, RTSDEN); DIS(bas, UCR2, RTSEN); DIS(bas, UCR3, DTREN); DIS(bas, UCR3, RI); DIS(bas, UCR3, DCD); DIS(bas, UCR3, DTRDEN); ENA(bas, UCR2, IRTS); ENA(bas, UCR3, RXDMUXSEL); return (0); } static int imx_uart_bus_detach(struct uart_softc *sc) { SETREG(&sc->sc_bas, REG(UCR4), 0); return (0); } static int imx_uart_bus_flush(struct uart_softc *sc, int what) { /* TODO */ return (0); } static int imx_uart_bus_getsig(struct uart_softc *sc) { uint32_t new, old, sig; uint8_t bes; do { old = sc->sc_hwsig; sig = old; uart_lock(sc->sc_hwmtx); bes = GETREG(&sc->sc_bas, REG(USR2)); uart_unlock(sc->sc_hwmtx); /* XXX: chip can show delta */ SIGCHG(bes & FLD(USR2, DCDIN), sig, SER_DCD, SER_DDCD); new = sig & ~SER_MASK_DELTA; } while (!atomic_cmpset_32(&sc->sc_hwsig, old, new)); return (sig); } static int imx_uart_bus_ioctl(struct uart_softc *sc, int request, intptr_t data) { struct uart_bas *bas; int error; bas = &sc->sc_bas; error = 0; uart_lock(sc->sc_hwmtx); switch (request) { case UART_IOCTL_BREAK: /* TODO */ break; case UART_IOCTL_BAUD: *(u_int*)data = imx_uart_getbaud(bas); break; default: error = EINVAL; break; } uart_unlock(sc->sc_hwmtx); return (error); } static int imx_uart_bus_ipend(struct uart_softc *sc) { struct uart_bas *bas; int ipend; uint32_t usr1, usr2; uint32_t ucr1, ucr2, ucr4; bas = &sc->sc_bas; ipend = 0; uart_lock(sc->sc_hwmtx); /* Read pending interrupts */ usr1 = GETREG(bas, REG(USR1)); usr2 = GETREG(bas, REG(USR2)); /* ACK interrupts */ SETREG(bas, REG(USR1), usr1); SETREG(bas, REG(USR2), usr2); ucr1 = GETREG(bas, REG(UCR1)); ucr2 = GETREG(bas, REG(UCR2)); ucr4 = GETREG(bas, REG(UCR4)); /* If we have reached tx low-water, we can tx some more now. */ if ((usr1 & FLD(USR1, TRDY)) && (ucr1 & FLD(UCR1, TRDYEN))) { DIS(bas, UCR1, TRDYEN); ipend |= SER_INT_TXIDLE; } /* * If we have reached the rx high-water, or if there are bytes in the rx * fifo and no new data has arrived for 8 character periods (aging * timer), we have input data to process. */ if (((usr1 & FLD(USR1, RRDY)) && (ucr1 & FLD(UCR1, RRDYEN))) || ((usr1 & FLD(USR1, AGTIM)) && (ucr2 & FLD(UCR2, ATEN)))) { DIS(bas, UCR1, RRDYEN); DIS(bas, UCR2, ATEN); ipend |= SER_INT_RXREADY; } /* A break can come in at any time, it never gets disabled. */ if ((usr2 & FLD(USR2, BRCD)) && (ucr4 & FLD(UCR4, BKEN))) ipend |= SER_INT_BREAK; uart_unlock(sc->sc_hwmtx); return (ipend); } static int imx_uart_bus_param(struct uart_softc *sc, int baudrate, int databits, int stopbits, int parity) { uart_lock(sc->sc_hwmtx); imx_uart_init(&sc->sc_bas, baudrate, databits, stopbits, parity); uart_unlock(sc->sc_hwmtx); return (0); } static int imx_uart_bus_probe(struct uart_softc *sc) { int error; error = imx_uart_probe(&sc->sc_bas); if (error) return (error); /* * On input we can read up to the full fifo size at once. On output, we * want to write only as much as the programmed tx low water level, * because that's all we can be certain we have room for in the fifo * when we get a tx-ready interrupt. */ sc->sc_rxfifosz = IMX_FIFOSZ; sc->sc_txfifosz = IMX_TXFIFO_LEVEL; device_set_desc(sc->sc_dev, "Freescale i.MX UART"); return (0); } static int imx_uart_bus_receive(struct uart_softc *sc) { struct uart_bas *bas; int xc, out; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* * Empty the rx fifo. We get the RRDY interrupt when IMX_RXFIFO_LEVEL * (the rx high-water level) is reached, but we set sc_rxfifosz to the * full hardware fifo size, so we can safely process however much is * there, not just the highwater size. */ while (IS(bas, USR2, RDR)) { if (uart_rx_full(sc)) { /* No space left in input buffer */ sc->sc_rxbuf[sc->sc_rxput] = UART_STAT_OVERRUN; break; } xc = GETREG(bas, REG(URXD)); out = xc & 0x000000ff; if (xc & FLD(URXD, FRMERR)) out |= UART_STAT_FRAMERR; if (xc & FLD(URXD, PRERR)) out |= UART_STAT_PARERR; if (xc & FLD(URXD, OVRRUN)) out |= UART_STAT_OVERRUN; if (xc & FLD(URXD, BRK)) out |= UART_STAT_BREAK; uart_rx_put(sc, out); } ENA(bas, UCR1, RRDYEN); ENA(bas, UCR2, ATEN); uart_unlock(sc->sc_hwmtx); return (0); } static int imx_uart_bus_setsig(struct uart_softc *sc, int sig) { return (0); } static int imx_uart_bus_transmit(struct uart_softc *sc) { struct uart_bas *bas = &sc->sc_bas; int i; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* * Fill the tx fifo. The uart core puts at most IMX_TXFIFO_LEVEL bytes * into the txbuf (because that's what sc_txfifosz is set to), and * because we got the TRDY (low-water reached) interrupt we know at * least that much space is available in the fifo. */ for (i = 0; i < sc->sc_txdatasz; i++) { SETREG(bas, REG(UTXD), sc->sc_txbuf[i] & 0xff); } sc->sc_txbusy = 1; ENA(bas, UCR1, TRDYEN); uart_unlock(sc->sc_hwmtx); return (0); } static void imx_uart_bus_grab(struct uart_softc *sc) { struct uart_bas *bas = &sc->sc_bas; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); DIS(bas, UCR1, RRDYEN); DIS(bas, UCR2, ATEN); uart_unlock(sc->sc_hwmtx); } static void imx_uart_bus_ungrab(struct uart_softc *sc) { struct uart_bas *bas = &sc->sc_bas; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); ENA(bas, UCR1, RRDYEN); ENA(bas, UCR2, ATEN); uart_unlock(sc->sc_hwmtx); } diff --git a/sys/dev/uart/uart_dev_imx.h b/sys/dev/uart/uart_dev_imx.h index 9a3962be2335..97d5dc4d5b9c 100644 --- a/sys/dev/uart/uart_dev_imx.h +++ b/sys/dev/uart/uart_dev_imx.h @@ -1,220 +1,219 @@ /*- * Copyright (c) 2012 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Oleksandr Rybalko under sponsorship * from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _UART_DEV_IMX5XX_H #define _UART_DEV_IMX5XX_H #define IMXUART_URXD_REG 0x0000 /* UART Receiver Register */ #define IMXUART_URXD_CHARRDY (1 << 15) #define IMXUART_URXD_ERR (1 << 14) #define IMXUART_URXD_OVRRUN (1 << 13) #define IMXUART_URXD_FRMERR (1 << 12) #define IMXUART_URXD_BRK (1 << 11) #define IMXUART_URXD_PRERR (1 << 10) #define IMXUART_URXD_RX_DATA_MASK 0xff #define IMXUART_UTXD_REG 0x0040 /* UART Transmitter Register */ #define IMXUART_UTXD_TX_DATA_MASK 0xff #define IMXUART_UCR1_REG 0x0080 /* UART Control Register 1 */ #define IMXUART_UCR1_ADEN (1 << 15) #define IMXUART_UCR1_ADBR (1 << 14) #define IMXUART_UCR1_TRDYEN (1 << 13) #define IMXUART_UCR1_IDEN (1 << 12) #define IMXUART_UCR1_ICD_MASK (3 << 10) #define IMXUART_UCR1_ICD_IDLE4 (0 << 10) #define IMXUART_UCR1_ICD_IDLE8 (1 << 10) #define IMXUART_UCR1_ICD_IDLE16 (2 << 10) #define IMXUART_UCR1_ICD_IDLE32 (3 << 10) #define IMXUART_UCR1_RRDYEN (1 << 9) #define IMXUART_UCR1_RXDMAEN (1 << 8) #define IMXUART_UCR1_IREN (1 << 7) #define IMXUART_UCR1_TXMPTYEN (1 << 6) #define IMXUART_UCR1_RTSDEN (1 << 5) #define IMXUART_UCR1_SNDBRK (1 << 4) #define IMXUART_UCR1_TXDMAEN (1 << 3) #define IMXUART_UCR1_ATDMAEN (1 << 2) #define IMXUART_UCR1_DOZE (1 << 1) #define IMXUART_UCR1_UARTEN (1 << 0) #define IMXUART_UCR2_REG 0x0084 /* UART Control Register 2 */ #define IMXUART_UCR2_ESCI (1 << 15) #define IMXUART_UCR2_IRTS (1 << 14) #define IMXUART_UCR2_CTSC (1 << 13) #define IMXUART_UCR2_CTS (1 << 12) #define IMXUART_UCR2_ESCEN (1 << 11) #define IMXUART_UCR2_RTEC_MASK (3 << 9) #define IMXUART_UCR2_RTEC_REDGE (0 << 9) #define IMXUART_UCR2_RTEC_FEDGE (1 << 9) #define IMXUART_UCR2_RTEC_EDGE (2 << 9) #define IMXUART_UCR2_PREN (1 << 8) #define IMXUART_UCR2_PROE (1 << 7) #define IMXUART_UCR2_STPB (1 << 6) #define IMXUART_UCR2_WS (1 << 5) #define IMXUART_UCR2_RTSEN (1 << 4) #define IMXUART_UCR2_ATEN (1 << 3) #define IMXUART_UCR2_TXEN (1 << 2) #define IMXUART_UCR2_RXEN (1 << 1) #define IMXUART_UCR2_N_SRST (1 << 0) #define IMXUART_UCR3_REG 0x0088 /* UART Control Register 3 */ #define IMXUART_UCR3_DPEC_MASK (3 << 14) #define IMXUART_UCR3_DPEC_REDGE (0 << 14) #define IMXUART_UCR3_DPEC_FEDGE (1 << 14) #define IMXUART_UCR3_DPEC_EDGE (2 << 14) #define IMXUART_UCR3_DTREN (1 << 13) #define IMXUART_UCR3_PARERREN (1 << 12) #define IMXUART_UCR3_FRAERREN (1 << 11) #define IMXUART_UCR3_DSR (1 << 10) #define IMXUART_UCR3_DCD (1 << 9) #define IMXUART_UCR3_RI (1 << 8) #define IMXUART_UCR3_ADNIMP (1 << 7) #define IMXUART_UCR3_RXDSEN (1 << 6) #define IMXUART_UCR3_AIRINTEN (1 << 5) #define IMXUART_UCR3_AWAKEN (1 << 4) #define IMXUART_UCR3_DTRDEN (1 << 3) #define IMXUART_UCR3_RXDMUXSEL (1 << 2) #define IMXUART_UCR3_INVT (1 << 1) #define IMXUART_UCR3_ACIEN (1 << 0) #define IMXUART_UCR4_REG 0x008c /* UART Control Register 4 */ #define IMXUART_UCR4_CTSTL_MASK (0x3f << 10) #define IMXUART_UCR4_CTSTL_SHIFT 10 #define IMXUART_UCR4_INVR (1 << 9) #define IMXUART_UCR4_ENIRI (1 << 8) #define IMXUART_UCR4_WKEN (1 << 7) #define IMXUART_UCR4_IDDMAEN (1 << 6) #define IMXUART_UCR4_IRSC (1 << 5) #define IMXUART_UCR4_LPBYP (1 << 4) #define IMXUART_UCR4_TCEN (1 << 3) #define IMXUART_UCR4_BKEN (1 << 2) #define IMXUART_UCR4_OREN (1 << 1) #define IMXUART_UCR4_DREN (1 << 0) #define IMXUART_UFCR_REG 0x0090 /* UART FIFO Control Register */ #define IMXUART_UFCR_TXTL_MASK (0x3f << 10) #define IMXUART_UFCR_TXTL_SHIFT 10 #define IMXUART_UFCR_RFDIV_MASK (0x07 << 7) #define IMXUART_UFCR_RFDIV_SHIFT 7 #define IMXUART_UFCR_RFDIV_DIV6 (0 << 7) #define IMXUART_UFCR_RFDIV_DIV5 (1 << 7) #define IMXUART_UFCR_RFDIV_DIV4 (2 << 7) #define IMXUART_UFCR_RFDIV_DIV3 (3 << 7) #define IMXUART_UFCR_RFDIV_DIV2 (4 << 7) #define IMXUART_UFCR_RFDIV_DIV1 (5 << 7) #define IMXUART_UFCR_RFDIV_DIV7 (6 << 7) #define IMXUART_UFCR_DCEDTE (1 << 6) #define IMXUART_UFCR_RXTL_MASK 0x0000003f #define IMXUART_UFCR_RXTL_SHIFT 0 #define IMXUART_USR1_REG 0x0094 /* UART Status Register 1 */ #define IMXUART_USR1_PARITYERR (1 << 15) #define IMXUART_USR1_RTSS (1 << 14) #define IMXUART_USR1_TRDY (1 << 13) #define IMXUART_USR1_RTSD (1 << 12) #define IMXUART_USR1_ESCF (1 << 11) #define IMXUART_USR1_FRAMERR (1 << 10) #define IMXUART_USR1_RRDY (1 << 9) #define IMXUART_USR1_AGTIM (1 << 8) #define IMXUART_USR1_DTRD (1 << 7) #define IMXUART_USR1_RXDS (1 << 6) #define IMXUART_USR1_AIRINT (1 << 5) #define IMXUART_USR1_AWAKE (1 << 4) /* 6040 5008 XXX */ #define IMXUART_USR2_REG 0x0098 /* UART Status Register 2 */ #define IMXUART_USR2_ADET (1 << 15) #define IMXUART_USR2_TXFE (1 << 14) #define IMXUART_USR2_DTRF (1 << 13) #define IMXUART_USR2_IDLE (1 << 12) #define IMXUART_USR2_ACST (1 << 11) #define IMXUART_USR2_RIDELT (1 << 10) #define IMXUART_USR2_RIIN (1 << 9) #define IMXUART_USR2_IRINT (1 << 8) #define IMXUART_USR2_WAKE (1 << 7) #define IMXUART_USR2_DCDDELT (1 << 6) #define IMXUART_USR2_DCDIN (1 << 5) #define IMXUART_USR2_RTSF (1 << 4) #define IMXUART_USR2_TXDC (1 << 3) #define IMXUART_USR2_BRCD (1 << 2) #define IMXUART_USR2_ORE (1 << 1) #define IMXUART_USR2_RDR (1 << 0) #define IMXUART_UESC_REG 0x009c /* UART Escape Character Register */ #define IMXUART_UESC_ESC_CHAR_MASK 0x000000ff #define IMXUART_UTIM_REG 0x00a0 /* UART Escape Timer Register */ #define IMXUART_UTIM_TIM_MASK 0x00000fff #define IMXUART_UBIR_REG 0x00a4 /* UART BRM Incremental Register */ #define IMXUART_UBIR_INC_MASK 0x0000ffff #define IMXUART_UBMR_REG 0x00a8 /* UART BRM Modulator Register */ #define IMXUART_UBMR_MOD_MASK 0x0000ffff #define IMXUART_UBRC_REG 0x00ac /* UART Baud Rate Count Register */ #define IMXUART_UBRC_BCNT_MASK 0x0000ffff #define IMXUART_ONEMS_REG 0x00b0 /* UART One Millisecond Register */ #define IMXUART_ONEMS_ONEMS_MASK 0x00ffffff #define IMXUART_UTS_REG 0x00b4 /* UART Test Register */ #define IMXUART_UTS_FRCPERR (1 << 13) #define IMXUART_UTS_LOOP (1 << 12) #define IMXUART_UTS_DBGEN (1 << 11) #define IMXUART_UTS_LOOPIR (1 << 10) #define IMXUART_UTS_RXDBG (1 << 9) #define IMXUART_UTS_TXEMPTY (1 << 6) #define IMXUART_UTS_RXEMPTY (1 << 5) #define IMXUART_UTS_TXFULL (1 << 4) #define IMXUART_UTS_RXFULL (1 << 3) #define IMXUART_UTS_SOFTRST (1 << 0) #define REG(_r) IMXUART_ ## _r ## _REG #define FLD(_r, _v) IMXUART_ ## _r ## _ ## _v #define GETREG(bas, reg) \ bus_space_read_4((bas)->bst, (bas)->bsh, (reg)) #define SETREG(bas, reg, value) \ bus_space_write_4((bas)->bst, (bas)->bsh, (reg), (value)) #define CLR(_bas, _r, _b) \ SETREG((_bas), (_r), GETREG((_bas), (_r)) & ~(_b)) #define SET(_bas, _r, _b) \ SETREG((_bas), (_r), GETREG((_bas), (_r)) | (_b)) #define IS_SET(_bas, _r, _b) \ ((GETREG((_bas), (_r)) & (_b)) ? 1 : 0) #define ENA(_bas, _r, _b) SET((_bas), REG(_r), FLD(_r, _b)) #define DIS(_bas, _r, _b) CLR((_bas), REG(_r), FLD(_r, _b)) #define IS(_bas, _r, _b) IS_SET((_bas), REG(_r), FLD(_r, _b)) #endif /* _UART_DEV_IMX5XX_H */ diff --git a/sys/dev/vnic/lmac_if.m b/sys/dev/vnic/lmac_if.m index e8770f22ce11..59fa635c5a4d 100644 --- a/sys/dev/vnic/lmac_if.m +++ b/sys/dev/vnic/lmac_if.m @@ -1,102 +1,101 @@ #- # Copyright (c) 2015 The FreeBSD Foundation -# All rights reserved. # # This software was developed by Semihalf under # the sponsorship of the FreeBSD Foundation. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # LMAC (BGX controller) interface description # INTERFACE lmac; CODE { static int null_lmac_media_status(device_t dev, int lmacid, int *link, int *duplex, int *speed) { return (ENXIO); } static int null_lmac_media_change(device_t dev, int lmacid, int link, int duplex, int speed) { return (ENXIO); } static int null_lmac_phy_connect(device_t dev, int lmacid, int phy) { return (ENXIO); } static int null_lmac_phy_disconnect(device_t dev, int lmacid, int phy) { return (ENXIO); } }; # Get link status # # 0 : Success # METHOD int media_status { device_t dev; int lmacid; int * link; int * duplex; int * speed; } DEFAULT null_lmac_media_status; # Change link status # # 0 : Success # METHOD int media_change { device_t dev; int lmacid; int link; int duplex; int speed; } DEFAULT null_lmac_media_change; # Connect PHY # # 0 : Success # METHOD int phy_connect { device_t dev; int lmacid; int phy; } DEFAULT null_lmac_phy_connect; # Disconnect PHY # # 0 : Success # METHOD int phy_disconnect { device_t dev; int lmacid; int phy; } DEFAULT null_lmac_phy_disconnect; diff --git a/sys/dev/vnic/thunder_bgx_fdt.c b/sys/dev/vnic/thunder_bgx_fdt.c index 37ebcad37b4d..4ff38fc7f90d 100644 --- a/sys/dev/vnic/thunder_bgx_fdt.c +++ b/sys/dev/vnic/thunder_bgx_fdt.c @@ -1,464 +1,463 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "thunder_bgx.h" #include "thunder_bgx_var.h" #define CONN_TYPE_MAXLEN 16 #define CONN_TYPE_OFFSET 2 #define BGX_NODE_NAME "bgx" #define BGX_MAXID 9 /* BGX func. 0, i.e.: reg = <0x8000 0 0 0 0>; DEVFN = 0x80 */ #define BGX_DEVFN_0 0x80 #define FDT_NAME_MAXLEN 31 int bgx_fdt_init_phy(struct bgx *); static void bgx_fdt_get_macaddr(phandle_t phy, uint8_t *hwaddr) { uint8_t addr[ETHER_ADDR_LEN]; if (OF_getprop(phy, "local-mac-address", addr, ETHER_ADDR_LEN) == -1) { /* Missing MAC address should be marked by clearing it */ memset(hwaddr, 0, ETHER_ADDR_LEN); } else memcpy(hwaddr, addr, ETHER_ADDR_LEN); } static boolean_t bgx_fdt_phy_mode_match(struct bgx *bgx, char *qlm_mode, ssize_t size) { const char *type; ssize_t sz; ssize_t offset; switch (bgx->qlm_mode) { case QLM_MODE_SGMII: type = "sgmii"; sz = sizeof("sgmii"); offset = size - sz; break; case QLM_MODE_XAUI_1X4: type = "xaui"; sz = sizeof("xaui"); offset = size - sz; if (offset < 0) return (FALSE); if (strncmp(&qlm_mode[offset], type, sz) == 0) return (TRUE); type = "dxaui"; sz = sizeof("dxaui"); offset = size - sz; break; case QLM_MODE_RXAUI_2X2: type = "raui"; sz = sizeof("raui"); offset = size - sz; break; case QLM_MODE_XFI_4X1: type = "xfi"; sz = sizeof("xfi"); offset = size - sz; break; case QLM_MODE_XLAUI_1X4: type = "xlaui"; sz = sizeof("xlaui"); offset = size - sz; break; case QLM_MODE_10G_KR_4X1: type = "xfi-10g-kr"; sz = sizeof("xfi-10g-kr"); offset = size - sz; break; case QLM_MODE_40G_KR4_1X4: type = "xlaui-40g-kr"; sz = sizeof("xlaui-40g-kr"); offset = size - sz; break; default: return (FALSE); } if (offset < 0) return (FALSE); if (strncmp(&qlm_mode[offset], type, sz) == 0) return (TRUE); return (FALSE); } static boolean_t bgx_fdt_phy_name_match(struct bgx *bgx, char *phy_name, ssize_t size) { const char *type; ssize_t sz; char last; switch (bgx->qlm_mode) { case QLM_MODE_SGMII: type = "sgmii"; sz = sizeof("sgmii"); break; case QLM_MODE_XAUI_1X4: type = "xaui"; sz = sizeof("xaui"); if (sz < size) return (FALSE); if (strncmp(phy_name, type, sz) == 0) return (TRUE); type = "dxaui"; sz = sizeof("dxaui"); break; case QLM_MODE_RXAUI_2X2: type = "raui"; sz = sizeof("raui"); break; case QLM_MODE_XFI_4X1: type = "xfi"; sz = sizeof("xfi"); break; case QLM_MODE_XLAUI_1X4: type = "xlaui"; sz = sizeof("xlaui"); break; case QLM_MODE_10G_KR_4X1: type = "xfi-10g-kr"; sz = sizeof("xfi-10g-kr"); break; case QLM_MODE_40G_KR4_1X4: type = "xlaui-40g-kr"; sz = sizeof("xlaui-40g-kr"); break; default: return (FALSE); } if (sz > size) return (FALSE); if (strncmp(phy_name, type, sz - 1) == 0) { last = phy_name[sz - 1]; if (last == '\0' || last == '@' || isdigit(last)) return (TRUE); } return (FALSE); } static phandle_t bgx_fdt_traverse_nodes(uint8_t unit, phandle_t start, char *name, size_t len) { phandle_t node, ret; uint32_t *reg; size_t buf_size; ssize_t proplen; char *node_name; int err; /* * Traverse all subordinate nodes of 'start' to find BGX instance. * This supports both old (by name) and new (by reg) methods. */ buf_size = sizeof(*node_name) * FDT_NAME_MAXLEN; if (len > buf_size) { /* * This is an erroneous situation since the string * to compare cannot be longer than FDT_NAME_MAXLEN. */ return (0); } node_name = malloc(buf_size, M_BGX, M_WAITOK); for (node = OF_child(start); node != 0; node = OF_peer(node)) { /* Clean-up the buffer */ memset(node_name, 0, buf_size); /* Recurse to children */ if (OF_child(node) != 0) { ret = bgx_fdt_traverse_nodes(unit, node, name, len); if (ret != 0) { free(node_name, M_BGX); return (ret); } } /* * Old way - by name */ proplen = OF_getproplen(node, "name"); if ((proplen <= 0) || (proplen < len)) continue; err = OF_getprop(node, "name", node_name, proplen); if (err <= 0) continue; if (strncmp(node_name, name, len) == 0) { free(node_name, M_BGX); return (node); } /* * New way - by reg */ /* Check if even BGX */ if (strncmp(node_name, BGX_NODE_NAME, sizeof(BGX_NODE_NAME) - 1) != 0) continue; /* Get reg */ err = OF_getencprop_alloc_multi(node, "reg", sizeof(*reg), (void **)®); if (err == -1) { free(reg, M_OFWPROP); continue; } /* Match BGX device function */ if ((BGX_DEVFN_0 + unit) == (reg[0] >> 8)) { free(reg, M_OFWPROP); free(node_name, M_BGX); return (node); } free(reg, M_OFWPROP); } free(node_name, M_BGX); return (0); } /* * Similar functionality to pci_find_pcie_root_port() * but this one works for ThunderX. */ static device_t bgx_find_root_pcib(device_t dev) { devclass_t pci_class; device_t pcib, bus; pci_class = devclass_find("pci"); KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class, ("%s: non-pci device %s", __func__, device_get_nameunit(dev))); /* Walk the bridge hierarchy until we find a non-PCI device */ for (;;) { bus = device_get_parent(dev); KASSERT(bus != NULL, ("%s: null parent of %s", __func__, device_get_nameunit(dev))); if (device_get_devclass(bus) != pci_class) return (NULL); pcib = device_get_parent(bus); KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__, device_get_nameunit(bus))); /* * If the parent of this PCIB is not PCI * then we found our root PCIB. */ if (device_get_devclass(device_get_parent(pcib)) != pci_class) return (pcib); dev = pcib; } } static __inline phandle_t bgx_fdt_find_node(struct bgx *bgx) { device_t root_pcib; phandle_t node; char *bgx_sel; size_t len; KASSERT(bgx->bgx_id <= BGX_MAXID, ("Invalid BGX ID: %d, max: %d", bgx->bgx_id, BGX_MAXID)); len = sizeof(BGX_NODE_NAME) + 1; /* ++<\0> */ /* Allocate memory for BGX node name + "/" character */ bgx_sel = malloc(sizeof(*bgx_sel) * (len + 1), M_BGX, M_ZERO | M_WAITOK); /* Prepare node's name */ snprintf(bgx_sel, len + 1, "/"BGX_NODE_NAME"%d", bgx->bgx_id); /* First try the root node */ node = OF_finddevice(bgx_sel); if (node != -1) { /* Found relevant node */ goto out; } /* * Clean-up and try to find BGX in DT * starting from the parent PCI bridge node. */ memset(bgx_sel, 0, sizeof(*bgx_sel) * (len + 1)); snprintf(bgx_sel, len, BGX_NODE_NAME"%d", bgx->bgx_id); /* Find PCI bridge that we are connected to */ root_pcib = bgx_find_root_pcib(bgx->dev); if (root_pcib == NULL) { device_printf(bgx->dev, "Unable to find BGX root bridge\n"); node = 0; goto out; } node = ofw_bus_get_node(root_pcib); if ((int)node <= 0) { device_printf(bgx->dev, "No parent FDT node for BGX\n"); goto out; } node = bgx_fdt_traverse_nodes(bgx->bgx_id, node, bgx_sel, len); out: free(bgx_sel, M_BGX); return (node); } int bgx_fdt_init_phy(struct bgx *bgx) { char *node_name; phandle_t node, child; phandle_t phy, mdio; ssize_t len; uint8_t lmac; char qlm_mode[CONN_TYPE_MAXLEN]; node = bgx_fdt_find_node(bgx); if (node == 0) { device_printf(bgx->dev, "Could not find bgx%d node in FDT\n", bgx->bgx_id); return (ENXIO); } lmac = 0; for (child = OF_child(node); child > 0; child = OF_peer(child)) { len = OF_getprop(child, "qlm-mode", qlm_mode, sizeof(qlm_mode)); if (len > 0) { if (!bgx_fdt_phy_mode_match(bgx, qlm_mode, len)) { /* * Connection type not match with BGX mode. */ continue; } } else { len = OF_getprop_alloc(child, "name", (void **)&node_name); if (len <= 0) { continue; } if (!bgx_fdt_phy_name_match(bgx, node_name, len)) { free(node_name, M_OFWPROP); continue; } free(node_name, M_OFWPROP); } /* Acquire PHY address */ if (OF_getencprop(child, "reg", &bgx->lmac[lmac].phyaddr, sizeof(bgx->lmac[lmac].phyaddr)) <= 0) { if (bootverbose) { device_printf(bgx->dev, "Could not retrieve PHY address\n"); } bgx->lmac[lmac].phyaddr = MII_PHY_ANY; } if (OF_getencprop(child, "phy-handle", &phy, sizeof(phy)) <= 0) { if (bootverbose) { device_printf(bgx->dev, "No phy-handle in PHY node. Skipping...\n"); } continue; } phy = OF_instance_to_package(phy); /* * Get PHY interface (MDIO bus) device. * Driver must be already attached. */ mdio = OF_parent(phy); bgx->lmac[lmac].phy_if_dev = OF_device_from_xref(OF_xref_from_node(mdio)); if (bgx->lmac[lmac].phy_if_dev == NULL) { if (bootverbose) { device_printf(bgx->dev, "Could not find interface to PHY\n"); } continue; } /* Get mac address from FDT */ bgx_fdt_get_macaddr(child, bgx->lmac[lmac].mac); bgx->lmac[lmac].lmacid = lmac; lmac++; if (lmac == MAX_LMAC_PER_BGX) break; } if (lmac == 0) { device_printf(bgx->dev, "Could not find matching PHY\n"); return (ENXIO); } return (0); } diff --git a/sys/dev/vnic/thunder_mdio.c b/sys/dev/vnic/thunder_mdio.c index 92a4fcdabc0e..118a01a533ea 100644 --- a/sys/dev/vnic/thunder_mdio.c +++ b/sys/dev/vnic/thunder_mdio.c @@ -1,513 +1,512 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "thunder_mdio_var.h" #include "lmac_if.h" #include "miibus_if.h" #define REG_BASE_RID 0 #define SMI_CMD 0x00 #define SMI_CMD_PHY_REG_ADR_SHIFT (0) #define SMI_CMD_PHY_REG_ADR_MASK (0x1FUL << SMI_CMD_PHY_REG_ADR_SHIFT) #define SMI_CMD_PHY_ADR_SHIFT (8) #define SMI_CMD_PHY_ADR_MASK (0x1FUL << SMI_CMD_PHY_ADR_SHIFT) #define SMI_CMD_PHY_OP_MASK (0x3UL << 16) #define SMI_CMD_PHY_OP_C22_READ (0x1UL << 16) #define SMI_CMD_PHY_OP_C22_WRITE (0x0UL << 16) #define SMI_CMD_PHY_OP_C45_READ (0x3UL << 16) #define SMI_CMD_PHY_OP_C45_WRITE (0x1UL << 16) #define SMI_CMD_PHY_OP_C45_ADDR (0x0UL << 16) #define SMI_WR_DAT 0x08 #define SMI_WR_DAT_PENDING (1UL << 17) #define SMI_WR_DAT_VAL (1UL << 16) #define SMI_WR_DAT_DAT_MASK (0xFFFFUL << 0) #define SMI_RD_DAT 0x10 #define SMI_RD_DAT_PENDING (1UL << 17) #define SMI_RD_DAT_VAL (1UL << 16) #define SMI_RD_DAT_DAT_MASK (0xFFFFUL << 0) #define SMI_CLK 0x18 #define SMI_CLK_PREAMBLE (1UL << 12) #define SMI_CLK_MODE (1UL << 24) #define SMI_EN 0x20 #define SMI_EN_EN (1UL << 0) /* Enabele interface */ #define SMI_DRV_CTL 0x28 static int thunder_mdio_detach(device_t); static int thunder_mdio_read(device_t, int, int); static int thunder_mdio_write(device_t, int, int, int); static int thunder_ifmedia_change_stub(struct ifnet *); static void thunder_ifmedia_status_stub(struct ifnet *, struct ifmediareq *); static int thunder_mdio_media_status(device_t, int, int *, int *, int *); static int thunder_mdio_media_change(device_t, int, int, int, int); static int thunder_mdio_phy_connect(device_t, int, int); static int thunder_mdio_phy_disconnect(device_t, int, int); static device_method_t thunder_mdio_methods[] = { /* Device interface */ DEVMETHOD(device_detach, thunder_mdio_detach), /* LMAC interface */ DEVMETHOD(lmac_media_status, thunder_mdio_media_status), DEVMETHOD(lmac_media_change, thunder_mdio_media_change), DEVMETHOD(lmac_phy_connect, thunder_mdio_phy_connect), DEVMETHOD(lmac_phy_disconnect, thunder_mdio_phy_disconnect), /* MII interface */ DEVMETHOD(miibus_readreg, thunder_mdio_read), DEVMETHOD(miibus_writereg, thunder_mdio_write), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(thunder_mdio, thunder_mdio_driver, thunder_mdio_methods, sizeof(struct thunder_mdio_softc)); DRIVER_MODULE(miibus, thunder_mdio, miibus_driver, miibus_devclass, 0, 0); MODULE_VERSION(thunder_mdio, 1); MODULE_DEPEND(thunder_mdio, ether, 1, 1, 1); MODULE_DEPEND(thunder_mdio, miibus, 1, 1, 1); MODULE_DEPEND(thunder_mdio, mrmlbus, 1, 1, 1); MALLOC_DEFINE(M_THUNDER_MDIO, "ThunderX MDIO", "Cavium ThunderX MDIO dynamic memory"); #define MDIO_LOCK_INIT(sc, name) \ mtx_init(&(sc)->mtx, name, NULL, MTX_DEF) #define MDIO_LOCK_DESTROY(sc) \ mtx_destroy(&(sc)->mtx) #define MDIO_LOCK(sc) mtx_lock(&(sc)->mtx) #define MDIO_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define MDIO_LOCK_ASSERT(sc) \ mtx_assert(&(sc)->mtx, MA_OWNED) #define mdio_reg_read(sc, reg) \ bus_read_8((sc)->reg_base, (reg)) #define mdio_reg_write(sc, reg, val) \ bus_write_8((sc)->reg_base, (reg), (val)) int thunder_mdio_attach(device_t dev) { struct thunder_mdio_softc *sc; int rid; sc = device_get_softc(dev); sc->dev = dev; /* Allocate memory resources */ rid = REG_BASE_RID; sc->reg_base = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->reg_base == NULL) { device_printf(dev, "Could not allocate memory\n"); return (ENXIO); } TAILQ_INIT(&sc->phy_desc_head); MDIO_LOCK_INIT(sc, "ThunderX MDIO lock"); /* Enable SMI/MDIO interface */ mdio_reg_write(sc, SMI_EN, SMI_EN_EN); return (0); } static int thunder_mdio_detach(device_t dev) { struct thunder_mdio_softc *sc; sc = device_get_softc(dev); if (sc->reg_base != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, REG_BASE_RID, sc->reg_base); } return (0); } static __inline void thunder_mdio_set_mode(struct thunder_mdio_softc *sc, enum thunder_mdio_mode mode) { uint64_t smi_clk; if (sc->mode == mode) return; /* Set mode, IEEE CLAUSE 22 or IEEE CAUSE 45 */ smi_clk = mdio_reg_read(sc, SMI_CLK); if (mode == MODE_IEEE_C22) smi_clk &= ~SMI_CLK_MODE; else smi_clk |= SMI_CLK_MODE; /* Enable sending 32 bit preable on SMI transactions */ smi_clk |= SMI_CLK_PREAMBLE; /* Saved setings */ mdio_reg_write(sc, SMI_CLK, smi_clk); sc->mode = mode; } static int thunder_mdio_c45_addr(struct thunder_mdio_softc *sc, int phy, int reg) { uint64_t smi_cmd, smi_wr_dat; ssize_t timeout; thunder_mdio_set_mode(sc, MODE_IEEE_C45); /* Prepare data for transmission */ mdio_reg_write(sc, SMI_WR_DAT, reg & SMI_WR_DAT_DAT_MASK); /* * Assemble command */ smi_cmd = 0; /* Set opcode */ smi_cmd |= SMI_CMD_PHY_OP_C45_WRITE; /* Set PHY address */ smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK); /* Set PHY register offset */ smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) & SMI_CMD_PHY_REG_ADR_MASK); mdio_reg_write(sc, SMI_CMD, smi_cmd); for (timeout = 1000; timeout > 0; timeout--) { smi_wr_dat = mdio_reg_read(sc, SMI_WR_DAT); if (smi_wr_dat & SMI_WR_DAT_PENDING) DELAY(1000); else break; } if (timeout <= 0) return (EIO); else { /* Return 0 on success */ return (0); } } static int thunder_mdio_read(device_t dev, int phy, int reg) { struct thunder_mdio_softc *sc; uint64_t smi_cmd, smi_rd_dat; ssize_t timeout; int err; sc = device_get_softc(dev); /* XXX Always C22 - for <= 1Gbps only */ thunder_mdio_set_mode(sc, MODE_IEEE_C22); /* * Assemble command */ smi_cmd = 0; /* Set opcode */ if (sc->mode == MODE_IEEE_C22) smi_cmd |= SMI_CMD_PHY_OP_C22_READ; else { smi_cmd |= SMI_CMD_PHY_OP_C45_READ; err = thunder_mdio_c45_addr(sc, phy, reg); if (err != 0) return (err); reg = (reg >> 16) & 0x1F; } /* Set PHY address */ smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK); /* Set PHY register offset */ smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) & SMI_CMD_PHY_REG_ADR_MASK); mdio_reg_write(sc, SMI_CMD, smi_cmd); for (timeout = 1000; timeout > 0; timeout--) { smi_rd_dat = mdio_reg_read(sc, SMI_RD_DAT); if (smi_rd_dat & SMI_RD_DAT_PENDING) DELAY(1000); else break; } if (smi_rd_dat & SMI_RD_DAT_VAL) return (smi_rd_dat & SMI_RD_DAT_DAT_MASK); else { /* Return 0 on error */ return (0); } } static int thunder_mdio_write(device_t dev, int phy, int reg, int data) { struct thunder_mdio_softc *sc; uint64_t smi_cmd, smi_wr_dat; ssize_t timeout; sc = device_get_softc(dev); /* XXX Always C22 - for <= 1Gbps only */ thunder_mdio_set_mode(sc, MODE_IEEE_C22); /* Prepare data for transmission */ mdio_reg_write(sc, SMI_WR_DAT, data & SMI_WR_DAT_DAT_MASK); /* * Assemble command */ smi_cmd = 0; /* Set opcode */ if (sc->mode == MODE_IEEE_C22) smi_cmd |= SMI_CMD_PHY_OP_C22_WRITE; else smi_cmd |= SMI_CMD_PHY_OP_C45_WRITE; /* Set PHY address */ smi_cmd |= ((phy << SMI_CMD_PHY_ADR_SHIFT) & SMI_CMD_PHY_ADR_MASK); /* Set PHY register offset */ smi_cmd |= ((reg << SMI_CMD_PHY_REG_ADR_SHIFT) & SMI_CMD_PHY_REG_ADR_MASK); mdio_reg_write(sc, SMI_CMD, smi_cmd); for (timeout = 1000; timeout > 0; timeout--) { smi_wr_dat = mdio_reg_read(sc, SMI_WR_DAT); if (smi_wr_dat & SMI_WR_DAT_PENDING) DELAY(1000); else break; } if (timeout <= 0) return (EIO); else { /* Return 0 on success */ return (0); } } static int thunder_ifmedia_change_stub(struct ifnet *ifp __unused) { /* Will never be called by if_media */ return (0); } static void thunder_ifmedia_status_stub(struct ifnet *ifp __unused, struct ifmediareq *ifmr __unused) { /* Will never be called by if_media */ } static __inline struct phy_desc * get_phy_desc(struct thunder_mdio_softc *sc, int lmacid) { struct phy_desc *pd = NULL; MDIO_LOCK_ASSERT(sc); TAILQ_FOREACH(pd, &sc->phy_desc_head, phy_desc_list) { if (pd->lmacid == lmacid) break; } return (pd); } static int thunder_mdio_media_status(device_t dev, int lmacid, int *link, int *duplex, int *speed) { struct thunder_mdio_softc *sc; struct mii_data *mii_sc; struct phy_desc *pd; sc = device_get_softc(dev); MDIO_LOCK(sc); pd = get_phy_desc(sc, lmacid); if (pd == NULL) { /* Panic when invariants are enabled, fail otherwise. */ KASSERT(0, ("%s: no PHY descriptor for LMAC%d", __func__, lmacid)); MDIO_UNLOCK(sc); return (ENXIO); } mii_sc = device_get_softc(pd->miibus); mii_tick(mii_sc); if ((mii_sc->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == (IFM_ACTIVE | IFM_AVALID)) { /* Link is up */ *link = 1; } else *link = 0; switch (IFM_SUBTYPE(mii_sc->mii_media_active)) { case IFM_10_T: *speed = 10; break; case IFM_100_TX: *speed = 100; break; case IFM_1000_T: *speed = 1000; break; default: /* IFM_NONE */ *speed = 0; } if ((IFM_OPTIONS(mii_sc->mii_media_active) & IFM_FDX) != 0) *duplex = 1; else *duplex = 0; MDIO_UNLOCK(sc); return (0); } static int thunder_mdio_media_change(device_t dev, int lmacid, int link, int duplex, int speed) { return (EIO); } static int thunder_mdio_phy_connect(device_t dev, int lmacid, int phy) { struct thunder_mdio_softc *sc; struct phy_desc *pd; int err; sc = device_get_softc(dev); MDIO_LOCK(sc); pd = get_phy_desc(sc, lmacid); MDIO_UNLOCK(sc); if (pd == NULL) { pd = malloc(sizeof(*pd), M_THUNDER_MDIO, (M_NOWAIT | M_ZERO)); if (pd == NULL) return (ENOMEM); pd->ifp = if_alloc(IFT_ETHER); if (pd->ifp == NULL) { free(pd, M_THUNDER_MDIO); return (ENOMEM); } pd->lmacid = lmacid; } err = mii_attach(dev, &pd->miibus, pd->ifp, thunder_ifmedia_change_stub, thunder_ifmedia_status_stub, BMSR_DEFCAPMASK, phy, MII_OFFSET_ANY, 0); if (err != 0) { device_printf(dev, "Could not attach PHY%d\n", phy); if_free(pd->ifp); free(pd, M_THUNDER_MDIO); return (ENXIO); } MDIO_LOCK(sc); TAILQ_INSERT_TAIL(&sc->phy_desc_head, pd, phy_desc_list); MDIO_UNLOCK(sc); return (0); } static int thunder_mdio_phy_disconnect(device_t dev, int lmacid, int phy) { struct thunder_mdio_softc *sc; struct phy_desc *pd; sc = device_get_softc(dev); MDIO_LOCK(sc); pd = get_phy_desc(sc, lmacid); if (pd == NULL) { MDIO_UNLOCK(sc); return (EINVAL); } /* Remove this PHY descriptor from the list */ TAILQ_REMOVE(&sc->phy_desc_head, pd, phy_desc_list); /* Detach miibus */ bus_generic_detach(dev); device_delete_child(dev, pd->miibus); /* Free fake ifnet */ if_free(pd->ifp); /* Free memory under phy descriptor */ free(pd, M_THUNDER_MDIO); MDIO_UNLOCK(sc); return (0); } diff --git a/sys/dev/vnic/thunder_mdio_fdt.c b/sys/dev/vnic/thunder_mdio_fdt.c index cc9415050388..4012950255ee 100644 --- a/sys/dev/vnic/thunder_mdio_fdt.c +++ b/sys/dev/vnic/thunder_mdio_fdt.c @@ -1,289 +1,288 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "thunder_mdio_var.h" static int thunder_mdio_fdt_probe(device_t); static int thunder_mdio_fdt_attach(device_t); static device_method_t thunder_mdio_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, thunder_mdio_fdt_probe), DEVMETHOD(device_attach, thunder_mdio_fdt_attach), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(thunder_mdio, thunder_mdio_fdt_driver, thunder_mdio_fdt_methods, sizeof(struct thunder_mdio_softc), thunder_mdio_driver); static devclass_t thunder_mdio_fdt_devclass; EARLY_DRIVER_MODULE(thunder_mdio, ofwbus, thunder_mdio_fdt_driver, thunder_mdio_fdt_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(thunder_mdio, mdionexus, thunder_mdio_fdt_driver, thunder_mdio_fdt_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); static struct ofw_compat_data mdio_compat_data[] = { {"cavium,octeon-3860-mdio", true}, {"cavium,thunder-8890-mdio", true}, {NULL, false} }; static int thunder_mdio_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, mdio_compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, THUNDER_MDIO_DEVSTR); return (BUS_PROBE_DEFAULT); } static int thunder_mdio_fdt_attach(device_t dev) { phandle_t node; int ret; /* Call core attach */ ret = thunder_mdio_attach(dev); if (ret != 0) return (ret); /* * Register device to this node/xref. * Thanks to that we will be able to retrieve device_t structure * while holding only node reference acquired from FDT. */ node = ofw_bus_get_node(dev); OF_device_register_xref(OF_xref_from_node(node), dev); return (0); } struct mdionexus_softc { struct simplebus_softc simplebus_sc; }; static device_probe_t mdionexus_fdt_probe; static device_attach_t mdionexus_fdt_attach; static const struct ofw_bus_devinfo * mdionexus_ofw_get_devinfo(device_t, device_t); static device_method_t mdionexus_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, mdionexus_fdt_probe), DEVMETHOD(device_attach, mdionexus_fdt_attach), /* Bus interface */ DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, mdionexus_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END }; DEFINE_CLASS_0(mdionexus, mdionexus_fdt_driver, mdionexus_fdt_methods, sizeof(struct mdionexus_softc)); static devclass_t mdionexus_fdt_devclass; EARLY_DRIVER_MODULE(mdionexus, mrmlbus, mdionexus_fdt_driver, mdionexus_fdt_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); static int mdionexus_ofw_fill_ranges(phandle_t, struct simplebus_softc *); static int mdionexus_ofw_bus_attach(device_t); static int mdionexus_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "cavium,thunder-8890-mdio-nexus")) return (ENXIO); device_set_desc(dev, "Cavium ThunderX MDIO nexus"); return (BUS_PROBE_SPECIFIC); } static int mdionexus_fdt_attach(device_t dev) { int err; err = mdionexus_ofw_bus_attach(dev); if (err != 0) return (err); return (bus_generic_attach(dev)); } /* OFW bus interface */ struct mdionexus_ofw_devinfo { struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; static const struct ofw_bus_devinfo * mdionexus_ofw_get_devinfo(device_t bus __unused, device_t child) { struct mdionexus_ofw_devinfo *di; di = device_get_ivars(child); return (&di->di_dinfo); } /* Helper functions */ static int mdionexus_ofw_fill_ranges(phandle_t node, struct simplebus_softc *sc) { int host_address_cells; cell_t *base_ranges; ssize_t nbase_ranges; int err; int i, j, k; err = OF_searchencprop(OF_parent(node), "#address-cells", &host_address_cells, sizeof(host_address_cells)); if (err <= 0) return (-1); nbase_ranges = OF_getproplen(node, "ranges"); if (nbase_ranges < 0) return (-1); sc->nranges = nbase_ranges / sizeof(cell_t) / (sc->acells + host_address_cells + sc->scells); if (sc->nranges == 0) return (0); sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), M_THUNDER_MDIO, M_WAITOK); base_ranges = malloc(nbase_ranges, M_THUNDER_MDIO, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { sc->ranges[i].bus = 0; for (k = 0; k < sc->acells; k++) { sc->ranges[i].bus <<= 32; sc->ranges[i].bus |= base_ranges[j++]; } sc->ranges[i].host = 0; for (k = 0; k < host_address_cells; k++) { sc->ranges[i].host <<= 32; sc->ranges[i].host |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < sc->scells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_THUNDER_MDIO); return (sc->nranges); } static int mdionexus_ofw_bus_attach(device_t dev) { struct simplebus_softc *sc; struct mdionexus_ofw_devinfo *di; device_t child; phandle_t parent, node; parent = ofw_bus_get_node(dev); simplebus_init(dev, parent); sc = (struct simplebus_softc *)device_get_softc(dev); if (mdionexus_ofw_fill_ranges(parent, sc) < 0) { device_printf(dev, "could not get ranges\n"); return (ENXIO); } /* Iterate through all bus subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_THUNDER_MDIO, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node) != 0) { free(di, M_THUNDER_MDIO); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, sc->acells, sc->scells, &di->di_rl); ofw_bus_intr_to_rl(dev, node, &di->di_rl, NULL); /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (child == NULL) { resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_THUNDER_MDIO); continue; } device_set_ivars(child, di); } return (0); } diff --git a/sys/dev/vnic/thunder_mdio_var.h b/sys/dev/vnic/thunder_mdio_var.h index 73fedcc77811..acebec0dbcc0 100644 --- a/sys/dev/vnic/thunder_mdio_var.h +++ b/sys/dev/vnic/thunder_mdio_var.h @@ -1,64 +1,63 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef __THUNDER_MDIO_VAR_H__ #define __THUNDER_MDIO_VAR_H__ #define THUNDER_MDIO_DEVSTR "Cavium ThunderX SMI/MDIO driver" MALLOC_DECLARE(M_THUNDER_MDIO); DECLARE_CLASS(thunder_mdio_driver); enum thunder_mdio_mode { MODE_NONE = 0, MODE_IEEE_C22, MODE_IEEE_C45 }; struct phy_desc { device_t miibus; /* One miibus per LMAC */ struct ifnet * ifp; /* Fake ifp to satisfy miibus */ int lmacid; /* ID number of LMAC connected */ TAILQ_ENTRY(phy_desc) phy_desc_list; }; struct thunder_mdio_softc { device_t dev; struct mtx mtx; struct resource * reg_base; enum thunder_mdio_mode mode; TAILQ_HEAD(,phy_desc) phy_desc_head; }; int thunder_mdio_attach(device_t); #endif diff --git a/sys/dev/vt/colors/vt_termcolors.c b/sys/dev/vt/colors/vt_termcolors.c index 941701f68cf2..dff276a86100 100644 --- a/sys/dev/vt/colors/vt_termcolors.c +++ b/sys/dev/vt/colors/vt_termcolors.c @@ -1,179 +1,178 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include static struct { unsigned char r; /* Red percentage value. */ unsigned char g; /* Green percentage value. */ unsigned char b; /* Blue percentage value. */ } color_def[NCOLORS] = { {0, 0, 0}, /* black */ {50, 0, 0}, /* dark red */ {0, 50, 0}, /* dark green */ {77, 63, 0}, /* dark yellow */ {20, 40, 64}, /* dark blue */ {50, 0, 50}, /* dark magenta */ {0, 50, 50}, /* dark cyan */ {75, 75, 75}, /* light gray */ {18, 20, 21}, /* dark gray */ {100, 0, 0}, /* light red */ {0, 100, 0}, /* light green */ {100, 100, 0}, /* light yellow */ {45, 62, 81}, /* light blue */ {100, 0, 100}, /* light magenta */ {0, 100, 100}, /* light cyan */ {100, 100, 100}, /* white */ }; static int vt_parse_rgb_triplet(const char *rgb, unsigned char *r, unsigned char *g, unsigned char *b) { unsigned long v; const char *ptr; char *endptr; ptr = rgb; /* Handle #rrggbb case */ if (*ptr == '#') { if (strlen(ptr) != 7) return (-1); v = strtoul(ptr + 1, &endptr, 16); if (*endptr != '\0') return (-1); *r = (v >> 16) & 0xff; *g = (v >> 8) & 0xff; *b = v & 0xff; return (0); } /* "r, g, b" case */ v = strtoul(ptr, &endptr, 10); if (ptr == endptr) return (-1); if (v > 255) return (-1); *r = v & 0xff; ptr = endptr; /* skip separator */ while (*ptr == ',' || *ptr == ' ') ptr++; v = strtoul(ptr, &endptr, 10); if (ptr == endptr) return (-1); if (v > 255) return (-1); *g = v & 0xff; ptr = endptr; /* skip separator */ while (*ptr == ',' || *ptr == ' ') ptr++; v = strtoul(ptr, &endptr, 10); if (ptr == endptr) return (-1); if (v > 255) return (-1); *b = v & 0xff; ptr = endptr; /* skip trailing spaces */ while (*ptr == ' ') ptr++; /* unexpected characters at the end of the string */ if (*ptr != 0) return (-1); return (0); } static void vt_palette_init(void) { int i; char rgb[32]; char tunable[32]; unsigned char r, g, b; for (i = 0; i < NCOLORS; i++) { snprintf(tunable, sizeof(tunable), "kern.vt.color.%d.rgb", i); if (TUNABLE_STR_FETCH(tunable, rgb, sizeof(rgb))) { if (vt_parse_rgb_triplet(rgb, &r, &g, &b) == 0) { /* convert to percentages */ color_def[i].r = r*100/255; color_def[i].g = g*100/255; color_def[i].b = b*100/255; } } } } int vt_generate_cons_palette(uint32_t *palette, int format, uint32_t rmax, int roffset, uint32_t gmax, int goffset, uint32_t bmax, int boffset) { int i; switch (format) { case COLOR_FORMAT_VGA: for (i = 0; i < NCOLORS; i++) palette[i] = cons_to_vga_colors[i]; break; case COLOR_FORMAT_RGB: vt_palette_init(); #define CF(_f, _i) ((_f ## max * color_def[(_i)]._f / 100) << _f ## offset) for (i = 0; i < NCOLORS; i++) palette[i] = CF(r, i) | CF(g, i) | CF(b, i); #undef CF break; default: return (ENODEV); } return (0); } diff --git a/sys/dev/vt/colors/vt_termcolors.h b/sys/dev/vt/colors/vt_termcolors.h index 8a8c7b65b31a..3fc99ee9743f 100644 --- a/sys/dev/vt/colors/vt_termcolors.h +++ b/sys/dev/vt/colors/vt_termcolors.h @@ -1,63 +1,62 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ enum vt_color_format { COLOR_FORMAT_BW = 0, COLOR_FORMAT_GRAY, COLOR_FORMAT_VGA, /* Color Index. */ COLOR_FORMAT_RGB, COLOR_FORMAT_ARGB, COLOR_FORMAT_CMYK, COLOR_FORMAT_HSL, COLOR_FORMAT_YUV, COLOR_FORMAT_YCbCr, COLOR_FORMAT_YPbPr, COLOR_FORMAT_MAX = 15, }; #define NCOLORS 16 /* * Between console's palette and VGA's one: * - blue and red are swapped (1 <-> 4) * - yellow and cyan are swapped (3 <-> 6) */ static const int cons_to_vga_colors[NCOLORS] = { 0, 4, 2, 6, 1, 5, 3, 7, 8, 12, 10, 14, 9, 13, 11, 15 }; /* Helper to fill color map used by driver */ int vt_generate_cons_palette(uint32_t *palette, int format, uint32_t rmax, int roffset, uint32_t gmax, int goffset, uint32_t bmax, int boffset); diff --git a/sys/dev/vt/font/vt_mouse_cursor.c b/sys/dev/vt/font/vt_mouse_cursor.c index 386fe99a1f48..96bbd8b45b0a 100644 --- a/sys/dev/vt/font/vt_mouse_cursor.c +++ b/sys/dev/vt/font/vt_mouse_cursor.c @@ -1,78 +1,77 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #ifndef SC_NO_CUTPASTE struct vt_mouse_cursor vt_default_mouse_pointer = { .map = { 0x00, 0x00, /* "__ " */ 0x40, 0x00, /* "_*_ " */ 0x60, 0x00, /* "_**_ " */ 0x70, 0x00, /* "_***_ " */ 0x78, 0x00, /* "_****_ " */ 0x7c, 0x00, /* "_*****_ " */ 0x7e, 0x00, /* "_******_ " */ 0x7f, 0x00, /* "_*******_ " */ 0x7f, 0x80, /* "_********_" */ 0x7c, 0x00, /* "_*****____" */ 0x6c, 0x00, /* "_**_**_ " */ 0x46, 0x00, /* "_*_ _**_ " */ 0x06, 0x00, /* "__ _**_ " */ 0x03, 0x00, /* " _**_ " */ 0x03, 0x00, /* " _**_ " */ 0x00, 0x00, /* " __ " */ }, .mask = { 0xc0, 0x00, /* "__ " */ 0xe0, 0x00, /* "___ " */ 0xf0, 0x00, /* "____ " */ 0xf8, 0x00, /* "_____ " */ 0xfc, 0x00, /* "______ " */ 0xfe, 0x00, /* "_______ " */ 0xff, 0x00, /* "________ " */ 0xff, 0x80, /* "_________ " */ 0xff, 0xc0, /* "__________" */ 0xff, 0xc0, /* "__________" */ 0xfe, 0x00, /* "_______ " */ 0xef, 0x00, /* "___ ____ " */ 0xcf, 0x00, /* "__ ____ " */ 0x07, 0x80, /* " ____ " */ 0x07, 0x80, /* " ____ " */ 0x03, 0x00, /* " __ " */ }, .width = 10, .height = 16, }; #endif diff --git a/sys/dev/vt/hw/efifb/efifb.c b/sys/dev/vt/hw/efifb/efifb.c index 1d08bdd99ce7..e4deb1b9d9e5 100644 --- a/sys/dev/vt/hw/efifb/efifb.c +++ b/sys/dev/vt/hw/efifb/efifb.c @@ -1,158 +1,157 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "opt_platform.h" #include #include #include #include #include #include #include static vd_init_t vt_efifb_init; static vd_fini_t vt_efifb_fini; static vd_probe_t vt_efifb_probe; static struct vt_driver vt_efifb_driver = { .vd_name = "efifb", .vd_probe = vt_efifb_probe, .vd_init = vt_efifb_init, .vd_fini = vt_efifb_fini, .vd_blank = vt_fb_blank, .vd_bitblt_text = vt_fb_bitblt_text, .vd_invalidate_text = vt_fb_invalidate_text, .vd_bitblt_bmp = vt_fb_bitblt_bitmap, .vd_drawrect = vt_fb_drawrect, .vd_setpixel = vt_fb_setpixel, .vd_fb_ioctl = vt_fb_ioctl, .vd_fb_mmap = vt_fb_mmap, .vd_suspend = vt_suspend, .vd_resume = vt_resume, /* Better than VGA, but still generic driver. */ .vd_priority = VD_PRIORITY_GENERIC + 1, }; static struct fb_info local_info; VT_DRIVER_DECLARE(vt_efifb, vt_efifb_driver); static int vt_efifb_probe(struct vt_device *vd) { int disabled; struct efi_fb *efifb; caddr_t kmdp; disabled = 0; TUNABLE_INT_FETCH("hw.syscons.disable", &disabled); if (disabled != 0) return (CN_DEAD); kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); efifb = (struct efi_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_FB); if (efifb == NULL) return (CN_DEAD); return (CN_INTERNAL); } static int vt_efifb_init(struct vt_device *vd) { struct fb_info *info; struct efi_fb *efifb; caddr_t kmdp; int roff, goff, boff; info = vd->vd_softc; if (info == NULL) info = vd->vd_softc = (void *)&local_info; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); efifb = (struct efi_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_FB); if (efifb == NULL) return (CN_DEAD); info->fb_height = efifb->fb_height; info->fb_width = efifb->fb_width; info->fb_depth = fls(efifb->fb_mask_red | efifb->fb_mask_green | efifb->fb_mask_blue | efifb->fb_mask_reserved); /* Round to a multiple of the bits in a byte. */ info->fb_bpp = roundup2(info->fb_depth, NBBY); /* Stride in bytes, not pixels */ info->fb_stride = efifb->fb_stride * (info->fb_bpp / NBBY); roff = ffs(efifb->fb_mask_red) - 1; goff = ffs(efifb->fb_mask_green) - 1; boff = ffs(efifb->fb_mask_blue) - 1; vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, efifb->fb_mask_red >> roff, roff, efifb->fb_mask_green >> goff, goff, efifb->fb_mask_blue >> boff, boff); info->fb_cmsize = NCOLORS; info->fb_size = info->fb_height * info->fb_stride; info->fb_pbase = efifb->fb_addr; info->fb_vbase = (intptr_t)pmap_mapdev_attr(info->fb_pbase, info->fb_size, VM_MEMATTR_WRITE_COMBINING); vt_fb_init(vd); return (CN_INTERNAL); } static void vt_efifb_fini(struct vt_device *vd, void *softc) { struct fb_info *info = softc; vt_fb_fini(vd, softc); pmap_unmapdev(info->fb_vbase, info->fb_size); } diff --git a/sys/dev/vt/hw/fb/vt_early_fb.c b/sys/dev/vt/hw/fb/vt_early_fb.c index 12db4368d919..9d9635835202 100644 --- a/sys/dev/vt/hw/fb/vt_early_fb.c +++ b/sys/dev/vt/hw/fb/vt_early_fb.c @@ -1,297 +1,296 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include "opt_platform.h" #ifdef FDT #include #include #include #include #include #endif #include #include #include static vd_init_t vt_efb_init; static vd_probe_t vt_efb_probe; static struct vt_driver vt_fb_early_driver = { .vd_name = "efb", .vd_probe = vt_efb_probe, .vd_init = vt_efb_init, .vd_blank = vt_fb_blank, .vd_bitblt_text = vt_fb_bitblt_text, .vd_invalidate_text = vt_fb_invalidate_text, .vd_bitblt_bmp = vt_fb_bitblt_bitmap, .vd_drawrect = vt_fb_drawrect, .vd_setpixel = vt_fb_setpixel, .vd_priority = VD_PRIORITY_GENERIC, }; static struct fb_info local_info; VT_DRIVER_DECLARE(vt_efb, vt_fb_early_driver); static void #ifdef FDT vt_efb_initialize(struct fb_info *info, phandle_t node) #else vt_efb_initialize(struct fb_info *info) #endif { #ifdef FDT char name[64]; cell_t retval; ihandle_t ih; int i; /* Open display device, thereby initializing it */ memset(name, 0, sizeof(name)); OF_package_to_path(node, name, sizeof(name)); ih = OF_open(name); #endif /* * Set up the color map */ switch (info->fb_depth) { case 8: vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, 0x7, 5, 0x7, 2, 0x3, 0); break; case 15: vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, 0x1f, 10, 0x1f, 5, 0x1f, 0); break; case 16: vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, 0x1f, 11, 0x3f, 5, 0x1f, 0); break; case 24: case 32: #if BYTE_ORDER == BIG_ENDIAN vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, 255, 0, 255, 8, 255, 16); #else vt_generate_cons_palette(info->fb_cmap, COLOR_FORMAT_RGB, 255, 16, 255, 8, 255, 0); #endif #ifdef FDT for (i = 0; i < 16; i++) { OF_call_method("color!", ih, 4, 1, (cell_t)((info->fb_cmap[i] >> 16) & 0xff), (cell_t)((info->fb_cmap[i] >> 8) & 0xff), (cell_t)((info->fb_cmap[i] >> 0) & 0xff), (cell_t)i, &retval); } #endif break; default: panic("Unknown color space fb_depth %d", info->fb_depth); break; } } static phandle_t vt_efb_get_fbnode() { phandle_t chosen, node; ihandle_t stdout; char type[64]; chosen = OF_finddevice("/chosen"); OF_getprop(chosen, "stdout", &stdout, sizeof(stdout)); node = OF_instance_to_package(stdout); if (node != -1) { /* The "/chosen/stdout" present. */ OF_getprop(node, "device_type", type, sizeof(type)); /* Check if it has "display" type. */ if (strcmp(type, "display") == 0) return (node); } /* Try device with name "screen". */ node = OF_finddevice("screen"); return (node); } static int vt_efb_probe(struct vt_device *vd) { phandle_t node; node = vt_efb_get_fbnode(); if (node == -1) return (CN_DEAD); if ((OF_getproplen(node, "height") <= 0) || (OF_getproplen(node, "width") <= 0) || (OF_getproplen(node, "depth") <= 0) || (OF_getproplen(node, "linebytes") <= 0)) return (CN_DEAD); return (CN_INTERNAL); } static int vt_efb_init(struct vt_device *vd) { struct ofw_pci_register pciaddrs[8]; struct fb_info *info; int i, len, n_pciaddrs; phandle_t node; if (vd->vd_softc == NULL) vd->vd_softc = (void *)&local_info; info = vd->vd_softc; node = vt_efb_get_fbnode(); if (node == -1) return (CN_DEAD); #define GET(name, var) \ if (OF_getproplen(node, (name)) != sizeof(info->fb_##var)) \ return (CN_DEAD); \ OF_getencprop(node, (name), &info->fb_##var, sizeof(info->fb_##var)); \ if (info->fb_##var == 0) \ return (CN_DEAD); GET("height", height) GET("width", width) GET("depth", depth) GET("linebytes", stride) #undef GET info->fb_size = info->fb_height * info->fb_stride; /* * Get the PCI addresses of the adapter, if present. The node may be the * child of the PCI device: in that case, try the parent for * the assigned-addresses property. */ len = OF_getprop(node, "assigned-addresses", pciaddrs, sizeof(pciaddrs)); if (len == -1) { len = OF_getprop(OF_parent(node), "assigned-addresses", pciaddrs, sizeof(pciaddrs)); } if (len == -1) len = 0; n_pciaddrs = len / sizeof(struct ofw_pci_register); /* * Grab the physical address of the framebuffer, and then map it * into our memory space. If the MMU is not yet up, it will be * remapped for us when relocation turns on. */ if (OF_getproplen(node, "address") == sizeof(info->fb_pbase)) { /* XXX We assume #address-cells is 1 at this point. */ OF_getencprop(node, "address", &info->fb_pbase, sizeof(info->fb_pbase)); #if defined(__powerpc__) sc->sc_memt = &bs_be_tag; bus_space_map(sc->sc_memt, info->fb_pbase, info->fb_size, BUS_SPACE_MAP_PREFETCHABLE, &info->fb_vbase); #else bus_space_map(fdtbus_bs_tag, info->fb_pbase, info->fb_size, BUS_SPACE_MAP_PREFETCHABLE, (bus_space_handle_t *)&info->fb_vbase); #endif } else { /* * Some IBM systems don't have an address property. Try to * guess the framebuffer region from the assigned addresses. * This is ugly, but there doesn't seem to be an alternative. * Linux does the same thing. */ info->fb_pbase = n_pciaddrs; for (i = 0; i < n_pciaddrs; i++) { /* If it is too small, not the framebuffer */ if (pciaddrs[i].size_lo < info->fb_size) continue; /* If it is not memory, it isn't either */ if (!(pciaddrs[i].phys_hi & OFW_PCI_PHYS_HI_SPACE_MEM32)) continue; /* This could be the framebuffer */ info->fb_pbase = i; /* If it is prefetchable, it certainly is */ if (pciaddrs[i].phys_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) break; } if (info->fb_pbase == n_pciaddrs) /* No candidates found */ return (CN_DEAD); #if defined(__powerpc__) OF_decode_addr(node, info->fb_pbase, &sc->sc_memt, &info->fb_vbase); #else bus_space_map(fdtbus_bs_tag, info->fb_pbase, info->fb_size, BUS_SPACE_MAP_PREFETCHABLE, (bus_space_handle_t *)&info->fb_vbase); #endif } /* blank full size */ len = info->fb_size / 4; for (i = 0; i < len; i++) { ((uint32_t *)info->fb_vbase)[i] = 0; } /* Get pixel storage size. */ info->fb_bpp = info->fb_stride / info->fb_width * 8; #ifdef FDT vt_efb_initialize(info, node); #else vt_efb_initialize(info); #endif vt_fb_init(vd); return (CN_INTERNAL); } diff --git a/sys/dev/vt/hw/fb/vt_fb.c b/sys/dev/vt/hw/fb/vt_fb.c index 01850f789d29..bdf2bbfcc7ce 100644 --- a/sys/dev/vt/hw/fb/vt_fb.c +++ b/sys/dev/vt/hw/fb/vt_fb.c @@ -1,534 +1,533 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include static struct vt_driver vt_fb_driver = { .vd_name = "fb", .vd_init = vt_fb_init, .vd_fini = vt_fb_fini, .vd_blank = vt_fb_blank, .vd_bitblt_text = vt_fb_bitblt_text, .vd_invalidate_text = vt_fb_invalidate_text, .vd_bitblt_bmp = vt_fb_bitblt_bitmap, .vd_drawrect = vt_fb_drawrect, .vd_setpixel = vt_fb_setpixel, .vd_postswitch = vt_fb_postswitch, .vd_priority = VD_PRIORITY_GENERIC+10, .vd_fb_ioctl = vt_fb_ioctl, .vd_fb_mmap = vt_fb_mmap, .vd_suspend = vt_fb_suspend, .vd_resume = vt_fb_resume, }; VT_DRIVER_DECLARE(vt_fb, vt_fb_driver); static void vt_fb_mem_wr1(struct fb_info *sc, uint32_t o, uint8_t v) { KASSERT((o < sc->fb_size), ("Offset %#08x out of fb size", o)); *(uint8_t *)(sc->fb_vbase + o) = v; } static void vt_fb_mem_wr2(struct fb_info *sc, uint32_t o, uint16_t v) { KASSERT((o < sc->fb_size), ("Offset %#08x out of fb size", o)); *(uint16_t *)(sc->fb_vbase + o) = v; } static void vt_fb_mem_wr4(struct fb_info *sc, uint32_t o, uint32_t v) { KASSERT((o < sc->fb_size), ("Offset %#08x out of fb size", o)); *(uint32_t *)(sc->fb_vbase + o) = v; } int vt_fb_ioctl(struct vt_device *vd, u_long cmd, caddr_t data, struct thread *td) { struct fb_info *info; int error = 0; info = vd->vd_softc; switch (cmd) { case FBIOGTYPE: bcopy(info, (struct fbtype *)data, sizeof(struct fbtype)); break; case FBIO_GETWINORG: /* get frame buffer window origin */ *(u_int *)data = 0; break; case FBIO_GETDISPSTART: /* get display start address */ ((video_display_start_t *)data)->x = 0; ((video_display_start_t *)data)->y = 0; break; case FBIO_GETLINEWIDTH: /* get scan line width in bytes */ *(u_int *)data = info->fb_stride; break; case FBIO_BLANK: /* blank display */ if (vd->vd_driver->vd_blank == NULL) return (ENODEV); vd->vd_driver->vd_blank(vd, TC_BLACK); break; default: error = ENOIOCTL; break; } return (error); } int vt_fb_mmap(struct vt_device *vd, vm_ooffset_t offset, vm_paddr_t *paddr, int prot, vm_memattr_t *memattr) { struct fb_info *info; info = vd->vd_softc; if (info->fb_flags & FB_FLAG_NOMMAP) return (ENODEV); if (offset < info->fb_size) { if (info->fb_pbase == 0) { *paddr = vtophys((uint8_t *)info->fb_vbase + offset); } else { *paddr = info->fb_pbase + offset; if (info->fb_flags & FB_FLAG_MEMATTR) *memattr = info->fb_memattr; #ifdef VM_MEMATTR_WRITE_COMBINING else *memattr = VM_MEMATTR_WRITE_COMBINING; #endif } return (0); } return (EINVAL); } void vt_fb_setpixel(struct vt_device *vd, int x, int y, term_color_t color) { struct fb_info *info; uint32_t c; u_int o; info = vd->vd_softc; c = info->fb_cmap[color]; o = info->fb_stride * y + x * FBTYPE_GET_BYTESPP(info); if (info->fb_flags & FB_FLAG_NOWRITE) return; KASSERT((info->fb_vbase != 0), ("Unmapped framebuffer")); switch (FBTYPE_GET_BYTESPP(info)) { case 1: vt_fb_mem_wr1(info, o, c); break; case 2: vt_fb_mem_wr2(info, o, c); break; case 3: vt_fb_mem_wr1(info, o, (c >> 16) & 0xff); vt_fb_mem_wr1(info, o + 1, (c >> 8) & 0xff); vt_fb_mem_wr1(info, o + 2, c & 0xff); break; case 4: vt_fb_mem_wr4(info, o, c); break; default: /* panic? */ return; } } void vt_fb_drawrect(struct vt_device *vd, int x1, int y1, int x2, int y2, int fill, term_color_t color) { int x, y; for (y = y1; y <= y2; y++) { if (fill || (y == y1) || (y == y2)) { for (x = x1; x <= x2; x++) vt_fb_setpixel(vd, x, y, color); } else { vt_fb_setpixel(vd, x1, y, color); vt_fb_setpixel(vd, x2, y, color); } } } void vt_fb_blank(struct vt_device *vd, term_color_t color) { struct fb_info *info; uint32_t c; u_int o, h; info = vd->vd_softc; c = info->fb_cmap[color]; if (info->fb_flags & FB_FLAG_NOWRITE) return; KASSERT((info->fb_vbase != 0), ("Unmapped framebuffer")); switch (FBTYPE_GET_BYTESPP(info)) { case 1: for (h = 0; h < info->fb_height; h++) for (o = 0; o < info->fb_stride; o++) vt_fb_mem_wr1(info, h*info->fb_stride + o, c); break; case 2: for (h = 0; h < info->fb_height; h++) for (o = 0; o < info->fb_stride - 1; o += 2) vt_fb_mem_wr2(info, h*info->fb_stride + o, c); break; case 3: for (h = 0; h < info->fb_height; h++) for (o = 0; o < info->fb_stride - 2; o += 3) { vt_fb_mem_wr1(info, h*info->fb_stride + o, (c >> 16) & 0xff); vt_fb_mem_wr1(info, h*info->fb_stride + o + 1, (c >> 8) & 0xff); vt_fb_mem_wr1(info, h*info->fb_stride + o + 2, c & 0xff); } break; case 4: for (h = 0; h < info->fb_height; h++) for (o = 0; o < info->fb_stride - 3; o += 4) vt_fb_mem_wr4(info, h*info->fb_stride + o, c); break; default: /* panic? */ return; } } void vt_fb_bitblt_bitmap(struct vt_device *vd, const struct vt_window *vw, const uint8_t *pattern, const uint8_t *mask, unsigned int width, unsigned int height, unsigned int x, unsigned int y, term_color_t fg, term_color_t bg) { struct fb_info *info; uint32_t fgc, bgc, cc, o; int bpp, bpl, xi, yi; int bit, byte; info = vd->vd_softc; bpp = FBTYPE_GET_BYTESPP(info); fgc = info->fb_cmap[fg]; bgc = info->fb_cmap[bg]; bpl = (width + 7) / 8; /* Bytes per source line. */ if (info->fb_flags & FB_FLAG_NOWRITE) return; KASSERT((info->fb_vbase != 0), ("Unmapped framebuffer")); /* Bound by right and bottom edges. */ if (y + height > vw->vw_draw_area.tr_end.tp_row) { if (y >= vw->vw_draw_area.tr_end.tp_row) return; height = vw->vw_draw_area.tr_end.tp_row - y; } if (x + width > vw->vw_draw_area.tr_end.tp_col) { if (x >= vw->vw_draw_area.tr_end.tp_col) return; width = vw->vw_draw_area.tr_end.tp_col - x; } for (yi = 0; yi < height; yi++) { for (xi = 0; xi < width; xi++) { byte = yi * bpl + xi / 8; bit = 0x80 >> (xi % 8); /* Skip pixel write, if mask bit not set. */ if (mask != NULL && (mask[byte] & bit) == 0) continue; o = (y + yi) * info->fb_stride + (x + xi) * bpp; o += vd->vd_transpose; cc = pattern[byte] & bit ? fgc : bgc; switch(bpp) { case 1: vt_fb_mem_wr1(info, o, cc); break; case 2: vt_fb_mem_wr2(info, o, cc); break; case 3: /* Packed mode, so unaligned. Byte access. */ vt_fb_mem_wr1(info, o, (cc >> 16) & 0xff); vt_fb_mem_wr1(info, o + 1, (cc >> 8) & 0xff); vt_fb_mem_wr1(info, o + 2, cc & 0xff); break; case 4: vt_fb_mem_wr4(info, o, cc); break; default: /* panic? */ break; } } } } void vt_fb_bitblt_text(struct vt_device *vd, const struct vt_window *vw, const term_rect_t *area) { unsigned int col, row, x, y; struct vt_font *vf; term_char_t c; term_color_t fg, bg; const uint8_t *pattern; size_t z; vf = vw->vw_font; for (row = area->tr_begin.tp_row; row < area->tr_end.tp_row; ++row) { for (col = area->tr_begin.tp_col; col < area->tr_end.tp_col; ++col) { x = col * vf->vf_width + vw->vw_draw_area.tr_begin.tp_col; y = row * vf->vf_height + vw->vw_draw_area.tr_begin.tp_row; c = VTBUF_GET_FIELD(&vw->vw_buf, row, col); pattern = vtfont_lookup(vf, c); vt_determine_colors(c, VTBUF_ISCURSOR(&vw->vw_buf, row, col), &fg, &bg); z = row * PIXEL_WIDTH(VT_FB_MAX_WIDTH) + col; if (vd->vd_drawn && (vd->vd_drawn[z] == c) && vd->vd_drawnfg && (vd->vd_drawnfg[z] == fg) && vd->vd_drawnbg && (vd->vd_drawnbg[z] == bg)) continue; vt_fb_bitblt_bitmap(vd, vw, pattern, NULL, vf->vf_width, vf->vf_height, x, y, fg, bg); if (vd->vd_drawn) vd->vd_drawn[z] = c; if (vd->vd_drawnfg) vd->vd_drawnfg[z] = fg; if (vd->vd_drawnbg) vd->vd_drawnbg[z] = bg; } } #ifndef SC_NO_CUTPASTE if (!vd->vd_mshown) return; term_rect_t drawn_area; drawn_area.tr_begin.tp_col = area->tr_begin.tp_col * vf->vf_width; drawn_area.tr_begin.tp_row = area->tr_begin.tp_row * vf->vf_height; drawn_area.tr_end.tp_col = area->tr_end.tp_col * vf->vf_width; drawn_area.tr_end.tp_row = area->tr_end.tp_row * vf->vf_height; if (vt_is_cursor_in_area(vd, &drawn_area)) { vt_fb_bitblt_bitmap(vd, vw, vd->vd_mcursor->map, vd->vd_mcursor->mask, vd->vd_mcursor->width, vd->vd_mcursor->height, vd->vd_mx_drawn + vw->vw_draw_area.tr_begin.tp_col, vd->vd_my_drawn + vw->vw_draw_area.tr_begin.tp_row, vd->vd_mcursor_fg, vd->vd_mcursor_bg); } #endif } void vt_fb_invalidate_text(struct vt_device *vd, const term_rect_t *area) { unsigned int col, row; size_t z; for (row = area->tr_begin.tp_row; row < area->tr_end.tp_row; ++row) { for (col = area->tr_begin.tp_col; col < area->tr_end.tp_col; ++col) { z = row * PIXEL_WIDTH(VT_FB_MAX_WIDTH) + col; if (vd->vd_drawn) vd->vd_drawn[z] = 0; if (vd->vd_drawnfg) vd->vd_drawnfg[z] = 0; if (vd->vd_drawnbg) vd->vd_drawnbg[z] = 0; } } } void vt_fb_postswitch(struct vt_device *vd) { struct fb_info *info; info = vd->vd_softc; if (info->enter != NULL) info->enter(info->fb_priv); } static int vt_fb_init_cmap(uint32_t *cmap, int depth) { switch (depth) { case 8: return (vt_generate_cons_palette(cmap, COLOR_FORMAT_RGB, 0x7, 5, 0x7, 2, 0x3, 0)); case 15: return (vt_generate_cons_palette(cmap, COLOR_FORMAT_RGB, 0x1f, 10, 0x1f, 5, 0x1f, 0)); case 16: return (vt_generate_cons_palette(cmap, COLOR_FORMAT_RGB, 0x1f, 11, 0x3f, 5, 0x1f, 0)); case 24: case 32: /* Ignore alpha. */ return (vt_generate_cons_palette(cmap, COLOR_FORMAT_RGB, 0xff, 16, 0xff, 8, 0xff, 0)); default: return (1); } } int vt_fb_init(struct vt_device *vd) { struct fb_info *info; u_int margin; int bg, err; term_color_t c; info = vd->vd_softc; vd->vd_height = MIN(VT_FB_MAX_HEIGHT, info->fb_height); margin = (info->fb_height - vd->vd_height) >> 1; vd->vd_transpose = margin * info->fb_stride; vd->vd_width = MIN(VT_FB_MAX_WIDTH, info->fb_width); margin = (info->fb_width - vd->vd_width) >> 1; vd->vd_transpose += margin * (info->fb_bpp / NBBY); vd->vd_video_dev = info->fb_video_dev; if (info->fb_size == 0) return (CN_DEAD); if (info->fb_pbase == 0 && info->fb_vbase == 0) info->fb_flags |= FB_FLAG_NOMMAP; if (info->fb_cmsize <= 0) { err = vt_fb_init_cmap(info->fb_cmap, FBTYPE_GET_BPP(info)); if (err) return (CN_DEAD); info->fb_cmsize = 16; } c = TC_BLACK; if (TUNABLE_INT_FETCH("teken.bg_color", &bg) != 0) { if (bg == TC_WHITE) bg |= TC_LIGHT; c = bg; } /* Clear the screen. */ vd->vd_driver->vd_blank(vd, c); /* Wakeup screen. KMS need this. */ vt_fb_postswitch(vd); return (CN_INTERNAL); } void vt_fb_fini(struct vt_device *vd, void *softc) { vd->vd_video_dev = NULL; } int vt_fb_attach(struct fb_info *info) { vt_allocate(&vt_fb_driver, info); return (0); } int vt_fb_detach(struct fb_info *info) { vt_deallocate(&vt_fb_driver, info); return (0); } void vt_fb_suspend(struct vt_device *vd) { vt_suspend(vd); } void vt_fb_resume(struct vt_device *vd) { vt_resume(vd); } diff --git a/sys/dev/vt/hw/fb/vt_fb.h b/sys/dev/vt/hw/fb/vt_fb.h index 42b395e8ff39..bd42bc176bab 100644 --- a/sys/dev/vt/hw/fb/vt_fb.h +++ b/sys/dev/vt/hw/fb/vt_fb.h @@ -1,54 +1,53 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_VT_HW_FB_VT_FB_H_ #define _DEV_VT_HW_FB_VT_FB_H_ /* Generic framebuffer interface call vt_fb_attach to init VT(9) */ int vt_fb_attach(struct fb_info *info); void vt_fb_resume(struct vt_device *vd); void vt_fb_suspend(struct vt_device *vd); int vt_fb_detach(struct fb_info *info); vd_init_t vt_fb_init; vd_fini_t vt_fb_fini; vd_blank_t vt_fb_blank; vd_bitblt_text_t vt_fb_bitblt_text; vd_invalidate_text_t vt_fb_invalidate_text; vd_bitblt_bmp_t vt_fb_bitblt_bitmap; vd_drawrect_t vt_fb_drawrect; vd_setpixel_t vt_fb_setpixel; vd_postswitch_t vt_fb_postswitch; vd_fb_ioctl_t vt_fb_ioctl; vd_fb_mmap_t vt_fb_mmap; #endif /* _DEV_VT_HW_FB_VT_FB_H_ */ diff --git a/sys/dev/vt/hw/vbefb/vbefb.c b/sys/dev/vt/hw/vbefb/vbefb.c index 85f594e4ed21..2e8fe8330e56 100644 --- a/sys/dev/vt/hw/vbefb/vbefb.c +++ b/sys/dev/vt/hw/vbefb/vbefb.c @@ -1,164 +1,163 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Aleksandr Rybalko under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "opt_platform.h" #include #include #include #include #include #include #include static vd_init_t vt_vbefb_init; static vd_fini_t vt_vbefb_fini; static vd_probe_t vt_vbefb_probe; static struct vt_driver vt_vbefb_driver = { .vd_name = "vbefb", .vd_probe = vt_vbefb_probe, .vd_init = vt_vbefb_init, .vd_fini = vt_vbefb_fini, .vd_blank = vt_fb_blank, .vd_bitblt_text = vt_fb_bitblt_text, .vd_invalidate_text = vt_fb_invalidate_text, .vd_bitblt_bmp = vt_fb_bitblt_bitmap, .vd_drawrect = vt_fb_drawrect, .vd_setpixel = vt_fb_setpixel, .vd_fb_ioctl = vt_fb_ioctl, .vd_fb_mmap = vt_fb_mmap, .vd_suspend = vt_suspend, .vd_resume = vt_resume, /* Better than VGA, but still generic driver. */ .vd_priority = VD_PRIORITY_GENERIC + 1, }; static struct fb_info local_vbe_info; VT_DRIVER_DECLARE(vt_vbefb, vt_vbefb_driver); static int vt_vbefb_probe(struct vt_device *vd) { int disabled; struct vbe_fb *vbefb; caddr_t kmdp; disabled = 0; TUNABLE_INT_FETCH("hw.syscons.disable", &disabled); if (disabled != 0) return (CN_DEAD); kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); vbefb = (struct vbe_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_VBE_FB); if (vbefb == NULL) return (CN_DEAD); return (CN_INTERNAL); } static int vt_vbefb_init(struct vt_device *vd) { struct fb_info *info; struct vbe_fb *vbefb; caddr_t kmdp; int format, roff, goff, boff; info = vd->vd_softc; if (info == NULL) info = vd->vd_softc = (void *)&local_vbe_info; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); vbefb = (struct vbe_fb *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_VBE_FB); if (vbefb == NULL) return (CN_DEAD); info->fb_height = vbefb->fb_height; info->fb_width = vbefb->fb_width; info->fb_depth = vbefb->fb_bpp; /* Round to a multiple of the bits in a byte. */ info->fb_bpp = roundup2(vbefb->fb_bpp, NBBY); /* Stride in bytes, not pixels */ info->fb_stride = vbefb->fb_stride * (info->fb_bpp / NBBY); if (info->fb_depth == 8) format = COLOR_FORMAT_VGA; else format = COLOR_FORMAT_RGB; roff = ffs(vbefb->fb_mask_red) - 1; goff = ffs(vbefb->fb_mask_green) - 1; boff = ffs(vbefb->fb_mask_blue) - 1; vt_generate_cons_palette(info->fb_cmap, format, vbefb->fb_mask_red >> roff, roff, vbefb->fb_mask_green >> goff, goff, vbefb->fb_mask_blue >> boff, boff); /* Mark cmap initialized. */ info->fb_cmsize = NCOLORS; info->fb_size = info->fb_height * info->fb_stride; info->fb_pbase = vbefb->fb_addr; info->fb_vbase = (intptr_t)pmap_mapdev_attr(info->fb_pbase, info->fb_size, VM_MEMATTR_WRITE_COMBINING); vt_fb_init(vd); return (CN_INTERNAL); } static void vt_vbefb_fini(struct vt_device *vd, void *softc) { struct fb_info *info = softc; vt_fb_fini(vd, softc); pmap_unmapdev(info->fb_vbase, info->fb_size); } diff --git a/sys/dev/vt/logo/logo_freebsd.c b/sys/dev/vt/logo/logo_freebsd.c index f6d141f09cfc..605d1a8591bd 100644 --- a/sys/dev/vt/logo/logo_freebsd.c +++ b/sys/dev/vt/logo/logo_freebsd.c @@ -1,643 +1,642 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); unsigned int vt_logo_width = 257; unsigned int vt_logo_height = 219; unsigned int vt_logo_depth = 1; unsigned char vt_logo_image[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x07, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xfe, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xfc, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc3, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xe0, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xc0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x7f, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0x81, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xfe, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xfc, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xf0, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xc1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xfe, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xfc, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xf8, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xf8, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xe1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xc1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x87, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8f, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8f, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe3, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf1, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x7f, 0xff, 0xff, 0xf1, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x3f, 0xff, 0xff, 0xf3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x1f, 0xff, 0xff, 0xe3, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x8f, 0xff, 0xff, 0xc7, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc7, 0xff, 0xff, 0xc7, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe1, 0xff, 0xff, 0x8f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0xff, 0xff, 0x0f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x3f, 0xff, 0x07, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x0f, 0xfe, 0x07, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0xf8, 0x07, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x07, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x07, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x0f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x0f, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x1f, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x7f, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x7f, 0x80, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xc0, 0x00, 0x07, 0xff, 0xf0, 0x00, 0x01, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xe0, 0x00, 0x0f, 0xff, 0xfe, 0x00, 0x07, 0xff, 0xff, 0x80, 0x00, 0x00, 0x07, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xff, 0xff, 0xf8, 0x00, 0x3f, 0xff, 0xff, 0x80, 0x0f, 0xff, 0xff, 0xe0, 0x00, 0x00, 0x0f, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xf8, 0x1f, 0xf8, 0x00, 0x7f, 0xff, 0xff, 0xc0, 0x1f, 0xff, 0xff, 0xf8, 0x00, 0x00, 0x1f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xc0, 0x03, 0xfc, 0x00, 0xff, 0xe0, 0xff, 0xc0, 0x1f, 0x80, 0x0f, 0xfc, 0x00, 0x00, 0x3f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x80, 0x00, 0xfe, 0x00, 0xfe, 0x00, 0x1f, 0xc0, 0x1f, 0x00, 0x01, 0xff, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x7e, 0x01, 0xfc, 0x00, 0x07, 0x80, 0x3e, 0x00, 0x00, 0x7f, 0x80, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x3f, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x1f, 0xc0, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x3f, 0x03, 0xf0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x0f, 0xe0, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x1f, 0x03, 0xe0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x07, 0xf0, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x03, 0xe0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x03, 0xf0, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x1f, 0x03, 0xe0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x01, 0xf8, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x1f, 0x03, 0xe0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0xfc, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x07, 0xfc, 0x00, 0x00, 0x03, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xc0, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x1f, 0x03, 0xe0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x7c, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x1f, 0xff, 0x00, 0x00, 0x1f, 0xff, 0xc0, 0x00, 0x00, 0x03, 0xff, 0xf8, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x1f, 0x03, 0xf0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x7e, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xc0, 0x00, 0x7f, 0xff, 0xf0, 0x00, 0x00, 0x0f, 0xff, 0xfe, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x3f, 0x03, 0xf0, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0xfc, 0x00, 0x00, 0x00, 0xff, 0xff, 0xe0, 0x01, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x1f, 0xff, 0xff, 0x80, 0x00, 0xf8, 0x00, 0x00, 0x3f, 0x03, 0xf8, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0xfc, 0x00, 0x00, 0x01, 0xff, 0xff, 0xf0, 0x03, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xc0, 0x00, 0xf8, 0x00, 0x00, 0x7e, 0x01, 0xfc, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x03, 0xfc, 0x07, 0xf8, 0x07, 0xfe, 0x03, 0xff, 0x00, 0x00, 0xff, 0xc0, 0x3f, 0xe0, 0x00, 0xfc, 0x00, 0x00, 0xfe, 0x00, 0xfe, 0x00, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x07, 0xf0, 0x01, 0xf8, 0x0f, 0xf0, 0x00, 0x7f, 0x80, 0x01, 0xfe, 0x00, 0x0f, 0xf0, 0x00, 0xfc, 0x00, 0x01, 0xfc, 0x00, 0xff, 0x80, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x07, 0xe0, 0x00, 0xf8, 0x1f, 0xc0, 0x00, 0x3f, 0xc0, 0x03, 0xfc, 0x00, 0x03, 0xf8, 0x00, 0xfe, 0x00, 0x0f, 0xfc, 0x00, 0x7f, 0xe0, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x00, 0xfe, 0x00, 0x00, 0x0f, 0xc0, 0x00, 0x78, 0x1f, 0x80, 0x00, 0x0f, 0xc0, 0x03, 0xf0, 0x00, 0x01, 0xfc, 0x00, 0xff, 0xff, 0xff, 0xf8, 0x00, 0x3f, 0xf0, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0x00, 0x00, 0x0f, 0x80, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x07, 0xe0, 0x07, 0xe0, 0x00, 0x00, 0xfc, 0x00, 0xff, 0xff, 0xff, 0xfc, 0x00, 0x0f, 0xfc, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0xff, 0xfc, 0x0f, 0x80, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x03, 0xf0, 0x07, 0xe0, 0x00, 0x00, 0x7e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x07, 0xff, 0x00, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0xff, 0xfc, 0x1f, 0x80, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x03, 0xf0, 0x0f, 0xc0, 0x00, 0x00, 0x7e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x80, 0x01, 0xff, 0xc0, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0xff, 0xfe, 0x1f, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x01, 0xf0, 0x0f, 0x80, 0x00, 0x00, 0x3e, 0x00, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x00, 0x7f, 0xf0, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0xff, 0xfc, 0x1f, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x01, 0xf8, 0x1f, 0x80, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0xe0, 0x00, 0x1f, 0xfc, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xff, 0xff, 0xf8, 0x1f, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xf8, 0x1f, 0x80, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x07, 0xf0, 0x00, 0x07, 0xff, 0x00, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xfe, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x01, 0xf8, 0x1f, 0x80, 0x00, 0x00, 0x1f, 0x00, 0xf8, 0x00, 0x00, 0x03, 0xf8, 0x00, 0x01, 0xff, 0x80, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x80, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x80, 0xf8, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x7f, 0xc0, 0x3e, 0x00, 0x00, 0x00, 0x0f, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x80, 0xf8, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0xe0, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xfc, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x80, 0xf8, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x0f, 0xf0, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x80, 0xf8, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x03, 0xf8, 0x3e, 0x00, 0x00, 0x00, 0x1f, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x01, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x1f, 0xff, 0xff, 0xff, 0xff, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x01, 0xf8, 0x3e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x1f, 0xff, 0xff, 0xff, 0xfc, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x00, 0x3e, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x00, 0x7c, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x00, 0x7c, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x00, 0xf8, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x80, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x01, 0xf8, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xc0, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0x7c, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x03, 0xf0, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x7e, 0x00, 0x00, 0x00, 0x00, 0x07, 0xe0, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0xfc, 0x3e, 0x00, 0x00, 0x07, 0xe0, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x07, 0xe0, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x01, 0xfc, 0x3e, 0x00, 0x00, 0x0f, 0xe0, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x1f, 0x80, 0x00, 0x0c, 0x00, 0x03, 0xf0, 0x00, 0x01, 0x80, 0x00, 0xf8, 0x00, 0x00, 0x01, 0xf8, 0x70, 0x00, 0x03, 0xf8, 0x3e, 0x00, 0x00, 0x1f, 0xc0, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x1f, 0xc0, 0x00, 0x3e, 0x00, 0x03, 0xfc, 0x00, 0x03, 0xc0, 0x00, 0xf8, 0x00, 0x00, 0x03, 0xf8, 0xfc, 0x00, 0x07, 0xf8, 0x3f, 0x00, 0x00, 0x7f, 0x80, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x0f, 0xf0, 0x00, 0x7e, 0x00, 0x01, 0xfe, 0x00, 0x0f, 0xe0, 0x00, 0xf8, 0x00, 0x00, 0x07, 0xf0, 0xff, 0x80, 0x0f, 0xf0, 0x3f, 0x00, 0x01, 0xff, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x07, 0xfe, 0x03, 0xfe, 0x00, 0x00, 0xff, 0xc0, 0x3f, 0xc0, 0x00, 0xf8, 0x00, 0x00, 0x3f, 0xe0, 0xff, 0xf8, 0x7f, 0xe0, 0x3f, 0x80, 0x1f, 0xfc, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x03, 0xff, 0xff, 0xfe, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xc0, 0x00, 0xff, 0xff, 0xff, 0xff, 0xc0, 0xff, 0xff, 0xff, 0xc0, 0x3f, 0xff, 0xff, 0xf8, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x01, 0xff, 0xff, 0xfc, 0x00, 0x00, 0x1f, 0xff, 0xff, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0x80, 0x7f, 0xff, 0xff, 0x80, 0x3f, 0xff, 0xff, 0xe0, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xff, 0xf0, 0x00, 0x00, 0x0f, 0xff, 0xfe, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x0f, 0xff, 0xff, 0x00, 0x1f, 0xff, 0xff, 0x80, 0x00, 0x00, 0x78, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xff, 0xc0, 0x00, 0x00, 0x03, 0xff, 0xf8, 0x00, 0x00, 0x7f, 0xff, 0xff, 0xfc, 0x00, 0x03, 0xff, 0xfc, 0x00, 0x1f, 0xff, 0xfc, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x03, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x7f, 0xc0, 0x00, 0x00, 0x3f, 0xff, 0xff, 0xf0, 0x00, 0x00, 0x3f, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, }; diff --git a/sys/dev/vt/vt.h b/sys/dev/vt/vt.h index 2d671d692384..8a74fd97a4e0 100644 --- a/sys/dev/vt/vt.h +++ b/sys/dev/vt/vt.h @@ -1,453 +1,452 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Portions of this software were developed by Oleksandr Rybalko * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_VT_VT_H_ #define _DEV_VT_VT_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include "opt_syscons.h" #include "opt_splash.h" #ifndef VT_MAXWINDOWS #ifdef MAXCONS #define VT_MAXWINDOWS MAXCONS #else #define VT_MAXWINDOWS 12 #endif #endif #ifndef VT_ALT_TO_ESC_HACK #define VT_ALT_TO_ESC_HACK 1 #endif #define VT_CONSWINDOW 0 #if defined(SC_TWOBUTTON_MOUSE) || defined(VT_TWOBUTTON_MOUSE) #define VT_MOUSE_PASTEBUTTON MOUSE_BUTTON3DOWN /* right button */ #define VT_MOUSE_EXTENDBUTTON MOUSE_BUTTON2DOWN /* not really used */ #else #define VT_MOUSE_PASTEBUTTON MOUSE_BUTTON2DOWN /* middle button */ #define VT_MOUSE_EXTENDBUTTON MOUSE_BUTTON3DOWN /* right button */ #endif /* defined(SC_TWOBUTTON_MOUSE) || defined(VT_TWOBUTTON_MOUSE) */ #define SC_DRIVER_NAME "vt" #ifdef VT_DEBUG #define DPRINTF(_l, ...) if (vt_debug > (_l)) printf( __VA_ARGS__ ) #define VT_CONSOLECTL_DEBUG #define VT_SYSMOUSE_DEBUG #else #define DPRINTF(_l, ...) do {} while (0) #endif #define ISSIGVALID(sig) ((sig) > 0 && (sig) < NSIG) #define VT_SYSCTL_INT(_name, _default, _descr) \ int vt_##_name = (_default); \ SYSCTL_INT(_kern_vt, OID_AUTO, _name, CTLFLAG_RWTUN, &vt_##_name, 0, _descr) struct vt_driver; void vt_allocate(const struct vt_driver *, void *); void vt_deallocate(const struct vt_driver *, void *); typedef unsigned int vt_axis_t; /* * List of locks * (d) locked by vd_lock * (b) locked by vb_lock * (G) locked by Giant * (u) unlocked, locked by higher levels * (c) const until freeing * (?) yet to be determined */ /* * Per-device datastructure. */ #ifndef SC_NO_CUTPASTE struct vt_mouse_cursor; #endif struct vt_pastebuf { term_char_t *vpb_buf; /* Copy-paste buffer. */ unsigned int vpb_bufsz; /* Buffer size. */ unsigned int vpb_len; /* Length of a last selection. */ }; struct vt_device { struct vt_window *vd_windows[VT_MAXWINDOWS]; /* (c) Windows. */ struct vt_window *vd_curwindow; /* (d) Current window. */ struct vt_window *vd_savedwindow;/* (?) Saved for suspend. */ struct vt_window *vd_grabwindow; /* (?) Saved before cngrab. */ struct vt_pastebuf vd_pastebuf; /* (?) Copy/paste buf. */ const struct vt_driver *vd_driver; /* (c) Graphics driver. */ void *vd_softc; /* (u) Driver data. */ const struct vt_driver *vd_prev_driver;/* (?) Previous driver. */ void *vd_prev_softc; /* (?) Previous driver data. */ device_t vd_video_dev; /* (?) Video adapter. */ #ifndef SC_NO_CUTPASTE struct vt_mouse_cursor *vd_mcursor; /* (?) Cursor bitmap. */ term_color_t vd_mcursor_fg; /* (?) Cursor fg color. */ term_color_t vd_mcursor_bg; /* (?) Cursor bg color. */ vt_axis_t vd_mx_drawn; /* (?) Mouse X and Y */ vt_axis_t vd_my_drawn; /* as of last redraw. */ int vd_mshown; /* (?) Mouse shown during */ #endif /* last redrawn. */ uint16_t vd_mx; /* (?) Current mouse X. */ uint16_t vd_my; /* (?) current mouse Y. */ uint32_t vd_mstate; /* (?) Mouse state. */ vt_axis_t vd_width; /* (?) Screen width. */ vt_axis_t vd_height; /* (?) Screen height. */ size_t vd_transpose; /* (?) Screen offset in FB */ struct mtx vd_lock; /* Per-device lock. */ struct cv vd_winswitch; /* (d) Window switch notify. */ struct callout vd_timer; /* (d) Display timer. */ volatile unsigned int vd_timer_armed;/* (?) Display timer started.*/ int vd_flags; /* (d) Device flags. */ #define VDF_TEXTMODE 0x01 /* Do text mode rendering. */ #define VDF_SPLASH 0x02 /* Splash screen active. */ #define VDF_ASYNC 0x04 /* vt_timer() running. */ #define VDF_INVALID 0x08 /* Entire screen should be re-rendered. */ #define VDF_DEAD 0x10 /* Early probing found nothing. */ #define VDF_INITIALIZED 0x20 /* vtterm_cnprobe already done. */ #define VDF_MOUSECURSOR 0x40 /* Mouse cursor visible. */ #define VDF_QUIET_BELL 0x80 /* Disable bell. */ #define VDF_SUSPENDED 0x100 /* Device has been suspended. */ #define VDF_DOWNGRADE 0x8000 /* The driver is being downgraded. */ struct keyboard *vd_keyboard; /* (G) Keyboard. */ unsigned int vd_kbstate; /* (?) Device unit. */ unsigned int vd_unit; /* (c) Device unit. */ int vd_altbrk; /* (?) Alt break seq. state */ term_char_t *vd_drawn; /* (?) Most recent char drawn. */ term_color_t *vd_drawnfg; /* (?) Most recent fg color drawn. */ term_color_t *vd_drawnbg; /* (?) Most recent bg color drawn. */ }; #define VD_PASTEBUF(vd) ((vd)->vd_pastebuf.vpb_buf) #define VD_PASTEBUFSZ(vd) ((vd)->vd_pastebuf.vpb_bufsz) #define VD_PASTEBUFLEN(vd) ((vd)->vd_pastebuf.vpb_len) #define VT_LOCK(vd) mtx_lock(&(vd)->vd_lock) #define VT_UNLOCK(vd) mtx_unlock(&(vd)->vd_lock) #define VT_LOCK_ASSERT(vd, what) mtx_assert(&(vd)->vd_lock, what) void vt_resume(struct vt_device *vd); void vt_resume_flush_timer(struct vt_window *vw, int ms); void vt_suspend(struct vt_device *vd); /* * Per-window terminal screen buffer. * * Because redrawing is performed asynchronously, the buffer keeps track * of a rectangle that needs to be redrawn (vb_dirtyrect). Because this * approach seemed to cause suboptimal performance (when the top left * and the bottom right of the screen are modified), it also uses a set * of bitmasks to keep track of the rows and columns (mod 64) that have * been modified. */ struct vt_buf { struct mtx vb_lock; /* Buffer lock. */ struct terminal *vb_terminal; term_pos_t vb_scr_size; /* (b) Screen dimensions. */ int vb_flags; /* (b) Flags. */ #define VBF_CURSOR 0x1 /* Cursor visible. */ #define VBF_STATIC 0x2 /* Buffer is statically allocated. */ #define VBF_MTX_INIT 0x4 /* Mutex initialized. */ #define VBF_SCROLL 0x8 /* scroll locked mode. */ #define VBF_HISTORY_FULL 0x10 /* All rows filled. */ unsigned int vb_history_size; unsigned int vb_roffset; /* (b) History rows offset. */ unsigned int vb_curroffset; /* (b) Saved rows offset. */ term_pos_t vb_cursor; /* (u) Cursor position. */ term_pos_t vb_mark_start; /* (b) Copy region start. */ term_pos_t vb_mark_end; /* (b) Copy region end. */ int vb_mark_last; /* Last mouse event. */ term_rect_t vb_dirtyrect; /* (b) Dirty rectangle. */ term_char_t *vb_buffer; /* (u) Data buffer. */ term_char_t **vb_rows; /* (u) Array of rows */ }; #ifdef SC_HISTORY_SIZE #define VBF_DEFAULT_HISTORY_SIZE SC_HISTORY_SIZE #else #define VBF_DEFAULT_HISTORY_SIZE 500 #endif void vtbuf_lock(struct vt_buf *); void vtbuf_unlock(struct vt_buf *); void vtbuf_copy(struct vt_buf *, const term_rect_t *, const term_pos_t *); void vtbuf_fill(struct vt_buf *, const term_rect_t *, term_char_t); void vtbuf_init_early(struct vt_buf *); void vtbuf_init(struct vt_buf *, const term_pos_t *); void vtbuf_grow(struct vt_buf *, const term_pos_t *, unsigned int); void vtbuf_putchar(struct vt_buf *, const term_pos_t *, term_char_t); void vtbuf_cursor_position(struct vt_buf *, const term_pos_t *); void vtbuf_scroll_mode(struct vt_buf *vb, int yes); void vtbuf_dirty(struct vt_buf *vb, const term_rect_t *area); void vtbuf_undirty(struct vt_buf *, term_rect_t *); void vtbuf_sethistory_size(struct vt_buf *, unsigned int); void vtbuf_clearhistory(struct vt_buf *); int vtbuf_iscursor(const struct vt_buf *vb, int row, int col); void vtbuf_cursor_visibility(struct vt_buf *, int); #ifndef SC_NO_CUTPASTE int vtbuf_set_mark(struct vt_buf *vb, int type, int col, int row); int vtbuf_get_marked_len(struct vt_buf *vb); void vtbuf_extract_marked(struct vt_buf *vb, term_char_t *buf, int sz); #endif #define VTB_MARK_NONE 0 #define VTB_MARK_END 1 #define VTB_MARK_START 2 #define VTB_MARK_WORD 3 #define VTB_MARK_ROW 4 #define VTB_MARK_EXTEND 5 #define VTB_MARK_MOVE 6 #define VTBUF_SLCK_ENABLE(vb) vtbuf_scroll_mode((vb), 1) #define VTBUF_SLCK_DISABLE(vb) vtbuf_scroll_mode((vb), 0) #define VTBUF_MAX_HEIGHT(vb) \ ((vb)->vb_history_size) #define VTBUF_GET_ROW(vb, r) \ ((vb)->vb_rows[((vb)->vb_roffset + (r)) % VTBUF_MAX_HEIGHT(vb)]) #define VTBUF_GET_FIELD(vb, r, c) \ ((vb)->vb_rows[((vb)->vb_roffset + (r)) % VTBUF_MAX_HEIGHT(vb)][(c)]) #define VTBUF_FIELD(vb, r, c) \ ((vb)->vb_rows[((vb)->vb_curroffset + (r)) % VTBUF_MAX_HEIGHT(vb)][(c)]) #define VTBUF_ISCURSOR(vb, r, c) \ vtbuf_iscursor((vb), (r), (c)) #define VTBUF_DIRTYROW(mask, row) \ ((mask)->vbm_row & ((uint64_t)1 << ((row) % 64))) #define VTBUF_DIRTYCOL(mask, col) \ ((mask)->vbm_col & ((uint64_t)1 << ((col) % 64))) #define VTBUF_SPACE_CHAR(attr) (' ' | (attr)) #define VHS_SET 0 #define VHS_CUR 1 #define VHS_END 2 int vthistory_seek(struct vt_buf *, int offset, int whence); void vthistory_addlines(struct vt_buf *vb, int offset); void vthistory_getpos(const struct vt_buf *, unsigned int *offset); /* * Per-window datastructure. */ struct vt_window { struct vt_device *vw_device; /* (c) Device. */ struct terminal *vw_terminal; /* (c) Terminal. */ struct vt_buf vw_buf; /* (u) Screen buffer. */ struct vt_font *vw_font; /* (d) Graphical font. */ term_rect_t vw_draw_area; /* (?) Drawable area. */ unsigned int vw_number; /* (c) Window number. */ int vw_kbdmode; /* (?) Keyboard mode. */ int vw_prev_kbdmode;/* (?) Previous mode. */ int vw_kbdstate; /* (?) Keyboard state. */ int vw_grabbed; /* (?) Grab count. */ char *vw_kbdsq; /* Escape sequence queue*/ unsigned int vw_flags; /* (d) Per-window flags. */ int vw_mouse_level;/* Mouse op mode. */ #define VWF_BUSY 0x1 /* Busy reconfiguring device. */ #define VWF_OPENED 0x2 /* TTY in use. */ #define VWF_SCROLL 0x4 /* Keys influence scrollback. */ #define VWF_CONSOLE 0x8 /* Kernel message console window. */ #define VWF_VTYLOCK 0x10 /* Prevent window switch. */ #define VWF_MOUSE_HIDE 0x20 /* Disable mouse events processing. */ #define VWF_READY 0x40 /* Window fully initialized. */ #define VWF_GRAPHICS 0x80 /* Window in graphics mode (KDSETMODE). */ #define VWF_SWWAIT_REL 0x10000 /* Program wait for VT acquire is done. */ #define VWF_SWWAIT_ACQ 0x20000 /* Program wait for VT release is done. */ pid_t vw_pid; /* Terminal holding process */ struct proc *vw_proc; struct vt_mode vw_smode; /* switch mode */ struct callout vw_proc_dead_timer; struct vt_window *vw_switch_to; }; #define VT_AUTO 0 /* switching is automatic */ #define VT_PROCESS 1 /* switching controlled by prog */ #define VT_KERNEL 255 /* switching controlled in kernel */ #define IS_VT_PROC_MODE(vw) ((vw)->vw_smode.mode == VT_PROCESS) /* * Per-device driver routines. */ typedef int vd_init_t(struct vt_device *vd); typedef int vd_probe_t(struct vt_device *vd); typedef void vd_fini_t(struct vt_device *vd, void *softc); typedef void vd_postswitch_t(struct vt_device *vd); typedef void vd_blank_t(struct vt_device *vd, term_color_t color); typedef void vd_bitblt_text_t(struct vt_device *vd, const struct vt_window *vw, const term_rect_t *area); typedef void vd_invalidate_text_t(struct vt_device *vd, const term_rect_t *area); typedef void vd_bitblt_bmp_t(struct vt_device *vd, const struct vt_window *vw, const uint8_t *pattern, const uint8_t *mask, unsigned int width, unsigned int height, unsigned int x, unsigned int y, term_color_t fg, term_color_t bg); typedef int vd_fb_ioctl_t(struct vt_device *, u_long, caddr_t, struct thread *); typedef int vd_fb_mmap_t(struct vt_device *, vm_ooffset_t, vm_paddr_t *, int, vm_memattr_t *); typedef void vd_drawrect_t(struct vt_device *, int, int, int, int, int, term_color_t); typedef void vd_setpixel_t(struct vt_device *, int, int, term_color_t); typedef void vd_suspend_t(struct vt_device *); typedef void vd_resume_t(struct vt_device *); struct vt_driver { char vd_name[16]; /* Console attachment. */ vd_probe_t *vd_probe; vd_init_t *vd_init; vd_fini_t *vd_fini; /* Drawing. */ vd_blank_t *vd_blank; vd_drawrect_t *vd_drawrect; vd_setpixel_t *vd_setpixel; vd_bitblt_text_t *vd_bitblt_text; vd_invalidate_text_t *vd_invalidate_text; vd_bitblt_bmp_t *vd_bitblt_bmp; /* Framebuffer ioctls, if present. */ vd_fb_ioctl_t *vd_fb_ioctl; /* Framebuffer mmap, if present. */ vd_fb_mmap_t *vd_fb_mmap; /* Update display setting on vt switch. */ vd_postswitch_t *vd_postswitch; /* Suspend/resume handlers. */ vd_suspend_t *vd_suspend; vd_resume_t *vd_resume; /* Priority to know which one can override */ int vd_priority; #define VD_PRIORITY_DUMB 10 #define VD_PRIORITY_GENERIC 100 #define VD_PRIORITY_SPECIFIC 1000 }; /* * Console device madness. * * Utility macro to make early vt(4) instances work. */ extern struct vt_device vt_consdev; extern struct terminal vt_consterm; extern const struct terminal_class vt_termclass; void vt_upgrade(struct vt_device *vd); #define PIXEL_WIDTH(w) ((w) / 8) #define PIXEL_HEIGHT(h) ((h) / 16) #ifndef VT_FB_MAX_WIDTH #define VT_FB_MAX_WIDTH 4096 #endif #ifndef VT_FB_MAX_HEIGHT #define VT_FB_MAX_HEIGHT 2400 #endif /* name argument is not used yet. */ #define VT_DRIVER_DECLARE(name, drv) DATA_SET(vt_drv_set, drv) #ifndef SC_NO_CUTPASTE struct vt_mouse_cursor { uint8_t map[64 * 64 / 8]; uint8_t mask[64 * 64 / 8]; uint8_t width; uint8_t height; }; #endif const uint8_t *vtfont_lookup(const struct vt_font *vf, term_char_t c); struct vt_font *vtfont_ref(struct vt_font *vf); void vtfont_unref(struct vt_font *vf); int vtfont_load(vfnt_t *f, struct vt_font **ret); /* Sysmouse. */ void sysmouse_process_event(mouse_info_t *mi); #ifndef SC_NO_CUTPASTE void vt_mouse_event(int type, int x, int y, int event, int cnt, int mlevel); void vt_mouse_state(int show); #endif #define VT_MOUSE_SHOW 1 #define VT_MOUSE_HIDE 0 /* Utilities. */ void vt_compute_drawable_area(struct vt_window *); void vt_determine_colors(term_char_t c, int cursor, term_color_t *fg, term_color_t *bg); int vt_is_cursor_in_area(const struct vt_device *vd, const term_rect_t *area); void vt_termsize(struct vt_device *, struct vt_font *, term_pos_t *); void vt_winsize(struct vt_device *, struct vt_font *, struct winsize *); /* Logos-on-boot. */ #define VT_LOGOS_DRAW_BEASTIE 0 #define VT_LOGOS_DRAW_ALT_BEASTIE 1 #define VT_LOGOS_DRAW_ORB 2 extern int vt_draw_logo_cpus; extern int vt_splash_cpu; extern int vt_splash_ncpu; extern int vt_splash_cpu_style; extern int vt_splash_cpu_duration; extern const unsigned int vt_logo_sprite_height; extern const unsigned int vt_logo_sprite_width; void vtterm_draw_cpu_logos(struct vt_device *); #endif /* !_DEV_VT_VT_H_ */ diff --git a/sys/dev/vt/vt_buf.c b/sys/dev/vt/vt_buf.c index f147449b20e2..9c9417d6eca4 100644 --- a/sys/dev/vt/vt_buf.c +++ b/sys/dev/vt/vt_buf.c @@ -1,883 +1,882 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Portions of this software were developed by Oleksandr Rybalko * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_VTBUF, "vtbuf", "vt buffer"); #define VTBUF_LOCK(vb) mtx_lock_spin(&(vb)->vb_lock) #define VTBUF_UNLOCK(vb) mtx_unlock_spin(&(vb)->vb_lock) #define POS_INDEX(c, r) (((r) << 12) + (c)) #define POS_COPY(d, s) do { \ (d).tp_col = (s).tp_col; \ (d).tp_row = (s).tp_row; \ } while (0) #ifndef SC_NO_CUTPASTE static int vtbuf_htw(const struct vt_buf *vb, int row); static int vtbuf_wth(const struct vt_buf *vb, int row); static int vtbuf_in_this_range(int begin, int test, int end, int sz); #endif /* * line4 * line5 <--- curroffset (terminal output to that line) * line0 * line1 <--- roffset (history display from that point) * line2 * line3 */ int vthistory_seek(struct vt_buf *vb, int offset, int whence) { int diff, top, bottom, roffset; /* No scrolling if not enabled. */ if ((vb->vb_flags & VBF_SCROLL) == 0) { if (vb->vb_roffset != vb->vb_curroffset) { vb->vb_roffset = vb->vb_curroffset; return (0xffff); } return (0); /* No changes */ } /* "top" may be a negative integer. */ bottom = vb->vb_curroffset; top = (vb->vb_flags & VBF_HISTORY_FULL) ? bottom + vb->vb_scr_size.tp_row - vb->vb_history_size : 0; roffset = 0; /* Make gcc happy. */ switch (whence) { case VHS_SET: if (offset < 0) offset = 0; roffset = top + offset; break; case VHS_CUR: /* * Operate on copy of offset value, since it temporary * can be bigger than amount of rows in buffer. */ roffset = vb->vb_roffset; if (roffset >= bottom + vb->vb_scr_size.tp_row) roffset -= vb->vb_history_size; roffset += offset; roffset = MAX(roffset, top); roffset = MIN(roffset, bottom); if (roffset < 0) roffset = vb->vb_history_size + roffset; break; case VHS_END: /* Go to current offset. */ roffset = vb->vb_curroffset; break; } diff = vb->vb_roffset != roffset; vb->vb_roffset = roffset; return (diff); } void vthistory_addlines(struct vt_buf *vb, int offset) { #ifndef SC_NO_CUTPASTE int cur, sz; #endif vb->vb_curroffset += offset; if (vb->vb_curroffset + vb->vb_scr_size.tp_row >= vb->vb_history_size) { vb->vb_flags |= VBF_HISTORY_FULL; vb->vb_curroffset %= vb->vb_history_size; } if ((vb->vb_flags & VBF_SCROLL) == 0) { vb->vb_roffset = vb->vb_curroffset; } #ifndef SC_NO_CUTPASTE sz = vb->vb_history_size; cur = vb->vb_roffset + vb->vb_scr_size.tp_row + sz - 1; if (vtbuf_in_this_range(cur, vb->vb_mark_start.tp_row, cur + offset, sz) || vtbuf_in_this_range(cur, vb->vb_mark_end.tp_row, cur + offset, sz)) { /* clear screen selection */ vb->vb_mark_start.tp_row = vb->vb_mark_end.tp_row; vb->vb_mark_start.tp_col = vb->vb_mark_end.tp_col; } #endif } void vthistory_getpos(const struct vt_buf *vb, unsigned int *offset) { *offset = vb->vb_roffset; } #ifndef SC_NO_CUTPASTE /* Only mouse support use it now. */ /* Translate history row to current view row number. */ static int vtbuf_htw(const struct vt_buf *vb, int row) { /* * total 1000 rows. * History offset roffset winrow * 205 200 ((205 - 200 + 1000) % 1000) = 5 * 90 990 ((90 - 990 + 1000) % 1000) = 100 */ return ((row - vb->vb_roffset + vb->vb_history_size) % vb->vb_history_size); } /* Translate current view row number to history row. */ static int vtbuf_wth(const struct vt_buf *vb, int row) { return ((vb->vb_roffset + row) % vb->vb_history_size); } /* * Test if an index in a circular buffer is within a range. * * begin - start index * end - end index * test - test index * sz - size of circular buffer when it turns over */ static int vtbuf_in_this_range(int begin, int test, int end, int sz) { begin %= sz; end %= sz; /* check for inversion */ if (begin > end) return (test >= begin || test < end); else return (test >= begin && test < end); } #endif int vtbuf_iscursor(const struct vt_buf *vb, int row, int col) { #ifndef SC_NO_CUTPASTE int sc, sr, sz, ec, er, tmp; #endif if ((vb->vb_flags & (VBF_CURSOR|VBF_SCROLL)) == VBF_CURSOR && (vb->vb_cursor.tp_row == row) && (vb->vb_cursor.tp_col == col)) return (1); #ifndef SC_NO_CUTPASTE /* Mark cut/paste region. */ if (vb->vb_mark_start.tp_col == vb->vb_mark_end.tp_col && vb->vb_mark_start.tp_row == vb->vb_mark_end.tp_row) return (0); sc = vb->vb_mark_start.tp_col; sr = vb->vb_mark_start.tp_row; ec = vb->vb_mark_end.tp_col; er = vb->vb_mark_end.tp_row; /* * Information about if the selection was made bottom-top or * top-bottom is lost due to modulo arithmetics and needs to * be recovered: */ sz = vb->vb_history_size; tmp = (sz + er - sr) % sz; row = vtbuf_wth(vb, row); /* Swap start and end if start > end */ if ((2 * tmp) > sz || (tmp == 0 && sc > ec)) { tmp = sc; sc = ec; ec = tmp; tmp = sr; sr = er; er = tmp; } if (vtbuf_in_this_range(POS_INDEX(sc, sr), POS_INDEX(col, row), POS_INDEX(ec, er), POS_INDEX(0, sz))) return (1); #endif return (0); } void vtbuf_lock(struct vt_buf *vb) { VTBUF_LOCK(vb); } void vtbuf_unlock(struct vt_buf *vb) { VTBUF_UNLOCK(vb); } void vtbuf_dirty(struct vt_buf *vb, const term_rect_t *area) { if (vb->vb_dirtyrect.tr_begin.tp_row > area->tr_begin.tp_row) vb->vb_dirtyrect.tr_begin.tp_row = area->tr_begin.tp_row; if (vb->vb_dirtyrect.tr_begin.tp_col > area->tr_begin.tp_col) vb->vb_dirtyrect.tr_begin.tp_col = area->tr_begin.tp_col; if (vb->vb_dirtyrect.tr_end.tp_row < area->tr_end.tp_row) vb->vb_dirtyrect.tr_end.tp_row = area->tr_end.tp_row; if (vb->vb_dirtyrect.tr_end.tp_col < area->tr_end.tp_col) vb->vb_dirtyrect.tr_end.tp_col = area->tr_end.tp_col; } static inline void vtbuf_dirty_cell(struct vt_buf *vb, const term_pos_t *p) { term_rect_t area; area.tr_begin = *p; area.tr_end.tp_row = p->tp_row + 1; area.tr_end.tp_col = p->tp_col + 1; vtbuf_dirty(vb, &area); } static void vtbuf_make_undirty(struct vt_buf *vb) { vb->vb_dirtyrect.tr_begin = vb->vb_scr_size; vb->vb_dirtyrect.tr_end.tp_row = vb->vb_dirtyrect.tr_end.tp_col = 0; } void vtbuf_undirty(struct vt_buf *vb, term_rect_t *r) { *r = vb->vb_dirtyrect; vtbuf_make_undirty(vb); } void vtbuf_copy(struct vt_buf *vb, const term_rect_t *r, const term_pos_t *p2) { const term_pos_t *p1 = &r->tr_begin; term_rect_t area; unsigned int rows, cols; int pr, rdiff; KASSERT(r->tr_begin.tp_row < vb->vb_scr_size.tp_row, ("vtbuf_copy begin.tp_row %d must be less than screen width %d", r->tr_begin.tp_row, vb->vb_scr_size.tp_row)); KASSERT(r->tr_begin.tp_col < vb->vb_scr_size.tp_col, ("vtbuf_copy begin.tp_col %d must be less than screen height %d", r->tr_begin.tp_col, vb->vb_scr_size.tp_col)); KASSERT(r->tr_end.tp_row <= vb->vb_scr_size.tp_row, ("vtbuf_copy end.tp_row %d must be less than screen width %d", r->tr_end.tp_row, vb->vb_scr_size.tp_row)); KASSERT(r->tr_end.tp_col <= vb->vb_scr_size.tp_col, ("vtbuf_copy end.tp_col %d must be less than screen height %d", r->tr_end.tp_col, vb->vb_scr_size.tp_col)); KASSERT(p2->tp_row < vb->vb_scr_size.tp_row, ("vtbuf_copy tp_row %d must be less than screen width %d", p2->tp_row, vb->vb_scr_size.tp_row)); KASSERT(p2->tp_col < vb->vb_scr_size.tp_col, ("vtbuf_copy tp_col %d must be less than screen height %d", p2->tp_col, vb->vb_scr_size.tp_col)); rows = r->tr_end.tp_row - r->tr_begin.tp_row; rdiff = r->tr_begin.tp_row - p2->tp_row; cols = r->tr_end.tp_col - r->tr_begin.tp_col; if (r->tr_begin.tp_row > p2->tp_row && r->tr_begin.tp_col == 0 && r->tr_end.tp_col == vb->vb_scr_size.tp_col && /* Full row. */ (rows + rdiff) == vb->vb_scr_size.tp_row && /* Whole screen. */ rdiff > 0) { /* Only forward direction. Do not eat history. */ vthistory_addlines(vb, rdiff); } else if (p2->tp_row < p1->tp_row) { /* Handle overlapping copies of line segments. */ /* Move data up. */ for (pr = 0; pr < rows; pr++) memmove( &VTBUF_FIELD(vb, p2->tp_row + pr, p2->tp_col), &VTBUF_FIELD(vb, p1->tp_row + pr, p1->tp_col), cols * sizeof(term_char_t)); } else { /* Move data down. */ for (pr = rows - 1; pr >= 0; pr--) memmove( &VTBUF_FIELD(vb, p2->tp_row + pr, p2->tp_col), &VTBUF_FIELD(vb, p1->tp_row + pr, p1->tp_col), cols * sizeof(term_char_t)); } area.tr_begin = *p2; area.tr_end.tp_row = MIN(p2->tp_row + rows, vb->vb_scr_size.tp_row); area.tr_end.tp_col = MIN(p2->tp_col + cols, vb->vb_scr_size.tp_col); vtbuf_dirty(vb, &area); } static void vtbuf_do_fill(struct vt_buf *vb, const term_rect_t *r, term_char_t c) { unsigned int pr, pc; term_char_t *row; for (pr = r->tr_begin.tp_row; pr < r->tr_end.tp_row; pr++) { row = vb->vb_rows[(vb->vb_curroffset + pr) % VTBUF_MAX_HEIGHT(vb)]; for (pc = r->tr_begin.tp_col; pc < r->tr_end.tp_col; pc++) { row[pc] = c; } } } void vtbuf_fill(struct vt_buf *vb, const term_rect_t *r, term_char_t c) { KASSERT(r->tr_begin.tp_row < vb->vb_scr_size.tp_row, ("vtbuf_fill begin.tp_row %d must be < screen height %d", r->tr_begin.tp_row, vb->vb_scr_size.tp_row)); KASSERT(r->tr_begin.tp_col < vb->vb_scr_size.tp_col, ("vtbuf_fill begin.tp_col %d must be < screen width %d", r->tr_begin.tp_col, vb->vb_scr_size.tp_col)); KASSERT(r->tr_end.tp_row <= vb->vb_scr_size.tp_row, ("vtbuf_fill end.tp_row %d must be <= screen height %d", r->tr_end.tp_row, vb->vb_scr_size.tp_row)); KASSERT(r->tr_end.tp_col <= vb->vb_scr_size.tp_col, ("vtbuf_fill end.tp_col %d must be <= screen width %d", r->tr_end.tp_col, vb->vb_scr_size.tp_col)); vtbuf_do_fill(vb, r, c); vtbuf_dirty(vb, r); } static void vtbuf_init_rows(struct vt_buf *vb) { int r; vb->vb_history_size = MAX(vb->vb_history_size, vb->vb_scr_size.tp_row); for (r = 0; r < vb->vb_history_size; r++) vb->vb_rows[r] = &vb->vb_buffer[r * vb->vb_scr_size.tp_col]; } static void vtbuf_do_clearhistory(struct vt_buf *vb) { term_rect_t rect; const teken_attr_t *a; term_char_t ch; a = teken_get_curattr(&vb->vb_terminal->tm_emulator); ch = TCOLOR_FG(a->ta_fgcolor) | TCOLOR_BG(a->ta_bgcolor); rect.tr_begin.tp_row = rect.tr_begin.tp_col = 0; rect.tr_end.tp_col = vb->vb_scr_size.tp_col; rect.tr_end.tp_row = vb->vb_history_size; vtbuf_do_fill(vb, &rect, VTBUF_SPACE_CHAR(ch)); } static void vtbuf_reset_scrollback(struct vt_buf *vb) { vb->vb_roffset = 0; vb->vb_curroffset = 0; vb->vb_mark_start.tp_row = 0; vb->vb_mark_start.tp_col = 0; vb->vb_mark_end.tp_row = 0; vb->vb_mark_end.tp_col = 0; } void vtbuf_init_early(struct vt_buf *vb) { vb->vb_flags |= VBF_CURSOR; vtbuf_reset_scrollback(vb); vtbuf_init_rows(vb); vtbuf_do_clearhistory(vb); vtbuf_make_undirty(vb); if ((vb->vb_flags & VBF_MTX_INIT) == 0) { mtx_init(&vb->vb_lock, "vtbuf", NULL, MTX_SPIN); vb->vb_flags |= VBF_MTX_INIT; } } void vtbuf_init(struct vt_buf *vb, const term_pos_t *p) { int sz; vb->vb_scr_size = *p; vb->vb_history_size = VBF_DEFAULT_HISTORY_SIZE; if ((vb->vb_flags & VBF_STATIC) == 0) { sz = vb->vb_history_size * p->tp_col * sizeof(term_char_t); vb->vb_buffer = malloc(sz, M_VTBUF, M_WAITOK | M_ZERO); sz = vb->vb_history_size * sizeof(term_char_t *); vb->vb_rows = malloc(sz, M_VTBUF, M_WAITOK | M_ZERO); } vtbuf_init_early(vb); } void vtbuf_clearhistory(struct vt_buf *vb) { VTBUF_LOCK(vb); vtbuf_do_clearhistory(vb); vtbuf_reset_scrollback(vb); vb->vb_flags &= ~VBF_HISTORY_FULL; VTBUF_UNLOCK(vb); } void vtbuf_sethistory_size(struct vt_buf *vb, unsigned int size) { term_pos_t p; /* With same size */ p.tp_row = vb->vb_scr_size.tp_row; p.tp_col = vb->vb_scr_size.tp_col; vtbuf_grow(vb, &p, size); } void vtbuf_grow(struct vt_buf *vb, const term_pos_t *p, unsigned int history_size) { term_char_t *old, *new, **rows, **oldrows, **copyrows, *row, *oldrow; unsigned int w, h, c, r, old_history_size; size_t bufsize, rowssize; int history_full; const teken_attr_t *a; term_char_t ch; a = teken_get_curattr(&vb->vb_terminal->tm_emulator); ch = TCOLOR_FG(a->ta_fgcolor) | TCOLOR_BG(a->ta_bgcolor); history_size = MAX(history_size, p->tp_row); /* Allocate new buffer. */ bufsize = history_size * p->tp_col * sizeof(term_char_t); new = malloc(bufsize, M_VTBUF, M_WAITOK | M_ZERO); rowssize = history_size * sizeof(term_pos_t *); rows = malloc(rowssize, M_VTBUF, M_WAITOK | M_ZERO); /* Toggle it. */ VTBUF_LOCK(vb); old = vb->vb_flags & VBF_STATIC ? NULL : vb->vb_buffer; oldrows = vb->vb_flags & VBF_STATIC ? NULL : vb->vb_rows; copyrows = vb->vb_rows; w = vb->vb_scr_size.tp_col; h = vb->vb_scr_size.tp_row; old_history_size = vb->vb_history_size; history_full = vb->vb_flags & VBF_HISTORY_FULL || vb->vb_curroffset + h >= history_size; vb->vb_history_size = history_size; vb->vb_buffer = new; vb->vb_rows = rows; vb->vb_flags &= ~VBF_STATIC; vb->vb_scr_size = *p; vtbuf_init_rows(vb); /* * Copy rows to the new buffer. The first row in the history * is back to index 0, ie. the new buffer doesn't cycle. */ if (history_size > old_history_size) { for (r = 0; r < old_history_size; r ++) { row = rows[r]; /* Compute the corresponding row in the old buffer. */ if (history_full) /* * The buffer is full, the "top" row is * the one just after the viewable area * (curroffset + viewable height) in the * cycling buffer. The corresponding row * is computed from this top row. */ oldrow = copyrows[ (vb->vb_curroffset + h + r) % old_history_size]; else /* * The buffer is not full, therefore, * we didn't cycle already. The * corresponding rows are the same in * both buffers. */ oldrow = copyrows[r]; memmove(row, oldrow, MIN(p->tp_col, w) * sizeof(term_char_t)); /* * XXX VTBUF_SPACE_CHAR(TERMINAL_NORM_ATTR) will * extended lines of kernel text using the wrong * background color. */ for (c = MIN(p->tp_col, w); c < p->tp_col; c++) { row[c] = VTBUF_SPACE_CHAR(ch); } } /* Fill remaining rows. */ for (r = old_history_size; r < history_size; r++) { row = rows[r]; for (c = MIN(p->tp_col, w); c < p->tp_col; c++) { row[c] = VTBUF_SPACE_CHAR(ch); } } vb->vb_flags &= ~VBF_HISTORY_FULL; /* * If the screen is already filled (there are non-visible lines * above the current viewable area), adjust curroffset to the * new viewable area. * * If the old buffer was full, set curroffset to the * th most recent line of history in the new, non-cycled * buffer. Otherwise, it didn't cycle, so the old curroffset * is the same in the new buffer. */ if (history_full) vb->vb_curroffset = old_history_size - h; } else { /* * (old_history_size - history_size) lines of history are * dropped. */ for (r = 0; r < history_size; r ++) { row = rows[r]; /* * Compute the corresponding row in the old buffer. * * See the equivalent if{} block above for an * explanation. */ if (history_full) oldrow = copyrows[ (vb->vb_curroffset + h + r + (old_history_size - history_size)) % old_history_size]; else oldrow = copyrows[r]; memmove(row, oldrow, MIN(p->tp_col, w) * sizeof(term_char_t)); /* * XXX VTBUF_SPACE_CHAR(TERMINAL_NORM_ATTR) will * extended lines of kernel text using the wrong * background color. */ for (c = MIN(p->tp_col, w); c < p->tp_col; c++) { row[c] = VTBUF_SPACE_CHAR(ch); } } if (history_full) { vb->vb_curroffset = history_size - h; vb->vb_flags |= VBF_HISTORY_FULL; } } vb->vb_roffset = vb->vb_curroffset; /* Adjust cursor position. */ if (vb->vb_cursor.tp_col > p->tp_col - 1) /* * Move cursor to the last column, in case its previous * position is outside of the new screen area. */ vb->vb_cursor.tp_col = p->tp_col - 1; if (vb->vb_curroffset > 0 || vb->vb_cursor.tp_row > p->tp_row - 1) /* Move cursor to the last line on the screen. */ vb->vb_cursor.tp_row = p->tp_row - 1; VTBUF_UNLOCK(vb); /* Deallocate old buffer. */ free(old, M_VTBUF); free(oldrows, M_VTBUF); } void vtbuf_putchar(struct vt_buf *vb, const term_pos_t *p, term_char_t c) { term_char_t *row; KASSERT(p->tp_row < vb->vb_scr_size.tp_row, ("vtbuf_putchar tp_row %d must be less than screen width %d", p->tp_row, vb->vb_scr_size.tp_row)); KASSERT(p->tp_col < vb->vb_scr_size.tp_col, ("vtbuf_putchar tp_col %d must be less than screen height %d", p->tp_col, vb->vb_scr_size.tp_col)); row = vb->vb_rows[(vb->vb_curroffset + p->tp_row) % VTBUF_MAX_HEIGHT(vb)]; if (row[p->tp_col] != c) { row[p->tp_col] = c; vtbuf_dirty_cell(vb, p); } } void vtbuf_cursor_position(struct vt_buf *vb, const term_pos_t *p) { if (vb->vb_flags & VBF_CURSOR) { vtbuf_dirty_cell(vb, &vb->vb_cursor); vb->vb_cursor = *p; vtbuf_dirty_cell(vb, &vb->vb_cursor); } else { vb->vb_cursor = *p; } } #ifndef SC_NO_CUTPASTE static void vtbuf_flush_mark(struct vt_buf *vb) { term_rect_t area; int s, e; /* Notify renderer to update marked region. */ if ((vb->vb_mark_start.tp_col != vb->vb_mark_end.tp_col) || (vb->vb_mark_start.tp_row != vb->vb_mark_end.tp_row)) { s = vtbuf_htw(vb, vb->vb_mark_start.tp_row); e = vtbuf_htw(vb, vb->vb_mark_end.tp_row); area.tr_begin.tp_col = 0; area.tr_begin.tp_row = MIN(s, e); area.tr_end.tp_col = vb->vb_scr_size.tp_col; area.tr_end.tp_row = MAX(s, e) + 1; VTBUF_LOCK(vb); vtbuf_dirty(vb, &area); VTBUF_UNLOCK(vb); } } int vtbuf_get_marked_len(struct vt_buf *vb) { int ei, si, sz; term_pos_t s, e; /* Swap according to window coordinates. */ if (POS_INDEX(vtbuf_htw(vb, vb->vb_mark_start.tp_row), vb->vb_mark_start.tp_col) > POS_INDEX(vtbuf_htw(vb, vb->vb_mark_end.tp_row), vb->vb_mark_end.tp_col)) { POS_COPY(e, vb->vb_mark_start); POS_COPY(s, vb->vb_mark_end); } else { POS_COPY(s, vb->vb_mark_start); POS_COPY(e, vb->vb_mark_end); } si = s.tp_row * vb->vb_scr_size.tp_col + s.tp_col; ei = e.tp_row * vb->vb_scr_size.tp_col + e.tp_col; /* Number symbols and number of rows to inject \n */ sz = ei - si + ((e.tp_row - s.tp_row) * 2); return (sz * sizeof(term_char_t)); } void vtbuf_extract_marked(struct vt_buf *vb, term_char_t *buf, int sz) { int i, r, c, cs, ce; term_pos_t s, e; /* Swap according to window coordinates. */ if (POS_INDEX(vtbuf_htw(vb, vb->vb_mark_start.tp_row), vb->vb_mark_start.tp_col) > POS_INDEX(vtbuf_htw(vb, vb->vb_mark_end.tp_row), vb->vb_mark_end.tp_col)) { POS_COPY(e, vb->vb_mark_start); POS_COPY(s, vb->vb_mark_end); } else { POS_COPY(s, vb->vb_mark_start); POS_COPY(e, vb->vb_mark_end); } i = 0; for (r = s.tp_row; r <= e.tp_row; r ++) { cs = (r == s.tp_row)?s.tp_col:0; ce = (r == e.tp_row)?e.tp_col:vb->vb_scr_size.tp_col; for (c = cs; c < ce; c ++) { buf[i++] = vb->vb_rows[r][c]; } /* Add new line for all rows, but not for last one. */ if (r != e.tp_row) { buf[i++] = '\r'; buf[i++] = '\n'; } } } int vtbuf_set_mark(struct vt_buf *vb, int type, int col, int row) { term_char_t *r; int i; switch (type) { case VTB_MARK_END: /* B1 UP */ if (vb->vb_mark_last != VTB_MARK_MOVE) return (0); /* FALLTHROUGH */ case VTB_MARK_MOVE: case VTB_MARK_EXTEND: vtbuf_flush_mark(vb); /* Clean old mark. */ vb->vb_mark_end.tp_col = col; vb->vb_mark_end.tp_row = vtbuf_wth(vb, row); break; case VTB_MARK_START: vtbuf_flush_mark(vb); /* Clean old mark. */ vb->vb_mark_start.tp_col = col; vb->vb_mark_start.tp_row = vtbuf_wth(vb, row); /* Start again, so clear end point. */ vb->vb_mark_end.tp_col = col; vb->vb_mark_end.tp_row = vtbuf_wth(vb, row); break; case VTB_MARK_WORD: vtbuf_flush_mark(vb); /* Clean old mark. */ vb->vb_mark_start.tp_row = vb->vb_mark_end.tp_row = vtbuf_wth(vb, row); r = vb->vb_rows[vb->vb_mark_start.tp_row]; for (i = col; i >= 0; i --) { if (TCHAR_CHARACTER(r[i]) == ' ') { vb->vb_mark_start.tp_col = i + 1; break; } } for (i = col; i < vb->vb_scr_size.tp_col; i ++) { if (TCHAR_CHARACTER(r[i]) == ' ') { vb->vb_mark_end.tp_col = i; break; } } if (vb->vb_mark_start.tp_col > vb->vb_mark_end.tp_col) vb->vb_mark_start.tp_col = vb->vb_mark_end.tp_col; break; case VTB_MARK_ROW: vtbuf_flush_mark(vb); /* Clean old mark. */ vb->vb_mark_start.tp_col = 0; vb->vb_mark_end.tp_col = vb->vb_scr_size.tp_col; vb->vb_mark_start.tp_row = vb->vb_mark_end.tp_row = vtbuf_wth(vb, row); break; case VTB_MARK_NONE: vb->vb_mark_last = type; /* FALLTHROUGH */ default: /* panic? */ return (0); } vb->vb_mark_last = type; /* Draw new marked region. */ vtbuf_flush_mark(vb); return (1); } #endif void vtbuf_cursor_visibility(struct vt_buf *vb, int yes) { int oflags, nflags; oflags = vb->vb_flags; if (yes) vb->vb_flags |= VBF_CURSOR; else vb->vb_flags &= ~VBF_CURSOR; nflags = vb->vb_flags; if (oflags != nflags) vtbuf_dirty_cell(vb, &vb->vb_cursor); } void vtbuf_scroll_mode(struct vt_buf *vb, int yes) { int oflags, nflags; VTBUF_LOCK(vb); oflags = vb->vb_flags; if (yes) vb->vb_flags |= VBF_SCROLL; else vb->vb_flags &= ~VBF_SCROLL; nflags = vb->vb_flags; if (oflags != nflags) vtbuf_dirty_cell(vb, &vb->vb_cursor); VTBUF_UNLOCK(vb); } diff --git a/sys/dev/vt/vt_consolectl.c b/sys/dev/vt/vt_consolectl.c index 4353271255ff..2751f0037e5d 100644 --- a/sys/dev/vt/vt_consolectl.c +++ b/sys/dev/vt/vt_consolectl.c @@ -1,84 +1,83 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include static d_ioctl_t consolectl_ioctl; static struct cdevsw consolectl_cdevsw = { .d_version = D_VERSION, .d_ioctl = consolectl_ioctl, .d_name = "consolectl", }; static int consolectl_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) { switch (cmd) { case CONS_GETVERS: *(int*)data = 0x200; return 0; case CONS_MOUSECTL: { mouse_info_t *mi = (mouse_info_t*)data; sysmouse_process_event(mi); return (0); } default: #ifdef VT_CONSOLECTL_DEBUG printf("consolectl: unknown ioctl: %c:%lx\n", (char)IOCGROUP(cmd), IOCBASECMD(cmd)); #endif return (ENOIOCTL); } } static void consolectl_drvinit(void *unused) { if (!vty_enabled(VTY_VT)) return; make_dev(&consolectl_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "consolectl"); } SYSINIT(consolectl, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, consolectl_drvinit, NULL); diff --git a/sys/dev/vt/vt_core.c b/sys/dev/vt/vt_core.c index e4a8288962c4..a8a1bfa2a8e5 100644 --- a/sys/dev/vt/vt_core.c +++ b/sys/dev/vt/vt_core.c @@ -1,3174 +1,3173 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Portions of this software were developed by Oleksandr Rybalko * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__i386__) || defined(__amd64__) #include #include #endif static int vtterm_cngrab_noswitch(struct vt_device *, struct vt_window *); static int vtterm_cnungrab_noswitch(struct vt_device *, struct vt_window *); static tc_bell_t vtterm_bell; static tc_cursor_t vtterm_cursor; static tc_putchar_t vtterm_putchar; static tc_fill_t vtterm_fill; static tc_copy_t vtterm_copy; static tc_pre_input_t vtterm_pre_input; static tc_post_input_t vtterm_post_input; static tc_param_t vtterm_param; static tc_done_t vtterm_done; static tc_cnprobe_t vtterm_cnprobe; static tc_cngetc_t vtterm_cngetc; static tc_cngrab_t vtterm_cngrab; static tc_cnungrab_t vtterm_cnungrab; static tc_opened_t vtterm_opened; static tc_ioctl_t vtterm_ioctl; static tc_mmap_t vtterm_mmap; const struct terminal_class vt_termclass = { .tc_bell = vtterm_bell, .tc_cursor = vtterm_cursor, .tc_putchar = vtterm_putchar, .tc_fill = vtterm_fill, .tc_copy = vtterm_copy, .tc_pre_input = vtterm_pre_input, .tc_post_input = vtterm_post_input, .tc_param = vtterm_param, .tc_done = vtterm_done, .tc_cnprobe = vtterm_cnprobe, .tc_cngetc = vtterm_cngetc, .tc_cngrab = vtterm_cngrab, .tc_cnungrab = vtterm_cnungrab, .tc_opened = vtterm_opened, .tc_ioctl = vtterm_ioctl, .tc_mmap = vtterm_mmap, }; /* * Use a constant timer of 25 Hz to redraw the screen. * * XXX: In theory we should only fire up the timer when there is really * activity. Unfortunately we cannot always start timers. We really * don't want to process kernel messages synchronously, because it * really slows down the system. */ #define VT_TIMERFREQ 25 /* Bell pitch/duration. */ #define VT_BELLDURATION ((5 * hz + 99) / 100) #define VT_BELLPITCH 800 #define VT_UNIT(vw) ((vw)->vw_device->vd_unit * VT_MAXWINDOWS + \ (vw)->vw_number) static SYSCTL_NODE(_kern, OID_AUTO, vt, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "vt(9) parameters"); static VT_SYSCTL_INT(enable_altgr, 1, "Enable AltGr key (Do not assume R.Alt as Alt)"); static VT_SYSCTL_INT(enable_bell, 1, "Enable bell"); static VT_SYSCTL_INT(debug, 0, "vt(9) debug level"); static VT_SYSCTL_INT(deadtimer, 15, "Time to wait busy process in VT_PROCESS mode"); static VT_SYSCTL_INT(suspendswitch, 1, "Switch to VT0 before suspend"); /* Allow to disable some keyboard combinations. */ static VT_SYSCTL_INT(kbd_halt, 1, "Enable halt keyboard combination. " "See kbdmap(5) to configure."); static VT_SYSCTL_INT(kbd_poweroff, 1, "Enable Power Off keyboard combination. " "See kbdmap(5) to configure."); static VT_SYSCTL_INT(kbd_reboot, 1, "Enable reboot keyboard combination. " "See kbdmap(5) to configure (typically Ctrl-Alt-Delete)."); static VT_SYSCTL_INT(kbd_debug, 1, "Enable key combination to enter debugger. " "See kbdmap(5) to configure (typically Ctrl-Alt-Esc)."); static VT_SYSCTL_INT(kbd_panic, 0, "Enable request to panic. " "See kbdmap(5) to configure."); /* Used internally, not a tunable. */ int vt_draw_logo_cpus; VT_SYSCTL_INT(splash_cpu, 0, "Show logo CPUs during boot"); VT_SYSCTL_INT(splash_ncpu, 0, "Override number of logos displayed " "(0 = do not override)"); VT_SYSCTL_INT(splash_cpu_style, 2, "Draw logo style " "(0 = Alternate beastie, 1 = Beastie, 2 = Orb)"); VT_SYSCTL_INT(splash_cpu_duration, 10, "Hide logos after (seconds)"); static unsigned int vt_unit = 0; static MALLOC_DEFINE(M_VT, "vt", "vt device"); struct vt_device *main_vd = &vt_consdev; /* Boot logo. */ extern unsigned int vt_logo_width; extern unsigned int vt_logo_height; extern unsigned int vt_logo_depth; extern unsigned char vt_logo_image[]; #ifndef DEV_SPLASH #define vtterm_draw_cpu_logos(...) do {} while (0) const unsigned int vt_logo_sprite_height; #endif /* * Console font. vt_font_loader will be filled with font data passed * by loader. If there is no font passed by boot loader, we use built in * default. */ extern struct vt_font vt_font_default; static struct vt_font vt_font_loader; static struct vt_font *vt_font_assigned = &vt_font_default; #ifndef SC_NO_CUTPASTE extern struct vt_mouse_cursor vt_default_mouse_pointer; #endif static int signal_vt_rel(struct vt_window *); static int signal_vt_acq(struct vt_window *); static int finish_vt_rel(struct vt_window *, int, int *); static int finish_vt_acq(struct vt_window *); static int vt_window_switch(struct vt_window *); static int vt_late_window_switch(struct vt_window *); static int vt_proc_alive(struct vt_window *); static void vt_resize(struct vt_device *); static void vt_update_static(void *); #ifndef SC_NO_CUTPASTE static void vt_mouse_paste(void); #endif static void vt_suspend_handler(void *priv); static void vt_resume_handler(void *priv); SET_DECLARE(vt_drv_set, struct vt_driver); #define _VTDEFH MAX(100, PIXEL_HEIGHT(VT_FB_MAX_HEIGHT)) #define _VTDEFW MAX(200, PIXEL_WIDTH(VT_FB_MAX_WIDTH)) struct terminal vt_consterm; static struct vt_window vt_conswindow; #ifndef SC_NO_CONSDRAWN static term_char_t vt_consdrawn[PIXEL_HEIGHT(VT_FB_MAX_HEIGHT) * PIXEL_WIDTH(VT_FB_MAX_WIDTH)]; static term_color_t vt_consdrawnfg[PIXEL_HEIGHT(VT_FB_MAX_HEIGHT) * PIXEL_WIDTH(VT_FB_MAX_WIDTH)]; static term_color_t vt_consdrawnbg[PIXEL_HEIGHT(VT_FB_MAX_HEIGHT) * PIXEL_WIDTH(VT_FB_MAX_WIDTH)]; #endif struct vt_device vt_consdev = { .vd_driver = NULL, .vd_softc = NULL, .vd_prev_driver = NULL, .vd_prev_softc = NULL, .vd_flags = VDF_INVALID, .vd_windows = { [VT_CONSWINDOW] = &vt_conswindow, }, .vd_curwindow = &vt_conswindow, .vd_kbstate = 0, #ifndef SC_NO_CUTPASTE .vd_pastebuf = { .vpb_buf = NULL, .vpb_bufsz = 0, .vpb_len = 0 }, .vd_mcursor = &vt_default_mouse_pointer, .vd_mcursor_fg = TC_WHITE, .vd_mcursor_bg = TC_BLACK, #endif #ifndef SC_NO_CONSDRAWN .vd_drawn = vt_consdrawn, .vd_drawnfg = vt_consdrawnfg, .vd_drawnbg = vt_consdrawnbg, #endif }; static term_char_t vt_constextbuf[(_VTDEFW) * (VBF_DEFAULT_HISTORY_SIZE)]; static term_char_t *vt_constextbufrows[VBF_DEFAULT_HISTORY_SIZE]; static struct vt_window vt_conswindow = { .vw_number = VT_CONSWINDOW, .vw_flags = VWF_CONSOLE, .vw_buf = { .vb_buffer = &vt_constextbuf[0], .vb_rows = &vt_constextbufrows[0], .vb_history_size = VBF_DEFAULT_HISTORY_SIZE, .vb_curroffset = 0, .vb_roffset = 0, .vb_flags = VBF_STATIC, .vb_mark_start = {.tp_row = 0, .tp_col = 0,}, .vb_mark_end = {.tp_row = 0, .tp_col = 0,}, .vb_scr_size = { .tp_row = _VTDEFH, .tp_col = _VTDEFW, }, }, .vw_device = &vt_consdev, .vw_terminal = &vt_consterm, .vw_kbdmode = K_XLATE, .vw_grabbed = 0, }; struct terminal vt_consterm = { .tm_class = &vt_termclass, .tm_softc = &vt_conswindow, .tm_flags = TF_CONS, }; static struct consdev vt_consterm_consdev = { .cn_ops = &termcn_cnops, .cn_arg = &vt_consterm, .cn_name = "ttyv0", }; /* Add to set of consoles. */ DATA_SET(cons_set, vt_consterm_consdev); /* * Right after kmem is done to allow early drivers to use locking and allocate * memory. */ SYSINIT(vt_update_static, SI_SUB_KMEM, SI_ORDER_ANY, vt_update_static, &vt_consdev); /* Delay until all devices attached, to not waste time. */ SYSINIT(vt_early_cons, SI_SUB_INT_CONFIG_HOOKS, SI_ORDER_ANY, vt_upgrade, &vt_consdev); /* Initialize locks/mem depended members. */ static void vt_update_static(void *dummy) { if (!vty_enabled(VTY_VT)) return; if (main_vd->vd_driver != NULL) printf("VT(%s): %s %ux%u\n", main_vd->vd_driver->vd_name, (main_vd->vd_flags & VDF_TEXTMODE) ? "text" : "resolution", main_vd->vd_width, main_vd->vd_height); else printf("VT: init without driver.\n"); mtx_init(&main_vd->vd_lock, "vtdev", NULL, MTX_DEF); cv_init(&main_vd->vd_winswitch, "vtwswt"); } static void vt_schedule_flush(struct vt_device *vd, int ms) { if (ms <= 0) /* Default to initial value. */ ms = 1000 / VT_TIMERFREQ; callout_schedule(&vd->vd_timer, hz / (1000 / ms)); } void vt_resume_flush_timer(struct vt_window *vw, int ms) { struct vt_device *vd = vw->vw_device; if (vd->vd_curwindow != vw) return; if (!(vd->vd_flags & VDF_ASYNC) || !atomic_cmpset_int(&vd->vd_timer_armed, 0, 1)) return; vt_schedule_flush(vd, ms); } static void vt_suspend_flush_timer(struct vt_device *vd) { /* * As long as this function is called locked, callout_stop() * has the same effect like callout_drain() with regard to * preventing the callback function from executing. */ VT_LOCK_ASSERT(vd, MA_OWNED); if (!(vd->vd_flags & VDF_ASYNC) || !atomic_cmpset_int(&vd->vd_timer_armed, 1, 0)) return; callout_stop(&vd->vd_timer); } static void vt_switch_timer(void *arg) { (void)vt_late_window_switch((struct vt_window *)arg); } static int vt_save_kbd_mode(struct vt_window *vw, keyboard_t *kbd) { int mode, ret; mode = 0; ret = kbdd_ioctl(kbd, KDGKBMODE, (caddr_t)&mode); if (ret == ENOIOCTL) ret = ENODEV; if (ret != 0) return (ret); vw->vw_kbdmode = mode; return (0); } static int vt_update_kbd_mode(struct vt_window *vw, keyboard_t *kbd) { int ret; ret = kbdd_ioctl(kbd, KDSKBMODE, (caddr_t)&vw->vw_kbdmode); if (ret == ENOIOCTL) ret = ENODEV; return (ret); } static int vt_save_kbd_state(struct vt_window *vw, keyboard_t *kbd) { int state, ret; state = 0; ret = kbdd_ioctl(kbd, KDGKBSTATE, (caddr_t)&state); if (ret == ENOIOCTL) ret = ENODEV; if (ret != 0) return (ret); vw->vw_kbdstate &= ~LOCK_MASK; vw->vw_kbdstate |= state & LOCK_MASK; return (0); } static int vt_update_kbd_state(struct vt_window *vw, keyboard_t *kbd) { int state, ret; state = vw->vw_kbdstate & LOCK_MASK; ret = kbdd_ioctl(kbd, KDSKBSTATE, (caddr_t)&state); if (ret == ENOIOCTL) ret = ENODEV; return (ret); } static int vt_save_kbd_leds(struct vt_window *vw, keyboard_t *kbd) { int leds, ret; leds = 0; ret = kbdd_ioctl(kbd, KDGETLED, (caddr_t)&leds); if (ret == ENOIOCTL) ret = ENODEV; if (ret != 0) return (ret); vw->vw_kbdstate &= ~LED_MASK; vw->vw_kbdstate |= leds & LED_MASK; return (0); } static int vt_update_kbd_leds(struct vt_window *vw, keyboard_t *kbd) { int leds, ret; leds = vw->vw_kbdstate & LED_MASK; ret = kbdd_ioctl(kbd, KDSETLED, (caddr_t)&leds); if (ret == ENOIOCTL) ret = ENODEV; return (ret); } static int vt_window_preswitch(struct vt_window *vw, struct vt_window *curvw) { DPRINTF(40, "%s\n", __func__); curvw->vw_switch_to = vw; /* Set timer to allow switch in case when process hang. */ callout_reset(&vw->vw_proc_dead_timer, hz * vt_deadtimer, vt_switch_timer, (void *)vw); /* Notify process about vt switch attempt. */ DPRINTF(30, "%s: Notify process.\n", __func__); signal_vt_rel(curvw); return (0); } static int vt_window_postswitch(struct vt_window *vw) { signal_vt_acq(vw); return (0); } /* vt_late_window_switch will do VT switching for regular case. */ static int vt_late_window_switch(struct vt_window *vw) { struct vt_window *curvw; int ret; callout_stop(&vw->vw_proc_dead_timer); ret = vt_window_switch(vw); if (ret != 0) { /* * If the switch hasn't happened, then return the VT * to the current owner, if any. */ curvw = vw->vw_device->vd_curwindow; if (curvw->vw_smode.mode == VT_PROCESS) (void)vt_window_postswitch(curvw); return (ret); } /* Notify owner process about terminal availability. */ if (vw->vw_smode.mode == VT_PROCESS) { ret = vt_window_postswitch(vw); } return (ret); } /* Switch window. */ static int vt_proc_window_switch(struct vt_window *vw) { struct vt_window *curvw; struct vt_device *vd; int ret; /* Prevent switching to NULL */ if (vw == NULL) { DPRINTF(30, "%s: Cannot switch: vw is NULL.", __func__); return (EINVAL); } vd = vw->vw_device; curvw = vd->vd_curwindow; /* Check if virtual terminal is locked */ if (curvw->vw_flags & VWF_VTYLOCK) return (EBUSY); /* Check if switch already in progress */ if (curvw->vw_flags & VWF_SWWAIT_REL) { /* Check if switching to same window */ if (curvw->vw_switch_to == vw) { DPRINTF(30, "%s: Switch in progress to same vw.", __func__); return (0); /* success */ } DPRINTF(30, "%s: Switch in progress to different vw.", __func__); return (EBUSY); } /* Avoid switching to already selected window */ if (vw == curvw) { DPRINTF(30, "%s: Cannot switch: vw == curvw.", __func__); return (0); /* success */ } /* * Early check for an attempt to switch to a non-functional VT. * The same check is done in vt_window_switch(), but it's better * to fail as early as possible to avoid needless pre-switch * actions. */ VT_LOCK(vd); if ((vw->vw_flags & (VWF_OPENED|VWF_CONSOLE)) == 0) { VT_UNLOCK(vd); return (EINVAL); } VT_UNLOCK(vd); /* Ask current process permission to switch away. */ if (curvw->vw_smode.mode == VT_PROCESS) { DPRINTF(30, "%s: VT_PROCESS ", __func__); if (vt_proc_alive(curvw) == FALSE) { DPRINTF(30, "Dead. Cleaning."); /* Dead */ } else { DPRINTF(30, "%s: Signaling process.\n", __func__); /* Alive, try to ask him. */ ret = vt_window_preswitch(vw, curvw); /* Wait for process answer or timeout. */ return (ret); } DPRINTF(30, "\n"); } ret = vt_late_window_switch(vw); return (ret); } /* Switch window ignoring process locking. */ static int vt_window_switch(struct vt_window *vw) { struct vt_device *vd = vw->vw_device; struct vt_window *curvw = vd->vd_curwindow; keyboard_t *kbd; if (kdb_active) { /* * When grabbing the console for the debugger, avoid * locks as that can result in deadlock. While this * could use try locks, that wouldn't really make a * difference as there are sufficient barriers in * debugger entry/exit to be equivalent to * successfully try-locking here. */ if (curvw == vw) return (0); if (!(vw->vw_flags & (VWF_OPENED|VWF_CONSOLE))) return (EINVAL); vd->vd_curwindow = vw; vd->vd_flags |= VDF_INVALID; if (vd->vd_driver->vd_postswitch) vd->vd_driver->vd_postswitch(vd); return (0); } VT_LOCK(vd); if (curvw == vw) { /* Nothing to do. */ VT_UNLOCK(vd); return (0); } if (!(vw->vw_flags & (VWF_OPENED|VWF_CONSOLE))) { VT_UNLOCK(vd); return (EINVAL); } vt_suspend_flush_timer(vd); vd->vd_curwindow = vw; vd->vd_flags |= VDF_INVALID; cv_broadcast(&vd->vd_winswitch); VT_UNLOCK(vd); if (vd->vd_driver->vd_postswitch) vd->vd_driver->vd_postswitch(vd); vt_resume_flush_timer(vw, 0); /* Restore per-window keyboard mode. */ mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) { if (curvw->vw_kbdmode == K_XLATE) vt_save_kbd_state(curvw, kbd); vt_update_kbd_mode(vw, kbd); vt_update_kbd_state(vw, kbd); } mtx_unlock(&Giant); DPRINTF(10, "%s(ttyv%d) done\n", __func__, vw->vw_number); return (0); } void vt_termsize(struct vt_device *vd, struct vt_font *vf, term_pos_t *size) { size->tp_row = vd->vd_height; if (vt_draw_logo_cpus) size->tp_row -= vt_logo_sprite_height; size->tp_col = vd->vd_width; if (vf != NULL) { size->tp_row = MIN(size->tp_row / vf->vf_height, PIXEL_HEIGHT(VT_FB_MAX_HEIGHT)); size->tp_col = MIN(size->tp_col / vf->vf_width, PIXEL_WIDTH(VT_FB_MAX_WIDTH)); } } static inline void vt_termrect(struct vt_device *vd, struct vt_font *vf, term_rect_t *rect) { rect->tr_begin.tp_row = rect->tr_begin.tp_col = 0; if (vt_draw_logo_cpus) rect->tr_begin.tp_row = vt_logo_sprite_height; rect->tr_end.tp_row = vd->vd_height; rect->tr_end.tp_col = vd->vd_width; if (vf != NULL) { rect->tr_begin.tp_row = howmany(rect->tr_begin.tp_row, vf->vf_height); rect->tr_end.tp_row = MIN(rect->tr_end.tp_row / vf->vf_height, PIXEL_HEIGHT(VT_FB_MAX_HEIGHT)); rect->tr_end.tp_col = MIN(rect->tr_end.tp_col / vf->vf_width, PIXEL_WIDTH(VT_FB_MAX_WIDTH)); } } void vt_winsize(struct vt_device *vd, struct vt_font *vf, struct winsize *size) { size->ws_ypixel = vd->vd_height; if (vt_draw_logo_cpus) size->ws_ypixel -= vt_logo_sprite_height; size->ws_row = size->ws_ypixel; size->ws_col = size->ws_xpixel = vd->vd_width; if (vf != NULL) { size->ws_row = MIN(size->ws_row / vf->vf_height, PIXEL_HEIGHT(VT_FB_MAX_HEIGHT)); size->ws_col = MIN(size->ws_col / vf->vf_width, PIXEL_WIDTH(VT_FB_MAX_WIDTH)); } } void vt_compute_drawable_area(struct vt_window *vw) { struct vt_device *vd; struct vt_font *vf; vt_axis_t height; vd = vw->vw_device; if (vw->vw_font == NULL) { vw->vw_draw_area.tr_begin.tp_col = 0; vw->vw_draw_area.tr_begin.tp_row = 0; if (vt_draw_logo_cpus) vw->vw_draw_area.tr_begin.tp_row = vt_logo_sprite_height; vw->vw_draw_area.tr_end.tp_col = vd->vd_width; vw->vw_draw_area.tr_end.tp_row = vd->vd_height; return; } vf = vw->vw_font; /* * Compute the drawable area, so that the text is centered on * the screen. */ height = vd->vd_height; if (vt_draw_logo_cpus) height -= vt_logo_sprite_height; vw->vw_draw_area.tr_begin.tp_col = (vd->vd_width % vf->vf_width) / 2; vw->vw_draw_area.tr_begin.tp_row = (height % vf->vf_height) / 2; if (vt_draw_logo_cpus) vw->vw_draw_area.tr_begin.tp_row += vt_logo_sprite_height; vw->vw_draw_area.tr_end.tp_col = vw->vw_draw_area.tr_begin.tp_col + rounddown(vd->vd_width, vf->vf_width); vw->vw_draw_area.tr_end.tp_row = vw->vw_draw_area.tr_begin.tp_row + rounddown(height, vf->vf_height); } static void vt_scroll(struct vt_window *vw, int offset, int whence) { int diff; term_pos_t size; if ((vw->vw_flags & VWF_SCROLL) == 0) return; vt_termsize(vw->vw_device, vw->vw_font, &size); diff = vthistory_seek(&vw->vw_buf, offset, whence); if (diff) vw->vw_device->vd_flags |= VDF_INVALID; vt_resume_flush_timer(vw, 0); } static int vt_machine_kbdevent(struct vt_device *vd, int c) { switch (c) { case SPCLKEY | DBG: /* kbdmap(5) keyword `debug`. */ if (vt_kbd_debug) { kdb_enter(KDB_WHY_BREAK, "manual escape to debugger"); #if VT_ALT_TO_ESC_HACK /* * There's an unfortunate conflict between SPCLKEY|DBG * and VT_ALT_TO_ESC_HACK. Just assume they didn't mean * it if we got to here. */ vd->vd_kbstate &= ~ALKED; #endif } return (1); case SPCLKEY | HALT: /* kbdmap(5) keyword `halt`. */ if (vt_kbd_halt) shutdown_nice(RB_HALT); return (1); case SPCLKEY | PASTE: /* kbdmap(5) keyword `paste`. */ #ifndef SC_NO_CUTPASTE /* Insert text from cut-paste buffer. */ vt_mouse_paste(); #endif break; case SPCLKEY | PDWN: /* kbdmap(5) keyword `pdwn`. */ if (vt_kbd_poweroff) shutdown_nice(RB_HALT|RB_POWEROFF); return (1); case SPCLKEY | PNC: /* kbdmap(5) keyword `panic`. */ /* * Request to immediate panic if sysctl * kern.vt.enable_panic_key allow it. */ if (vt_kbd_panic) panic("Forced by the panic key"); return (1); case SPCLKEY | RBT: /* kbdmap(5) keyword `boot`. */ if (vt_kbd_reboot) shutdown_nice(RB_AUTOBOOT); return (1); case SPCLKEY | SPSC: /* kbdmap(5) keyword `spsc`. */ /* Force activatation/deactivation of the screen saver. */ /* TODO */ return (1); case SPCLKEY | STBY: /* XXX Not present in kbdcontrol parser. */ /* Put machine into Stand-By mode. */ power_pm_suspend(POWER_SLEEP_STATE_STANDBY); return (1); case SPCLKEY | SUSP: /* kbdmap(5) keyword `susp`. */ /* Suspend machine. */ power_pm_suspend(POWER_SLEEP_STATE_SUSPEND); return (1); } return (0); } static void vt_scrollmode_kbdevent(struct vt_window *vw, int c, int console) { struct vt_device *vd; term_pos_t size; vd = vw->vw_device; /* Only special keys handled in ScrollLock mode */ if ((c & SPCLKEY) == 0) return; c &= ~SPCLKEY; if (console == 0) { if (c >= F_SCR && c <= MIN(L_SCR, F_SCR + VT_MAXWINDOWS - 1)) { vw = vd->vd_windows[c - F_SCR]; vt_proc_window_switch(vw); return; } VT_LOCK(vd); } switch (c) { case SLK: { /* Turn scrolling off. */ vt_scroll(vw, 0, VHS_END); VTBUF_SLCK_DISABLE(&vw->vw_buf); vw->vw_flags &= ~VWF_SCROLL; break; } case FKEY | F(49): /* Home key. */ vt_scroll(vw, 0, VHS_SET); break; case FKEY | F(50): /* Arrow up. */ vt_scroll(vw, -1, VHS_CUR); break; case FKEY | F(51): /* Page up. */ vt_termsize(vd, vw->vw_font, &size); vt_scroll(vw, -size.tp_row, VHS_CUR); break; case FKEY | F(57): /* End key. */ vt_scroll(vw, 0, VHS_END); break; case FKEY | F(58): /* Arrow down. */ vt_scroll(vw, 1, VHS_CUR); break; case FKEY | F(59): /* Page down. */ vt_termsize(vd, vw->vw_font, &size); vt_scroll(vw, size.tp_row, VHS_CUR); break; } if (console == 0) VT_UNLOCK(vd); } static int vt_processkey(keyboard_t *kbd, struct vt_device *vd, int c) { struct vt_window *vw = vd->vd_curwindow; random_harvest_queue(&c, sizeof(c), RANDOM_KEYBOARD); #if VT_ALT_TO_ESC_HACK if (c & RELKEY) { switch (c & ~RELKEY) { case (SPCLKEY | RALT): if (vt_enable_altgr != 0) break; case (SPCLKEY | LALT): vd->vd_kbstate &= ~ALKED; } /* Other keys ignored for RELKEY event. */ return (0); } else { switch (c & ~RELKEY) { case (SPCLKEY | RALT): if (vt_enable_altgr != 0) break; case (SPCLKEY | LALT): vd->vd_kbstate |= ALKED; } } #else if (c & RELKEY) /* Other keys ignored for RELKEY event. */ return (0); #endif if (vt_machine_kbdevent(vd, c)) return (0); if (vw->vw_flags & VWF_SCROLL) { vt_scrollmode_kbdevent(vw, c, 0/* Not a console */); /* Scroll mode keys handled, nothing to do more. */ return (0); } if (c & SPCLKEY) { c &= ~SPCLKEY; if (c >= F_SCR && c <= MIN(L_SCR, F_SCR + VT_MAXWINDOWS - 1)) { vw = vd->vd_windows[c - F_SCR]; vt_proc_window_switch(vw); return (0); } switch (c) { case NEXT: /* Switch to next VT. */ c = (vw->vw_number + 1) % VT_MAXWINDOWS; vw = vd->vd_windows[c]; vt_proc_window_switch(vw); return (0); case PREV: /* Switch to previous VT. */ c = (vw->vw_number + VT_MAXWINDOWS - 1) % VT_MAXWINDOWS; vw = vd->vd_windows[c]; vt_proc_window_switch(vw); return (0); case SLK: { vt_save_kbd_state(vw, kbd); VT_LOCK(vd); if (vw->vw_kbdstate & SLKED) { /* Turn scrolling on. */ vw->vw_flags |= VWF_SCROLL; VTBUF_SLCK_ENABLE(&vw->vw_buf); } else { /* Turn scrolling off. */ vw->vw_flags &= ~VWF_SCROLL; VTBUF_SLCK_DISABLE(&vw->vw_buf); vt_scroll(vw, 0, VHS_END); } VT_UNLOCK(vd); break; } case FKEY | F(1): case FKEY | F(2): case FKEY | F(3): case FKEY | F(4): case FKEY | F(5): case FKEY | F(6): case FKEY | F(7): case FKEY | F(8): case FKEY | F(9): case FKEY | F(10): case FKEY | F(11): case FKEY | F(12): /* F1 through F12 keys. */ terminal_input_special(vw->vw_terminal, TKEY_F1 + c - (FKEY | F(1))); break; case FKEY | F(49): /* Home key. */ terminal_input_special(vw->vw_terminal, TKEY_HOME); break; case FKEY | F(50): /* Arrow up. */ terminal_input_special(vw->vw_terminal, TKEY_UP); break; case FKEY | F(51): /* Page up. */ terminal_input_special(vw->vw_terminal, TKEY_PAGE_UP); break; case FKEY | F(53): /* Arrow left. */ terminal_input_special(vw->vw_terminal, TKEY_LEFT); break; case FKEY | F(55): /* Arrow right. */ terminal_input_special(vw->vw_terminal, TKEY_RIGHT); break; case FKEY | F(57): /* End key. */ terminal_input_special(vw->vw_terminal, TKEY_END); break; case FKEY | F(58): /* Arrow down. */ terminal_input_special(vw->vw_terminal, TKEY_DOWN); break; case FKEY | F(59): /* Page down. */ terminal_input_special(vw->vw_terminal, TKEY_PAGE_DOWN); break; case FKEY | F(60): /* Insert key. */ terminal_input_special(vw->vw_terminal, TKEY_INSERT); break; case FKEY | F(61): /* Delete key. */ terminal_input_special(vw->vw_terminal, TKEY_DELETE); break; } } else if (KEYFLAGS(c) == 0) { /* Don't do UTF-8 conversion when doing raw mode. */ if (vw->vw_kbdmode == K_XLATE) { #if VT_ALT_TO_ESC_HACK if (vd->vd_kbstate & ALKED) { /* * Prepend ESC sequence if one of ALT keys down. */ terminal_input_char(vw->vw_terminal, 0x1b); } #endif #if defined(KDB) kdb_alt_break(c, &vd->vd_altbrk); #endif terminal_input_char(vw->vw_terminal, KEYCHAR(c)); } else terminal_input_raw(vw->vw_terminal, c); } return (0); } static int vt_kbdevent(keyboard_t *kbd, int event, void *arg) { struct vt_device *vd = arg; int c; switch (event) { case KBDIO_KEYINPUT: break; case KBDIO_UNLOADING: mtx_lock(&Giant); vd->vd_keyboard = NULL; kbd_release(kbd, (void *)vd); mtx_unlock(&Giant); return (0); default: return (EINVAL); } while ((c = kbdd_read_char(kbd, 0)) != NOKEY) vt_processkey(kbd, vd, c); return (0); } static int vt_allocate_keyboard(struct vt_device *vd) { int grabbed, i, idx0, idx; keyboard_t *k0, *k; keyboard_info_t ki; /* * If vt_upgrade() happens while the console is grabbed, we are * potentially going to switch keyboard devices while the keyboard is in * use. Unwind the grabbing of the current keyboard first, then we will * re-grab the new keyboard below, before we return. */ if (vd->vd_curwindow == &vt_conswindow) { grabbed = vd->vd_curwindow->vw_grabbed; for (i = 0; i < grabbed; ++i) vtterm_cnungrab_noswitch(vd, vd->vd_curwindow); } idx0 = kbd_allocate("kbdmux", -1, vd, vt_kbdevent, vd); if (idx0 >= 0) { DPRINTF(20, "%s: kbdmux allocated, idx = %d\n", __func__, idx0); k0 = kbd_get_keyboard(idx0); for (idx = kbd_find_keyboard2("*", -1, 0); idx != -1; idx = kbd_find_keyboard2("*", -1, idx + 1)) { k = kbd_get_keyboard(idx); if (idx == idx0 || KBD_IS_BUSY(k)) continue; bzero(&ki, sizeof(ki)); strncpy(ki.kb_name, k->kb_name, sizeof(ki.kb_name)); ki.kb_name[sizeof(ki.kb_name) - 1] = '\0'; ki.kb_unit = k->kb_unit; kbdd_ioctl(k0, KBADDKBD, (caddr_t) &ki); } } else { DPRINTF(20, "%s: no kbdmux allocated\n", __func__); idx0 = kbd_allocate("*", -1, vd, vt_kbdevent, vd); if (idx0 < 0) { DPRINTF(10, "%s: No keyboard found.\n", __func__); return (-1); } k0 = kbd_get_keyboard(idx0); } vd->vd_keyboard = k0; DPRINTF(20, "%s: vd_keyboard = %d\n", __func__, vd->vd_keyboard->kb_index); if (vd->vd_curwindow == &vt_conswindow) { for (i = 0; i < grabbed; ++i) vtterm_cngrab_noswitch(vd, vd->vd_curwindow); } return (idx0); } static void vtterm_bell(struct terminal *tm) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; if (!vt_enable_bell) return; if (vd->vd_flags & VDF_QUIET_BELL) return; sysbeep(1193182 / VT_BELLPITCH, VT_BELLDURATION); } static void vtterm_beep(struct terminal *tm, u_int param) { u_int freq, period; if (!vt_enable_bell) return; if ((param == 0) || ((param & 0xffff) == 0)) { vtterm_bell(tm); return; } period = ((param >> 16) & 0xffff) * hz / 1000; freq = 1193182 / (param & 0xffff); sysbeep(freq, period); } static void vtterm_cursor(struct terminal *tm, const term_pos_t *p) { struct vt_window *vw = tm->tm_softc; vtbuf_cursor_position(&vw->vw_buf, p); } static void vtterm_putchar(struct terminal *tm, const term_pos_t *p, term_char_t c) { struct vt_window *vw = tm->tm_softc; vtbuf_putchar(&vw->vw_buf, p, c); } static void vtterm_fill(struct terminal *tm, const term_rect_t *r, term_char_t c) { struct vt_window *vw = tm->tm_softc; vtbuf_fill(&vw->vw_buf, r, c); } static void vtterm_copy(struct terminal *tm, const term_rect_t *r, const term_pos_t *p) { struct vt_window *vw = tm->tm_softc; vtbuf_copy(&vw->vw_buf, r, p); } static void vtterm_param(struct terminal *tm, int cmd, unsigned int arg) { struct vt_window *vw = tm->tm_softc; switch (cmd) { case TP_SETLOCALCURSOR: /* * 0 means normal (usually block), 1 means hidden, and * 2 means blinking (always block) for compatibility with * syscons. We don't support any changes except hiding, * so must map 2 to 0. */ arg = (arg == 1) ? 0 : 1; /* FALLTHROUGH */ case TP_SHOWCURSOR: vtbuf_cursor_visibility(&vw->vw_buf, arg); vt_resume_flush_timer(vw, 0); break; case TP_MOUSE: vw->vw_mouse_level = arg; break; } } void vt_determine_colors(term_char_t c, int cursor, term_color_t *fg, term_color_t *bg) { term_color_t tmp; int invert; invert = 0; *fg = TCHAR_FGCOLOR(c); if (TCHAR_FORMAT(c) & TF_BOLD) *fg = TCOLOR_LIGHT(*fg); *bg = TCHAR_BGCOLOR(c); if (TCHAR_FORMAT(c) & TF_BLINK) *bg = TCOLOR_LIGHT(*bg); if (TCHAR_FORMAT(c) & TF_REVERSE) invert ^= 1; if (cursor) invert ^= 1; if (invert) { tmp = *fg; *fg = *bg; *bg = tmp; } } #ifndef SC_NO_CUTPASTE int vt_is_cursor_in_area(const struct vt_device *vd, const term_rect_t *area) { unsigned int mx, my; /* * We use the cursor position saved during the current refresh, * in case the cursor moved since. */ mx = vd->vd_mx_drawn + vd->vd_curwindow->vw_draw_area.tr_begin.tp_col; my = vd->vd_my_drawn + vd->vd_curwindow->vw_draw_area.tr_begin.tp_row; if (mx >= area->tr_end.tp_col || mx + vd->vd_mcursor->width <= area->tr_begin.tp_col || my >= area->tr_end.tp_row || my + vd->vd_mcursor->height <= area->tr_begin.tp_row) return (0); return (1); } static void vt_mark_mouse_position_as_dirty(struct vt_device *vd, int locked) { term_rect_t area; struct vt_window *vw; struct vt_font *vf; int x, y; vw = vd->vd_curwindow; vf = vw->vw_font; x = vd->vd_mx_drawn; y = vd->vd_my_drawn; if (vf != NULL) { area.tr_begin.tp_col = x / vf->vf_width; area.tr_begin.tp_row = y / vf->vf_height; area.tr_end.tp_col = ((x + vd->vd_mcursor->width) / vf->vf_width) + 1; area.tr_end.tp_row = ((y + vd->vd_mcursor->height) / vf->vf_height) + 1; } else { /* * No font loaded (ie. vt_vga operating in textmode). * * FIXME: This fake area needs to be revisited once the * mouse cursor is supported in vt_vga's textmode. */ area.tr_begin.tp_col = x; area.tr_begin.tp_row = y; area.tr_end.tp_col = x + 2; area.tr_end.tp_row = y + 2; } if (!locked) vtbuf_lock(&vw->vw_buf); if (vd->vd_driver->vd_invalidate_text) vd->vd_driver->vd_invalidate_text(vd, &area); vtbuf_dirty(&vw->vw_buf, &area); if (!locked) vtbuf_unlock(&vw->vw_buf); } #endif static void vt_set_border(struct vt_device *vd, const term_rect_t *area, term_color_t c) { vd_drawrect_t *drawrect = vd->vd_driver->vd_drawrect; if (drawrect == NULL) return; /* Top bar */ if (area->tr_begin.tp_row > 0) drawrect(vd, 0, 0, vd->vd_width - 1, area->tr_begin.tp_row - 1, 1, c); /* Left bar */ if (area->tr_begin.tp_col > 0) drawrect(vd, 0, area->tr_begin.tp_row, area->tr_begin.tp_col - 1, area->tr_end.tp_row - 1, 1, c); /* Right bar */ if (area->tr_end.tp_col < vd->vd_width) drawrect(vd, area->tr_end.tp_col, area->tr_begin.tp_row, vd->vd_width - 1, area->tr_end.tp_row - 1, 1, c); /* Bottom bar */ if (area->tr_end.tp_row < vd->vd_height) drawrect(vd, 0, area->tr_end.tp_row, vd->vd_width - 1, vd->vd_height - 1, 1, c); } static int vt_flush(struct vt_device *vd) { struct vt_window *vw; struct vt_font *vf; term_rect_t tarea; #ifndef SC_NO_CUTPASTE int cursor_was_shown, cursor_moved; #endif vw = vd->vd_curwindow; if (vw == NULL) return (0); if (vd->vd_flags & VDF_SPLASH || vw->vw_flags & VWF_BUSY) return (0); vf = vw->vw_font; if (((vd->vd_flags & VDF_TEXTMODE) == 0) && (vf == NULL)) return (0); vtbuf_lock(&vw->vw_buf); #ifndef SC_NO_CUTPASTE cursor_was_shown = vd->vd_mshown; cursor_moved = (vd->vd_mx != vd->vd_mx_drawn || vd->vd_my != vd->vd_my_drawn); /* Check if the cursor should be displayed or not. */ if ((vd->vd_flags & VDF_MOUSECURSOR) && /* Mouse support enabled. */ !(vw->vw_flags & VWF_MOUSE_HIDE) && /* Cursor displayed. */ !kdb_active && !KERNEL_PANICKED()) { /* DDB inactive. */ vd->vd_mshown = 1; } else { vd->vd_mshown = 0; } /* * If the cursor changed display state or moved, we must mark * the old position as dirty, so that it's erased. */ if (cursor_was_shown != vd->vd_mshown || (vd->vd_mshown && cursor_moved)) vt_mark_mouse_position_as_dirty(vd, true); /* * Save position of the mouse cursor. It's used by backends to * know where to draw the cursor and during the next refresh to * erase the previous position. */ vd->vd_mx_drawn = vd->vd_mx; vd->vd_my_drawn = vd->vd_my; /* * If the cursor is displayed and has moved since last refresh, * mark the new position as dirty. */ if (vd->vd_mshown && cursor_moved) vt_mark_mouse_position_as_dirty(vd, true); #endif vtbuf_undirty(&vw->vw_buf, &tarea); /* Force a full redraw when the screen contents might be invalid. */ if (vd->vd_flags & (VDF_INVALID | VDF_SUSPENDED)) { const teken_attr_t *a; vd->vd_flags &= ~VDF_INVALID; a = teken_get_curattr(&vw->vw_terminal->tm_emulator); vt_set_border(vd, &vw->vw_draw_area, a->ta_bgcolor); vt_termrect(vd, vf, &tarea); if (vd->vd_driver->vd_invalidate_text) vd->vd_driver->vd_invalidate_text(vd, &tarea); if (vt_draw_logo_cpus) vtterm_draw_cpu_logos(vd); } if (tarea.tr_begin.tp_col < tarea.tr_end.tp_col) { vd->vd_driver->vd_bitblt_text(vd, vw, &tarea); vtbuf_unlock(&vw->vw_buf); return (1); } vtbuf_unlock(&vw->vw_buf); return (0); } static void vt_timer(void *arg) { struct vt_device *vd; int changed; vd = arg; /* Update screen if required. */ changed = vt_flush(vd); /* Schedule for next update. */ if (changed) vt_schedule_flush(vd, 0); else vd->vd_timer_armed = 0; } static void vtterm_pre_input(struct terminal *tm) { struct vt_window *vw = tm->tm_softc; vtbuf_lock(&vw->vw_buf); } static void vtterm_post_input(struct terminal *tm) { struct vt_window *vw = tm->tm_softc; vtbuf_unlock(&vw->vw_buf); vt_resume_flush_timer(vw, 0); } static void vtterm_done(struct terminal *tm) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; if (kdb_active || KERNEL_PANICKED()) { /* Switch to the debugger. */ if (vd->vd_curwindow != vw) { vd->vd_curwindow = vw; vd->vd_flags |= VDF_INVALID; if (vd->vd_driver->vd_postswitch) vd->vd_driver->vd_postswitch(vd); } vd->vd_flags &= ~VDF_SPLASH; vt_flush(vd); } else if (!(vd->vd_flags & VDF_ASYNC)) { vt_flush(vd); } } #ifdef DEV_SPLASH static void vtterm_splash(struct vt_device *vd) { vt_axis_t top, left; /* Display a nice boot splash. */ if (!(vd->vd_flags & VDF_TEXTMODE) && (boothowto & RB_MUTE)) { top = (vd->vd_height - vt_logo_height) / 2; left = (vd->vd_width - vt_logo_width) / 2; switch (vt_logo_depth) { case 1: /* XXX: Unhardcode colors! */ vd->vd_driver->vd_bitblt_bmp(vd, vd->vd_curwindow, vt_logo_image, NULL, vt_logo_width, vt_logo_height, left, top, TC_WHITE, TC_BLACK); } vd->vd_flags |= VDF_SPLASH; } } #endif static struct vt_font * parse_font_info_static(struct font_info *fi) { struct vt_font *vfp; uintptr_t ptr; uint32_t checksum; if (fi == NULL) return (NULL); ptr = (uintptr_t)fi; /* * Compute and verify checksum. The total sum of all the fields * must be 0. */ checksum = fi->fi_width; checksum += fi->fi_height; checksum += fi->fi_bitmap_size; for (unsigned i = 0; i < VFNT_MAPS; i++) checksum += fi->fi_map_count[i]; if (checksum + fi->fi_checksum != 0) return (NULL); ptr += sizeof(struct font_info); ptr = roundup2(ptr, 8); vfp = &vt_font_loader; vfp->vf_height = fi->fi_height; vfp->vf_width = fi->fi_width; /* This is default font, set refcount 1 to disable removal. */ vfp->vf_refcount = 1; for (unsigned i = 0; i < VFNT_MAPS; i++) { if (fi->fi_map_count[i] == 0) continue; vfp->vf_map_count[i] = fi->fi_map_count[i]; vfp->vf_map[i] = (vfnt_map_t *)ptr; ptr += (fi->fi_map_count[i] * sizeof(vfnt_map_t)); ptr = roundup2(ptr, 8); } vfp->vf_bytes = (uint8_t *)ptr; return (vfp); } /* * Set up default font with allocated data structures. * However, we can not set refcount here, because it is already set and * incremented in vtterm_cnprobe() to avoid being released by font load from * userland. */ static struct vt_font * parse_font_info(struct font_info *fi) { struct vt_font *vfp; uintptr_t ptr; uint32_t checksum; size_t size; if (fi == NULL) return (NULL); ptr = (uintptr_t)fi; /* * Compute and verify checksum. The total sum of all the fields * must be 0. */ checksum = fi->fi_width; checksum += fi->fi_height; checksum += fi->fi_bitmap_size; for (unsigned i = 0; i < VFNT_MAPS; i++) checksum += fi->fi_map_count[i]; if (checksum + fi->fi_checksum != 0) return (NULL); ptr += sizeof(struct font_info); ptr = roundup2(ptr, 8); vfp = &vt_font_loader; vfp->vf_height = fi->fi_height; vfp->vf_width = fi->fi_width; for (unsigned i = 0; i < VFNT_MAPS; i++) { if (fi->fi_map_count[i] == 0) continue; vfp->vf_map_count[i] = fi->fi_map_count[i]; size = fi->fi_map_count[i] * sizeof(vfnt_map_t); vfp->vf_map[i] = malloc(size, M_VT, M_WAITOK | M_ZERO); bcopy((vfnt_map_t *)ptr, vfp->vf_map[i], size); ptr += size; ptr = roundup2(ptr, 8); } vfp->vf_bytes = malloc(fi->fi_bitmap_size, M_VT, M_WAITOK | M_ZERO); bcopy((uint8_t *)ptr, vfp->vf_bytes, fi->fi_bitmap_size); return (vfp); } static void vt_init_font(void *arg) { caddr_t kmdp; struct font_info *fi; struct vt_font *font; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); fi = MD_FETCH(kmdp, MODINFOMD_FONT, struct font_info *); font = parse_font_info(fi); if (font != NULL) vt_font_assigned = font; } SYSINIT(vt_init_font, SI_SUB_KMEM, SI_ORDER_ANY, vt_init_font, &vt_consdev); static void vt_init_font_static(void) { caddr_t kmdp; struct font_info *fi; struct vt_font *font; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) kmdp = preload_search_by_type("elf64 kernel"); fi = MD_FETCH(kmdp, MODINFOMD_FONT, struct font_info *); font = parse_font_info_static(fi); if (font != NULL) vt_font_assigned = font; } static void vtterm_cnprobe(struct terminal *tm, struct consdev *cp) { struct vt_driver *vtd, **vtdlist, *vtdbest = NULL; struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; struct winsize wsz; const term_attr_t *a; if (!vty_enabled(VTY_VT)) return; if (vd->vd_flags & VDF_INITIALIZED) /* Initialization already done. */ return; SET_FOREACH(vtdlist, vt_drv_set) { vtd = *vtdlist; if (vtd->vd_probe == NULL) continue; if (vtd->vd_probe(vd) == CN_DEAD) continue; if ((vtdbest == NULL) || (vtd->vd_priority > vtdbest->vd_priority)) vtdbest = vtd; } if (vtdbest == NULL) { cp->cn_pri = CN_DEAD; vd->vd_flags |= VDF_DEAD; } else { vd->vd_driver = vtdbest; cp->cn_pri = vd->vd_driver->vd_init(vd); } /* Check if driver's vt_init return CN_DEAD. */ if (cp->cn_pri == CN_DEAD) { vd->vd_flags |= VDF_DEAD; } /* Initialize any early-boot keyboard drivers */ kbd_configure(KB_CONF_PROBE_ONLY); vd->vd_unit = atomic_fetchadd_int(&vt_unit, 1); vd->vd_windows[VT_CONSWINDOW] = vw; sprintf(cp->cn_name, "ttyv%r", VT_UNIT(vw)); vt_init_font_static(); /* Attach default font if not in TEXTMODE. */ if ((vd->vd_flags & VDF_TEXTMODE) == 0) { vw->vw_font = vtfont_ref(vt_font_assigned); vt_compute_drawable_area(vw); } /* * The original screen size was faked (_VTDEFW x _VTDEFH). Now * that we have the real viewable size, fix it in the static * buffer. */ if (vd->vd_width != 0 && vd->vd_height != 0) vt_termsize(vd, vw->vw_font, &vw->vw_buf.vb_scr_size); /* We need to access terminal attributes from vtbuf */ vw->vw_buf.vb_terminal = tm; vtbuf_init_early(&vw->vw_buf); vt_winsize(vd, vw->vw_font, &wsz); a = teken_get_curattr(&tm->tm_emulator); terminal_set_winsize_blank(tm, &wsz, 1, a); if (vtdbest != NULL) { #ifdef DEV_SPLASH if (!vt_splash_cpu) vtterm_splash(vd); #endif vd->vd_flags |= VDF_INITIALIZED; } } static int vtterm_cngetc(struct terminal *tm) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; keyboard_t *kbd; u_int c; if (vw->vw_kbdsq && *vw->vw_kbdsq) return (*vw->vw_kbdsq++); /* Make sure the splash screen is not there. */ if (vd->vd_flags & VDF_SPLASH) { /* Remove splash */ vd->vd_flags &= ~VDF_SPLASH; /* Mark screen as invalid to force update */ vd->vd_flags |= VDF_INVALID; vt_flush(vd); } /* Stripped down keyboard handler. */ if ((kbd = vd->vd_keyboard) == NULL) return (-1); /* Force keyboard input mode to K_XLATE */ vw->vw_kbdmode = K_XLATE; vt_update_kbd_mode(vw, kbd); /* Switch the keyboard to polling to make it work here. */ kbdd_poll(kbd, TRUE); c = kbdd_read_char(kbd, 0); kbdd_poll(kbd, FALSE); if (c & RELKEY) return (-1); if (vw->vw_flags & VWF_SCROLL) { vt_scrollmode_kbdevent(vw, c, 1/* Console mode */); vt_flush(vd); return (-1); } /* Stripped down handling of vt_kbdevent(), without locking, etc. */ if (c & SPCLKEY) { switch (c) { case SPCLKEY | SLK: vt_save_kbd_state(vw, kbd); if (vw->vw_kbdstate & SLKED) { /* Turn scrolling on. */ vw->vw_flags |= VWF_SCROLL; VTBUF_SLCK_ENABLE(&vw->vw_buf); } else { /* Turn scrolling off. */ vt_scroll(vw, 0, VHS_END); vw->vw_flags &= ~VWF_SCROLL; VTBUF_SLCK_DISABLE(&vw->vw_buf); } break; /* XXX: KDB can handle history. */ case SPCLKEY | FKEY | F(50): /* Arrow up. */ vw->vw_kbdsq = "\x1b[A"; break; case SPCLKEY | FKEY | F(58): /* Arrow down. */ vw->vw_kbdsq = "\x1b[B"; break; case SPCLKEY | FKEY | F(55): /* Arrow right. */ vw->vw_kbdsq = "\x1b[C"; break; case SPCLKEY | FKEY | F(53): /* Arrow left. */ vw->vw_kbdsq = "\x1b[D"; break; } /* Force refresh to make scrollback work. */ vt_flush(vd); } else if (KEYFLAGS(c) == 0) { return (KEYCHAR(c)); } if (vw->vw_kbdsq && *vw->vw_kbdsq) return (*vw->vw_kbdsq++); return (-1); } /* * These two do most of what we want to do in vtterm_cnungrab, but without * actually switching windows. This is necessary for, e.g., * vt_allocate_keyboard() to get the current keyboard into the state it needs to * be in without damaging the device's window state. * * Both return the current grab count, though it's only used in vtterm_cnungrab. */ static int vtterm_cngrab_noswitch(struct vt_device *vd, struct vt_window *vw) { keyboard_t *kbd; if (vw->vw_grabbed++ > 0) return (vw->vw_grabbed); if ((kbd = vd->vd_keyboard) == NULL) return (1); /* * Make sure the keyboard is accessible even when the kbd device * driver is disabled. */ kbdd_enable(kbd); /* We shall always use the keyboard in the XLATE mode here. */ vw->vw_prev_kbdmode = vw->vw_kbdmode; vw->vw_kbdmode = K_XLATE; vt_update_kbd_mode(vw, kbd); kbdd_poll(kbd, TRUE); return (1); } static int vtterm_cnungrab_noswitch(struct vt_device *vd, struct vt_window *vw) { keyboard_t *kbd; if (--vw->vw_grabbed > 0) return (vw->vw_grabbed); if ((kbd = vd->vd_keyboard) == NULL) return (0); kbdd_poll(kbd, FALSE); vw->vw_kbdmode = vw->vw_prev_kbdmode; vt_update_kbd_mode(vw, kbd); kbdd_disable(kbd); return (0); } static void vtterm_cngrab(struct terminal *tm) { struct vt_device *vd; struct vt_window *vw; vw = tm->tm_softc; vd = vw->vw_device; /* To be restored after we ungrab. */ if (vd->vd_grabwindow == NULL) vd->vd_grabwindow = vd->vd_curwindow; if (!cold) vt_window_switch(vw); vtterm_cngrab_noswitch(vd, vw); } static void vtterm_cnungrab(struct terminal *tm) { struct vt_device *vd; struct vt_window *vw; vw = tm->tm_softc; vd = vw->vw_device; MPASS(vd->vd_grabwindow != NULL); if (vtterm_cnungrab_noswitch(vd, vw) != 0) return; if (!cold && vd->vd_grabwindow != vw) vt_window_switch(vd->vd_grabwindow); vd->vd_grabwindow = NULL; } static void vtterm_opened(struct terminal *tm, int opened) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; VT_LOCK(vd); vd->vd_flags &= ~VDF_SPLASH; if (opened) vw->vw_flags |= VWF_OPENED; else { vw->vw_flags &= ~VWF_OPENED; /* TODO: finish ACQ/REL */ } VT_UNLOCK(vd); } static int vt_change_font(struct vt_window *vw, struct vt_font *vf) { struct vt_device *vd = vw->vw_device; struct terminal *tm = vw->vw_terminal; term_pos_t size; struct winsize wsz; /* * Changing fonts. * * Changing fonts is a little tricky. We must prevent * simultaneous access to the device, so we must stop * the display timer and the terminal from accessing. * We need to switch fonts and grow our screen buffer. * * XXX: Right now the code uses terminal_mute() to * prevent data from reaching the console driver while * resizing the screen buffer. This isn't elegant... */ VT_LOCK(vd); if (vw->vw_flags & VWF_BUSY) { /* Another process is changing the font. */ VT_UNLOCK(vd); return (EBUSY); } vw->vw_flags |= VWF_BUSY; VT_UNLOCK(vd); vt_termsize(vd, vf, &size); vt_winsize(vd, vf, &wsz); /* Grow the screen buffer and terminal. */ terminal_mute(tm, 1); vtbuf_grow(&vw->vw_buf, &size, vw->vw_buf.vb_history_size); terminal_set_winsize_blank(tm, &wsz, 0, NULL); terminal_set_cursor(tm, &vw->vw_buf.vb_cursor); terminal_mute(tm, 0); /* Actually apply the font to the current window. */ VT_LOCK(vd); if (vw->vw_font != vf && vw->vw_font != NULL && vf != NULL) { /* * In case vt_change_font called to update size we don't need * to update font link. */ vtfont_unref(vw->vw_font); vw->vw_font = vtfont_ref(vf); } /* * Compute the drawable area and move the mouse cursor inside * it, in case the new area is smaller than the previous one. */ vt_compute_drawable_area(vw); vd->vd_mx = min(vd->vd_mx, vw->vw_draw_area.tr_end.tp_col - vw->vw_draw_area.tr_begin.tp_col - 1); vd->vd_my = min(vd->vd_my, vw->vw_draw_area.tr_end.tp_row - vw->vw_draw_area.tr_begin.tp_row - 1); /* Force a full redraw the next timer tick. */ if (vd->vd_curwindow == vw) { vd->vd_flags |= VDF_INVALID; vt_resume_flush_timer(vw, 0); } vw->vw_flags &= ~VWF_BUSY; VT_UNLOCK(vd); return (0); } static int vt_proc_alive(struct vt_window *vw) { struct proc *p; if (vw->vw_smode.mode != VT_PROCESS) return (FALSE); if (vw->vw_proc) { if ((p = pfind(vw->vw_pid)) != NULL) PROC_UNLOCK(p); if (vw->vw_proc == p) return (TRUE); vw->vw_proc = NULL; vw->vw_smode.mode = VT_AUTO; DPRINTF(1, "vt controlling process %d died\n", vw->vw_pid); vw->vw_pid = 0; } return (FALSE); } static int signal_vt_rel(struct vt_window *vw) { if (vw->vw_smode.mode != VT_PROCESS) return (FALSE); if (vw->vw_proc == NULL || vt_proc_alive(vw) == FALSE) { vw->vw_proc = NULL; vw->vw_pid = 0; return (TRUE); } vw->vw_flags |= VWF_SWWAIT_REL; PROC_LOCK(vw->vw_proc); kern_psignal(vw->vw_proc, vw->vw_smode.relsig); PROC_UNLOCK(vw->vw_proc); DPRINTF(1, "sending relsig to %d\n", vw->vw_pid); return (TRUE); } static int signal_vt_acq(struct vt_window *vw) { if (vw->vw_smode.mode != VT_PROCESS) return (FALSE); if (vw == vw->vw_device->vd_windows[VT_CONSWINDOW]) cnavailable(vw->vw_terminal->consdev, FALSE); if (vw->vw_proc == NULL || vt_proc_alive(vw) == FALSE) { vw->vw_proc = NULL; vw->vw_pid = 0; return (TRUE); } vw->vw_flags |= VWF_SWWAIT_ACQ; PROC_LOCK(vw->vw_proc); kern_psignal(vw->vw_proc, vw->vw_smode.acqsig); PROC_UNLOCK(vw->vw_proc); DPRINTF(1, "sending acqsig to %d\n", vw->vw_pid); return (TRUE); } static int finish_vt_rel(struct vt_window *vw, int release, int *s) { if (vw->vw_flags & VWF_SWWAIT_REL) { vw->vw_flags &= ~VWF_SWWAIT_REL; if (release) { callout_drain(&vw->vw_proc_dead_timer); (void)vt_late_window_switch(vw->vw_switch_to); } return (0); } return (EINVAL); } static int finish_vt_acq(struct vt_window *vw) { if (vw->vw_flags & VWF_SWWAIT_ACQ) { vw->vw_flags &= ~VWF_SWWAIT_ACQ; return (0); } return (EINVAL); } #ifndef SC_NO_CUTPASTE static void vt_mouse_terminput_button(struct vt_device *vd, int button) { struct vt_window *vw; struct vt_font *vf; char mouseb[6] = "\x1B[M"; int i, x, y; vw = vd->vd_curwindow; vf = vw->vw_font; /* Translate to char position. */ x = vd->vd_mx / vf->vf_width; y = vd->vd_my / vf->vf_height; /* Avoid overflow. */ x = MIN(x, 255 - '!'); y = MIN(y, 255 - '!'); mouseb[3] = ' ' + button; mouseb[4] = '!' + x; mouseb[5] = '!' + y; for (i = 0; i < sizeof(mouseb); i++) terminal_input_char(vw->vw_terminal, mouseb[i]); } static void vt_mouse_terminput(struct vt_device *vd, int type, int x, int y, int event, int cnt) { switch (type) { case MOUSE_BUTTON_EVENT: if (cnt > 0) { /* Mouse button pressed. */ if (event & MOUSE_BUTTON1DOWN) vt_mouse_terminput_button(vd, 0); if (event & MOUSE_BUTTON2DOWN) vt_mouse_terminput_button(vd, 1); if (event & MOUSE_BUTTON3DOWN) vt_mouse_terminput_button(vd, 2); } else { /* Mouse button released. */ vt_mouse_terminput_button(vd, 3); } break; #ifdef notyet case MOUSE_MOTION_EVENT: if (mouse->u.data.z < 0) { /* Scroll up. */ sc_mouse_input_button(vd, 64); } else if (mouse->u.data.z > 0) { /* Scroll down. */ sc_mouse_input_button(vd, 65); } break; #endif } } static void vt_mouse_paste() { term_char_t *buf; int i, len; len = VD_PASTEBUFLEN(main_vd); buf = VD_PASTEBUF(main_vd); len /= sizeof(term_char_t); for (i = 0; i < len; i++) { if (buf[i] == '\0') continue; terminal_input_char(main_vd->vd_curwindow->vw_terminal, buf[i]); } } void vt_mouse_event(int type, int x, int y, int event, int cnt, int mlevel) { struct vt_device *vd; struct vt_window *vw; struct vt_font *vf; term_pos_t size; int len, mark; vd = main_vd; vw = vd->vd_curwindow; vf = vw->vw_font; mark = 0; if (vw->vw_flags & (VWF_MOUSE_HIDE | VWF_GRAPHICS)) /* * Either the mouse is disabled, or the window is in * "graphics mode". The graphics mode is usually set by * an X server, using the KDSETMODE ioctl. */ return; if (vf == NULL) /* Text mode. */ return; /* * TODO: add flag about pointer position changed, to not redraw chars * under mouse pointer when nothing changed. */ if (vw->vw_mouse_level > 0) vt_mouse_terminput(vd, type, x, y, event, cnt); switch (type) { case MOUSE_ACTION: case MOUSE_MOTION_EVENT: /* Movement */ x += vd->vd_mx; y += vd->vd_my; vt_termsize(vd, vf, &size); /* Apply limits. */ x = MAX(x, 0); y = MAX(y, 0); x = MIN(x, (size.tp_col * vf->vf_width) - 1); y = MIN(y, (size.tp_row * vf->vf_height) - 1); vd->vd_mx = x; vd->vd_my = y; if (vd->vd_mstate & MOUSE_BUTTON1DOWN) vtbuf_set_mark(&vw->vw_buf, VTB_MARK_MOVE, vd->vd_mx / vf->vf_width, vd->vd_my / vf->vf_height); vt_resume_flush_timer(vw, 0); return; /* Done */ case MOUSE_BUTTON_EVENT: /* Buttons */ break; default: return; /* Done */ } switch (event) { case MOUSE_BUTTON1DOWN: switch (cnt % 4) { case 0: /* up */ mark = VTB_MARK_END; break; case 1: /* single click: start cut operation */ mark = VTB_MARK_START; break; case 2: /* double click: cut a word */ mark = VTB_MARK_WORD; break; case 3: /* triple click: cut a line */ mark = VTB_MARK_ROW; break; } break; case VT_MOUSE_PASTEBUTTON: switch (cnt) { case 0: /* up */ break; default: vt_mouse_paste(); break; } return; /* Done */ case VT_MOUSE_EXTENDBUTTON: switch (cnt) { case 0: /* up */ if (!(vd->vd_mstate & MOUSE_BUTTON1DOWN)) mark = VTB_MARK_EXTEND; else mark = 0; break; default: mark = VTB_MARK_EXTEND; break; } break; default: return; /* Done */ } /* Save buttons state. */ if (cnt > 0) vd->vd_mstate |= event; else vd->vd_mstate &= ~event; if (vtbuf_set_mark(&vw->vw_buf, mark, vd->vd_mx / vf->vf_width, vd->vd_my / vf->vf_height) == 1) { /* * We have something marked to copy, so update pointer to * window with selection. */ vt_resume_flush_timer(vw, 0); switch (mark) { case VTB_MARK_END: case VTB_MARK_WORD: case VTB_MARK_ROW: case VTB_MARK_EXTEND: break; default: /* Other types of mark do not require to copy data. */ return; } /* Get current selection size in bytes. */ len = vtbuf_get_marked_len(&vw->vw_buf); if (len <= 0) return; /* Reallocate buffer only if old one is too small. */ if (len > VD_PASTEBUFSZ(vd)) { VD_PASTEBUF(vd) = realloc(VD_PASTEBUF(vd), len, M_VT, M_WAITOK | M_ZERO); /* Update buffer size. */ VD_PASTEBUFSZ(vd) = len; } /* Request copy/paste buffer data, no more than `len' */ vtbuf_extract_marked(&vw->vw_buf, VD_PASTEBUF(vd), VD_PASTEBUFSZ(vd)); VD_PASTEBUFLEN(vd) = len; /* XXX VD_PASTEBUF(vd) have to be freed on shutdown/unload. */ } } void vt_mouse_state(int show) { struct vt_device *vd; struct vt_window *vw; vd = main_vd; vw = vd->vd_curwindow; switch (show) { case VT_MOUSE_HIDE: vw->vw_flags |= VWF_MOUSE_HIDE; break; case VT_MOUSE_SHOW: vw->vw_flags &= ~VWF_MOUSE_HIDE; break; } /* Mark mouse position as dirty. */ vt_mark_mouse_position_as_dirty(vd, false); vt_resume_flush_timer(vw, 0); } #endif static int vtterm_mmap(struct terminal *tm, vm_ooffset_t offset, vm_paddr_t * paddr, int nprot, vm_memattr_t *memattr) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; if (vd->vd_driver->vd_fb_mmap) return (vd->vd_driver->vd_fb_mmap(vd, offset, paddr, nprot, memattr)); return (ENXIO); } static int vtterm_ioctl(struct terminal *tm, u_long cmd, caddr_t data, struct thread *td) { struct vt_window *vw = tm->tm_softc; struct vt_device *vd = vw->vw_device; keyboard_t *kbd; int error, i, s; #if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD4) || defined(COMPAT_43) int ival; switch (cmd) { case _IO('v', 4): cmd = VT_RELDISP; break; case _IO('v', 5): cmd = VT_ACTIVATE; break; case _IO('v', 6): cmd = VT_WAITACTIVE; break; case _IO('K', 20): cmd = KDSKBSTATE; break; case _IO('K', 67): cmd = KDSETRAD; break; case _IO('K', 7): cmd = KDSKBMODE; break; case _IO('K', 8): cmd = KDMKTONE; break; case _IO('K', 10): cmd = KDSETMODE; break; case _IO('K', 13): cmd = KDSBORDER; break; case _IO('K', 63): cmd = KIOCSOUND; break; case _IO('K', 66): cmd = KDSETLED; break; case _IO('c', 104): cmd = CONS_SETWINORG; break; case _IO('c', 110): cmd = CONS_SETKBD; break; default: goto skip_thunk; } ival = IOCPARM_IVAL(data); data = (caddr_t)&ival; skip_thunk: #endif switch (cmd) { case KDSETRAD: /* set keyboard repeat & delay rates (old) */ if (*(int *)data & ~0x7f) return (EINVAL); /* FALLTHROUGH */ case GIO_KEYMAP: case PIO_KEYMAP: case GIO_DEADKEYMAP: case PIO_DEADKEYMAP: case GETFKEY: case SETFKEY: case KDGKBINFO: case KDGKBTYPE: case KDGETREPEAT: /* get keyboard repeat & delay rates */ case KDSETREPEAT: /* set keyboard repeat & delay rates (new) */ case KBADDKBD: /* add/remove keyboard to/from mux */ case KBRELKBD: { error = 0; mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = kbdd_ioctl(kbd, cmd, data); mtx_unlock(&Giant); if (error == ENOIOCTL) { if (cmd == KDGKBTYPE) { /* always return something? XXX */ *(int *)data = 0; } else { return (ENODEV); } } return (error); } case KDGKBSTATE: { /* get keyboard state (locks) */ error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_save_kbd_state(vw, kbd); mtx_unlock(&Giant); if (error != 0) return (error); } *(int *)data = vw->vw_kbdstate & LOCK_MASK; return (error); } case KDSKBSTATE: { /* set keyboard state (locks) */ int state; state = *(int *)data; if (state & ~LOCK_MASK) return (EINVAL); vw->vw_kbdstate &= ~LOCK_MASK; vw->vw_kbdstate |= state; error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_update_kbd_state(vw, kbd); mtx_unlock(&Giant); } return (error); } case KDGETLED: { /* get keyboard LED status */ error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_save_kbd_leds(vw, kbd); mtx_unlock(&Giant); if (error != 0) return (error); } *(int *)data = vw->vw_kbdstate & LED_MASK; return (error); } case KDSETLED: { /* set keyboard LED status */ int leds; leds = *(int *)data; if (leds & ~LED_MASK) return (EINVAL); vw->vw_kbdstate &= ~LED_MASK; vw->vw_kbdstate |= leds; error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_update_kbd_leds(vw, kbd); mtx_unlock(&Giant); } return (error); } case KDGETMODE: *(int *)data = (vw->vw_flags & VWF_GRAPHICS) ? KD_GRAPHICS : KD_TEXT; return (0); case KDGKBMODE: { error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_save_kbd_mode(vw, kbd); mtx_unlock(&Giant); if (error != 0) return (error); } *(int *)data = vw->vw_kbdmode; return (error); } case KDSKBMODE: { int mode; mode = *(int *)data; switch (mode) { case K_XLATE: case K_RAW: case K_CODE: vw->vw_kbdmode = mode; error = 0; if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) error = vt_update_kbd_mode(vw, kbd); mtx_unlock(&Giant); } return (error); default: return (EINVAL); } } case FBIOGTYPE: case FBIO_GETWINORG: /* get frame buffer window origin */ case FBIO_GETDISPSTART: /* get display start address */ case FBIO_GETLINEWIDTH: /* get scan line width in bytes */ case FBIO_BLANK: /* blank display */ if (vd->vd_driver->vd_fb_ioctl) return (vd->vd_driver->vd_fb_ioctl(vd, cmd, data, td)); break; case CONS_BLANKTIME: /* XXX */ return (0); case CONS_HISTORY: if (*(int *)data < 0) return EINVAL; if (*(int *)data != vw->vw_buf.vb_history_size) vtbuf_sethistory_size(&vw->vw_buf, *(int *)data); return (0); case CONS_CLRHIST: vtbuf_clearhistory(&vw->vw_buf); /* * Invalidate the entire visible window; it is not guaranteed * that this operation will be immediately followed by a scroll * event, so it would otherwise be possible for prior artifacts * to remain visible. */ VT_LOCK(vd); if (vw == vd->vd_curwindow) { vd->vd_flags |= VDF_INVALID; vt_resume_flush_timer(vw, 0); } VT_UNLOCK(vd); return (0); case CONS_GET: /* XXX */ *(int *)data = M_CG640x480; return (0); case CONS_BELLTYPE: /* set bell type sound */ if ((*(int *)data) & CONS_QUIET_BELL) vd->vd_flags |= VDF_QUIET_BELL; else vd->vd_flags &= ~VDF_QUIET_BELL; return (0); case CONS_GETINFO: { vid_info_t *vi = (vid_info_t *)data; if (vi->size != sizeof(struct vid_info)) return (EINVAL); if (vw == vd->vd_curwindow) { mtx_lock(&Giant); if ((kbd = vd->vd_keyboard) != NULL) vt_save_kbd_state(vw, kbd); mtx_unlock(&Giant); } vi->m_num = vd->vd_curwindow->vw_number + 1; vi->mk_keylock = vw->vw_kbdstate & LOCK_MASK; /* XXX: other fields! */ return (0); } case CONS_GETVERS: *(int *)data = 0x200; return (0); case CONS_MODEINFO: /* XXX */ return (0); case CONS_MOUSECTL: { mouse_info_t *mouse = (mouse_info_t*)data; /* * All the commands except MOUSE_SHOW nd MOUSE_HIDE * should not be applied to individual TTYs, but only to * consolectl. */ switch (mouse->operation) { case MOUSE_HIDE: if (vd->vd_flags & VDF_MOUSECURSOR) { vd->vd_flags &= ~VDF_MOUSECURSOR; #ifndef SC_NO_CUTPASTE vt_mouse_state(VT_MOUSE_HIDE); #endif } return (0); case MOUSE_SHOW: if (!(vd->vd_flags & VDF_MOUSECURSOR)) { vd->vd_flags |= VDF_MOUSECURSOR; vd->vd_mx = vd->vd_width / 2; vd->vd_my = vd->vd_height / 2; #ifndef SC_NO_CUTPASTE vt_mouse_state(VT_MOUSE_SHOW); #endif } return (0); default: return (EINVAL); } } case PIO_VFONT: { struct vt_font *vf; if (vd->vd_flags & VDF_TEXTMODE) return (ENOTSUP); error = vtfont_load((void *)data, &vf); if (error != 0) return (error); error = vt_change_font(vw, vf); vtfont_unref(vf); return (error); } case PIO_VFONT_DEFAULT: { /* Reset to default font. */ error = vt_change_font(vw, vt_font_assigned); return (error); } case GIO_SCRNMAP: { scrmap_t *sm = (scrmap_t *)data; /* We don't have screen maps, so return a handcrafted one. */ for (i = 0; i < 256; i++) sm->scrmap[i] = i; return (0); } case KDSETMODE: /* * FIXME: This implementation is incomplete compared to * syscons. */ switch (*(int *)data) { case KD_TEXT: case KD_TEXT1: case KD_PIXEL: vw->vw_flags &= ~VWF_GRAPHICS; break; case KD_GRAPHICS: vw->vw_flags |= VWF_GRAPHICS; break; } return (0); case KDENABIO: /* allow io operations */ error = priv_check(td, PRIV_IO); if (error != 0) return (error); error = securelevel_gt(td->td_ucred, 0); if (error != 0) return (error); #if defined(__i386__) td->td_frame->tf_eflags |= PSL_IOPL; #elif defined(__amd64__) td->td_frame->tf_rflags |= PSL_IOPL; #endif return (0); case KDDISABIO: /* disallow io operations (default) */ #if defined(__i386__) td->td_frame->tf_eflags &= ~PSL_IOPL; #elif defined(__amd64__) td->td_frame->tf_rflags &= ~PSL_IOPL; #endif return (0); case KDMKTONE: /* sound the bell */ vtterm_beep(tm, *(u_int *)data); return (0); case KIOCSOUND: /* make tone (*data) hz */ /* TODO */ return (0); case CONS_SETKBD: /* set the new keyboard */ mtx_lock(&Giant); error = 0; if (vd->vd_keyboard == NULL || vd->vd_keyboard->kb_index != *(int *)data) { kbd = kbd_get_keyboard(*(int *)data); if (kbd == NULL) { mtx_unlock(&Giant); return (EINVAL); } i = kbd_allocate(kbd->kb_name, kbd->kb_unit, (void *)vd, vt_kbdevent, vd); if (i >= 0) { if ((kbd = vd->vd_keyboard) != NULL) { vt_save_kbd_state(vd->vd_curwindow, kbd); kbd_release(kbd, (void *)vd); } kbd = vd->vd_keyboard = kbd_get_keyboard(i); vt_update_kbd_mode(vd->vd_curwindow, kbd); vt_update_kbd_state(vd->vd_curwindow, kbd); } else { error = EPERM; /* XXX */ } } mtx_unlock(&Giant); return (error); case CONS_RELKBD: /* release the current keyboard */ mtx_lock(&Giant); error = 0; if ((kbd = vd->vd_keyboard) != NULL) { vt_save_kbd_state(vd->vd_curwindow, kbd); error = kbd_release(kbd, (void *)vd); if (error == 0) { vd->vd_keyboard = NULL; } } mtx_unlock(&Giant); return (error); case VT_ACTIVATE: { int win; win = *(int *)data - 1; DPRINTF(5, "%s%d: VT_ACTIVATE ttyv%d ", SC_DRIVER_NAME, VT_UNIT(vw), win); if ((win >= VT_MAXWINDOWS) || (win < 0)) return (EINVAL); return (vt_proc_window_switch(vd->vd_windows[win])); } case VT_GETACTIVE: *(int *)data = vd->vd_curwindow->vw_number + 1; return (0); case VT_GETINDEX: *(int *)data = vw->vw_number + 1; return (0); case VT_LOCKSWITCH: /* TODO: Check current state, switching can be in progress. */ if ((*(int *)data) == 0x01) vw->vw_flags |= VWF_VTYLOCK; else if ((*(int *)data) == 0x02) vw->vw_flags &= ~VWF_VTYLOCK; else return (EINVAL); return (0); case VT_OPENQRY: VT_LOCK(vd); for (i = 0; i < VT_MAXWINDOWS; i++) { vw = vd->vd_windows[i]; if (vw == NULL) continue; if (!(vw->vw_flags & VWF_OPENED)) { *(int *)data = vw->vw_number + 1; VT_UNLOCK(vd); return (0); } } VT_UNLOCK(vd); return (EINVAL); case VT_WAITACTIVE: { unsigned int idx; error = 0; idx = *(unsigned int *)data; if (idx > VT_MAXWINDOWS) return (EINVAL); if (idx > 0) vw = vd->vd_windows[idx - 1]; VT_LOCK(vd); while (vd->vd_curwindow != vw && error == 0) error = cv_wait_sig(&vd->vd_winswitch, &vd->vd_lock); VT_UNLOCK(vd); return (error); } case VT_SETMODE: { /* set screen switcher mode */ struct vt_mode *mode; struct proc *p1; mode = (struct vt_mode *)data; DPRINTF(5, "%s%d: VT_SETMODE ", SC_DRIVER_NAME, VT_UNIT(vw)); if (vw->vw_smode.mode == VT_PROCESS) { p1 = pfind(vw->vw_pid); if (vw->vw_proc == p1 && vw->vw_proc != td->td_proc) { if (p1) PROC_UNLOCK(p1); DPRINTF(5, "error EPERM\n"); return (EPERM); } if (p1) PROC_UNLOCK(p1); } if (mode->mode == VT_AUTO) { vw->vw_smode.mode = VT_AUTO; vw->vw_proc = NULL; vw->vw_pid = 0; DPRINTF(5, "VT_AUTO, "); if (vw == vw->vw_device->vd_windows[VT_CONSWINDOW]) cnavailable(vw->vw_terminal->consdev, TRUE); /* were we in the middle of the vty switching process? */ if (finish_vt_rel(vw, TRUE, &s) == 0) DPRINTF(5, "reset WAIT_REL, "); if (finish_vt_acq(vw) == 0) DPRINTF(5, "reset WAIT_ACQ, "); return (0); } else if (mode->mode == VT_PROCESS) { if (!ISSIGVALID(mode->relsig) || !ISSIGVALID(mode->acqsig) || !ISSIGVALID(mode->frsig)) { DPRINTF(5, "error EINVAL\n"); return (EINVAL); } DPRINTF(5, "VT_PROCESS %d, ", td->td_proc->p_pid); bcopy(data, &vw->vw_smode, sizeof(struct vt_mode)); vw->vw_proc = td->td_proc; vw->vw_pid = vw->vw_proc->p_pid; if (vw == vw->vw_device->vd_windows[VT_CONSWINDOW]) cnavailable(vw->vw_terminal->consdev, FALSE); } else { DPRINTF(5, "VT_SETMODE failed, unknown mode %d\n", mode->mode); return (EINVAL); } DPRINTF(5, "\n"); return (0); } case VT_GETMODE: /* get screen switcher mode */ bcopy(&vw->vw_smode, data, sizeof(struct vt_mode)); return (0); case VT_RELDISP: /* screen switcher ioctl */ /* * This must be the current vty which is in the VT_PROCESS * switching mode... */ if ((vw != vd->vd_curwindow) || (vw->vw_smode.mode != VT_PROCESS)) { return (EINVAL); } /* ...and this process is controlling it. */ if (vw->vw_proc != td->td_proc) { return (EPERM); } error = EINVAL; switch(*(int *)data) { case VT_FALSE: /* user refuses to release screen, abort */ if ((error = finish_vt_rel(vw, FALSE, &s)) == 0) DPRINTF(5, "%s%d: VT_RELDISP: VT_FALSE\n", SC_DRIVER_NAME, VT_UNIT(vw)); break; case VT_TRUE: /* user has released screen, go on */ /* finish_vt_rel(..., TRUE, ...) should not be locked */ if (vw->vw_flags & VWF_SWWAIT_REL) { if ((error = finish_vt_rel(vw, TRUE, &s)) == 0) DPRINTF(5, "%s%d: VT_RELDISP: VT_TRUE\n", SC_DRIVER_NAME, VT_UNIT(vw)); } else { error = EINVAL; } return (error); case VT_ACKACQ: /* acquire acknowledged, switch completed */ if ((error = finish_vt_acq(vw)) == 0) DPRINTF(5, "%s%d: VT_RELDISP: VT_ACKACQ\n", SC_DRIVER_NAME, VT_UNIT(vw)); break; default: break; } return (error); } return (ENOIOCTL); } static struct vt_window * vt_allocate_window(struct vt_device *vd, unsigned int window) { struct vt_window *vw; struct terminal *tm; term_pos_t size; struct winsize wsz; vw = malloc(sizeof *vw, M_VT, M_WAITOK|M_ZERO); vw->vw_device = vd; vw->vw_number = window; vw->vw_kbdmode = K_XLATE; if ((vd->vd_flags & VDF_TEXTMODE) == 0) { vw->vw_font = vtfont_ref(vt_font_assigned); vt_compute_drawable_area(vw); } vt_termsize(vd, vw->vw_font, &size); vt_winsize(vd, vw->vw_font, &wsz); tm = vw->vw_terminal = terminal_alloc(&vt_termclass, vw); vw->vw_buf.vb_terminal = tm; /* must be set before vtbuf_init() */ vtbuf_init(&vw->vw_buf, &size); terminal_set_winsize(tm, &wsz); vd->vd_windows[window] = vw; callout_init(&vw->vw_proc_dead_timer, 0); return (vw); } void vt_upgrade(struct vt_device *vd) { struct vt_window *vw; unsigned int i; int register_handlers; if (!vty_enabled(VTY_VT)) return; if (main_vd->vd_driver == NULL) return; for (i = 0; i < VT_MAXWINDOWS; i++) { vw = vd->vd_windows[i]; if (vw == NULL) { /* New window. */ vw = vt_allocate_window(vd, i); } if (!(vw->vw_flags & VWF_READY)) { callout_init(&vw->vw_proc_dead_timer, 0); terminal_maketty(vw->vw_terminal, "v%r", VT_UNIT(vw)); vw->vw_flags |= VWF_READY; if (vw->vw_flags & VWF_CONSOLE) { /* For existing console window. */ EVENTHANDLER_REGISTER(shutdown_pre_sync, vt_window_switch, vw, SHUTDOWN_PRI_DEFAULT); } } } VT_LOCK(vd); if (vd->vd_curwindow == NULL) vd->vd_curwindow = vd->vd_windows[VT_CONSWINDOW]; register_handlers = 0; if (!(vd->vd_flags & VDF_ASYNC)) { /* Attach keyboard. */ vt_allocate_keyboard(vd); /* Init 25 Hz timer. */ callout_init_mtx(&vd->vd_timer, &vd->vd_lock, 0); /* * Start timer when everything ready. * Note that the operations here are purposefully ordered. * We need to ensure vd_timer_armed is non-zero before we set * the VDF_ASYNC flag. That prevents this function from * racing with vt_resume_flush_timer() to update the * callout structure. */ atomic_add_acq_int(&vd->vd_timer_armed, 1); vd->vd_flags |= VDF_ASYNC; callout_reset(&vd->vd_timer, hz / VT_TIMERFREQ, vt_timer, vd); register_handlers = 1; } VT_UNLOCK(vd); /* Refill settings with new sizes. */ vt_resize(vd); if (register_handlers) { /* Register suspend/resume handlers. */ EVENTHANDLER_REGISTER(power_suspend_early, vt_suspend_handler, vd, EVENTHANDLER_PRI_ANY); EVENTHANDLER_REGISTER(power_resume, vt_resume_handler, vd, EVENTHANDLER_PRI_ANY); } } static void vt_resize(struct vt_device *vd) { struct vt_window *vw; int i; for (i = 0; i < VT_MAXWINDOWS; i++) { vw = vd->vd_windows[i]; VT_LOCK(vd); /* Assign default font to window, if not textmode. */ if (!(vd->vd_flags & VDF_TEXTMODE) && vw->vw_font == NULL) vw->vw_font = vtfont_ref(vt_font_assigned); VT_UNLOCK(vd); /* Resize terminal windows */ while (vt_change_font(vw, vw->vw_font) == EBUSY) { DPRINTF(100, "%s: vt_change_font() is busy, " "window %d\n", __func__, i); } } } static void vt_replace_backend(const struct vt_driver *drv, void *softc) { struct vt_device *vd; vd = main_vd; if (vd->vd_flags & VDF_ASYNC) { /* Stop vt_flush periodic task. */ VT_LOCK(vd); vt_suspend_flush_timer(vd); VT_UNLOCK(vd); /* * Mute current terminal until we done. vt_change_font (called * from vt_resize) will unmute it. */ terminal_mute(vd->vd_curwindow->vw_terminal, 1); } /* * Reset VDF_TEXTMODE flag, driver who require that flag (vt_vga) will * set it. */ VT_LOCK(vd); vd->vd_flags &= ~VDF_TEXTMODE; if (drv != NULL) { /* * We want to upgrade from the current driver to the * given driver. */ vd->vd_prev_driver = vd->vd_driver; vd->vd_prev_softc = vd->vd_softc; vd->vd_driver = drv; vd->vd_softc = softc; vd->vd_driver->vd_init(vd); } else if (vd->vd_prev_driver != NULL && vd->vd_prev_softc != NULL) { /* * No driver given: we want to downgrade to the previous * driver. */ const struct vt_driver *old_drv; void *old_softc; old_drv = vd->vd_driver; old_softc = vd->vd_softc; vd->vd_driver = vd->vd_prev_driver; vd->vd_softc = vd->vd_prev_softc; vd->vd_prev_driver = NULL; vd->vd_prev_softc = NULL; vd->vd_flags |= VDF_DOWNGRADE; vd->vd_driver->vd_init(vd); if (old_drv->vd_fini) old_drv->vd_fini(vd, old_softc); vd->vd_flags &= ~VDF_DOWNGRADE; } VT_UNLOCK(vd); /* Update windows sizes and initialize last items. */ vt_upgrade(vd); #ifdef DEV_SPLASH if (vd->vd_flags & VDF_SPLASH) vtterm_splash(vd); #endif if (vd->vd_flags & VDF_ASYNC) { /* Allow to put chars now. */ terminal_mute(vd->vd_curwindow->vw_terminal, 0); /* Rerun timer for screen updates. */ vt_resume_flush_timer(vd->vd_curwindow, 0); } /* * Register as console. If it already registered, cnadd() will ignore * it. */ termcn_cnregister(vd->vd_windows[VT_CONSWINDOW]->vw_terminal); } static void vt_suspend_handler(void *priv) { struct vt_device *vd; vd = priv; vd->vd_flags |= VDF_SUSPENDED; if (vd->vd_driver != NULL && vd->vd_driver->vd_suspend != NULL) vd->vd_driver->vd_suspend(vd); } static void vt_resume_handler(void *priv) { struct vt_device *vd; vd = priv; if (vd->vd_driver != NULL && vd->vd_driver->vd_resume != NULL) vd->vd_driver->vd_resume(vd); vd->vd_flags &= ~VDF_SUSPENDED; } void vt_allocate(const struct vt_driver *drv, void *softc) { if (!vty_enabled(VTY_VT)) return; if (main_vd->vd_driver == NULL) { main_vd->vd_driver = drv; printf("VT: initialize with new VT driver \"%s\".\n", drv->vd_name); } else { /* * Check if have rights to replace current driver. For example: * it is bad idea to replace KMS driver with generic VGA one. */ if (drv->vd_priority <= main_vd->vd_driver->vd_priority) { printf("VT: Driver priority %d too low. Current %d\n ", drv->vd_priority, main_vd->vd_driver->vd_priority); return; } printf("VT: Replacing driver \"%s\" with new \"%s\".\n", main_vd->vd_driver->vd_name, drv->vd_name); } vt_replace_backend(drv, softc); } void vt_deallocate(const struct vt_driver *drv, void *softc) { if (!vty_enabled(VTY_VT)) return; if (main_vd->vd_prev_driver == NULL || main_vd->vd_driver != drv || main_vd->vd_softc != softc) return; printf("VT: Switching back from \"%s\" to \"%s\".\n", main_vd->vd_driver->vd_name, main_vd->vd_prev_driver->vd_name); vt_replace_backend(NULL, NULL); } void vt_suspend(struct vt_device *vd) { int error; if (vt_suspendswitch == 0) return; /* Save current window. */ vd->vd_savedwindow = vd->vd_curwindow; /* Ask holding process to free window and switch to console window */ vt_proc_window_switch(vd->vd_windows[VT_CONSWINDOW]); /* Wait for the window switch to complete. */ error = 0; VT_LOCK(vd); while (vd->vd_curwindow != vd->vd_windows[VT_CONSWINDOW] && error == 0) error = cv_wait_sig(&vd->vd_winswitch, &vd->vd_lock); VT_UNLOCK(vd); } void vt_resume(struct vt_device *vd) { if (vt_suspendswitch == 0) return; /* Switch back to saved window, if any */ vt_proc_window_switch(vd->vd_savedwindow); vd->vd_savedwindow = NULL; } diff --git a/sys/dev/vt/vt_font.c b/sys/dev/vt/vt_font.c index 298ac4c31014..c230342ecacb 100644 --- a/sys/dev/vt/vt_font.c +++ b/sys/dev/vt/vt_font.c @@ -1,223 +1,222 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include static MALLOC_DEFINE(M_VTFONT, "vtfont", "vt font"); /* Some limits to prevent abnormal fonts from being loaded. */ #define VTFONT_MAXMAPPINGS 65536 #define VTFONT_MAXGLYPHS 131072 #define VTFONT_MAXGLYPHSIZE 2097152 #define VTFONT_MAXDIMENSION 128 static uint16_t vtfont_bisearch(const vfnt_map_t *map, unsigned int len, uint32_t src) { int min, mid, max; min = 0; max = len - 1; /* Empty font map. */ if (len == 0) return (0); /* Character below minimal entry. */ if (src < map[0].vfm_src) return (0); /* Optimization: ASCII characters occur very often. */ if (src <= map[0].vfm_src + map[0].vfm_len) return (src - map[0].vfm_src + map[0].vfm_dst); /* Character above maximum entry. */ if (src > map[max].vfm_src + map[max].vfm_len) return (0); /* Binary search. */ while (max >= min) { mid = (min + max) / 2; if (src < map[mid].vfm_src) max = mid - 1; else if (src > map[mid].vfm_src + map[mid].vfm_len) min = mid + 1; else return (src - map[mid].vfm_src + map[mid].vfm_dst); } return (0); } const uint8_t * vtfont_lookup(const struct vt_font *vf, term_char_t c) { uint32_t src; uint16_t dst; size_t stride; unsigned int normal_map; unsigned int bold_map; src = TCHAR_CHARACTER(c); if (TCHAR_FORMAT(c) & TF_CJK_RIGHT) { normal_map = VFNT_MAP_NORMAL_RIGHT; bold_map = VFNT_MAP_BOLD_RIGHT; } else { normal_map = VFNT_MAP_NORMAL; bold_map = VFNT_MAP_BOLD; } if (TCHAR_FORMAT(c) & TF_BOLD) { dst = vtfont_bisearch(vf->vf_map[bold_map], vf->vf_map_count[bold_map], src); if (dst != 0) goto found; } dst = vtfont_bisearch(vf->vf_map[normal_map], vf->vf_map_count[normal_map], src); found: stride = howmany(vf->vf_width, 8) * vf->vf_height; return (&vf->vf_bytes[dst * stride]); } struct vt_font * vtfont_ref(struct vt_font *vf) { refcount_acquire(&vf->vf_refcount); return (vf); } void vtfont_unref(struct vt_font *vf) { unsigned int i; if (refcount_release(&vf->vf_refcount)) { for (i = 0; i < VFNT_MAPS; i++) free(vf->vf_map[i], M_VTFONT); free(vf->vf_bytes, M_VTFONT); free(vf, M_VTFONT); } } static int vtfont_validate_map(vfnt_map_t *vfm, unsigned int length, unsigned int glyph_count) { unsigned int i, last = 0; for (i = 0; i < length; i++) { /* Not ordered. */ if (i > 0 && vfm[i].vfm_src <= last) return (EINVAL); /* * Destination extends amount of glyphs. */ if (vfm[i].vfm_dst >= glyph_count || vfm[i].vfm_dst + vfm[i].vfm_len >= glyph_count) return (EINVAL); last = vfm[i].vfm_src + vfm[i].vfm_len; } return (0); } int vtfont_load(vfnt_t *f, struct vt_font **ret) { size_t glyphsize, mapsize; struct vt_font *vf; int error; unsigned int i; /* Make sure the dimensions are valid. */ if (f->width < 1 || f->height < 1) return (EINVAL); if (f->width > VTFONT_MAXDIMENSION || f->height > VTFONT_MAXDIMENSION || f->glyph_count > VTFONT_MAXGLYPHS) return (E2BIG); /* Not too many mappings. */ for (i = 0; i < VFNT_MAPS; i++) if (f->map_count[i] > VTFONT_MAXMAPPINGS) return (E2BIG); /* Character 0 must always be present. */ if (f->glyph_count < 1) return (EINVAL); glyphsize = howmany(f->width, 8) * f->height * f->glyph_count; if (glyphsize > VTFONT_MAXGLYPHSIZE) return (E2BIG); /* Allocate new font structure. */ vf = malloc(sizeof *vf, M_VTFONT, M_WAITOK | M_ZERO); vf->vf_bytes = malloc(glyphsize, M_VTFONT, M_WAITOK); vf->vf_height = f->height; vf->vf_width = f->width; vf->vf_refcount = 1; /* Allocate, copy in, and validate mappings. */ for (i = 0; i < VFNT_MAPS; i++) { vf->vf_map_count[i] = f->map_count[i]; if (f->map_count[i] == 0) continue; mapsize = f->map_count[i] * sizeof(vfnt_map_t); vf->vf_map[i] = malloc(mapsize, M_VTFONT, M_WAITOK); error = copyin(f->map[i], vf->vf_map[i], mapsize); if (error) goto bad; error = vtfont_validate_map(vf->vf_map[i], vf->vf_map_count[i], f->glyph_count); if (error) goto bad; } /* Copy in glyph data. */ error = copyin(f->glyphs, vf->vf_bytes, glyphsize); if (error) goto bad; /* Success. */ *ret = vf; return (0); bad: vtfont_unref(vf); return (error); } diff --git a/sys/dts/arm/efikamx.dts b/sys/dts/arm/efikamx.dts index e077fb78ad0f..6a2cfcae54b5 100644 --- a/sys/dts/arm/efikamx.dts +++ b/sys/dts/arm/efikamx.dts @@ -1,128 +1,127 @@ /* * Copyright (c) 2012 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Freescale i.MX515 Device Tree Source. * * $FreeBSD$ */ /dts-v1/; /include/ "imx51x.dtsi" / { model = "Genesi Efika MX"; compatible = "genesi,imx51-efikamx", "fsl,imx51"; memory { /* RAM 512M */ reg = <0x90000000 0x20000000>; }; localbus@5e000000 { ipu3@5e000000 { status = "okay"; }; }; soc@70000000 { aips@70000000 { spba@70000000 { esdhc@70004000 { clock-frequency = <216000000>; status = "okay"; }; esdhc@70008000 { clock-frequency = <216000000>; status = "okay"; }; SSI2: ssi@70014000 { status = "okay"; }; }; timer@73fa0000 { status = "okay"; }; /* UART1, console */ UART1: serial@73fbc000 { status = "okay"; clock-frequency = <3000000>; /* XXX */ }; clock@73fd4000 { status = "okay"; }; gpio@73f84000 { status = "okay"; }; gpio@73f88000 { status = "okay"; }; gpio@73f8c000 { status = "okay"; }; gpio@73f90000 { status = "okay"; }; usb@73f80000 /* OTG */ { status = "okay"; }; usb@73f80200 /* Host 1 */ { status = "okay"; }; wdog@73f98000 { status = "okay"; }; }; aips@80000000 { i2c@83fc4000 { status = "okay"; }; i2c@83fc8000 { status = "okay"; }; audmux@83fd4000 { status = "okay"; }; ide@83fe0000 { status = "okay"; }; }; }; aliases { UART1 = &UART1; SSI2 = &SSI2; }; chosen { bootargs = "-v"; stdin = "UART1"; stdout = "UART1"; }; }; diff --git a/sys/dts/arm/imx51x.dtsi b/sys/dts/arm/imx51x.dtsi index cc06f0f355ec..f759afbc4a01 100644 --- a/sys/dts/arm/imx51x.dtsi +++ b/sys/dts/arm/imx51x.dtsi @@ -1,622 +1,621 @@ /* * Copyright (c) 2012 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * Freescale i.MX515 Device Tree Source. * * $FreeBSD$ */ / { #address-cells = <1>; #size-cells = <1>; aliases { soc = &SOC; }; cpus { #address-cells = <1>; #size-cells = <0>; cpu@0 { device_type = "cpu"; compatible = "ARM,MCIMX515"; reg = <0x0>; d-cache-line-size = <32>; i-cache-line-size = <32>; d-cache-size = <0x8000>; i-cache-size = <0x8000>; /* TODO: describe L2 cache also */ timebase-frequency = <0>; bus-frequency = <0>; clock-frequency = <0>; }; }; localbus@e0000000 { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; /* This reflects CPU decode windows setup. */ ranges; tzic: tz-interrupt-controller@e0000000 { compatible = "fsl,imx51-tzic", "fsl,tzic"; interrupt-controller; #interrupt-cells = <1>; reg = <0xe0000000 0x00004000>; }; /* * 60000000 60000FFF 4K Debug ROM * 60001000 60001FFF 4K ETB * 60002000 60002FFF 4K ETM * 60003000 60003FFF 4K TPIU * 60004000 60004FFF 4K CTI0 * 60005000 60005FFF 4K CTI1 * 60006000 60006FFF 4K CTI2 * 60007000 60007FFF 4K CTI3 * 60008000 60008FFF 4K Cortex Debug Unit * * E0000000 E0003FFF 0x4000 TZIC */ }; SOC: soc@70000000 { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; interrupt-parent = <&tzic>; ranges = <0x70000000 0x70000000 0x14000000>; aips@70000000 { /* AIPS1 */ compatible = "fsl,aips-bus", "simple-bus"; #address-cells = <1>; #size-cells = <1>; interrupt-parent = <&tzic>; ranges; /* Required by many devices, so better to stay first */ /* 73FD4000 0x4000 CCM */ clock@73fd4000 { compatible = "fsl,imx51-ccm"; /* 83F80000 0x4000 DPLLIP1 */ /* 83F84000 0x4000 DPLLIP2 */ /* 83F88000 0x4000 DPLLIP3 */ reg = <0x73fd4000 0x4000 0x83F80000 0x4000 0x83F84000 0x4000 0x83F88000 0x4000>; interrupt-parent = <&tzic>; interrupts = <71 72>; status = "disabled"; }; /* * GPIO modules moved up - to have it attached for * drivers which rely on GPIO */ /* 73F84000 0x4000 GPIO1 */ gpio1: gpio@73f84000 { compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; reg = <0x73f84000 0x4000>; interrupt-parent = <&tzic>; interrupts = <50 51 42 43 44 45 46 47 48 49>; /* TODO: use <> also */ gpio-controller; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <1>; }; /* 73F88000 0x4000 GPIO2 */ gpio2: gpio@73f88000 { compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; reg = <0x73f88000 0x4000>; interrupt-parent = <&tzic>; interrupts = <52 53>; gpio-controller; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <1>; }; /* 73F8C000 0x4000 GPIO3 */ gpio3: gpio@73f8c000 { compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; reg = <0x73f8c000 0x4000>; interrupt-parent = <&tzic>; interrupts = <54 55>; gpio-controller; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <1>; }; /* 73F90000 0x4000 GPIO4 */ gpio4: gpio@73f90000 { compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; reg = <0x73f90000 0x4000>; interrupt-parent = <&tzic>; interrupts = <56 57>; gpio-controller; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <1>; }; spba@70000000 { compatible = "fsl,spba-bus", "simple-bus"; #address-cells = <1>; #size-cells = <1>; interrupt-parent = <&tzic>; ranges; /* 70004000 0x4000 ESDHC 1 */ esdhc@70004000 { compatible = "fsl,imx51-esdhc"; reg = <0x70004000 0x4000>; interrupt-parent = <&tzic>; interrupts = <1>; status = "disabled"; }; /* 70008000 0x4000 ESDHC 2 */ esdhc@70008000 { compatible = "fsl,imx51-esdhc"; reg = <0x70008000 0x4000>; interrupt-parent = <&tzic>; interrupts = <2>; status = "disabled"; }; /* 7000C000 0x4000 UART 3 */ uart3: serial@7000c000 { compatible = "fsl,imx51-uart", "fsl,imx-uart"; reg = <0x7000c000 0x4000>; interrupt-parent = <&tzic>; interrupts = <33>; status = "disabled"; }; /* 70010000 0x4000 eCSPI1 */ ecspi@70010000 { #address-cells = <1>; #size-cells = <0>; compatible = "fsl,imx51-ecspi"; reg = <0x70010000 0x4000>; interrupt-parent = <&tzic>; interrupts = <36>; status = "disabled"; }; /* 70014000 0x4000 SSI2 irq30 */ SSI2: ssi@70014000 { compatible = "fsl,imx51-ssi"; reg = <0x70014000 0x4000>; interrupt-parent = <&tzic>; interrupts = <30>; status = "disabled"; }; /* 70020000 0x4000 ESDHC 3 */ esdhc@70020000 { compatible = "fsl,imx51-esdhc"; reg = <0x70020000 0x4000>; interrupt-parent = <&tzic>; interrupts = <3>; status = "disabled"; }; /* 70024000 0x4000 ESDHC 4 */ esdhc@70024000 { compatible = "fsl,imx51-esdhc"; reg = <0x70024000 0x4000>; interrupt-parent = <&tzic>; interrupts = <4>; status = "disabled"; }; /* 70028000 0x4000 SPDIF */ /* 91 SPDIF */ /* 70030000 0x4000 PATA (PORT UDMA) irq70 */ /* 70034000 0x4000 SLM */ /* 70038000 0x4000 HSI2C */ /* 64 HS-I2C */ /* 7003C000 0x4000 SPBA */ }; usbphy0: usbphy@0 { compatible = "usb-nop-xceiv"; status = "okay"; }; usbotg: usb@73f80000 { compatible = "fsl,imx51-usb", "fsl,imx27-usb"; reg = <0x73f80000 0x0200>; interrupts = <18>; fsl,usbmisc = <&usbmisc 0>; fsl,usbphy = <&usbphy0>; status = "disabled"; }; usbh1: usb@73f80200 { compatible = "fsl,imx51-usb", "fsl,imx27-usb"; reg = <0x73f80200 0x0200>; interrupts = <14>; fsl,usbmisc = <&usbmisc 1>; status = "disabled"; }; usbh2: usb@73f80400 { compatible = "fsl,imx51-usb", "fsl,imx27-usb"; reg = <0x73f80400 0x0200>; interrupts = <16>; fsl,usbmisc = <&usbmisc 2>; status = "disabled"; }; usbh3: usb@73f80600 { compatible = "fsl,imx51-usb", "fsl,imx27-usb"; reg = <0x73f80600 0x0200>; interrupts = <17>; fsl,usbmisc = <&usbmisc 3>; status = "disabled"; }; usbmisc: usbmisc@73f80800 { #index-cells = <1>; compatible = "fsl,imx51-usbmisc"; reg = <0x73f80800 0x200>; }; /* 73F98000 0x4000 WDOG1 */ wdog@73f98000 { compatible = "fsl,imx51-wdt", "fsl,imx21-wdt"; reg = <0x73f98000 0x4000>; interrupt-parent = <&tzic>; interrupts = <58>; status = "disabled"; }; /* 73F9C000 0x4000 WDOG2 (TZ) */ wdog@73f9c000 { compatible = "fsl,imx51-wdt", "fsl,imx21-wdt"; reg = <0x73f9c000 0x4000>; interrupt-parent = <&tzic>; interrupts = <59>; status = "disabled"; }; /* 73F94000 0x4000 KPP */ keyboard@73f94000 { compatible = "fsl,imx51-kpp"; reg = <0x73f94000 0x4000>; interrupt-parent = <&tzic>; interrupts = <60>; status = "disabled"; }; /* 73FA0000 0x4000 GPT */ timer@73fa0000 { compatible = "fsl,imx51-gpt"; reg = <0x73fa0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <39>; status = "disabled"; }; /* 73FA4000 0x4000 SRTC */ rtc@73fa4000 { compatible = "fsl,imx51-srtc"; reg = <0x73fa4000 0x4000>; interrupt-parent = <&tzic>; interrupts = <24 25>; status = "disabled"; }; /* 73FA8000 0x4000 IOMUXC */ iomux@73fa8000 { compatible = "fsl,imx51-iomux"; reg = <0x73fa8000 0x4000>; interrupt-parent = <&tzic>; interrupts = <7>; }; /* 73FAC000 0x4000 EPIT1 */ epit1: timer@73fac000 { compatible = "fsl,imx51-epit"; reg = <0x73fac000 0x4000>; interrupt-parent = <&tzic>; interrupts = <40>; status = "disabled"; }; /* 73FB0000 0x4000 EPIT2 */ epit2: timer@73fb0000 { compatible = "fsl,imx51-epit"; reg = <0x73fb0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <41>; status = "disabled"; }; /* 73FB4000 0x4000 PWM1 */ pwm@73fb4000 { compatible = "fsl,imx51-pwm"; reg = <0x73fb4000 0x4000>; interrupt-parent = <&tzic>; interrupts = <61>; status = "disabled"; }; /* 73FB8000 0x4000 PWM2 */ pwm@73fb8000 { compatible = "fsl,imx51-pwm"; reg = <0x73fb8000 0x4000>; interrupt-parent = <&tzic>; interrupts = <94>; status = "disabled"; }; /* 73FBC000 0x4000 UART 1 */ uart1: serial@73fbc000 { compatible = "fsl,imx51-uart", "fsl,imx-uart"; reg = <0x73fbc000 0x4000>; interrupt-parent = <&tzic>; interrupts = <31>; status = "disabled"; }; /* 73FC0000 0x4000 UART 2 */ uart2: serial@73fc0000 { compatible = "fsl,imx51-uart", "fsl,imx-uart"; reg = <0x73fc0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <32>; status = "disabled"; }; /* 73FC4000 0x4000 USBOH3 */ /* NOTYET usb@73fc4000 { compatible = "fsl,imx51-otg"; reg = <0x73fc4000 0x4000>; interrupt-parent = <&tzic>; interrupts = <>; status = "disabled"; }; */ /* 73FD0000 0x4000 SRC */ reset@73fd0000 { compatible = "fsl,imx51-src"; reg = <0x73fd0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <75>; status = "disabled"; }; /* 73FD8000 0x4000 GPC */ power@73fd8000 { compatible = "fsl,imx51-gpc"; reg = <0x73fd8000 0x4000>; interrupt-parent = <&tzic>; interrupts = <73 74>; status = "disabled"; }; }; aips@80000000 { /* AIPS2 */ compatible = "fsl,aips-bus", "simple-bus"; #address-cells = <1>; #size-cells = <1>; interrupt-parent = <&tzic>; ranges; /* 83F94000 0x4000 AHBMAX */ /* 83F98000 0x4000 IIM */ /* * 69 IIM Interrupt request to the processor. * Indicates to the processor that program or * explicit. */ /* 83F9C000 0x4000 CSU */ /* * 27 CSU Interrupt Request 1. Indicates to the * processor that one or more alarm inputs were. */ /* 83FA0000 0x4000 TIGERP_PLATFORM_NE_32K_256K */ /* irq76 Neon Monitor Interrupt */ /* irq77 Performance Unit Interrupt */ /* irq78 CTI IRQ */ /* irq79 Debug Interrupt, Cross-Trigger Interface 1 */ /* irq80 Debug Interrupt, Cross-Trigger Interface 1 */ /* irq89 Debug Interrupt, Cross-Trigger Interface 2 */ /* irq98 Debug Interrupt, Cross-Trigger Interface 3 */ /* 83FA4000 0x4000 OWIRE irq88 */ /* 83FA8000 0x4000 FIRI irq93 */ /* 83FAC000 0x4000 eCSPI2 */ ecspi@83fac000 { #address-cells = <1>; #size-cells = <0>; compatible = "fsl,imx51-ecspi"; reg = <0x83fac000 0x4000>; interrupt-parent = <&tzic>; interrupts = <37>; status = "disabled"; }; /* 83FB0000 0x4000 SDMA */ sdma@83fb0000 { compatible = "fsl,imx51-sdma", "fsl,imx35-sdma"; reg = <0x83fb0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <6>; }; /* 83FB4000 0x4000 SCC */ /* 21 SCC Security Monitor High Priority Interrupt. */ /* 22 SCC Secure (TrustZone) Interrupt. */ /* 23 SCC Regular (Non-Secure) Interrupt. */ /* 83FB8000 0x4000 ROMCP */ /* 83FBC000 0x4000 RTIC */ /* * 26 RTIC RTIC (Trust Zone) Interrupt Request. * Indicates that the RTIC has completed hashing the */ /* 83FC0000 0x4000 CSPI */ cspi@83fc0000 { #address-cells = <1>; #size-cells = <0>; compatible = "fsl,imx51-cspi", "fsl,imx35-cspi"; reg = <0x83fc0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <38>; status = "disabled"; }; /* 83FC4000 0x4000 I2C2 */ i2c@83fc4000 { #address-cells = <1>; #size-cells = <0>; compatible = "fsl,imx51-i2c", "fsl,imx1-i2c", "fsl,imx-i2c"; reg = <0x83fc4000 0x4000>; interrupt-parent = <&tzic>; interrupts = <63>; status = "disabled"; }; /* 83FC8000 0x4000 I2C1 */ i2c@83fc8000 { #address-cells = <1>; #size-cells = <0>; compatible = "fsl,imx51-i2c", "fsl,imx1-i2c", "fsl,imx-i2c"; reg = <0x83fc8000 0x4000>; interrupt-parent = <&tzic>; interrupts = <62>; status = "disabled"; }; /* 83FCC000 0x4000 SSI1 */ /* 29 SSI1 SSI-1 Interrupt Request */ SSI1: ssi@83fcc000 { compatible = "fsl,imx51-ssi"; reg = <0x83fcc000 0x4000>; interrupt-parent = <&tzic>; interrupts = <29>; status = "disabled"; }; /* 83FD0000 0x4000 AUDMUX */ audmux@83fd4000 { compatible = "fsl,imx51-audmux"; reg = <0x83fd4000 0x4000>; status = "disabled"; }; /* 83FD8000 0x4000 EMI1 */ /* 8 EMI (NFC) */ /* 15 EMI */ /* 97 EMI Boot sequence completed interrupt */ /* * 101 EMI Indicates all pages have been transferred * to NFC during an auto program operation. */ /* 83FE0000 0x4000 PATA (PORT PIO) */ /* 70 PATA Parallel ATA host controller interrupt */ ide@83fe0000 { compatible = "fsl,imx51-ata"; reg = <0x83fe0000 0x4000>; interrupt-parent = <&tzic>; interrupts = <70>; status = "disabled"; }; /* 83FE4000 0x4000 SIM */ /* 67 SIM intr composed of oef, xte, sdi1, and sdi0 */ /* 68 SIM intr composed of tc, etc, tfe, and rdrf */ /* 83FE8000 0x4000 SSI3 */ /* 96 SSI3 SSI-3 Interrupt Request */ SSI3: ssi@83fe8000 { compatible = "fsl,imx51-ssi"; reg = <0x83fe8000 0x4000>; interrupt-parent = <&tzic>; interrupts = <96>; status = "disabled"; }; /* 83FEC000 0x4000 FEC */ ethernet@83fec000 { compatible = "fsl,imx51-fec"; reg = <0x83fec000 0x4000>; interrupt-parent = <&tzic>; interrupts = <87>; status = "disabled"; }; /* 83FF0000 0x4000 TVE */ /* 92 TVE */ /* 83FF4000 0x4000 VPU */ /* 9 VPU */ /* 100 VPU Idle interrupt from VPU */ /* 83FF8000 0x4000 SAHARA Lite */ /* 19 SAHARA SAHARA host 0 (TrustZone) Intr Lite */ /* 20 SAHARA SAHARA host 1 (non-TrustZone) Intr Lite */ }; }; localbus@5e000000 { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; ranges; vga: ipu3@5e000000 { compatible = "fsl,ipu3"; reg = < 0x5e000000 0x08000 /* CM */ 0x5e008000 0x08000 /* IDMAC */ 0x5e018000 0x08000 /* DP */ 0x5e020000 0x08000 /* IC */ 0x5e028000 0x08000 /* IRT */ 0x5e030000 0x08000 /* CSI0 */ 0x5e038000 0x08000 /* CSI1 */ 0x5e040000 0x08000 /* DI0 */ 0x5e048000 0x08000 /* DI1 */ 0x5e050000 0x08000 /* SMFC */ 0x5e058000 0x08000 /* DC */ 0x5e060000 0x08000 /* DMFC */ 0x5e068000 0x08000 /* VDI */ 0x5f000000 0x20000 /* CPMEM */ 0x5f020000 0x20000 /* LUT */ 0x5f040000 0x20000 /* SRM */ 0x5f060000 0x20000 /* TPM */ 0x5f080000 0x20000 /* DCTMPL */ >; interrupt-parent = <&tzic>; interrupts = < 10 /* IPUEX Error */ 11 /* IPUEX Sync */ >; status = "disabled"; }; }; }; /* TODO: Not mapped interrupts 5 DAP 84 GPU2D (OpenVG) general interrupt 85 GPU2D (OpenVG) busy signal (for S/W power gating feasibility) 12 GPU3D 102 GPU3D Idle interrupt from GPU3D (for S/W power gating) 90 SJC */ diff --git a/sys/i386/i386/copyout.c b/sys/i386/i386/copyout.c index 428acd616cde..ed875c09d65a 100644 --- a/sys/i386/i386/copyout.c +++ b/sys/i386/i386/copyout.c @@ -1,477 +1,476 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int); static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int); int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int); static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int); int fubyte_fast(volatile const void *base, u_int kcr3); static int (*fubyte_fast_tramp)(volatile const void *, u_int); int fuword16_fast(volatile const void *base, u_int kcr3); static int (*fuword16_fast_tramp)(volatile const void *, u_int); int fueword_fast(volatile const void *base, long *val, u_int kcr3); static int (*fueword_fast_tramp)(volatile const void *, long *, u_int); int subyte_fast(volatile void *base, int val, u_int kcr3); static int (*subyte_fast_tramp)(volatile void *, int, u_int); int suword16_fast(volatile void *base, int val, u_int kcr3); static int (*suword16_fast_tramp)(volatile void *, int, u_int); int suword_fast(volatile void *base, long val, u_int kcr3); static int (*suword_fast_tramp)(volatile void *, long, u_int); static int fast_copyout = 1; SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN, &fast_copyout, 0, ""); void copyout_init_tramp(void) { copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))( (uintptr_t)copyin_fast + setidt_disp); copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))( (uintptr_t)copyout_fast + setidt_disp); fubyte_fast_tramp = (int (*)(volatile const void *, u_int))( (uintptr_t)fubyte_fast + setidt_disp); fuword16_fast_tramp = (int (*)(volatile const void *, u_int))( (uintptr_t)fuword16_fast + setidt_disp); fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))( (uintptr_t)fueword_fast + setidt_disp); subyte_fast_tramp = (int (*)(volatile void *, int, u_int))( (uintptr_t)subyte_fast + setidt_disp); suword16_fast_tramp = (int (*)(volatile void *, int, u_int))( (uintptr_t)suword16_fast + setidt_disp); suword_fast_tramp = (int (*)(volatile void *, long, u_int))( (uintptr_t)suword_fast + setidt_disp); } int cp_slow0(vm_offset_t uva, size_t len, bool write, void (*f)(vm_offset_t, void *), void *arg) { struct pcpu *pc; vm_page_t m[2]; vm_offset_t kaddr; int error, i, plen; bool sleepable; plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE); MPASS(plen <= nitems(m)); error = 0; i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len, (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT, m, nitems(m)); if (i != plen) return (EFAULT); sched_pin(); pc = get_pcpu(); if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 || (curthread->td_pflags & TDP_NOFAULTING) != 0) { sleepable = false; mtx_lock(&pc->pc_copyout_mlock); kaddr = pc->pc_copyout_maddr; } else { sleepable = true; sx_xlock(&pc->pc_copyout_slock); kaddr = pc->pc_copyout_saddr; } pmap_cp_slow0_map(kaddr, plen, m); kaddr += uva - trunc_page(uva); f(kaddr, arg); sched_unpin(); if (sleepable) sx_xunlock(&pc->pc_copyout_slock); else mtx_unlock(&pc->pc_copyout_mlock); vm_page_unhold_pages(m, plen); return (error); } struct copyinstr_arg0 { vm_offset_t kc; size_t len; size_t alen; bool end; }; static void copyinstr_slow0(vm_offset_t kva, void *arg) { struct copyinstr_arg0 *ca; char c; ca = arg; MPASS(ca->alen == 0 && ca->len > 0 && !ca->end); while (ca->alen < ca->len && !ca->end) { c = *(char *)(kva + ca->alen); *(char *)ca->kc = c; ca->alen++; ca->kc++; if (c == '\0') ca->end = true; } } int copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied) { struct copyinstr_arg0 ca; vm_offset_t uc; size_t plen; int error; error = 0; ca.end = false; for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) { ca.len = round_page(uc) - uc; if (ca.len == 0) ca.len = PAGE_SIZE; if (plen + ca.len > maxlen) ca.len = maxlen - plen; ca.alen = 0; if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) { error = EFAULT; break; } } if (!ca.end && plen == maxlen && error == 0) error = ENAMETOOLONG; if (lencopied != NULL) *lencopied = plen; return (error); } struct copyin_arg0 { vm_offset_t kc; size_t len; }; static void copyin_slow0(vm_offset_t kva, void *arg) { struct copyin_arg0 *ca; ca = arg; bcopy((void *)kva, (void *)ca->kc, ca->len); } int copyin(const void *udaddr, void *kaddr, size_t len) { struct copyin_arg0 ca; vm_offset_t uc; size_t plen; if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) return (EFAULT); if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0)) return (0); for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { ca.len = round_page(uc) - uc; if (ca.len == 0) ca.len = PAGE_SIZE; if (plen + ca.len > len) ca.len = len - plen; if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0) return (EFAULT); } return (0); } static void copyout_slow0(vm_offset_t kva, void *arg) { struct copyin_arg0 *ca; ca = arg; bcopy((void *)ca->kc, (void *)kva, ca->len); } int copyout(const void *kaddr, void *udaddr, size_t len) { struct copyin_arg0 ca; vm_offset_t uc; size_t plen; if ((uintptr_t)udaddr + len < (uintptr_t)udaddr || (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS) return (EFAULT); if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ && copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0)) return (0); for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = (vm_offset_t)kaddr; plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) { ca.len = round_page(uc) - uc; if (ca.len == 0) ca.len = PAGE_SIZE; if (plen + ca.len > len) ca.len = len - plen; if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0) return (EFAULT); } return (0); } /* * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user * memory. */ static void fubyte_slow0(vm_offset_t kva, void *arg) { *(int *)arg = *(u_char *)kva; } int fubyte(volatile const void *base) { int res; if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout) { res = fubyte_fast_tramp(base, pmap_get_kcr3()); if (res != -1) return (res); } if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0, &res) != 0) return (-1); return (res); } static void fuword16_slow0(vm_offset_t kva, void *arg) { *(int *)arg = *(uint16_t *)kva; } int fuword16(volatile const void *base) { int res; if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout) { res = fuword16_fast_tramp(base, pmap_get_kcr3()); if (res != -1) return (res); } if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false, fuword16_slow0, &res) != 0) return (-1); return (res); } static void fueword_slow0(vm_offset_t kva, void *arg) { *(uint32_t *)arg = *(uint32_t *)kva; } int fueword(volatile const void *base, long *val) { uint32_t res; if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base || (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout) { if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0) return (0); } if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0, &res) != 0) return (-1); *val = res; return (0); } int fueword32(volatile const void *base, int32_t *val) { return (fueword(base, (long *)val)); } /* * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory. */ static void subyte_slow0(vm_offset_t kva, void *arg) { *(u_char *)kva = *(int *)arg; } int subyte(volatile void *base, int byte) { if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base || (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0) return (0); return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0, &byte) != 0 ? -1 : 0); } static void suword16_slow0(vm_offset_t kva, void *arg) { *(int *)kva = *(uint16_t *)arg; } int suword16(volatile void *base, int word) { if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base || (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3()) == 0) return (0); return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true, suword16_slow0, &word) != 0 ? -1 : 0); } static void suword_slow0(vm_offset_t kva, void *arg) { *(int *)kva = *(uint32_t *)arg; } int suword(volatile void *base, long word) { if ((uintptr_t)base + sizeof(word) < (uintptr_t)base || (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS) return (-1); if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0) return (0); return (cp_slow0((vm_offset_t)base, sizeof(long), true, suword_slow0, &word) != 0 ? -1 : 0); } int suword32(volatile void *base, int32_t word) { return (suword(base, word)); } struct casueword_arg0 { uint32_t oldval; uint32_t newval; int res; }; static void casueword_slow0(vm_offset_t kva, void *arg) { struct casueword_arg0 *ca; ca = arg; ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval, ca->newval); } int casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp, uint32_t newval) { struct casueword_arg0 ca; int res; ca.oldval = oldval; ca.newval = newval; res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true, casueword_slow0, &ca); if (res == 0) { *oldvalp = ca.oldval; return (ca.res); } return (-1); } int casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval) { struct casueword_arg0 ca; int res; ca.oldval = oldval; ca.newval = newval; res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true, casueword_slow0, &ca); if (res == 0) { *oldvalp = ca.oldval; return (ca.res); } return (-1); } diff --git a/sys/i386/i386/copyout_fast.s b/sys/i386/i386/copyout_fast.s index 81a9516eb37d..715952f5fe20 100644 --- a/sys/i386/i386/copyout_fast.s +++ b/sys/i386/i386/copyout_fast.s @@ -1,362 +1,361 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include #include #include #include #include "assym.inc" .text ENTRY(copyout_fast) pushl %ebp movl %esp, %ebp pushl %esi pushl %edi pushl %ebx movl $copyout_fault,%edx movl 20(%ebp),%ebx /* KCR3 */ movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%edi cli movl PCPU(TRAMPSTK),%esi movl PCPU(COPYOUT_BUF),%eax subl $4,%esi movl %eax,(%esi) movl 12(%ebp),%eax /* udaddr */ subl $4,%esi movl %eax,(%esi) movl 16(%ebp),%eax /* len */ subl $4,%esi movl %eax,(%esi) subl $4, %esi movl %edi, (%esi) movl 8(%ebp),%eax /* kaddr */ subl $4,%esi movl %eax,(%esi) movl PCPU(COPYOUT_BUF),%eax subl $4,%esi movl %eax,(%esi) movl 16(%ebp),%eax /* len */ subl $4,%esi movl %eax,(%esi) movl %esp,%eax movl %esi,%esp /* bcopy(%esi = kaddr, %edi = PCPU(copyout_buf), %ecx = len) */ popl %ecx popl %edi popl %esi rep; movsb popl %edi movl %edi,%cr3 /* bcopy(%esi = PCPU(copyout_buf), %edi = udaddr, %ecx = len) */ popl %ecx popl %edi popl %esi rep; movsb movl %ebx,%cr3 movl %eax,%esp sti xorl %eax,%eax popl %ebx popl %edi popl %esi leave ret END(copyout_fast) ENTRY(copyin_fast) pushl %ebp movl %esp, %ebp pushl %esi pushl %edi pushl %ebx movl $copyout_fault,%edx movl 20(%ebp),%ebx /* KCR3 */ movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%edi cli movl PCPU(TRAMPSTK),%esi movl PCPU(COPYOUT_BUF),%eax subl $4,%esi movl %eax,(%esi) movl 12(%ebp),%eax /* kaddr */ subl $4,%esi movl %eax,(%esi) movl 16(%ebp),%eax /* len */ subl $4,%esi movl %eax,(%esi) movl 8(%ebp),%eax /* udaddr */ subl $4,%esi movl %eax,(%esi) movl PCPU(COPYOUT_BUF),%eax subl $4,%esi movl %eax,(%esi) movl 16(%ebp),%eax /* len */ subl $4,%esi movl %eax,(%esi) movl %esp,%eax movl %esi,%esp movl %edi,%cr3 /* bcopy(%esi = udaddr, %edi = PCPU(copyout_buf), %ecx = len) */ popl %ecx popl %edi popl %esi rep; movsb movl %ebx,%cr3 /* bcopy(%esi = PCPU(copyout_buf), %edi = kaddr, %ecx = len) */ popl %ecx popl %edi popl %esi rep; movsb movl %eax,%esp sti xorl %eax,%eax popl %ebx popl %edi popl %esi leave ret END(copyin_fast) ALIGN_TEXT copyout_fault: movl %eax,%esp sti movl $EFAULT,%eax popl %ebx popl %edi popl %esi leave ret ENTRY(fueword_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl 8(%ebp),%ecx /* from */ movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 16(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movl (%ecx),%eax movl %ebx,%cr3 movl %esi,%esp sti movl 12(%ebp),%edx movl %eax,(%edx) xorl %eax,%eax popl %edi popl %esi popl %ebx leave ret END(fueword_fast) ENTRY(fuword16_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl 8(%ebp),%ecx /* from */ movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 12(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movzwl (%ecx),%eax movl %ebx,%cr3 movl %esi,%esp sti popl %edi popl %esi popl %ebx leave ret END(fuword16_fast) ENTRY(fubyte_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl 8(%ebp),%ecx /* from */ movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 12(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movzbl (%ecx),%eax movl %ebx,%cr3 movl %esi,%esp sti popl %edi popl %esi popl %ebx leave ret END(fubyte_fast) ALIGN_TEXT fusufault: movl %esi,%esp sti xorl %eax,%eax decl %eax popl %edi popl %esi popl %ebx leave ret ENTRY(suword_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 8(%ebp),%ecx /* to */ movl 12(%ebp),%edi /* val */ movl 16(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movl %edi,(%ecx) movl %ebx,%cr3 movl %esi,%esp sti xorl %eax,%eax popl %edi popl %esi popl %ebx leave ret END(suword_fast) ENTRY(suword16_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 8(%ebp),%ecx /* to */ movl 12(%ebp),%edi /* val */ movl 16(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movw %di,(%ecx) movl %ebx,%cr3 movl %esi,%esp sti xorl %eax,%eax popl %edi popl %esi popl %ebx leave ret END(suword16_fast) ENTRY(subyte_fast) pushl %ebp movl %esp,%ebp pushl %ebx pushl %esi pushl %edi movl PCPU(CURPCB),%eax movl PCB_CR3(%eax),%eax movl $fusufault,%edx movl 8(%ebp),%ecx /* to */ movl 12(%ebp),%edi /* val */ movl 16(%ebp),%ebx movl %esp,%esi cli movl PCPU(TRAMPSTK),%esp movl %eax,%cr3 movl %edi,%eax movb %al,(%ecx) movl %ebx,%cr3 movl %esi,%esp sti xorl %eax,%eax popl %edi popl %esi popl %ebx leave ret END(subyte_fast) diff --git a/sys/i386/i386/minidump_machdep_nopae.c b/sys/i386/i386/minidump_machdep_nopae.c index 52283881561f..ce1aa26cd016 100644 --- a/sys/i386/i386/minidump_machdep_nopae.c +++ b/sys/i386/i386/minidump_machdep_nopae.c @@ -1,40 +1,39 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "minidump_machdep_base.c" diff --git a/sys/i386/i386/minidump_machdep_pae.c b/sys/i386/i386/minidump_machdep_pae.c index bec842de5973..7c7f1d6a1cc2 100644 --- a/sys/i386/i386/minidump_machdep_pae.c +++ b/sys/i386/i386/minidump_machdep_pae.c @@ -1,41 +1,40 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #define PMAP_PAE_COMP #include #include #include #include #include #include "minidump_machdep_base.c" diff --git a/sys/i386/i386/pmap_nopae.c b/sys/i386/i386/pmap_nopae.c index 0527e1cf62da..a19885545c12 100644 --- a/sys/i386/i386/pmap_nopae.c +++ b/sys/i386/i386/pmap_nopae.c @@ -1,48 +1,47 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2019 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_apic.h" #include "opt_cpu.h" #include "opt_pmap.h" #include "opt_smp.h" #include "opt_vm.h" #include #include #include #define PMTYPE pmap_nopae_ #define pm_pdir pm_pdir_nopae #include #include #include "pmap.c" diff --git a/sys/i386/i386/pmap_pae.c b/sys/i386/i386/pmap_pae.c index 1e7abdccd313..6592e9641af6 100644 --- a/sys/i386/i386/pmap_pae.c +++ b/sys/i386/i386/pmap_pae.c @@ -1,50 +1,49 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_apic.h" #include "opt_cpu.h" #include "opt_pmap.h" #include "opt_smp.h" #include "opt_vm.h" #define PMAP_PAE_COMP #include #include #include #define PMTYPE pmap_pae_ #define pm_pdir pm_pdir_pae #define pm_pdpt pm_pdpt_pae #include #include #include "pmap.c" diff --git a/sys/i386/include/pmap_base.h b/sys/i386/include/pmap_base.h index 76bc9f28262d..322d749c4591 100644 --- a/sys/i386/include/pmap_base.h +++ b/sys/i386/include/pmap_base.h @@ -1,128 +1,127 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_PMAP_BASE_H_ #define _MACHINE_PMAP_BASE_H_ /* Internal flags for pmap_mapdev_attr(). */ #define MAPDEV_SETATTR 0x0000001 /* Modify existing attrs. */ struct pmap_methods { void (*pm_ksetrw)(vm_offset_t); void (*pm_remap_lower)(bool); void (*pm_remap_lowptdi)(bool); void (*pm_align_superpage)(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size); vm_offset_t (*pm_quick_enter_page)(vm_page_t m); void (*pm_quick_remove_page)(vm_offset_t addr); void *(*pm_trm_alloc)(size_t size, int flags); void (*pm_trm_free)(void *addr, size_t size); vm_offset_t (*pm_get_map_low)(void); vm_offset_t (*pm_get_vm_maxuser_address)(void); vm_paddr_t (*pm_kextract)(vm_offset_t va); vm_paddr_t (*pm_pg_frame)(vm_paddr_t pa); void (*pm_sf_buf_map)(struct sf_buf *sf); void (*pm_cp_slow0_map)(vm_offset_t kaddr, int plen, vm_page_t *ma); u_int (*pm_get_kcr3)(void); u_int (*pm_get_cr3)(pmap_t); caddr_t (*pm_cmap3)(vm_paddr_t pa, u_int pte_flags); void (*pm_basemem_setup)(u_int basemem); void (*pm_set_nx)(void); void *(*pm_bios16_enter)(void); void (*pm_bios16_leave)(void *handle); void (*pm_bootstrap)(vm_paddr_t firstaddr); boolean_t (*pm_is_valid_memattr)(pmap_t, vm_memattr_t); int (*pm_cache_bits)(pmap_t, int, boolean_t); bool (*pm_ps_enabled)(pmap_t); void (*pm_pinit0)(pmap_t); int (*pm_pinit)(pmap_t); void (*pm_activate)(struct thread *); void (*pm_activate_boot)(pmap_t); void (*pm_advise)(pmap_t, vm_offset_t, vm_offset_t, int); void (*pm_clear_modify)(vm_page_t); int (*pm_change_attr)(vm_offset_t, vm_size_t, int); int (*pm_mincore)(pmap_t, vm_offset_t, vm_paddr_t *); void (*pm_copy)(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); void (*pm_copy_page)(vm_page_t, vm_page_t); void (*pm_copy_pages)(vm_page_t [], vm_offset_t, vm_page_t [], vm_offset_t, int); void (*pm_zero_page)(vm_page_t); void (*pm_zero_page_area)(vm_page_t, int, int); int (*pm_enter)(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); void (*pm_enter_object)(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, vm_prot_t); void (*pm_enter_quick)(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); void *(*pm_kenter_temporary)(vm_paddr_t pa, int); void (*pm_object_init_pt)(pmap_t, vm_offset_t, vm_object_t, vm_pindex_t, vm_size_t); void (*pm_unwire)(pmap_t, vm_offset_t, vm_offset_t); boolean_t (*pm_page_exists_quick)(pmap_t, vm_page_t); int (*pm_page_wired_mappings)(vm_page_t); boolean_t (*pm_page_is_mapped)(vm_page_t); void (*pm_remove_pages)(pmap_t); boolean_t (*pm_is_modified)(vm_page_t); boolean_t (*pm_is_prefaultable)(pmap_t, vm_offset_t); boolean_t (*pm_is_referenced)(vm_page_t); void (*pm_remove_write)(vm_page_t); int (*pm_ts_referenced)(vm_page_t); void *(*pm_mapdev_attr)(vm_paddr_t, vm_size_t, int, int); void (*pm_unmapdev)(vm_offset_t, vm_size_t); void (*pm_page_set_memattr)(vm_page_t, vm_memattr_t); vm_paddr_t (*pm_extract)(pmap_t, vm_offset_t); vm_page_t (*pm_extract_and_hold)(pmap_t, vm_offset_t, vm_prot_t); vm_offset_t (*pm_map)(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); void (*pm_qenter)(vm_offset_t sva, vm_page_t *, int); void (*pm_qremove)(vm_offset_t, int); void (*pm_release)(pmap_t); void (*pm_protect)(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); void (*pm_remove)(pmap_t, vm_offset_t, vm_offset_t); void (*pm_remove_all)(vm_page_t); void (*pm_init)(void); void (*pm_init_pat)(void); void (*pm_growkernel)(vm_offset_t); void (*pm_invalidate_page)(pmap_t, vm_offset_t); void (*pm_invalidate_range)(pmap_t, vm_offset_t, vm_offset_t); void (*pm_invalidate_all)(pmap_t); void (*pm_invalidate_cache)(void); void (*pm_flush_page)(vm_page_t); void (*pm_kenter)(vm_offset_t, vm_paddr_t); void (*pm_kremove)(vm_offset_t); int (*pm_sysctl_kmaps)(SYSCTL_HANDLER_ARGS); }; void pmap_cold(void); void pmap_pae_cold(void); void pmap_nopae_cold(void); #endif diff --git a/sys/i386/linux/linux_copyout.c b/sys/i386/linux/linux_copyout.c index 0fb55d645c71..bf4c874c3cf2 100644 --- a/sys/i386/linux/linux_copyout.c +++ b/sys/i386/linux/linux_copyout.c @@ -1,178 +1,177 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_compat.h" #include #include #include #include #include #include #include #include #include #include #include #include #include struct futex_st0 { int oparg; int *oldval; }; static void futex_xchgl_slow0(vm_offset_t kva, void *arg) { struct futex_st0 *st; st = arg; *st->oldval = atomic_swap_int((int *)kva, st->oparg); } int futex_xchgl(int oparg, uint32_t *uaddr, int *oldval) { struct futex_st0 st; st.oparg = oparg; st.oldval = oldval; if (cp_slow0((vm_offset_t)uaddr, sizeof(uint32_t), true, futex_xchgl_slow0, &st) != 0) return (-EFAULT); return (0); } static void futex_addl_slow0(vm_offset_t kva, void *arg) { struct futex_st0 *st; st = arg; *st->oldval = atomic_fetchadd_int((int *)kva, st->oparg); } int futex_addl(int oparg, uint32_t *uaddr, int *oldval) { struct futex_st0 st; st.oparg = oparg; st.oldval = oldval; if (cp_slow0((vm_offset_t)uaddr, sizeof(uint32_t), true, futex_addl_slow0, &st) != 0) return (-EFAULT); return (0); } static void futex_orl_slow0(vm_offset_t kva, void *arg) { struct futex_st0 *st; int old; st = arg; old = *(int *)kva; while (!atomic_fcmpset_int((int *)kva, &old, old | st->oparg)) ; *st->oldval = old; } int futex_orl(int oparg, uint32_t *uaddr, int *oldval) { struct futex_st0 st; st.oparg = oparg; st.oldval = oldval; if (cp_slow0((vm_offset_t)uaddr, sizeof(uint32_t), true, futex_orl_slow0, &st) != 0) return (-EFAULT); return (0); } static void futex_andl_slow0(vm_offset_t kva, void *arg) { struct futex_st0 *st; int old; st = arg; old = *(int *)kva; while (!atomic_fcmpset_int((int *)kva, &old, old & st->oparg)) ; *st->oldval = old; } int futex_andl(int oparg, uint32_t *uaddr, int *oldval) { struct futex_st0 st; st.oparg = oparg; st.oldval = oldval; if (cp_slow0((vm_offset_t)uaddr, sizeof(uint32_t), true, futex_andl_slow0, &st) != 0) return (-EFAULT); return (0); } static void futex_xorl_slow0(vm_offset_t kva, void *arg) { struct futex_st0 *st; int old; st = arg; old = *(int *)kva; while (!atomic_fcmpset_int((int *)kva, &old, old ^ st->oparg)) ; *st->oldval = old; } int futex_xorl(int oparg, uint32_t *uaddr, int *oldval) { struct futex_st0 st; st.oparg = oparg; st.oldval = oldval; if (cp_slow0((vm_offset_t)uaddr, sizeof(uint32_t), true, futex_xorl_slow0, &st) != 0) return (-EFAULT); return (0); } diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index 4d4e20ef1934..81d373b3b1d0 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -1,2235 +1,2234 @@ /*- * SPDX-License-Identifier: Beerware * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation - * All rights reserved. * * Portions of this software were developed by Julien Ridoux at the University * of Melbourne under sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. */ #include __FBSDID("$FreeBSD$"); #include "opt_ntp.h" #include "opt_ffclock.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * A large step happens on boot. This constant detects such steps. * It is relatively small so that ntp_update_second gets called enough * in the typical 'missed a couple of seconds' case, but doesn't loop * forever when the time step is large. */ #define LARGE_STEP 200 /* * Implement a dummy timecounter which we can use until we get a real one * in the air. This allows the console and other early stuff to use * time services. */ static u_int dummy_get_timecount(struct timecounter *tc) { static u_int now; return (++now); } static struct timecounter dummy_timecounter = { dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 }; struct timehands { /* These fields must be initialized by the driver. */ struct timecounter *th_counter; int64_t th_adjustment; uint64_t th_scale; u_int th_large_delta; u_int th_offset_count; struct bintime th_offset; struct bintime th_bintime; struct timeval th_microtime; struct timespec th_nanotime; struct bintime th_boottime; /* Fields not to be copied in tc_windup start with th_generation. */ u_int th_generation; struct timehands *th_next; }; static struct timehands ths[16] = { [0] = { .th_counter = &dummy_timecounter, .th_scale = (uint64_t)-1 / 1000000, .th_large_delta = 1000000, .th_offset = { .sec = 1 }, .th_generation = 1, }, }; static struct timehands *volatile timehands = &ths[0]; struct timecounter *timecounter = &dummy_timecounter; static struct timecounter *timecounters = &dummy_timecounter; int tc_min_ticktock_freq = 1; volatile time_t time_second = 1; volatile time_t time_uptime = 1; /* * The system time is always computed by summing the estimated boot time and the * system uptime. The timehands track boot time, but it changes when the system * time is set by the user, stepped by ntpd or adjusted when resuming. It * is set to new_time - uptime. */ static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_boottime, "S,timeval", "Estimated system boottime"); SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, ""); static int timestepwarnings; SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, ×tepwarnings, 0, "Log time steps"); static int timehands_count = 2; SYSCTL_INT(_kern_timecounter, OID_AUTO, timehands_count, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &timehands_count, 0, "Count of timehands in rotation"); struct bintime bt_timethreshold; struct bintime bt_tickthreshold; sbintime_t sbt_timethreshold; sbintime_t sbt_tickthreshold; struct bintime tc_tick_bt; sbintime_t tc_tick_sbt; int tc_precexp; int tc_timepercentage = TC_DEFAULTPERC; static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS); SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0, sysctl_kern_timecounter_adjprecision, "I", "Allowed time interval deviation in percents"); volatile int rtc_generation = 1; static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */ static char tc_from_tunable[16]; static void tc_windup(struct bintime *new_boottimebin); static void cpu_tick_calibrate(int); void dtrace_getnanotime(struct timespec *tsp); void dtrace_getnanouptime(struct timespec *tsp); static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) { struct timeval boottime; getboottime(&boottime); /* i386 is the only arch which uses a 32bits time_t */ #ifdef __amd64__ #ifdef SCTL_MASK32 int tv[2]; if (req->flags & SCTL_MASK32) { tv[0] = boottime.tv_sec; tv[1] = boottime.tv_usec; return (SYSCTL_OUT(req, tv, sizeof(tv))); } #endif #endif return (SYSCTL_OUT(req, &boottime, sizeof(boottime))); } static int sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) { u_int ncount; struct timecounter *tc = arg1; ncount = tc->tc_get_timecount(tc); return (sysctl_handle_int(oidp, &ncount, 0, req)); } static int sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) { uint64_t freq; struct timecounter *tc = arg1; freq = tc->tc_frequency; return (sysctl_handle_64(oidp, &freq, 0, req)); } /* * Return the difference between the timehands' counter value now and what * was when we copied it to the timehands' offset_count. */ static __inline u_int tc_delta(struct timehands *th) { struct timecounter *tc; tc = th->th_counter; return ((tc->tc_get_timecount(tc) - th->th_offset_count) & tc->tc_counter_mask); } /* * Functions for reading the time. We have to loop until we are sure that * the timehands that we operated on was not updated under our feet. See * the comment in for a description of these 12 functions. */ static __inline void bintime_off(struct bintime *bt, u_int off) { struct timehands *th; struct bintime *btp; uint64_t scale, x; u_int delta, gen, large_delta; do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); btp = (struct bintime *)((vm_offset_t)th + off); *bt = *btp; scale = th->th_scale; delta = tc_delta(th); large_delta = th->th_large_delta; atomic_thread_fence_acq(); } while (gen == 0 || gen != th->th_generation); if (__predict_false(delta >= large_delta)) { /* Avoid overflow for scale * delta. */ x = (scale >> 32) * delta; bt->sec += x >> 32; bintime_addx(bt, x << 32); bintime_addx(bt, (scale & 0xffffffff) * delta); } else { bintime_addx(bt, scale * delta); } } #define GETTHBINTIME(dst, member) \ do { \ _Static_assert(_Generic(((struct timehands *)NULL)->member, \ struct bintime: 1, default: 0) == 1, \ "struct timehands member is not of struct bintime type"); \ bintime_off(dst, __offsetof(struct timehands, member)); \ } while (0) static __inline void getthmember(void *out, size_t out_size, u_int off) { struct timehands *th; u_int gen; do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); memcpy(out, (char *)th + off, out_size); atomic_thread_fence_acq(); } while (gen == 0 || gen != th->th_generation); } #define GETTHMEMBER(dst, member) \ do { \ _Static_assert(_Generic(*dst, \ __typeof(((struct timehands *)NULL)->member): 1, \ default: 0) == 1, \ "*dst and struct timehands member have different types"); \ getthmember(dst, sizeof(*dst), __offsetof(struct timehands, \ member)); \ } while (0) #ifdef FFCLOCK void fbclock_binuptime(struct bintime *bt) { GETTHBINTIME(bt, th_offset); } void fbclock_nanouptime(struct timespec *tsp) { struct bintime bt; fbclock_binuptime(&bt); bintime2timespec(&bt, tsp); } void fbclock_microuptime(struct timeval *tvp) { struct bintime bt; fbclock_binuptime(&bt); bintime2timeval(&bt, tvp); } void fbclock_bintime(struct bintime *bt) { GETTHBINTIME(bt, th_bintime); } void fbclock_nanotime(struct timespec *tsp) { struct bintime bt; fbclock_bintime(&bt); bintime2timespec(&bt, tsp); } void fbclock_microtime(struct timeval *tvp) { struct bintime bt; fbclock_bintime(&bt); bintime2timeval(&bt, tvp); } void fbclock_getbinuptime(struct bintime *bt) { GETTHMEMBER(bt, th_offset); } void fbclock_getnanouptime(struct timespec *tsp) { struct bintime bt; GETTHMEMBER(&bt, th_offset); bintime2timespec(&bt, tsp); } void fbclock_getmicrouptime(struct timeval *tvp) { struct bintime bt; GETTHMEMBER(&bt, th_offset); bintime2timeval(&bt, tvp); } void fbclock_getbintime(struct bintime *bt) { GETTHMEMBER(bt, th_bintime); } void fbclock_getnanotime(struct timespec *tsp) { GETTHMEMBER(tsp, th_nanotime); } void fbclock_getmicrotime(struct timeval *tvp) { GETTHMEMBER(tvp, th_microtime); } #else /* !FFCLOCK */ void binuptime(struct bintime *bt) { GETTHBINTIME(bt, th_offset); } void nanouptime(struct timespec *tsp) { struct bintime bt; binuptime(&bt); bintime2timespec(&bt, tsp); } void microuptime(struct timeval *tvp) { struct bintime bt; binuptime(&bt); bintime2timeval(&bt, tvp); } void bintime(struct bintime *bt) { GETTHBINTIME(bt, th_bintime); } void nanotime(struct timespec *tsp) { struct bintime bt; bintime(&bt); bintime2timespec(&bt, tsp); } void microtime(struct timeval *tvp) { struct bintime bt; bintime(&bt); bintime2timeval(&bt, tvp); } void getbinuptime(struct bintime *bt) { GETTHMEMBER(bt, th_offset); } void getnanouptime(struct timespec *tsp) { struct bintime bt; GETTHMEMBER(&bt, th_offset); bintime2timespec(&bt, tsp); } void getmicrouptime(struct timeval *tvp) { struct bintime bt; GETTHMEMBER(&bt, th_offset); bintime2timeval(&bt, tvp); } void getbintime(struct bintime *bt) { GETTHMEMBER(bt, th_bintime); } void getnanotime(struct timespec *tsp) { GETTHMEMBER(tsp, th_nanotime); } void getmicrotime(struct timeval *tvp) { GETTHMEMBER(tvp, th_microtime); } #endif /* FFCLOCK */ void getboottime(struct timeval *boottime) { struct bintime boottimebin; getboottimebin(&boottimebin); bintime2timeval(&boottimebin, boottime); } void getboottimebin(struct bintime *boottimebin) { GETTHMEMBER(boottimebin, th_boottime); } #ifdef FFCLOCK /* * Support for feed-forward synchronization algorithms. This is heavily inspired * by the timehands mechanism but kept independent from it. *_windup() functions * have some connection to avoid accessing the timecounter hardware more than * necessary. */ /* Feed-forward clock estimates kept updated by the synchronization daemon. */ struct ffclock_estimate ffclock_estimate; struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ uint32_t ffclock_status; /* Feed-forward clock status. */ int8_t ffclock_updated; /* New estimates are available. */ struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ struct fftimehands { struct ffclock_estimate cest; struct bintime tick_time; struct bintime tick_time_lerp; ffcounter tick_ffcount; uint64_t period_lerp; volatile uint8_t gen; struct fftimehands *next; }; #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) static struct fftimehands ffth[10]; static struct fftimehands *volatile fftimehands = ffth; static void ffclock_init(void) { struct fftimehands *cur; struct fftimehands *last; memset(ffth, 0, sizeof(ffth)); last = ffth + NUM_ELEMENTS(ffth) - 1; for (cur = ffth; cur < last; cur++) cur->next = cur + 1; last->next = ffth; ffclock_updated = 0; ffclock_status = FFCLOCK_STA_UNSYNC; mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); } /* * Reset the feed-forward clock estimates. Called from inittodr() to get things * kick started and uses the timecounter nominal frequency as a first period * estimate. Note: this function may be called several time just after boot. * Note: this is the only function that sets the value of boot time for the * monotonic (i.e. uptime) version of the feed-forward clock. */ void ffclock_reset_clock(struct timespec *ts) { struct timecounter *tc; struct ffclock_estimate cest; tc = timehands->th_counter; memset(&cest, 0, sizeof(struct ffclock_estimate)); timespec2bintime(ts, &ffclock_boottime); timespec2bintime(ts, &(cest.update_time)); ffclock_read_counter(&cest.update_ffcount); cest.leapsec_next = 0; cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; cest.errb_abs = 0; cest.errb_rate = 0; cest.status = FFCLOCK_STA_UNSYNC; cest.leapsec_total = 0; cest.leapsec = 0; mtx_lock(&ffclock_mtx); bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); ffclock_updated = INT8_MAX; mtx_unlock(&ffclock_mtx); printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, (unsigned long)ts->tv_nsec); } /* * Sub-routine to convert a time interval measured in RAW counter units to time * in seconds stored in bintime format. * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be * larger than the max value of u_int (on 32 bit architecture). Loop to consume * extra cycles. */ static void ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) { struct bintime bt2; ffcounter delta, delta_max; delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; bintime_clear(bt); do { if (ffdelta > delta_max) delta = delta_max; else delta = ffdelta; bt2.sec = 0; bt2.frac = period; bintime_mul(&bt2, (unsigned int)delta); bintime_add(bt, &bt2); ffdelta -= delta; } while (ffdelta > 0); } /* * Update the fftimehands. * Push the tick ffcount and time(s) forward based on current clock estimate. * The conversion from ffcounter to bintime relies on the difference clock * principle, whose accuracy relies on computing small time intervals. If a new * clock estimate has been passed by the synchronisation daemon, make it * current, and compute the linear interpolation for monotonic time if needed. */ static void ffclock_windup(unsigned int delta) { struct ffclock_estimate *cest; struct fftimehands *ffth; struct bintime bt, gap_lerp; ffcounter ffdelta; uint64_t frac; unsigned int polling; uint8_t forward_jump, ogen; /* * Pick the next timehand, copy current ffclock estimates and move tick * times and counter forward. */ forward_jump = 0; ffth = fftimehands->next; ogen = ffth->gen; ffth->gen = 0; cest = &ffth->cest; bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); ffdelta = (ffcounter)delta; ffth->period_lerp = fftimehands->period_lerp; ffth->tick_time = fftimehands->tick_time; ffclock_convert_delta(ffdelta, cest->period, &bt); bintime_add(&ffth->tick_time, &bt); ffth->tick_time_lerp = fftimehands->tick_time_lerp; ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); bintime_add(&ffth->tick_time_lerp, &bt); ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; /* * Assess the status of the clock, if the last update is too old, it is * likely the synchronisation daemon is dead and the clock is free * running. */ if (ffclock_updated == 0) { ffdelta = ffth->tick_ffcount - cest->update_ffcount; ffclock_convert_delta(ffdelta, cest->period, &bt); if (bt.sec > 2 * FFCLOCK_SKM_SCALE) ffclock_status |= FFCLOCK_STA_UNSYNC; } /* * If available, grab updated clock estimates and make them current. * Recompute time at this tick using the updated estimates. The clock * estimates passed the feed-forward synchronisation daemon may result * in time conversion that is not monotonically increasing (just after * the update). time_lerp is a particular linear interpolation over the * synchronisation algo polling period that ensures monotonicity for the * clock ids requesting it. */ if (ffclock_updated > 0) { bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); ffdelta = ffth->tick_ffcount - cest->update_ffcount; ffth->tick_time = cest->update_time; ffclock_convert_delta(ffdelta, cest->period, &bt); bintime_add(&ffth->tick_time, &bt); /* ffclock_reset sets ffclock_updated to INT8_MAX */ if (ffclock_updated == INT8_MAX) ffth->tick_time_lerp = ffth->tick_time; if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) forward_jump = 1; else forward_jump = 0; bintime_clear(&gap_lerp); if (forward_jump) { gap_lerp = ffth->tick_time; bintime_sub(&gap_lerp, &ffth->tick_time_lerp); } else { gap_lerp = ffth->tick_time_lerp; bintime_sub(&gap_lerp, &ffth->tick_time); } /* * The reset from the RTC clock may be far from accurate, and * reducing the gap between real time and interpolated time * could take a very long time if the interpolated clock insists * on strict monotonicity. The clock is reset under very strict * conditions (kernel time is known to be wrong and * synchronization daemon has been restarted recently. * ffclock_boottime absorbs the jump to ensure boot time is * correct and uptime functions stay consistent. */ if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { if (forward_jump) bintime_add(&ffclock_boottime, &gap_lerp); else bintime_sub(&ffclock_boottime, &gap_lerp); ffth->tick_time_lerp = ffth->tick_time; bintime_clear(&gap_lerp); } ffclock_status = cest->status; ffth->period_lerp = cest->period; /* * Compute corrected period used for the linear interpolation of * time. The rate of linear interpolation is capped to 5000PPM * (5ms/s). */ if (bintime_isset(&gap_lerp)) { ffdelta = cest->update_ffcount; ffdelta -= fftimehands->cest.update_ffcount; ffclock_convert_delta(ffdelta, cest->period, &bt); polling = bt.sec; bt.sec = 0; bt.frac = 5000000 * (uint64_t)18446744073LL; bintime_mul(&bt, polling); if (bintime_cmp(&gap_lerp, &bt, >)) gap_lerp = bt; /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ frac = 0; if (gap_lerp.sec > 0) { frac -= 1; frac /= ffdelta / gap_lerp.sec; } frac += gap_lerp.frac / ffdelta; if (forward_jump) ffth->period_lerp += frac; else ffth->period_lerp -= frac; } ffclock_updated = 0; } if (++ogen == 0) ogen = 1; ffth->gen = ogen; fftimehands = ffth; } /* * Adjust the fftimehands when the timecounter is changed. Stating the obvious, * the old and new hardware counter cannot be read simultaneously. tc_windup() * does read the two counters 'back to back', but a few cycles are effectively * lost, and not accumulated in tick_ffcount. This is a fairly radical * operation for a feed-forward synchronization daemon, and it is its job to not * pushing irrelevant data to the kernel. Because there is no locking here, * simply force to ignore pending or next update to give daemon a chance to * realize the counter has changed. */ static void ffclock_change_tc(struct timehands *th) { struct fftimehands *ffth; struct ffclock_estimate *cest; struct timecounter *tc; uint8_t ogen; tc = th->th_counter; ffth = fftimehands->next; ogen = ffth->gen; ffth->gen = 0; cest = &ffth->cest; bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; cest->errb_abs = 0; cest->errb_rate = 0; cest->status |= FFCLOCK_STA_UNSYNC; ffth->tick_ffcount = fftimehands->tick_ffcount; ffth->tick_time_lerp = fftimehands->tick_time_lerp; ffth->tick_time = fftimehands->tick_time; ffth->period_lerp = cest->period; /* Do not lock but ignore next update from synchronization daemon. */ ffclock_updated--; if (++ogen == 0) ogen = 1; ffth->gen = ogen; fftimehands = ffth; } /* * Retrieve feed-forward counter and time of last kernel tick. */ void ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) { struct fftimehands *ffth; uint8_t gen; /* * No locking but check generation has not changed. Also need to make * sure ffdelta is positive, i.e. ffcount > tick_ffcount. */ do { ffth = fftimehands; gen = ffth->gen; if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) *bt = ffth->tick_time_lerp; else *bt = ffth->tick_time; *ffcount = ffth->tick_ffcount; } while (gen == 0 || gen != ffth->gen); } /* * Absolute clock conversion. Low level function to convert ffcounter to * bintime. The ffcounter is converted using the current ffclock period estimate * or the "interpolated period" to ensure monotonicity. * NOTE: this conversion may have been deferred, and the clock updated since the * hardware counter has been read. */ void ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) { struct fftimehands *ffth; struct bintime bt2; ffcounter ffdelta; uint8_t gen; /* * No locking but check generation has not changed. Also need to make * sure ffdelta is positive, i.e. ffcount > tick_ffcount. */ do { ffth = fftimehands; gen = ffth->gen; if (ffcount > ffth->tick_ffcount) ffdelta = ffcount - ffth->tick_ffcount; else ffdelta = ffth->tick_ffcount - ffcount; if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { *bt = ffth->tick_time_lerp; ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); } else { *bt = ffth->tick_time; ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); } if (ffcount > ffth->tick_ffcount) bintime_add(bt, &bt2); else bintime_sub(bt, &bt2); } while (gen == 0 || gen != ffth->gen); } /* * Difference clock conversion. * Low level function to Convert a time interval measured in RAW counter units * into bintime. The difference clock allows measuring small intervals much more * reliably than the absolute clock. */ void ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) { struct fftimehands *ffth; uint8_t gen; /* No locking but check generation has not changed. */ do { ffth = fftimehands; gen = ffth->gen; ffclock_convert_delta(ffdelta, ffth->cest.period, bt); } while (gen == 0 || gen != ffth->gen); } /* * Access to current ffcounter value. */ void ffclock_read_counter(ffcounter *ffcount) { struct timehands *th; struct fftimehands *ffth; unsigned int gen, delta; /* * ffclock_windup() called from tc_windup(), safe to rely on * th->th_generation only, for correct delta and ffcounter. */ do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); ffth = fftimehands; delta = tc_delta(th); *ffcount = ffth->tick_ffcount; atomic_thread_fence_acq(); } while (gen == 0 || gen != th->th_generation); *ffcount += delta; } void binuptime(struct bintime *bt) { binuptime_fromclock(bt, sysclock_active); } void nanouptime(struct timespec *tsp) { nanouptime_fromclock(tsp, sysclock_active); } void microuptime(struct timeval *tvp) { microuptime_fromclock(tvp, sysclock_active); } void bintime(struct bintime *bt) { bintime_fromclock(bt, sysclock_active); } void nanotime(struct timespec *tsp) { nanotime_fromclock(tsp, sysclock_active); } void microtime(struct timeval *tvp) { microtime_fromclock(tvp, sysclock_active); } void getbinuptime(struct bintime *bt) { getbinuptime_fromclock(bt, sysclock_active); } void getnanouptime(struct timespec *tsp) { getnanouptime_fromclock(tsp, sysclock_active); } void getmicrouptime(struct timeval *tvp) { getmicrouptime_fromclock(tvp, sysclock_active); } void getbintime(struct bintime *bt) { getbintime_fromclock(bt, sysclock_active); } void getnanotime(struct timespec *tsp) { getnanotime_fromclock(tsp, sysclock_active); } void getmicrotime(struct timeval *tvp) { getmicrouptime_fromclock(tvp, sysclock_active); } #endif /* FFCLOCK */ /* * This is a clone of getnanotime and used for walltimestamps. * The dtrace_ prefix prevents fbt from creating probes for * it so walltimestamp can be safely used in all fbt probes. */ void dtrace_getnanotime(struct timespec *tsp) { GETTHMEMBER(tsp, th_nanotime); } /* * This is a clone of getnanouptime used for time since boot. * The dtrace_ prefix prevents fbt from creating probes for * it so an uptime that can be safely used in all fbt probes. */ void dtrace_getnanouptime(struct timespec *tsp) { struct bintime bt; GETTHMEMBER(&bt, th_offset); bintime2timespec(&bt, tsp); } /* * System clock currently providing time to the system. Modifiable via sysctl * when the FFCLOCK option is defined. */ int sysclock_active = SYSCLOCK_FBCK; /* Internal NTP status and error estimates. */ extern int time_status; extern long time_esterror; /* * Take a snapshot of sysclock data which can be used to compare system clocks * and generate timestamps after the fact. */ void sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) { struct fbclock_info *fbi; struct timehands *th; struct bintime bt; unsigned int delta, gen; #ifdef FFCLOCK ffcounter ffcount; struct fftimehands *ffth; struct ffclock_info *ffi; struct ffclock_estimate cest; ffi = &clock_snap->ff_info; #endif fbi = &clock_snap->fb_info; delta = 0; do { th = timehands; gen = atomic_load_acq_int(&th->th_generation); fbi->th_scale = th->th_scale; fbi->tick_time = th->th_offset; #ifdef FFCLOCK ffth = fftimehands; ffi->tick_time = ffth->tick_time_lerp; ffi->tick_time_lerp = ffth->tick_time_lerp; ffi->period = ffth->cest.period; ffi->period_lerp = ffth->period_lerp; clock_snap->ffcount = ffth->tick_ffcount; cest = ffth->cest; #endif if (!fast) delta = tc_delta(th); atomic_thread_fence_acq(); } while (gen == 0 || gen != th->th_generation); clock_snap->delta = delta; clock_snap->sysclock_active = sysclock_active; /* Record feedback clock status and error. */ clock_snap->fb_info.status = time_status; /* XXX: Very crude estimate of feedback clock error. */ bt.sec = time_esterror / 1000000; bt.frac = ((time_esterror - bt.sec) * 1000000) * (uint64_t)18446744073709ULL; clock_snap->fb_info.error = bt; #ifdef FFCLOCK if (!fast) clock_snap->ffcount += delta; /* Record feed-forward clock leap second adjustment. */ ffi->leapsec_adjustment = cest.leapsec_total; if (clock_snap->ffcount > cest.leapsec_next) ffi->leapsec_adjustment -= cest.leapsec; /* Record feed-forward clock status and error. */ clock_snap->ff_info.status = cest.status; ffcount = clock_snap->ffcount - cest.update_ffcount; ffclock_convert_delta(ffcount, cest.period, &bt); /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */ bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL); /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */ bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL); clock_snap->ff_info.error = bt; #endif } /* * Convert a sysclock snapshot into a struct bintime based on the specified * clock source and flags. */ int sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt, int whichclock, uint32_t flags) { struct bintime boottimebin; #ifdef FFCLOCK struct bintime bt2; uint64_t period; #endif switch (whichclock) { case SYSCLOCK_FBCK: *bt = cs->fb_info.tick_time; /* If snapshot was created with !fast, delta will be >0. */ if (cs->delta > 0) bintime_addx(bt, cs->fb_info.th_scale * cs->delta); if ((flags & FBCLOCK_UPTIME) == 0) { getboottimebin(&boottimebin); bintime_add(bt, &boottimebin); } break; #ifdef FFCLOCK case SYSCLOCK_FFWD: if (flags & FFCLOCK_LERP) { *bt = cs->ff_info.tick_time_lerp; period = cs->ff_info.period_lerp; } else { *bt = cs->ff_info.tick_time; period = cs->ff_info.period; } /* If snapshot was created with !fast, delta will be >0. */ if (cs->delta > 0) { ffclock_convert_delta(cs->delta, period, &bt2); bintime_add(bt, &bt2); } /* Leap second adjustment. */ if (flags & FFCLOCK_LEAPSEC) bt->sec -= cs->ff_info.leapsec_adjustment; /* Boot time adjustment, for uptime/monotonic clocks. */ if (flags & FFCLOCK_UPTIME) bintime_sub(bt, &ffclock_boottime); break; #endif default: return (EINVAL); break; } return (0); } /* * Initialize a new timecounter and possibly use it. */ void tc_init(struct timecounter *tc) { u_int u; struct sysctl_oid *tc_root; u = tc->tc_frequency / tc->tc_counter_mask; /* XXX: We need some margin here, 10% is a guess */ u *= 11; u /= 10; if (u > hz && tc->tc_quality >= 0) { tc->tc_quality = -2000; if (bootverbose) { printf("Timecounter \"%s\" frequency %ju Hz", tc->tc_name, (uintmax_t)tc->tc_frequency); printf(" -- Insufficient hz, needs at least %u\n", u); } } else if (tc->tc_quality >= 0 || bootverbose) { printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", tc->tc_name, (uintmax_t)tc->tc_frequency, tc->tc_quality); } tc->tc_next = timecounters; timecounters = tc; /* * Set up sysctl tree for this counter. */ tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL, SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "timecounter description", "timecounter"); SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, "mask for implemented bits"); SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "counter", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, tc, sizeof(*tc), sysctl_kern_timecounter_get, "IU", "current timecounter value"); SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "frequency", CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, tc, sizeof(*tc), sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, "quality", CTLFLAG_RD, &(tc->tc_quality), 0, "goodness of time counter"); /* * Do not automatically switch if the current tc was specifically * chosen. Never automatically use a timecounter with negative quality. * Even though we run on the dummy counter, switching here may be * worse since this timecounter may not be monotonic. */ if (tc_chosen) return; if (tc->tc_quality < 0) return; if (tc_from_tunable[0] != '\0' && strcmp(tc->tc_name, tc_from_tunable) == 0) { tc_chosen = 1; tc_from_tunable[0] = '\0'; } else { if (tc->tc_quality < timecounter->tc_quality) return; if (tc->tc_quality == timecounter->tc_quality && tc->tc_frequency < timecounter->tc_frequency) return; } (void)tc->tc_get_timecount(tc); timecounter = tc; } /* Report the frequency of the current timecounter. */ uint64_t tc_getfrequency(void) { return (timehands->th_counter->tc_frequency); } static bool sleeping_on_old_rtc(struct thread *td) { /* * td_rtcgen is modified by curthread when it is running, * and by other threads in this function. By finding the thread * on a sleepqueue and holding the lock on the sleepqueue * chain, we guarantee that the thread is not running and that * modifying td_rtcgen is safe. Setting td_rtcgen to zero informs * the thread that it was woken due to a real-time clock adjustment. * (The declaration of td_rtcgen refers to this comment.) */ if (td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation) { td->td_rtcgen = 0; return (true); } return (false); } static struct mtx tc_setclock_mtx; MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN); /* * Step our concept of UTC. This is done by modifying our estimate of * when we booted. */ void tc_setclock(struct timespec *ts) { struct timespec tbef, taft; struct bintime bt, bt2; timespec2bintime(ts, &bt); nanotime(&tbef); mtx_lock_spin(&tc_setclock_mtx); cpu_tick_calibrate(1); binuptime(&bt2); bintime_sub(&bt, &bt2); /* XXX fiddle all the little crinkly bits around the fiords... */ tc_windup(&bt); mtx_unlock_spin(&tc_setclock_mtx); /* Avoid rtc_generation == 0, since td_rtcgen == 0 is special. */ atomic_add_rel_int(&rtc_generation, 2); sleepq_chains_remove_matching(sleeping_on_old_rtc); if (timestepwarnings) { nanotime(&taft); log(LOG_INFO, "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", (intmax_t)tbef.tv_sec, tbef.tv_nsec, (intmax_t)taft.tv_sec, taft.tv_nsec, (intmax_t)ts->tv_sec, ts->tv_nsec); } } /* * Initialize the next struct timehands in the ring and make * it the active timehands. Along the way we might switch to a different * timecounter and/or do seconds processing in NTP. Slightly magic. */ static void tc_windup(struct bintime *new_boottimebin) { struct bintime bt; struct timehands *th, *tho; uint64_t scale; u_int delta, ncount, ogen; int i; time_t t; /* * Make the next timehands a copy of the current one, but do * not overwrite the generation or next pointer. While we * update the contents, the generation must be zero. We need * to ensure that the zero generation is visible before the * data updates become visible, which requires release fence. * For similar reasons, re-reading of the generation after the * data is read should use acquire fence. */ tho = timehands; th = tho->th_next; ogen = th->th_generation; th->th_generation = 0; atomic_thread_fence_rel(); memcpy(th, tho, offsetof(struct timehands, th_generation)); if (new_boottimebin != NULL) th->th_boottime = *new_boottimebin; /* * Capture a timecounter delta on the current timecounter and if * changing timecounters, a counter value from the new timecounter. * Update the offset fields accordingly. */ delta = tc_delta(th); if (th->th_counter != timecounter) ncount = timecounter->tc_get_timecount(timecounter); else ncount = 0; #ifdef FFCLOCK ffclock_windup(delta); #endif th->th_offset_count += delta; th->th_offset_count &= th->th_counter->tc_counter_mask; while (delta > th->th_counter->tc_frequency) { /* Eat complete unadjusted seconds. */ delta -= th->th_counter->tc_frequency; th->th_offset.sec++; } if ((delta > th->th_counter->tc_frequency / 2) && (th->th_scale * delta < ((uint64_t)1 << 63))) { /* The product th_scale * delta just barely overflows. */ th->th_offset.sec++; } bintime_addx(&th->th_offset, th->th_scale * delta); /* * Hardware latching timecounters may not generate interrupts on * PPS events, so instead we poll them. There is a finite risk that * the hardware might capture a count which is later than the one we * got above, and therefore possibly in the next NTP second which might * have a different rate than the current NTP second. It doesn't * matter in practice. */ if (tho->th_counter->tc_poll_pps) tho->th_counter->tc_poll_pps(tho->th_counter); /* * Deal with NTP second processing. The for loop normally * iterates at most once, but in extreme situations it might * keep NTP sane if timeouts are not run for several seconds. * At boot, the time step can be large when the TOD hardware * has been read, so on really large steps, we call * ntp_update_second only twice. We need to call it twice in * case we missed a leap second. */ bt = th->th_offset; bintime_add(&bt, &th->th_boottime); i = bt.sec - tho->th_microtime.tv_sec; if (i > LARGE_STEP) i = 2; for (; i > 0; i--) { t = bt.sec; ntp_update_second(&th->th_adjustment, &bt.sec); if (bt.sec != t) th->th_boottime.sec += bt.sec - t; } /* Update the UTC timestamps used by the get*() functions. */ th->th_bintime = bt; bintime2timeval(&bt, &th->th_microtime); bintime2timespec(&bt, &th->th_nanotime); /* Now is a good time to change timecounters. */ if (th->th_counter != timecounter) { #ifndef __arm__ if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) cpu_disable_c2_sleep++; if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) cpu_disable_c2_sleep--; #endif th->th_counter = timecounter; th->th_offset_count = ncount; tc_min_ticktock_freq = max(1, timecounter->tc_frequency / (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); #ifdef FFCLOCK ffclock_change_tc(th); #endif } /*- * Recalculate the scaling factor. We want the number of 1/2^64 * fractions of a second per period of the hardware counter, taking * into account the th_adjustment factor which the NTP PLL/adjtime(2) * processing provides us with. * * The th_adjustment is nanoseconds per second with 32 bit binary * fraction and we want 64 bit binary fraction of second: * * x = a * 2^32 / 10^9 = a * 4.294967296 * * The range of th_adjustment is +/- 5000PPM so inside a 64bit int * we can only multiply by about 850 without overflowing, that * leaves no suitably precise fractions for multiply before divide. * * Divide before multiply with a fraction of 2199/512 results in a * systematic undercompensation of 10PPM of th_adjustment. On a * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. * * We happily sacrifice the lowest of the 64 bits of our result * to the goddess of code clarity. * */ scale = (uint64_t)1 << 63; scale += (th->th_adjustment / 1024) * 2199; scale /= th->th_counter->tc_frequency; th->th_scale = scale * 2; th->th_large_delta = MIN(((uint64_t)1 << 63) / scale, UINT_MAX); /* * Now that the struct timehands is again consistent, set the new * generation number, making sure to not make it zero. */ if (++ogen == 0) ogen = 1; atomic_store_rel_int(&th->th_generation, ogen); /* Go live with the new struct timehands. */ #ifdef FFCLOCK switch (sysclock_active) { case SYSCLOCK_FBCK: #endif time_second = th->th_microtime.tv_sec; time_uptime = th->th_offset.sec; #ifdef FFCLOCK break; case SYSCLOCK_FFWD: time_second = fftimehands->tick_time_lerp.sec; time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; break; } #endif timehands = th; timekeep_push_vdso(); } /* Report or change the active timecounter hardware. */ static int sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) { char newname[32]; struct timecounter *newtc, *tc; int error; tc = timecounter; strlcpy(newname, tc->tc_name, sizeof(newname)); error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); if (error != 0 || req->newptr == NULL) return (error); /* Record that the tc in use now was specifically chosen. */ tc_chosen = 1; if (strcmp(newname, tc->tc_name) == 0) return (0); for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { if (strcmp(newname, newtc->tc_name) != 0) continue; /* Warm up new timecounter. */ (void)newtc->tc_get_timecount(newtc); timecounter = newtc; /* * The vdso timehands update is deferred until the next * 'tc_windup()'. * * This is prudent given that 'timekeep_push_vdso()' does not * use any locking and that it can be called in hard interrupt * context via 'tc_windup()'. */ return (0); } return (EINVAL); } SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 0, 0, sysctl_kern_timecounter_hardware, "A", "Timecounter hardware selected"); /* Report the available timecounter hardware. */ static int sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) { struct sbuf sb; struct timecounter *tc; int error; sbuf_new_for_sysctl(&sb, NULL, 0, req); for (tc = timecounters; tc != NULL; tc = tc->tc_next) { if (tc != timecounters) sbuf_putc(&sb, ' '); sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality); } error = sbuf_finish(&sb); sbuf_delete(&sb); return (error); } SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); /* * RFC 2783 PPS-API implementation. */ /* * Return true if the driver is aware of the abi version extensions in the * pps_state structure, and it supports at least the given abi version number. */ static inline int abi_aware(struct pps_state *pps, int vers) { return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers); } static int pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) { int err, timo; pps_seq_t aseq, cseq; struct timeval tv; if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) return (EINVAL); /* * If no timeout is requested, immediately return whatever values were * most recently captured. If timeout seconds is -1, that's a request * to block without a timeout. WITNESS won't let us sleep forever * without a lock (we really don't need a lock), so just repeatedly * sleep a long time. */ if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) { if (fapi->timeout.tv_sec == -1) timo = 0x7fffffff; else { tv.tv_sec = fapi->timeout.tv_sec; tv.tv_usec = fapi->timeout.tv_nsec / 1000; timo = tvtohz(&tv); } aseq = atomic_load_int(&pps->ppsinfo.assert_sequence); cseq = atomic_load_int(&pps->ppsinfo.clear_sequence); while (aseq == atomic_load_int(&pps->ppsinfo.assert_sequence) && cseq == atomic_load_int(&pps->ppsinfo.clear_sequence)) { if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { if (pps->flags & PPSFLAG_MTX_SPIN) { err = msleep_spin(pps, pps->driver_mtx, "ppsfch", timo); } else { err = msleep(pps, pps->driver_mtx, PCATCH, "ppsfch", timo); } } else { err = tsleep(pps, PCATCH, "ppsfch", timo); } if (err == EWOULDBLOCK) { if (fapi->timeout.tv_sec == -1) { continue; } else { return (ETIMEDOUT); } } else if (err != 0) { return (err); } } } pps->ppsinfo.current_mode = pps->ppsparam.mode; fapi->pps_info_buf = pps->ppsinfo; return (0); } int pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) { pps_params_t *app; struct pps_fetch_args *fapi; #ifdef FFCLOCK struct pps_fetch_ffc_args *fapi_ffc; #endif #ifdef PPS_SYNC struct pps_kcbind_args *kapi; #endif KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); switch (cmd) { case PPS_IOC_CREATE: return (0); case PPS_IOC_DESTROY: return (0); case PPS_IOC_SETPARAMS: app = (pps_params_t *)data; if (app->mode & ~pps->ppscap) return (EINVAL); #ifdef FFCLOCK /* Ensure only a single clock is selected for ffc timestamp. */ if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK) return (EINVAL); #endif pps->ppsparam = *app; return (0); case PPS_IOC_GETPARAMS: app = (pps_params_t *)data; *app = pps->ppsparam; app->api_version = PPS_API_VERS_1; return (0); case PPS_IOC_GETCAP: *(int*)data = pps->ppscap; return (0); case PPS_IOC_FETCH: fapi = (struct pps_fetch_args *)data; return (pps_fetch(fapi, pps)); #ifdef FFCLOCK case PPS_IOC_FETCH_FFCOUNTER: fapi_ffc = (struct pps_fetch_ffc_args *)data; if (fapi_ffc->tsformat && fapi_ffc->tsformat != PPS_TSFMT_TSPEC) return (EINVAL); if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec) return (EOPNOTSUPP); pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode; fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc; /* Overwrite timestamps if feedback clock selected. */ switch (pps->ppsparam.mode & PPS_TSCLK_MASK) { case PPS_TSCLK_FBCK: fapi_ffc->pps_info_buf_ffc.assert_timestamp = pps->ppsinfo.assert_timestamp; fapi_ffc->pps_info_buf_ffc.clear_timestamp = pps->ppsinfo.clear_timestamp; break; case PPS_TSCLK_FFWD: break; default: break; } return (0); #endif /* FFCLOCK */ case PPS_IOC_KCBIND: #ifdef PPS_SYNC kapi = (struct pps_kcbind_args *)data; /* XXX Only root should be able to do this */ if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) return (EINVAL); if (kapi->kernel_consumer != PPS_KC_HARDPPS) return (EINVAL); if (kapi->edge & ~pps->ppscap) return (EINVAL); pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) | (pps->kcmode & KCMODE_ABIFLAG); return (0); #else return (EOPNOTSUPP); #endif default: return (ENOIOCTL); } } void pps_init(struct pps_state *pps) { pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT; if (pps->ppscap & PPS_CAPTUREASSERT) pps->ppscap |= PPS_OFFSETASSERT; if (pps->ppscap & PPS_CAPTURECLEAR) pps->ppscap |= PPS_OFFSETCLEAR; #ifdef FFCLOCK pps->ppscap |= PPS_TSCLK_MASK; #endif pps->kcmode &= ~KCMODE_ABIFLAG; } void pps_init_abi(struct pps_state *pps) { pps_init(pps); if (pps->driver_abi > 0) { pps->kcmode |= KCMODE_ABIFLAG; pps->kernel_abi = PPS_ABI_VERSION; } } void pps_capture(struct pps_state *pps) { struct timehands *th; KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); th = timehands; pps->capgen = atomic_load_acq_int(&th->th_generation); pps->capth = th; #ifdef FFCLOCK pps->capffth = fftimehands; #endif pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); atomic_thread_fence_acq(); if (pps->capgen != th->th_generation) pps->capgen = 0; } void pps_event(struct pps_state *pps, int event) { struct bintime bt; struct timespec ts, *tsp, *osp; u_int tcount, *pcount; int foff; pps_seq_t *pseq; #ifdef FFCLOCK struct timespec *tsp_ffc; pps_seq_t *pseq_ffc; ffcounter *ffcount; #endif #ifdef PPS_SYNC int fhard; #endif KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); /* Nothing to do if not currently set to capture this event type. */ if ((event & pps->ppsparam.mode) == 0) return; /* If the timecounter was wound up underneath us, bail out. */ if (pps->capgen == 0 || pps->capgen != atomic_load_acq_int(&pps->capth->th_generation)) return; /* Things would be easier with arrays. */ if (event == PPS_CAPTUREASSERT) { tsp = &pps->ppsinfo.assert_timestamp; osp = &pps->ppsparam.assert_offset; foff = pps->ppsparam.mode & PPS_OFFSETASSERT; #ifdef PPS_SYNC fhard = pps->kcmode & PPS_CAPTUREASSERT; #endif pcount = &pps->ppscount[0]; pseq = &pps->ppsinfo.assert_sequence; #ifdef FFCLOCK ffcount = &pps->ppsinfo_ffc.assert_ffcount; tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp; pseq_ffc = &pps->ppsinfo_ffc.assert_sequence; #endif } else { tsp = &pps->ppsinfo.clear_timestamp; osp = &pps->ppsparam.clear_offset; foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; #ifdef PPS_SYNC fhard = pps->kcmode & PPS_CAPTURECLEAR; #endif pcount = &pps->ppscount[1]; pseq = &pps->ppsinfo.clear_sequence; #ifdef FFCLOCK ffcount = &pps->ppsinfo_ffc.clear_ffcount; tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp; pseq_ffc = &pps->ppsinfo_ffc.clear_sequence; #endif } /* * If the timecounter changed, we cannot compare the count values, so * we have to drop the rest of the PPS-stuff until the next event. */ if (pps->ppstc != pps->capth->th_counter) { pps->ppstc = pps->capth->th_counter; *pcount = pps->capcount; pps->ppscount[2] = pps->capcount; return; } /* Convert the count to a timespec. */ tcount = pps->capcount - pps->capth->th_offset_count; tcount &= pps->capth->th_counter->tc_counter_mask; bt = pps->capth->th_bintime; bintime_addx(&bt, pps->capth->th_scale * tcount); bintime2timespec(&bt, &ts); /* If the timecounter was wound up underneath us, bail out. */ atomic_thread_fence_acq(); if (pps->capgen != pps->capth->th_generation) return; *pcount = pps->capcount; (*pseq)++; *tsp = ts; if (foff) { timespecadd(tsp, osp, tsp); if (tsp->tv_nsec < 0) { tsp->tv_nsec += 1000000000; tsp->tv_sec -= 1; } } #ifdef FFCLOCK *ffcount = pps->capffth->tick_ffcount + tcount; bt = pps->capffth->tick_time; ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt); bintime_add(&bt, &pps->capffth->tick_time); bintime2timespec(&bt, &ts); (*pseq_ffc)++; *tsp_ffc = ts; #endif #ifdef PPS_SYNC if (fhard) { uint64_t scale; /* * Feed the NTP PLL/FLL. * The FLL wants to know how many (hardware) nanoseconds * elapsed since the previous event. */ tcount = pps->capcount - pps->ppscount[2]; pps->ppscount[2] = pps->capcount; tcount &= pps->capth->th_counter->tc_counter_mask; scale = (uint64_t)1 << 63; scale /= pps->capth->th_counter->tc_frequency; scale *= 2; bt.sec = 0; bt.frac = 0; bintime_addx(&bt, scale * tcount); bintime2timespec(&bt, &ts); hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); } #endif /* Wakeup anyone sleeping in pps_fetch(). */ wakeup(pps); } /* * Timecounters need to be updated every so often to prevent the hardware * counter from overflowing. Updating also recalculates the cached values * used by the get*() family of functions, so their precision depends on * the update frequency. */ static int tc_tick; SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, "Approximate number of hardclock ticks in a millisecond"); void tc_ticktock(int cnt) { static int count; if (mtx_trylock_spin(&tc_setclock_mtx)) { count += cnt; if (count >= tc_tick) { count = 0; tc_windup(NULL); } mtx_unlock_spin(&tc_setclock_mtx); } } static void __inline tc_adjprecision(void) { int t; if (tc_timepercentage > 0) { t = (99 + tc_timepercentage) / tc_timepercentage; tc_precexp = fls(t + (t >> 1)) - 1; FREQ2BT(hz / tc_tick, &bt_timethreshold); FREQ2BT(hz, &bt_tickthreshold); bintime_shift(&bt_timethreshold, tc_precexp); bintime_shift(&bt_tickthreshold, tc_precexp); } else { tc_precexp = 31; bt_timethreshold.sec = INT_MAX; bt_timethreshold.frac = ~(uint64_t)0; bt_tickthreshold = bt_timethreshold; } sbt_timethreshold = bttosbt(bt_timethreshold); sbt_tickthreshold = bttosbt(bt_tickthreshold); } static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS) { int error, val; val = tc_timepercentage; error = sysctl_handle_int(oidp, &val, 0, req); if (error != 0 || req->newptr == NULL) return (error); tc_timepercentage = val; if (cold) goto done; tc_adjprecision(); done: return (0); } /* Set up the requested number of timehands. */ static void inittimehands(void *dummy) { struct timehands *thp; int i; TUNABLE_INT_FETCH("kern.timecounter.timehands_count", &timehands_count); if (timehands_count < 1) timehands_count = 1; if (timehands_count > nitems(ths)) timehands_count = nitems(ths); for (i = 1, thp = &ths[0]; i < timehands_count; thp = &ths[i++]) thp->th_next = &ths[i]; thp->th_next = &ths[0]; TUNABLE_STR_FETCH("kern.timecounter.hardware", tc_from_tunable, sizeof(tc_from_tunable)); } SYSINIT(timehands, SI_SUB_TUNABLES, SI_ORDER_ANY, inittimehands, NULL); static void inittimecounter(void *dummy) { u_int p; int tick_rate; /* * Set the initial timeout to * max(1, ). * People should probably not use the sysctl to set the timeout * to smaller than its initial value, since that value is the * smallest reasonable one. If they want better timestamps they * should use the non-"get"* functions. */ if (hz > 1000) tc_tick = (hz + 500) / 1000; else tc_tick = 1; tc_adjprecision(); FREQ2BT(hz, &tick_bt); tick_sbt = bttosbt(tick_bt); tick_rate = hz / tc_tick; FREQ2BT(tick_rate, &tc_tick_bt); tc_tick_sbt = bttosbt(tc_tick_bt); p = (tc_tick * 1000000) / hz; printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); #ifdef FFCLOCK ffclock_init(); #endif /* warm up new timecounter (again) and get rolling. */ (void)timecounter->tc_get_timecount(timecounter); mtx_lock_spin(&tc_setclock_mtx); tc_windup(NULL); mtx_unlock_spin(&tc_setclock_mtx); } SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); /* Cpu tick handling -------------------------------------------------*/ static int cpu_tick_variable; static uint64_t cpu_tick_frequency; DPCPU_DEFINE_STATIC(uint64_t, tc_cpu_ticks_base); DPCPU_DEFINE_STATIC(unsigned, tc_cpu_ticks_last); static uint64_t tc_cpu_ticks(void) { struct timecounter *tc; uint64_t res, *base; unsigned u, *last; critical_enter(); base = DPCPU_PTR(tc_cpu_ticks_base); last = DPCPU_PTR(tc_cpu_ticks_last); tc = timehands->th_counter; u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; if (u < *last) *base += (uint64_t)tc->tc_counter_mask + 1; *last = u; res = u + *base; critical_exit(); return (res); } void cpu_tick_calibration(void) { static time_t last_calib; if (time_uptime != last_calib && !(time_uptime & 0xf)) { cpu_tick_calibrate(0); last_calib = time_uptime; } } /* * This function gets called every 16 seconds on only one designated * CPU in the system from hardclock() via cpu_tick_calibration()(). * * Whenever the real time clock is stepped we get called with reset=1 * to make sure we handle suspend/resume and similar events correctly. */ static void cpu_tick_calibrate(int reset) { static uint64_t c_last; uint64_t c_this, c_delta; static struct bintime t_last; struct bintime t_this, t_delta; uint32_t divi; if (reset) { /* The clock was stepped, abort & reset */ t_last.sec = 0; return; } /* we don't calibrate fixed rate cputicks */ if (!cpu_tick_variable) return; getbinuptime(&t_this); c_this = cpu_ticks(); if (t_last.sec != 0) { c_delta = c_this - c_last; t_delta = t_this; bintime_sub(&t_delta, &t_last); /* * Headroom: * 2^(64-20) / 16[s] = * 2^(44) / 16[s] = * 17.592.186.044.416 / 16 = * 1.099.511.627.776 [Hz] */ divi = t_delta.sec << 20; divi |= t_delta.frac >> (64 - 20); c_delta <<= 20; c_delta /= divi; if (c_delta > cpu_tick_frequency) { if (0 && bootverbose) printf("cpu_tick increased to %ju Hz\n", c_delta); cpu_tick_frequency = c_delta; } } c_last = c_this; t_last = t_this; } void set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) { if (func == NULL) { cpu_ticks = tc_cpu_ticks; } else { cpu_tick_frequency = freq; cpu_tick_variable = var; cpu_ticks = func; } } uint64_t cpu_tickrate(void) { if (cpu_ticks == tc_cpu_ticks) return (tc_getfrequency()); return (cpu_tick_frequency); } /* * We need to be slightly careful converting cputicks to microseconds. * There is plenty of margin in 64 bits of microseconds (half a million * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply * before divide conversion (to retain precision) we find that the * margin shrinks to 1.5 hours (one millionth of 146y). * With a three prong approach we never lose significant bits, no * matter what the cputick rate and length of timeinterval is. */ uint64_t cputick2usec(uint64_t tick) { if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ return (tick / (cpu_tickrate() / 1000000LL)); else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); else return ((tick * 1000000LL) / cpu_tickrate()); } cpu_tick_f *cpu_ticks = tc_cpu_ticks; static int vdso_th_enable = 1; static int sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) { int old_vdso_th_enable, error; old_vdso_th_enable = vdso_th_enable; error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); if (error != 0) return (error); vdso_th_enable = old_vdso_th_enable; return (0); } SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) { struct timehands *th; uint32_t enabled; th = timehands; vdso_th->th_scale = th->th_scale; vdso_th->th_offset_count = th->th_offset_count; vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; vdso_th->th_offset = th->th_offset; vdso_th->th_boottime = th->th_boottime; if (th->th_counter->tc_fill_vdso_timehands != NULL) { enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th, th->th_counter); } else enabled = 0; if (!vdso_th_enable) enabled = 0; return (enabled); } #ifdef COMPAT_FREEBSD32 uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) { struct timehands *th; uint32_t enabled; th = timehands; *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; vdso_th32->th_offset_count = th->th_offset_count; vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; vdso_th32->th_offset.sec = th->th_offset.sec; *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; vdso_th32->th_boottime.sec = th->th_boottime.sec; *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac; if (th->th_counter->tc_fill_vdso_timehands32 != NULL) { enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32, th->th_counter); } else enabled = 0; if (!vdso_th_enable) enabled = 0; return (enabled); } #endif #include "opt_ddb.h" #ifdef DDB #include DB_SHOW_COMMAND(timecounter, db_show_timecounter) { struct timehands *th; struct timecounter *tc; u_int val1, val2; th = timehands; tc = th->th_counter; val1 = tc->tc_get_timecount(tc); __compiler_membar(); val2 = tc->tc_get_timecount(tc); db_printf("timecounter %p %s\n", tc, tc->tc_name); db_printf(" mask %#x freq %ju qual %d flags %#x priv %p\n", tc->tc_counter_mask, (uintmax_t)tc->tc_frequency, tc->tc_quality, tc->tc_flags, tc->tc_priv); db_printf(" val %#x %#x\n", val1, val2); db_printf("timehands adj %#jx scale %#jx ldelta %d off_cnt %d gen %d\n", (uintmax_t)th->th_adjustment, (uintmax_t)th->th_scale, th->th_large_delta, th->th_offset_count, th->th_generation); db_printf(" offset %jd %jd boottime %jd %jd\n", (intmax_t)th->th_offset.sec, (uintmax_t)th->th_offset.frac, (intmax_t)th->th_boottime.sec, (uintmax_t)th->th_boottime.frac); } #endif diff --git a/sys/kern/msi_if.m b/sys/kern/msi_if.m index 8112450d7580..cb354379535f 100644 --- a/sys/kern/msi_if.m +++ b/sys/kern/msi_if.m @@ -1,103 +1,102 @@ #- # Copyright (c) 2016 The FreeBSD Foundation -# All rights reserved. # # This software was developed by Andrew Turner under # sponsorship from the FreeBSD Foundation. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # INTERFACE msi; HEADER { #include #include struct intr_irqsrc; }; # # Default implementations of some methods. # CODE { static int iommu_init(device_t dev, device_t child, struct iommu_domain **domain) { *domain = NULL; return (0); } static void iommu_deinit(device_t dev, device_t child) { } }; METHOD int alloc_msi { device_t dev; device_t child; int count; int maxcount; device_t *pic; struct intr_irqsrc **srcs; }; METHOD int release_msi { device_t dev; device_t child; int count; struct intr_irqsrc **srcs; }; METHOD int alloc_msix { device_t dev; device_t child; device_t *pic; struct intr_irqsrc **src; }; METHOD int release_msix { device_t dev; device_t child; struct intr_irqsrc *src; }; METHOD int map_msi { device_t dev; device_t child; struct intr_irqsrc *src; uint64_t *addr; uint32_t *data; }; METHOD int iommu_init { device_t dev; device_t child; struct iommu_domain **domain; } DEFAULT iommu_init; METHOD void iommu_deinit { device_t dev; device_t child; } DEFAULT iommu_deinit; diff --git a/sys/kern/subr_capability.c b/sys/kern/subr_capability.c index 6e6865ae7300..47a6f256fe7e 100644 --- a/sys/kern/subr_capability.c +++ b/sys/kern/subr_capability.c @@ -1,410 +1,409 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Note that this file is compiled into the kernel and into libc. */ #include #include #ifdef _KERNEL #include #include #include #else /* !_KERNEL */ #include #include #include #include #include #endif #ifdef _KERNEL #define assert(exp) KASSERT((exp), ("%s:%u", __func__, __LINE__)) __read_mostly cap_rights_t cap_accept_rights; __read_mostly cap_rights_t cap_bind_rights; __read_mostly cap_rights_t cap_chflags_rights; __read_mostly cap_rights_t cap_connect_rights; __read_mostly cap_rights_t cap_event_rights; __read_mostly cap_rights_t cap_fchdir_rights; __read_mostly cap_rights_t cap_fchflags_rights; __read_mostly cap_rights_t cap_fchmod_rights; __read_mostly cap_rights_t cap_fchown_rights; __read_mostly cap_rights_t cap_fcntl_rights; __read_mostly cap_rights_t cap_fexecve_rights; __read_mostly cap_rights_t cap_flock_rights; __read_mostly cap_rights_t cap_fpathconf_rights; __read_mostly cap_rights_t cap_fstat_rights; __read_mostly cap_rights_t cap_fstatfs_rights; __read_mostly cap_rights_t cap_fsync_rights; __read_mostly cap_rights_t cap_ftruncate_rights; __read_mostly cap_rights_t cap_futimes_rights; __read_mostly cap_rights_t cap_getpeername_rights; __read_mostly cap_rights_t cap_getsockopt_rights; __read_mostly cap_rights_t cap_getsockname_rights; __read_mostly cap_rights_t cap_ioctl_rights; __read_mostly cap_rights_t cap_listen_rights; __read_mostly cap_rights_t cap_linkat_source_rights; __read_mostly cap_rights_t cap_linkat_target_rights; __read_mostly cap_rights_t cap_mmap_rights; __read_mostly cap_rights_t cap_mkdirat_rights; __read_mostly cap_rights_t cap_mkfifoat_rights; __read_mostly cap_rights_t cap_mknodat_rights; __read_mostly cap_rights_t cap_pdgetpid_rights; __read_mostly cap_rights_t cap_pdkill_rights; __read_mostly cap_rights_t cap_pread_rights; __read_mostly cap_rights_t cap_pwrite_rights; __read_mostly cap_rights_t cap_read_rights; __read_mostly cap_rights_t cap_recv_rights; __read_mostly cap_rights_t cap_renameat_source_rights; __read_mostly cap_rights_t cap_renameat_target_rights; __read_mostly cap_rights_t cap_seek_rights; __read_mostly cap_rights_t cap_send_rights; __read_mostly cap_rights_t cap_send_connect_rights; __read_mostly cap_rights_t cap_setsockopt_rights; __read_mostly cap_rights_t cap_shutdown_rights; __read_mostly cap_rights_t cap_symlinkat_rights; __read_mostly cap_rights_t cap_unlinkat_rights; __read_mostly cap_rights_t cap_write_rights; __read_mostly cap_rights_t cap_no_rights; static void cap_rights_sysinit(void *arg) { cap_rights_init_one(&cap_accept_rights, CAP_ACCEPT); cap_rights_init_one(&cap_bind_rights, CAP_BIND); cap_rights_init_one(&cap_connect_rights, CAP_CONNECT); cap_rights_init_one(&cap_event_rights, CAP_EVENT); cap_rights_init_one(&cap_fchdir_rights, CAP_FCHDIR); cap_rights_init_one(&cap_fchflags_rights, CAP_FCHFLAGS); cap_rights_init_one(&cap_fchmod_rights, CAP_FCHMOD); cap_rights_init_one(&cap_fchown_rights, CAP_FCHOWN); cap_rights_init_one(&cap_fcntl_rights, CAP_FCNTL); cap_rights_init_one(&cap_fexecve_rights, CAP_FEXECVE); cap_rights_init_one(&cap_flock_rights, CAP_FLOCK); cap_rights_init_one(&cap_fpathconf_rights, CAP_FPATHCONF); cap_rights_init_one(&cap_fstat_rights, CAP_FSTAT); cap_rights_init_one(&cap_fstatfs_rights, CAP_FSTATFS); cap_rights_init_one(&cap_fsync_rights, CAP_FSYNC); cap_rights_init_one(&cap_ftruncate_rights, CAP_FTRUNCATE); cap_rights_init_one(&cap_futimes_rights, CAP_FUTIMES); cap_rights_init_one(&cap_getpeername_rights, CAP_GETPEERNAME); cap_rights_init_one(&cap_getsockname_rights, CAP_GETSOCKNAME); cap_rights_init_one(&cap_getsockopt_rights, CAP_GETSOCKOPT); cap_rights_init_one(&cap_ioctl_rights, CAP_IOCTL); cap_rights_init_one(&cap_linkat_source_rights, CAP_LINKAT_SOURCE); cap_rights_init_one(&cap_linkat_target_rights, CAP_LINKAT_TARGET); cap_rights_init_one(&cap_listen_rights, CAP_LISTEN); cap_rights_init_one(&cap_mkdirat_rights, CAP_MKDIRAT); cap_rights_init_one(&cap_mkfifoat_rights, CAP_MKFIFOAT); cap_rights_init_one(&cap_mknodat_rights, CAP_MKNODAT); cap_rights_init_one(&cap_mmap_rights, CAP_MMAP); cap_rights_init_one(&cap_pdgetpid_rights, CAP_PDGETPID); cap_rights_init_one(&cap_pdkill_rights, CAP_PDKILL); cap_rights_init_one(&cap_pread_rights, CAP_PREAD); cap_rights_init_one(&cap_pwrite_rights, CAP_PWRITE); cap_rights_init_one(&cap_read_rights, CAP_READ); cap_rights_init_one(&cap_recv_rights, CAP_RECV); cap_rights_init_one(&cap_renameat_source_rights, CAP_RENAMEAT_SOURCE); cap_rights_init_one(&cap_renameat_target_rights, CAP_RENAMEAT_TARGET); cap_rights_init_one(&cap_seek_rights, CAP_SEEK); cap_rights_init_one(&cap_send_rights, CAP_SEND); cap_rights_init(&cap_send_connect_rights, CAP_SEND, CAP_CONNECT); cap_rights_init_one(&cap_setsockopt_rights, CAP_SETSOCKOPT); cap_rights_init_one(&cap_shutdown_rights, CAP_SHUTDOWN); cap_rights_init_one(&cap_symlinkat_rights, CAP_SYMLINKAT); cap_rights_init_one(&cap_unlinkat_rights, CAP_UNLINKAT); cap_rights_init_one(&cap_write_rights, CAP_WRITE); cap_rights_init(&cap_no_rights); } SYSINIT(cap_rights_sysinit, SI_SUB_COPYRIGHT, SI_ORDER_ANY, cap_rights_sysinit, NULL); #endif #define CAPARSIZE_MIN (CAP_RIGHTS_VERSION_00 + 2) #define CAPARSIZE_MAX (CAP_RIGHTS_VERSION + 2) static __inline int right_to_index(uint64_t right) { static const int bit2idx[] = { -1, 0, 1, -1, 2, -1, -1, -1, 3, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; int idx; idx = CAPIDXBIT(right); assert(idx >= 0 && idx < sizeof(bit2idx) / sizeof(bit2idx[0])); return (bit2idx[idx]); } static void cap_rights_vset(cap_rights_t *rights, va_list ap) { uint64_t right; int i, n __unused; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); n = CAPARSIZE(rights); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (;;) { right = (uint64_t)va_arg(ap, unsigned long long); if (right == 0) break; assert(CAPRVER(right) == 0); i = right_to_index(right); assert(i >= 0); assert(i < n); assert(CAPIDXBIT(rights->cr_rights[i]) == CAPIDXBIT(right)); rights->cr_rights[i] |= right; assert(CAPIDXBIT(rights->cr_rights[i]) == CAPIDXBIT(right)); } } static void cap_rights_vclear(cap_rights_t *rights, va_list ap) { uint64_t right; int i, n __unused; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); n = CAPARSIZE(rights); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (;;) { right = (uint64_t)va_arg(ap, unsigned long long); if (right == 0) break; assert(CAPRVER(right) == 0); i = right_to_index(right); assert(i >= 0); assert(i < n); assert(CAPIDXBIT(rights->cr_rights[i]) == CAPIDXBIT(right)); rights->cr_rights[i] &= ~(right & 0x01FFFFFFFFFFFFFFULL); assert(CAPIDXBIT(rights->cr_rights[i]) == CAPIDXBIT(right)); } } static bool cap_rights_is_vset(const cap_rights_t *rights, va_list ap) { uint64_t right; int i, n __unused; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); n = CAPARSIZE(rights); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (;;) { right = (uint64_t)va_arg(ap, unsigned long long); if (right == 0) break; assert(CAPRVER(right) == 0); i = right_to_index(right); assert(i >= 0); assert(i < n); assert(CAPIDXBIT(rights->cr_rights[i]) == CAPIDXBIT(right)); if ((rights->cr_rights[i] & right) != right) return (false); } return (true); } cap_rights_t * __cap_rights_init(int version, cap_rights_t *rights, ...) { unsigned int n __unused; va_list ap; assert(version == CAP_RIGHTS_VERSION_00); n = version + 2; assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); CAP_NONE(rights); va_start(ap, rights); cap_rights_vset(rights, ap); va_end(ap); return (rights); } cap_rights_t * __cap_rights_set(cap_rights_t *rights, ...) { va_list ap; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); va_start(ap, rights); cap_rights_vset(rights, ap); va_end(ap); return (rights); } cap_rights_t * __cap_rights_clear(cap_rights_t *rights, ...) { va_list ap; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); va_start(ap, rights); cap_rights_vclear(rights, ap); va_end(ap); return (rights); } bool __cap_rights_is_set(const cap_rights_t *rights, ...) { va_list ap; bool ret; assert(CAPVER(rights) == CAP_RIGHTS_VERSION_00); va_start(ap, rights); ret = cap_rights_is_vset(rights, ap); va_end(ap); return (ret); } bool cap_rights_is_valid(const cap_rights_t *rights) { cap_rights_t allrights; int i, j; if (CAPVER(rights) != CAP_RIGHTS_VERSION_00) return (false); if (CAPARSIZE(rights) < CAPARSIZE_MIN || CAPARSIZE(rights) > CAPARSIZE_MAX) { return (false); } CAP_ALL(&allrights); if (!cap_rights_contains(&allrights, rights)) return (false); for (i = 0; i < CAPARSIZE(rights); i++) { j = right_to_index(rights->cr_rights[i]); if (i != j) return (false); if (i > 0) { if (CAPRVER(rights->cr_rights[i]) != 0) return (false); } } return (true); } cap_rights_t * cap_rights_merge(cap_rights_t *dst, const cap_rights_t *src) { unsigned int i, n; assert(CAPVER(dst) == CAP_RIGHTS_VERSION_00); assert(CAPVER(src) == CAP_RIGHTS_VERSION_00); assert(CAPVER(dst) == CAPVER(src)); assert(cap_rights_is_valid(src)); assert(cap_rights_is_valid(dst)); n = CAPARSIZE(dst); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (i = 0; i < n; i++) dst->cr_rights[i] |= src->cr_rights[i]; assert(cap_rights_is_valid(src)); assert(cap_rights_is_valid(dst)); return (dst); } cap_rights_t * cap_rights_remove(cap_rights_t *dst, const cap_rights_t *src) { unsigned int i, n; assert(CAPVER(dst) == CAP_RIGHTS_VERSION_00); assert(CAPVER(src) == CAP_RIGHTS_VERSION_00); assert(CAPVER(dst) == CAPVER(src)); assert(cap_rights_is_valid(src)); assert(cap_rights_is_valid(dst)); n = CAPARSIZE(dst); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (i = 0; i < n; i++) { dst->cr_rights[i] &= ~(src->cr_rights[i] & 0x01FFFFFFFFFFFFFFULL); } assert(cap_rights_is_valid(src)); assert(cap_rights_is_valid(dst)); return (dst); } #ifndef _KERNEL bool cap_rights_contains(const cap_rights_t *big, const cap_rights_t *little) { unsigned int i, n; assert(CAPVER(big) == CAP_RIGHTS_VERSION_00); assert(CAPVER(little) == CAP_RIGHTS_VERSION_00); assert(CAPVER(big) == CAPVER(little)); n = CAPARSIZE(big); assert(n >= CAPARSIZE_MIN && n <= CAPARSIZE_MAX); for (i = 0; i < n; i++) { if ((big->cr_rights[i] & little->cr_rights[i]) != little->cr_rights[i]) { return (false); } } return (true); } #endif diff --git a/sys/kern/subr_rangeset.c b/sys/kern/subr_rangeset.c index 1e77ab9c5311..3798b30e2240 100644 --- a/sys/kern/subr_rangeset.c +++ b/sys/kern/subr_rangeset.c @@ -1,365 +1,364 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2019 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #ifdef DIAGNOSTIC static void rangeset_check(struct rangeset *rs); #else #define rangeset_check(rs) #endif static uma_zone_t rs_node_zone; static void rs_rangeset_init(void *arg __unused) { rs_node_zone = uma_zcreate("rangeset pctrie nodes", pctrie_node_size(), NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR, 0); } SYSINIT(rs, SI_SUB_LOCK, SI_ORDER_ANY, rs_rangeset_init, NULL); static void * rs_node_alloc(struct pctrie *ptree) { struct rangeset *rs; rs = __containerof(ptree, struct rangeset, rs_trie); return (uma_zalloc(rs_node_zone, rs->rs_alloc_flags)); } static void rs_node_free(struct pctrie *ptree __unused, void *node) { uma_zfree(rs_node_zone, node); } void rangeset_init(struct rangeset *rs, rs_dup_data_t dup_data, rs_free_data_t free_data, void *data_ctx, u_int alloc_flags) { pctrie_init(&rs->rs_trie); rs->rs_dup_data = dup_data; rs->rs_free_data = free_data; rs->rs_data_ctx = data_ctx; rs->rs_alloc_flags = alloc_flags; } void rangeset_fini(struct rangeset *rs) { rangeset_check(rs); rangeset_remove_all(rs); } bool rangeset_check_empty(struct rangeset *rs, uint64_t start, uint64_t end) { struct rs_el *r; uint64_t *r1; rangeset_check(rs); r1 = pctrie_lookup_le(&rs->rs_trie, end); if (r1 != NULL) { r = __containerof(r1, struct rs_el, re_start); if (r->re_end > start) return (false); } return (true); } int rangeset_insert(struct rangeset *rs, uint64_t start, uint64_t end, void *data) { struct rs_el *r; int error; rangeset_check(rs); error = rangeset_remove(rs, start, end); if (error != 0) return (error); r = data; r->re_start = start; r->re_end = end; error = pctrie_insert(&rs->rs_trie, &r->re_start, rs_node_alloc); rangeset_check(rs); return (error); } int rangeset_remove_pred(struct rangeset *rs, uint64_t start, uint64_t end, rs_pred_t pred) { struct rs_el *r, *rn; uint64_t *r1; int error; rangeset_check(rs); error = 0; for (; end > 0 && start < end;) { r1 = pctrie_lookup_le(&rs->rs_trie, end - 1); if (r1 == NULL) break; r = __containerof(r1, struct rs_el, re_start); /* * ------============================--|-------|---- * rs re s e */ if (r->re_end <= start) break; if (r->re_end <= end) { if (r->re_start < start) { /* * ------========|==============-------|---- * rs s re e */ if (pred(rs->rs_data_ctx, r)) r->re_end = start; break; } /* * ------|--------===================----------|---- * s rs re e */ end = r->re_start; if (pred(rs->rs_data_ctx, r)) { pctrie_remove(&rs->rs_trie, r->re_start, rs_node_free); rs->rs_free_data(rs->rs_data_ctx, r); } continue; } /* * ------|--------====================|==========---- * s rs e re */ if (r->re_start >= start) { if (pred(rs->rs_data_ctx, r)) { pctrie_remove(&rs->rs_trie, r->re_start, rs_node_free); r->re_start = end; error = pctrie_insert(&rs->rs_trie, &r->re_start, rs_node_alloc); /* * The insert above must succeed * because rs_node zone is marked * nofree and we freed one element * just before. */ MPASS(error == 0); } else { end = r->re_start; } continue; } /* * ------=========|===================|==========---- * rs s e re */ if (pred(rs->rs_data_ctx, r)) { /* * Split. Can only happen once, and then if * any allocation fails, the rangeset is kept * intact. */ rn = rs->rs_dup_data(rs->rs_data_ctx, r); if (rn == NULL) { error = ENOMEM; break; } rn->re_start = end; rn->re_end = r->re_end; error = pctrie_insert(&rs->rs_trie, &rn->re_start, rs_node_alloc); if (error != 0) { rs->rs_free_data(rs->rs_data_ctx, rn); break; } r->re_end = start; } break; } rangeset_check(rs); return (error); } static bool rangeset_true_pred(void *ctx __unused, void *r __unused) { return (true); } int rangeset_remove(struct rangeset *rs, uint64_t start, uint64_t end) { return (rangeset_remove_pred(rs, start, end, rangeset_true_pred)); } void rangeset_remove_all(struct rangeset *rs) { struct rs_el *r; uint64_t *r1; for (;;) { r1 = pctrie_lookup_ge(&rs->rs_trie, 0); if (r1 == NULL) break; r = __containerof(r1, struct rs_el, re_start); pctrie_remove(&rs->rs_trie, r->re_start, rs_node_free); rs->rs_free_data(rs->rs_data_ctx, r); } } void * rangeset_lookup(struct rangeset *rs, uint64_t place) { struct rs_el *r; uint64_t *r1; rangeset_check(rs); r1 = pctrie_lookup_le(&rs->rs_trie, place); if (r1 == NULL) return (NULL); r = __containerof(r1, struct rs_el, re_start); if (r->re_end <= place) return (NULL); return (r); } int rangeset_copy(struct rangeset *dst_rs, struct rangeset *src_rs) { struct rs_el *src_r, *dst_r; uint64_t cursor, *r1; int error; MPASS(pctrie_is_empty(&dst_rs->rs_trie)); rangeset_check(src_rs); MPASS(dst_rs->rs_dup_data == src_rs->rs_dup_data); error = 0; for (cursor = 0;; cursor = src_r->re_start + 1) { r1 = pctrie_lookup_ge(&src_rs->rs_trie, cursor); if (r1 == NULL) break; src_r = __containerof(r1, struct rs_el, re_start); dst_r = dst_rs->rs_dup_data(dst_rs->rs_data_ctx, src_r); if (dst_r == NULL) { error = ENOMEM; break; } error = pctrie_insert(&dst_rs->rs_trie, &dst_r->re_start, rs_node_alloc); if (error != 0) break; } if (error != 0) rangeset_remove_all(dst_rs); return (error); } #ifdef DIAGNOSTIC static void rangeset_check(struct rangeset *rs) { struct rs_el *r, *rp; uint64_t cursor, *r1; for (cursor = 0, rp = NULL;; cursor = r->re_start + 1, rp = r) { r1 = pctrie_lookup_ge(&rs->rs_trie, cursor); if (r1 == NULL) break; r = __containerof(r1, struct rs_el, re_start); KASSERT(r->re_start < r->re_end, ("invalid interval rs %p elem %p (%#jx, %#jx)", rs, r, (uintmax_t)r->re_start, (uintmax_t)r->re_end)); if (rp != NULL) { KASSERT(rp->re_end <= r->re_start, ("non-ascending neighbors rs %p " "prev elem %p (%#jx, %#jx) elem %p (%#jx, %#jx)", rs, rp, (uintmax_t)rp->re_start, (uintmax_t)rp->re_end, r, (uintmax_t)r->re_start, (uintmax_t)r->re_end)); } } } #endif #include "opt_ddb.h" #ifdef DDB #include #include DB_SHOW_COMMAND(rangeset, rangeset_show_fn) { struct rangeset *rs; struct rs_el *r; uint64_t cursor, *r1; if (!have_addr) { db_printf("show rangeset addr\n"); return; } rs = (struct rangeset *)addr; db_printf("rangeset %p\n", rs); for (cursor = 0;; cursor = r->re_start + 1) { r1 = pctrie_lookup_ge(&rs->rs_trie, cursor); if (r1 == NULL) break; r = __containerof(r1, struct rs_el, re_start); db_printf(" el %p start %#jx end %#jx\n", r, r->re_start, r->re_end); } } #endif diff --git a/sys/kern/subr_terminal.c b/sys/kern/subr_terminal.c index fad2a4872a73..6a7163b63d15 100644 --- a/sys/kern/subr_terminal.c +++ b/sys/kern/subr_terminal.c @@ -1,714 +1,713 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_TERMINAL, "terminal", "terminal device"); /* * Locking. * * Normally we don't need to lock down the terminal emulator, because * the TTY lock is already held when calling teken_input(). * Unfortunately this is not the case when the terminal acts as a * console device, because cnputc() can be called at the same time. * This means terminals may need to be locked down using a spin lock. */ #define TERMINAL_LOCK(tm) do { \ if ((tm)->tm_flags & TF_CONS) \ mtx_lock_spin(&(tm)->tm_mtx); \ else if ((tm)->tm_tty != NULL) \ tty_lock((tm)->tm_tty); \ } while (0) #define TERMINAL_UNLOCK(tm) do { \ if ((tm)->tm_flags & TF_CONS) \ mtx_unlock_spin(&(tm)->tm_mtx); \ else if ((tm)->tm_tty != NULL) \ tty_unlock((tm)->tm_tty); \ } while (0) #define TERMINAL_LOCK_TTY(tm) do { \ if ((tm)->tm_flags & TF_CONS) \ mtx_lock_spin(&(tm)->tm_mtx); \ } while (0) #define TERMINAL_UNLOCK_TTY(tm) do { \ if ((tm)->tm_flags & TF_CONS) \ mtx_unlock_spin(&(tm)->tm_mtx); \ } while (0) #define TERMINAL_LOCK_CONS(tm) mtx_lock_spin(&(tm)->tm_mtx) #define TERMINAL_UNLOCK_CONS(tm) mtx_unlock_spin(&(tm)->tm_mtx) /* * TTY routines. */ static tsw_open_t termtty_open; static tsw_close_t termtty_close; static tsw_outwakeup_t termtty_outwakeup; static tsw_ioctl_t termtty_ioctl; static tsw_mmap_t termtty_mmap; static struct ttydevsw terminal_tty_class = { .tsw_open = termtty_open, .tsw_close = termtty_close, .tsw_outwakeup = termtty_outwakeup, .tsw_ioctl = termtty_ioctl, .tsw_mmap = termtty_mmap, }; /* * Terminal emulator routines. */ static tf_bell_t termteken_bell; static tf_cursor_t termteken_cursor; static tf_putchar_t termteken_putchar; static tf_fill_t termteken_fill; static tf_copy_t termteken_copy; static tf_pre_input_t termteken_pre_input; static tf_post_input_t termteken_post_input; static tf_param_t termteken_param; static tf_respond_t termteken_respond; static teken_funcs_t terminal_drawmethods = { .tf_bell = termteken_bell, .tf_cursor = termteken_cursor, .tf_putchar = termteken_putchar, .tf_fill = termteken_fill, .tf_copy = termteken_copy, .tf_pre_input = termteken_pre_input, .tf_post_input = termteken_post_input, .tf_param = termteken_param, .tf_respond = termteken_respond, }; /* Kernel message formatting. */ static teken_attr_t kernel_message = { .ta_fgcolor = TCHAR_FGCOLOR(TERMINAL_KERN_ATTR), .ta_bgcolor = TCHAR_BGCOLOR(TERMINAL_KERN_ATTR), .ta_format = TCHAR_FORMAT(TERMINAL_KERN_ATTR) }; static teken_attr_t default_message = { .ta_fgcolor = TCHAR_FGCOLOR(TERMINAL_NORM_ATTR), .ta_bgcolor = TCHAR_BGCOLOR(TERMINAL_NORM_ATTR), .ta_format = TCHAR_FORMAT(TERMINAL_NORM_ATTR) }; /* Fudge fg brightness as TF_BOLD (shifted). */ #define TCOLOR_FG_FUDGED(color) __extension__ ({ \ teken_color_t _c; \ \ _c = (color); \ TCOLOR_FG(_c & 7) | ((_c & 8) << 18); \ }) /* Fudge bg brightness as TF_BLINK (shifted). */ #define TCOLOR_BG_FUDGED(color) __extension__ ({ \ teken_color_t _c; \ \ _c = (color); \ TCOLOR_BG(_c & 7) | ((_c & 8) << 20); \ }) #define TCOLOR_256TO16(color) __extension__ ({ \ teken_color_t _c; \ \ _c = (color); \ if (_c >= 16) \ _c = teken_256to16(_c); \ _c; \ }) #define TCHAR_CREATE(c, a) ((c) | TFORMAT((a)->ta_format) | \ TCOLOR_FG_FUDGED(TCOLOR_256TO16((a)->ta_fgcolor)) | \ TCOLOR_BG_FUDGED(TCOLOR_256TO16((a)->ta_bgcolor))) static void terminal_init(struct terminal *tm) { int fg, bg; if (tm->tm_flags & TF_CONS) mtx_init(&tm->tm_mtx, "trmlck", NULL, MTX_SPIN); teken_init(&tm->tm_emulator, &terminal_drawmethods, tm); fg = bg = -1; TUNABLE_INT_FETCH("teken.fg_color", &fg); TUNABLE_INT_FETCH("teken.bg_color", &bg); if (fg != -1) { default_message.ta_fgcolor = fg; kernel_message.ta_fgcolor = fg; } if (bg != -1) { default_message.ta_bgcolor = bg; kernel_message.ta_bgcolor = bg; } if (default_message.ta_bgcolor == TC_WHITE) { default_message.ta_bgcolor |= TC_LIGHT; kernel_message.ta_bgcolor |= TC_LIGHT; } if (default_message.ta_bgcolor == TC_BLACK && default_message.ta_fgcolor < TC_NCOLORS) kernel_message.ta_fgcolor |= TC_LIGHT; teken_set_defattr(&tm->tm_emulator, &default_message); } struct terminal * terminal_alloc(const struct terminal_class *tc, void *softc) { struct terminal *tm; tm = malloc(sizeof(struct terminal), M_TERMINAL, M_WAITOK|M_ZERO); terminal_init(tm); tm->tm_class = tc; tm->tm_softc = softc; return (tm); } static void terminal_sync_ttysize(struct terminal *tm) { struct tty *tp; tp = tm->tm_tty; if (tp == NULL) return; tty_lock(tp); tty_set_winsize(tp, &tm->tm_winsize); tty_unlock(tp); } void terminal_maketty(struct terminal *tm, const char *fmt, ...) { struct tty *tp; char name[8]; va_list ap; va_start(ap, fmt); vsnrprintf(name, sizeof name, 32, fmt, ap); va_end(ap); tp = tty_alloc(&terminal_tty_class, tm); tty_makedev(tp, NULL, "%s", name); tm->tm_tty = tp; terminal_sync_ttysize(tm); } void terminal_set_cursor(struct terminal *tm, const term_pos_t *pos) { teken_set_cursor(&tm->tm_emulator, pos); } void terminal_set_winsize_blank(struct terminal *tm, const struct winsize *size, int blank, const term_attr_t *attr) { term_rect_t r; tm->tm_winsize = *size; r.tr_begin.tp_row = r.tr_begin.tp_col = 0; r.tr_end.tp_row = size->ws_row; r.tr_end.tp_col = size->ws_col; TERMINAL_LOCK(tm); if (blank == 0) teken_set_winsize_noreset(&tm->tm_emulator, &r.tr_end); else teken_set_winsize(&tm->tm_emulator, &r.tr_end); TERMINAL_UNLOCK(tm); if ((blank != 0) && !(tm->tm_flags & TF_MUTE)) tm->tm_class->tc_fill(tm, &r, TCHAR_CREATE((teken_char_t)' ', attr)); terminal_sync_ttysize(tm); } void terminal_set_winsize(struct terminal *tm, const struct winsize *size) { terminal_set_winsize_blank(tm, size, 1, (const term_attr_t *)&default_message); } /* * XXX: This function is a kludge. Drivers like vt(4) need to * temporarily stop input when resizing, etc. This should ideally be * handled within the driver. */ void terminal_mute(struct terminal *tm, int yes) { TERMINAL_LOCK(tm); if (yes) tm->tm_flags |= TF_MUTE; else tm->tm_flags &= ~TF_MUTE; TERMINAL_UNLOCK(tm); } void terminal_input_char(struct terminal *tm, term_char_t c) { struct tty *tp; tp = tm->tm_tty; if (tp == NULL) return; /* * Strip off any attributes. Also ignore input of second part of * CJK fullwidth characters, as we don't want to return these * characters twice. */ if (TCHAR_FORMAT(c) & TF_CJK_RIGHT) return; c = TCHAR_CHARACTER(c); tty_lock(tp); /* * Conversion to UTF-8. */ if (c < 0x80) { ttydisc_rint(tp, c, 0); } else if (c < 0x800) { char str[2] = { 0xc0 | (c >> 6), 0x80 | (c & 0x3f) }; ttydisc_rint_simple(tp, str, sizeof str); } else if (c < 0x10000) { char str[3] = { 0xe0 | (c >> 12), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f) }; ttydisc_rint_simple(tp, str, sizeof str); } else { char str[4] = { 0xf0 | (c >> 18), 0x80 | ((c >> 12) & 0x3f), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f) }; ttydisc_rint_simple(tp, str, sizeof str); } ttydisc_rint_done(tp); tty_unlock(tp); } void terminal_input_raw(struct terminal *tm, char c) { struct tty *tp; tp = tm->tm_tty; if (tp == NULL) return; tty_lock(tp); ttydisc_rint(tp, c, 0); ttydisc_rint_done(tp); tty_unlock(tp); } void terminal_input_special(struct terminal *tm, unsigned int k) { struct tty *tp; const char *str; tp = tm->tm_tty; if (tp == NULL) return; str = teken_get_sequence(&tm->tm_emulator, k); if (str == NULL) return; tty_lock(tp); ttydisc_rint_simple(tp, str, strlen(str)); ttydisc_rint_done(tp); tty_unlock(tp); } /* * Binding with the TTY layer. */ static int termtty_open(struct tty *tp) { struct terminal *tm = tty_softc(tp); tm->tm_class->tc_opened(tm, 1); return (0); } static void termtty_close(struct tty *tp) { struct terminal *tm = tty_softc(tp); tm->tm_class->tc_opened(tm, 0); } static void termtty_outwakeup(struct tty *tp) { struct terminal *tm = tty_softc(tp); char obuf[128]; size_t olen; unsigned int flags = 0; while ((olen = ttydisc_getc(tp, obuf, sizeof obuf)) > 0) { TERMINAL_LOCK_TTY(tm); if (!(tm->tm_flags & TF_MUTE)) { tm->tm_flags &= ~TF_BELL; teken_input(&tm->tm_emulator, obuf, olen); flags |= tm->tm_flags; } TERMINAL_UNLOCK_TTY(tm); } TERMINAL_LOCK_TTY(tm); if (!(tm->tm_flags & TF_MUTE)) tm->tm_class->tc_done(tm); TERMINAL_UNLOCK_TTY(tm); if (flags & TF_BELL) tm->tm_class->tc_bell(tm); } static int termtty_ioctl(struct tty *tp, u_long cmd, caddr_t data, struct thread *td) { struct terminal *tm = tty_softc(tp); int error; switch (cmd) { case CONS_GETINFO: { vid_info_t *vi = (vid_info_t *)data; const teken_pos_t *p; int fg, bg; if (vi->size != sizeof(vid_info_t)) return (EINVAL); /* Already help the console driver by filling in some data. */ p = teken_get_cursor(&tm->tm_emulator); vi->mv_row = p->tp_row; vi->mv_col = p->tp_col; p = teken_get_winsize(&tm->tm_emulator); vi->mv_rsz = p->tp_row; vi->mv_csz = p->tp_col; teken_get_defattr_cons25(&tm->tm_emulator, &fg, &bg); vi->mv_norm.fore = fg; vi->mv_norm.back = bg; /* XXX: keep vidcontrol happy; bold backgrounds. */ vi->mv_rev.fore = bg; vi->mv_rev.back = fg & 0x7; break; } } /* * Unlike various other drivers, this driver will never * deallocate TTYs. This means it's safe to temporarily unlock * the TTY when handling ioctls. */ tty_unlock(tp); error = tm->tm_class->tc_ioctl(tm, cmd, data, td); tty_lock(tp); if ((error == 0) && (cmd == CONS_CLRHIST)) { /* * Scrollback history has been successfully cleared, * so reset the cursor position to the top left of the screen. */ teken_pos_t p; p.tp_row = 0; p.tp_col = 0; teken_set_cursor(&tm->tm_emulator, &p); } return (error); } static int termtty_mmap(struct tty *tp, vm_ooffset_t offset, vm_paddr_t * paddr, int nprot, vm_memattr_t *memattr) { struct terminal *tm = tty_softc(tp); return (tm->tm_class->tc_mmap(tm, offset, paddr, nprot, memattr)); } /* * Binding with the kernel and debug console. */ static cn_probe_t termcn_cnprobe; static cn_init_t termcn_cninit; static cn_term_t termcn_cnterm; static cn_getc_t termcn_cngetc; static cn_putc_t termcn_cnputc; static cn_grab_t termcn_cngrab; static cn_ungrab_t termcn_cnungrab; const struct consdev_ops termcn_cnops = { .cn_probe = termcn_cnprobe, .cn_init = termcn_cninit, .cn_term = termcn_cnterm, .cn_getc = termcn_cngetc, .cn_putc = termcn_cnputc, .cn_grab = termcn_cngrab, .cn_ungrab = termcn_cnungrab, }; void termcn_cnregister(struct terminal *tm) { struct consdev *cp; cp = tm->consdev; if (cp == NULL) { cp = malloc(sizeof(struct consdev), M_TERMINAL, M_WAITOK|M_ZERO); cp->cn_ops = &termcn_cnops; cp->cn_arg = tm; cp->cn_pri = CN_INTERNAL; sprintf(cp->cn_name, "ttyv0"); tm->tm_flags = TF_CONS; tm->consdev = cp; terminal_init(tm); } /* Attach terminal as console. */ cnadd(cp); } static void termcn_cngrab(struct consdev *cp) { struct terminal *tm = cp->cn_arg; tm->tm_class->tc_cngrab(tm); } static void termcn_cnungrab(struct consdev *cp) { struct terminal *tm = cp->cn_arg; tm->tm_class->tc_cnungrab(tm); } static void termcn_cnprobe(struct consdev *cp) { struct terminal *tm = cp->cn_arg; if (tm == NULL) { cp->cn_pri = CN_DEAD; return; } tm->consdev = cp; terminal_init(tm); tm->tm_class->tc_cnprobe(tm, cp); } static void termcn_cninit(struct consdev *cp) { } static void termcn_cnterm(struct consdev *cp) { } static int termcn_cngetc(struct consdev *cp) { struct terminal *tm = cp->cn_arg; return (tm->tm_class->tc_cngetc(tm)); } static void termcn_cnputc(struct consdev *cp, int c) { struct terminal *tm = cp->cn_arg; teken_attr_t backup; char cv = c; TERMINAL_LOCK_CONS(tm); if (!(tm->tm_flags & TF_MUTE)) { backup = *teken_get_curattr(&tm->tm_emulator); teken_set_curattr(&tm->tm_emulator, &kernel_message); teken_input(&tm->tm_emulator, &cv, 1); teken_set_curattr(&tm->tm_emulator, &backup); tm->tm_class->tc_done(tm); } TERMINAL_UNLOCK_CONS(tm); } /* * Binding with the terminal emulator. */ static void termteken_bell(void *softc) { struct terminal *tm = softc; tm->tm_flags |= TF_BELL; } static void termteken_cursor(void *softc, const teken_pos_t *p) { struct terminal *tm = softc; tm->tm_class->tc_cursor(tm, p); } static void termteken_putchar(void *softc, const teken_pos_t *p, teken_char_t c, const teken_attr_t *a) { struct terminal *tm = softc; tm->tm_class->tc_putchar(tm, p, TCHAR_CREATE(c, a)); } static void termteken_fill(void *softc, const teken_rect_t *r, teken_char_t c, const teken_attr_t *a) { struct terminal *tm = softc; tm->tm_class->tc_fill(tm, r, TCHAR_CREATE(c, a)); } static void termteken_copy(void *softc, const teken_rect_t *r, const teken_pos_t *p) { struct terminal *tm = softc; tm->tm_class->tc_copy(tm, r, p); } static void termteken_pre_input(void *softc) { struct terminal *tm = softc; tm->tm_class->tc_pre_input(tm); } static void termteken_post_input(void *softc) { struct terminal *tm = softc; tm->tm_class->tc_post_input(tm); } static void termteken_param(void *softc, int cmd, unsigned int arg) { struct terminal *tm = softc; tm->tm_class->tc_param(tm, cmd, arg); } static void termteken_respond(void *softc, const void *buf, size_t len) { #if 0 struct terminal *tm = softc; struct tty *tp; /* * Only inject a response into the TTY if the data actually * originated from the TTY. * * XXX: This cannot be done right now. The TTY could pick up * other locks. It could also in theory cause loops, when the * TTY performs echoing of a command that generates even more * input. */ tp = tm->tm_tty; if (tp == NULL) return; ttydisc_rint_simple(tp, buf, len); ttydisc_rint_done(tp); #endif } diff --git a/sys/libkern/arc4random.c b/sys/libkern/arc4random.c index c5ab50a80015..a4bee71c0efd 100644 --- a/sys/libkern/arc4random.c +++ b/sys/libkern/arc4random.c @@ -1,253 +1,252 @@ /*- * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RANDOM_FENESTRASX #include #endif #define CHACHA20_RESEED_BYTES 65536 #define CHACHA20_RESEED_SECONDS 300 #define CHACHA20_KEYBYTES 32 #define CHACHA20_BUFFER_SIZE 64 CTASSERT(CHACHA20_KEYBYTES*8 >= CHACHA_MINKEYLEN); #ifndef RANDOM_FENESTRASX int arc4rand_iniseed_state = ARC4_ENTR_NONE; #endif MALLOC_DEFINE(M_CHACHA20RANDOM, "chacha20random", "chacha20random structures"); struct chacha20_s { struct mtx mtx; int numbytes; time_t t_reseed; u_int8_t m_buffer[CHACHA20_BUFFER_SIZE]; struct chacha_ctx ctx; #ifdef RANDOM_FENESTRASX uint64_t seed_version; #endif } __aligned(CACHE_LINE_SIZE); static struct chacha20_s *chacha20inst = NULL; #define CHACHA20_FOREACH(_chacha20) \ for (_chacha20 = &chacha20inst[0]; \ _chacha20 <= &chacha20inst[mp_maxid]; \ _chacha20++) /* * Mix up the current context. */ static void chacha20_randomstir(struct chacha20_s *chacha20) { struct timeval tv_now; u_int8_t key[CHACHA20_KEYBYTES]; #ifdef RANDOM_FENESTRASX uint64_t seed_version; #else if (__predict_false(random_bypass_before_seeding && !is_random_seeded())) { SHA256_CTX ctx; uint64_t cc; uint32_t fver; if (!arc4random_bypassed_before_seeding) { arc4random_bypassed_before_seeding = true; if (!random_bypass_disable_warnings) printf("arc4random: WARNING: initial seeding " "bypassed the cryptographic random device " "because it was not yet seeded and the " "knob 'bypass_before_seeding' was " "enabled.\n"); } /* Last ditch effort to inject something in a bad condition. */ cc = get_cyclecount(); SHA256_Init(&ctx); SHA256_Update(&ctx, key, sizeof(key)); SHA256_Update(&ctx, &cc, sizeof(cc)); fver = __FreeBSD_version; SHA256_Update(&ctx, &fver, sizeof(fver)); _Static_assert(sizeof(key) == SHA256_DIGEST_LENGTH, "make sure 256 bits is still 256 bits"); SHA256_Final(key, &ctx); } else { #endif #ifdef RANDOM_FENESTRASX read_random_key(key, CHACHA20_KEYBYTES, &seed_version); #else /* * If the loader(8) did not have an entropy stash from the * previous shutdown to load, then we will block. The answer is * to make sure there is an entropy stash at shutdown time. * * On the other hand, if the random_bypass_before_seeding knob * was set and we landed in this branch, we know this won't * block because we know the random device is seeded. */ read_random(key, CHACHA20_KEYBYTES); } #endif getmicrouptime(&tv_now); mtx_lock(&chacha20->mtx); chacha_keysetup(&chacha20->ctx, key, CHACHA20_KEYBYTES*8); chacha_ivsetup(&chacha20->ctx, (u_char *)&tv_now.tv_sec, (u_char *)&tv_now.tv_usec); /* Reset for next reseed cycle. */ chacha20->t_reseed = tv_now.tv_sec + CHACHA20_RESEED_SECONDS; chacha20->numbytes = 0; #ifdef RANDOM_FENESTRASX chacha20->seed_version = seed_version; #endif mtx_unlock(&chacha20->mtx); } /* * Initialize the contexts. */ static void chacha20_init(void) { struct chacha20_s *chacha20; chacha20inst = malloc((mp_maxid + 1) * sizeof(struct chacha20_s), M_CHACHA20RANDOM, M_NOWAIT | M_ZERO); KASSERT(chacha20inst != NULL, ("chacha20_init: memory allocation error")); CHACHA20_FOREACH(chacha20) { mtx_init(&chacha20->mtx, "chacha20_mtx", NULL, MTX_DEF); chacha20->t_reseed = -1; chacha20->numbytes = 0; explicit_bzero(chacha20->m_buffer, CHACHA20_BUFFER_SIZE); explicit_bzero(&chacha20->ctx, sizeof(chacha20->ctx)); } } SYSINIT(chacha20, SI_SUB_LOCK, SI_ORDER_ANY, chacha20_init, NULL); static void chacha20_uninit(void) { struct chacha20_s *chacha20; CHACHA20_FOREACH(chacha20) mtx_destroy(&chacha20->mtx); free(chacha20inst, M_CHACHA20RANDOM); } SYSUNINIT(chacha20, SI_SUB_LOCK, SI_ORDER_ANY, chacha20_uninit, NULL); /* * MPSAFE */ void arc4rand(void *ptr, u_int len, int reseed) { struct chacha20_s *chacha20; struct timeval tv; u_int length; u_int8_t *p; #ifdef RANDOM_FENESTRASX if (__predict_false(reseed)) #else if (__predict_false(reseed || (arc4rand_iniseed_state == ARC4_ENTR_HAVE && atomic_cmpset_int(&arc4rand_iniseed_state, ARC4_ENTR_HAVE, ARC4_ENTR_SEED)))) #endif CHACHA20_FOREACH(chacha20) chacha20_randomstir(chacha20); getmicrouptime(&tv); chacha20 = &chacha20inst[curcpu]; /* We may get unlucky and be migrated off this CPU, but that is expected to be infrequent */ if ((chacha20->numbytes > CHACHA20_RESEED_BYTES) || (tv.tv_sec > chacha20->t_reseed)) chacha20_randomstir(chacha20); mtx_lock(&chacha20->mtx); #ifdef RANDOM_FENESTRASX if (__predict_false( atomic_load_acq_64(&fxrng_root_generation) != chacha20->seed_version )) { mtx_unlock(&chacha20->mtx); chacha20_randomstir(chacha20); mtx_lock(&chacha20->mtx); } #endif p = ptr; while (len) { length = MIN(CHACHA20_BUFFER_SIZE, len); chacha_encrypt_bytes(&chacha20->ctx, chacha20->m_buffer, p, length); p += length; len -= length; chacha20->numbytes += length; if (chacha20->numbytes > CHACHA20_RESEED_BYTES) { mtx_unlock(&chacha20->mtx); chacha20_randomstir(chacha20); mtx_lock(&chacha20->mtx); } } mtx_unlock(&chacha20->mtx); } uint32_t arc4random(void) { uint32_t ret; arc4rand(&ret, sizeof(ret), 0); return ret; } void arc4random_buf(void *ptr, size_t len) { arc4rand(ptr, len, 0); } diff --git a/sys/mips/broadcom/bcm_mips.c b/sys/mips/broadcom/bcm_mips.c index a3540416dff9..2c4e8e696a8b 100644 --- a/sys/mips/broadcom/bcm_mips.c +++ b/sys/mips/broadcom/bcm_mips.c @@ -1,697 +1,696 @@ /*- * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #include "bcm_mipsvar.h" /* * Broadcom MIPS core driver. * * Abstract driver for Broadcom MIPS CPU/PIC cores. */ static uintptr_t bcm_mips_pic_xref(struct bcm_mips_softc *sc); static device_t bcm_mips_find_bhnd_parent(device_t dev); static int bcm_mips_retain_cpu_intr(struct bcm_mips_softc *sc, struct bcm_mips_irqsrc *isrc, struct resource *res); static int bcm_mips_release_cpu_intr(struct bcm_mips_softc *sc, struct bcm_mips_irqsrc *isrc, struct resource *res); static const int bcm_mips_debug = 0; #define DPRINTF(fmt, ...) do { \ if (bcm_mips_debug) \ printf("%s: " fmt, __FUNCTION__, ##__VA_ARGS__); \ } while (0) #define DENTRY(dev, fmt, ...) do { \ if (bcm_mips_debug) \ printf("%s(%s, " fmt ")\n", __FUNCTION__, \ device_get_nameunit(dev), ##__VA_ARGS__); \ } while (0) /** * Register all interrupt source definitions. */ static int bcm_mips_register_isrcs(struct bcm_mips_softc *sc) { const char *name; uintptr_t xref; int error; xref = bcm_mips_pic_xref(sc); name = device_get_nameunit(sc->dev); for (size_t ivec = 0; ivec < nitems(sc->isrcs); ivec++) { sc->isrcs[ivec].ivec = ivec; sc->isrcs[ivec].cpuirq = NULL; sc->isrcs[ivec].refs = 0; error = intr_isrc_register(&sc->isrcs[ivec].isrc, sc->dev, xref, "%s,%u", name, ivec); if (error) { for (size_t i = 0; i < ivec; i++) intr_isrc_deregister(&sc->isrcs[i].isrc); device_printf(sc->dev, "error registering IRQ %zu: " "%d\n", ivec, error); return (error); } } return (0); } /** * Initialize the given @p cpuirq state as unavailable. * * @param sc BHND MIPS driver instance state. * @param cpuirq The CPU IRQ state to be initialized. * * @retval 0 success * @retval non-zero if initializing @p cpuirq otherwise fails, a regular * unix error code will be returned. */ static int bcm_mips_init_cpuirq_unavail(struct bcm_mips_softc *sc, struct bcm_mips_cpuirq *cpuirq) { BCM_MIPS_LOCK(sc); KASSERT(cpuirq->sc == NULL, ("cpuirq already initialized")); cpuirq->sc = sc; cpuirq->mips_irq = 0; cpuirq->irq_rid = -1; cpuirq->irq_res = NULL; cpuirq->irq_cookie = NULL; cpuirq->isrc_solo = NULL; cpuirq->refs = 0; BCM_MIPS_UNLOCK(sc); return (0); } /** * Allocate required resources and initialize the given @p cpuirq state. * * @param sc BHND MIPS driver instance state. * @param cpuirq The CPU IRQ state to be initialized. * @param rid The resource ID to be assigned for the CPU IRQ resource, * or -1 if no resource should be assigned. * @param irq The MIPS HW IRQ# to be allocated. * @param filter The interrupt filter to be setup. * * @retval 0 success * @retval non-zero if initializing @p cpuirq otherwise fails, a regular * unix error code will be returned. */ static int bcm_mips_init_cpuirq(struct bcm_mips_softc *sc, struct bcm_mips_cpuirq *cpuirq, int rid, u_int irq, driver_filter_t filter) { struct resource *res; void *cookie; u_int irq_real; int error; /* Must fall within MIPS HW IRQ range */ if (irq >= NHARD_IRQS) return (EINVAL); /* HW IRQs are numbered relative to SW IRQs */ irq_real = irq + NSOFT_IRQS; /* Try to assign and allocate the resource */ BCM_MIPS_LOCK(sc); KASSERT(cpuirq->sc == NULL, ("cpuirq already initialized")); error = bus_set_resource(sc->dev, SYS_RES_IRQ, rid, irq_real, 1); if (error) { BCM_MIPS_UNLOCK(sc); device_printf(sc->dev, "failed to assign interrupt %u: " "%d\n", irq, error); return (error); } res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (res == NULL) { BCM_MIPS_UNLOCK(sc); device_printf(sc->dev, "failed to allocate interrupt " "%u resource\n", irq); bus_delete_resource(sc->dev, SYS_RES_IRQ, rid); return (ENXIO); } error = bus_setup_intr(sc->dev, res, INTR_TYPE_MISC | INTR_MPSAFE | INTR_EXCL, filter, NULL, cpuirq, &cookie); if (error) { BCM_MIPS_UNLOCK(sc); printf("failed to setup internal interrupt handler: %d\n", error); bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res); bus_delete_resource(sc->dev, SYS_RES_IRQ, rid); return (error); } /* Initialize CPU IRQ state */ cpuirq->sc = sc; cpuirq->mips_irq = irq; cpuirq->irq_rid = rid; cpuirq->irq_res = res; cpuirq->irq_cookie = cookie; cpuirq->isrc_solo = NULL; cpuirq->refs = 0; BCM_MIPS_UNLOCK(sc); return (0); } /** * Free any resources associated with the given @p cpuirq state. * * @param sc BHND MIPS driver instance state. * @param cpuirq A CPU IRQ instance previously successfully initialized * via bcm_mips_init_cpuirq(). * * @retval 0 success * @retval non-zero if finalizing @p cpuirq otherwise fails, a regular * unix error code will be returned. */ static int bcm_mips_fini_cpuirq(struct bcm_mips_softc *sc, struct bcm_mips_cpuirq *cpuirq) { int error; BCM_MIPS_LOCK(sc); if (cpuirq->sc == NULL) { KASSERT(cpuirq->irq_res == NULL, ("leaking cpuirq resource")); BCM_MIPS_UNLOCK(sc); return (0); /* not initialized */ } if (cpuirq->refs != 0) { BCM_MIPS_UNLOCK(sc); return (EBUSY); } if (cpuirq->irq_cookie != NULL) { KASSERT(cpuirq->irq_res != NULL, ("resource missing")); error = bus_teardown_intr(sc->dev, cpuirq->irq_res, cpuirq->irq_cookie); if (error) { BCM_MIPS_UNLOCK(sc); return (error); } cpuirq->irq_cookie = NULL; } if (cpuirq->irq_res != NULL) { bus_release_resource(sc->dev, SYS_RES_IRQ, cpuirq->irq_rid, cpuirq->irq_res); cpuirq->irq_res = NULL; } if (cpuirq->irq_rid != -1) { bus_delete_resource(sc->dev, SYS_RES_IRQ, cpuirq->irq_rid); cpuirq->irq_rid = -1; } BCM_MIPS_UNLOCK(sc); return (0); } static int bcm_mips_attach_default(device_t dev) { /* subclassing drivers must provide an implementation of * DEVICE_ATTACH() */ panic("device_attach() unimplemented"); } /** * BHND MIPS device attach. * * This must be called from subclass drivers' DEVICE_ATTACH(). * * @param dev BHND MIPS device. * @param num_cpuirqs The number of usable MIPS HW IRQs. * @param timer_irq The MIPS HW IRQ assigned to the MIPS CPU timer. * @param filter The subclass's core-specific IRQ dispatch filter. Will be * passed the associated bcm_mips_cpuirq instance as its argument. */ int bcm_mips_attach(device_t dev, u_int num_cpuirqs, u_int timer_irq, driver_filter_t filter) { struct bcm_mips_softc *sc; struct intr_pic *pic; uintptr_t xref; u_int irq_rid; rman_res_t irq; int error; sc = device_get_softc(dev); sc->dev = dev; sc->num_cpuirqs = num_cpuirqs; sc->timer_irq = timer_irq; /* Must not exceed the actual size of our fixed IRQ array */ if (sc->num_cpuirqs > nitems(sc->cpuirqs)) { device_printf(dev, "%u nirqs exceeds maximum supported %zu", sc->num_cpuirqs, nitems(sc->cpuirqs)); return (ENXIO); } pic = NULL; xref = bcm_mips_pic_xref(sc); BCM_MIPS_LOCK_INIT(sc); /* Register our interrupt sources */ if ((error = bcm_mips_register_isrcs(sc))) { BCM_MIPS_LOCK_DESTROY(sc); return (error); } /* Initialize our CPU interrupt state */ irq_rid = bhnd_get_intr_count(dev); /* last bhnd-assigned RID + 1 */ irq = 0; for (u_int i = 0; i < sc->num_cpuirqs; i++) { /* Must not overflow signed resource ID representation */ if (irq_rid >= INT_MAX) { device_printf(dev, "exhausted IRQ resource IDs\n"); error = ENOMEM; goto failed; } if (irq == sc->timer_irq) { /* Mark the CPU timer's IRQ as unavailable */ error = bcm_mips_init_cpuirq_unavail(sc, &sc->cpuirqs[i]); } else { /* Initialize state */ error = bcm_mips_init_cpuirq(sc, &sc->cpuirqs[i], irq_rid, irq, filter); } if (error) goto failed; /* Increment IRQ and resource ID for next allocation */ irq_rid++; irq++; } /* Sanity check; our shared IRQ must be available */ if (sc->num_cpuirqs <= BCM_MIPS_IRQ_SHARED) panic("missing shared interrupt %d\n", BCM_MIPS_IRQ_SHARED); if (sc->cpuirqs[BCM_MIPS_IRQ_SHARED].irq_rid == -1) panic("shared interrupt %d unavailable", BCM_MIPS_IRQ_SHARED); /* Register PIC */ if ((pic = intr_pic_register(dev, xref)) == NULL) { device_printf(dev, "error registering PIC\n"); error = ENXIO; goto failed; } return (0); failed: /* Deregister PIC before performing any other cleanup */ if (pic != NULL) intr_pic_deregister(dev, 0); /* Deregister all interrupt sources */ for (size_t i = 0; i < nitems(sc->isrcs); i++) intr_isrc_deregister(&sc->isrcs[i].isrc); /* Free our MIPS CPU interrupt handler state */ for (u_int i = 0; i < sc->num_cpuirqs; i++) bcm_mips_fini_cpuirq(sc, &sc->cpuirqs[i]); BCM_MIPS_LOCK_DESTROY(sc); return (error); } int bcm_mips_detach(device_t dev) { struct bcm_mips_softc *sc; sc = device_get_softc(dev); /* Deregister PIC before performing any other cleanup */ intr_pic_deregister(dev, 0); /* Deregister all interrupt sources */ for (size_t i = 0; i < nitems(sc->isrcs); i++) intr_isrc_deregister(&sc->isrcs[i].isrc); /* Free our MIPS CPU interrupt handler state */ for (u_int i = 0; i < sc->num_cpuirqs; i++) bcm_mips_fini_cpuirq(sc, &sc->cpuirqs[i]); return (0); } /* PIC_MAP_INTR() */ static int bcm_mips_pic_map_intr(device_t dev, struct intr_map_data *d, struct intr_irqsrc **isrcp) { struct bcm_mips_softc *sc; struct bcm_mips_intr_map_data *data; sc = device_get_softc(dev); if (d->type != INTR_MAP_DATA_BCM_MIPS) { DENTRY(dev, "type=%d", d->type); return (ENOTSUP); } data = (struct bcm_mips_intr_map_data *)d; DENTRY(dev, "type=%d, ivec=%u", d->type, data->ivec); if (data->ivec < 0 || data->ivec >= nitems(sc->isrcs)) return (EINVAL); *isrcp = &sc->isrcs[data->ivec].isrc; return (0); } /* PIC_SETUP_INTR() */ static int bcm_mips_pic_setup_intr(device_t dev, struct intr_irqsrc *irqsrc, struct resource *res, struct intr_map_data *data) { struct bcm_mips_softc *sc; struct bcm_mips_irqsrc *isrc; int error; sc = device_get_softc(dev); isrc = (struct bcm_mips_irqsrc *)irqsrc; /* Assign a CPU interrupt */ BCM_MIPS_LOCK(sc); error = bcm_mips_retain_cpu_intr(sc, isrc, res); BCM_MIPS_UNLOCK(sc); return (error); } /* PIC_TEARDOWN_INTR() */ static int bcm_mips_pic_teardown_intr(device_t dev, struct intr_irqsrc *irqsrc, struct resource *res, struct intr_map_data *data) { struct bcm_mips_softc *sc; struct bcm_mips_irqsrc *isrc; int error; sc = device_get_softc(dev); isrc = (struct bcm_mips_irqsrc *)irqsrc; /* Release the CPU interrupt */ BCM_MIPS_LOCK(sc); error = bcm_mips_release_cpu_intr(sc, isrc, res); BCM_MIPS_UNLOCK(sc); return (error); } /** return our PIC's xref */ static uintptr_t bcm_mips_pic_xref(struct bcm_mips_softc *sc) { uintptr_t xref; /* Determine our interrupt domain */ xref = BHND_BUS_GET_INTR_DOMAIN(device_get_parent(sc->dev), sc->dev, true); KASSERT(xref != 0, ("missing interrupt domain")); return (xref); } /** * Walk up the device tree from @p dev until we find a bhnd-attached core, * returning either the core, or NULL if @p dev is not attached under a bhnd * bus. */ static device_t bcm_mips_find_bhnd_parent(device_t dev) { device_t core, bus; devclass_t bhnd_class; bhnd_class = devclass_find("bhnd"); core = dev; while ((bus = device_get_parent(core)) != NULL) { if (device_get_devclass(bus) == bhnd_class) return (core); core = bus; } /* Not found */ return (NULL); } /** * Retain @p isrc and assign a MIPS CPU interrupt on behalf of @p res; if * the @p isrc already has a MIPS CPU interrupt assigned, the existing * reference will be left unmodified. * * @param sc BHND MIPS driver state. * @param isrc The interrupt source corresponding to @p res. * @param res The interrupt resource for which a MIPS CPU IRQ will be * assigned. */ static int bcm_mips_retain_cpu_intr(struct bcm_mips_softc *sc, struct bcm_mips_irqsrc *isrc, struct resource *res) { struct bcm_mips_cpuirq *cpuirq; bhnd_devclass_t devclass; device_t core; BCM_MIPS_LOCK_ASSERT(sc, MA_OWNED); /* Prefer existing assignment */ if (isrc->cpuirq != NULL) { KASSERT(isrc->cpuirq->refs > 0, ("assigned IRQ has no " "references")); /* Increment our reference count */ if (isrc->refs == UINT_MAX) return (ENOMEM); /* would overflow */ isrc->refs++; return (0); } /* Use the device class of the bhnd core to which the interrupt * vector is routed to determine whether a shared interrupt should * be preferred. */ devclass = BHND_DEVCLASS_OTHER; core = bcm_mips_find_bhnd_parent(rman_get_device(res)); if (core != NULL) devclass = bhnd_get_class(core); switch (devclass) { case BHND_DEVCLASS_CC: case BHND_DEVCLASS_CC_B: case BHND_DEVCLASS_PMU: case BHND_DEVCLASS_RAM: case BHND_DEVCLASS_MEMC: case BHND_DEVCLASS_CPU: case BHND_DEVCLASS_SOC_ROUTER: case BHND_DEVCLASS_SOC_BRIDGE: case BHND_DEVCLASS_EROM: case BHND_DEVCLASS_NVRAM: /* Always use a shared interrupt for these devices */ cpuirq = &sc->cpuirqs[BCM_MIPS_IRQ_SHARED]; break; case BHND_DEVCLASS_PCI: case BHND_DEVCLASS_PCIE: case BHND_DEVCLASS_PCCARD: case BHND_DEVCLASS_ENET: case BHND_DEVCLASS_ENET_MAC: case BHND_DEVCLASS_ENET_PHY: case BHND_DEVCLASS_WLAN: case BHND_DEVCLASS_WLAN_MAC: case BHND_DEVCLASS_WLAN_PHY: case BHND_DEVCLASS_USB_HOST: case BHND_DEVCLASS_USB_DEV: case BHND_DEVCLASS_USB_DUAL: case BHND_DEVCLASS_OTHER: case BHND_DEVCLASS_INVALID: default: /* Fall back on a shared interrupt */ cpuirq = &sc->cpuirqs[BCM_MIPS_IRQ_SHARED]; /* Try to assign a dedicated MIPS HW interrupt */ for (u_int i = 0; i < sc->num_cpuirqs; i++) { if (i == BCM_MIPS_IRQ_SHARED) continue; if (sc->cpuirqs[i].irq_rid == -1) continue; /* unavailable */ if (sc->cpuirqs[i].refs != 0) continue; /* already assigned */ /* Found an unused CPU IRQ */ cpuirq = &sc->cpuirqs[i]; break; } break; } DPRINTF("routing backplane interrupt vector %u to MIPS IRQ %u\n", isrc->ivec, cpuirq->mips_irq); KASSERT(isrc->cpuirq == NULL, ("CPU IRQ already assigned")); KASSERT(isrc->refs == 0, ("isrc has active references with no " "assigned CPU IRQ")); KASSERT(cpuirq->refs == 1 || cpuirq->isrc_solo == NULL, ("single isrc dispatch enabled on cpuirq with multiple refs")); /* Verify that bumping the cpuirq refcount below will not overflow */ if (cpuirq->refs == UINT_MAX) return (ENOMEM); /* Increment cpuirq refcount on behalf of the isrc */ cpuirq->refs++; /* Increment isrc refcount on behalf of the caller */ isrc->refs++; /* Assign the IRQ to the isrc */ isrc->cpuirq = cpuirq; /* Can we enable the single isrc dispatch path? */ if (cpuirq->refs == 1 && cpuirq->mips_irq != BCM_MIPS_IRQ_SHARED) cpuirq->isrc_solo = isrc; return (0); } /** * Release the MIPS CPU interrupt assigned to @p isrc on behalf of @p res. * * @param sc BHND MIPS driver state. * @param isrc The interrupt source corresponding to @p res. * @param res The interrupt resource being activated. */ static int bcm_mips_release_cpu_intr(struct bcm_mips_softc *sc, struct bcm_mips_irqsrc *isrc, struct resource *res) { struct bcm_mips_cpuirq *cpuirq; BCM_MIPS_LOCK_ASSERT(sc, MA_OWNED); /* Decrement the refcount */ KASSERT(isrc->refs > 0, ("isrc over-release")); isrc->refs--; /* Nothing else to do if the isrc is still actively referenced */ if (isrc->refs > 0) return (0); /* Otherwise, we need to release our CPU IRQ reference */ cpuirq = isrc->cpuirq; isrc->cpuirq = NULL; KASSERT(cpuirq != NULL, ("no assigned IRQ")); KASSERT(cpuirq->refs > 0, ("cpuirq over-release")); /* Disable single isrc dispatch path */ if (cpuirq->refs == 1 && cpuirq->isrc_solo != NULL) { KASSERT(cpuirq->isrc_solo == isrc, ("invalid solo isrc")); cpuirq->isrc_solo = NULL; } cpuirq->refs--; return (0); } static device_method_t bcm_mips_methods[] = { /* Device interface */ DEVMETHOD(device_attach, bcm_mips_attach_default), DEVMETHOD(device_detach, bcm_mips_detach), /* Interrupt controller interface */ DEVMETHOD(pic_map_intr, bcm_mips_pic_map_intr), DEVMETHOD(pic_setup_intr, bcm_mips_pic_setup_intr), DEVMETHOD(pic_teardown_intr, bcm_mips_pic_teardown_intr), DEVMETHOD_END }; DEFINE_CLASS_0(bcm_mips, bcm_mips_driver, bcm_mips_methods, sizeof(struct bcm_mips_softc)); MODULE_VERSION(bcm_mips, 1); MODULE_DEPEND(bcm_mips, bhnd, 1, 1, 1); diff --git a/sys/mips/broadcom/bcm_mipsvar.h b/sys/mips/broadcom/bcm_mipsvar.h index b2c10026895c..bb530ae6c3ee 100644 --- a/sys/mips/broadcom/bcm_mipsvar.h +++ b/sys/mips/broadcom/bcm_mipsvar.h @@ -1,112 +1,111 @@ /*- * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Landon Fuller under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ #ifndef _MIPS_BROADCOM_BCM_MIPSVAR_H_ #define _MIPS_BROADCOM_BCM_MIPSVAR_H_ #include #include #include #include #include DECLARE_CLASS(bcm_mips_driver); struct bcm_mips_irqsrc; struct bcm_mips_softc; #define BCM_MIPS_NINTR 32 /**< maximum number of addressable backplane interrupt vectors */ #define BCM_MIPS_IRQ_SHARED 0 /**< MIPS CPU IRQ reserved for shared interrupt handling */ #define INTR_MAP_DATA_BCM_MIPS INTR_MAP_DATA_PLAT_2 /**< Broadcom MIPS PIC interrupt map data type */ int bcm_mips_attach(device_t dev, u_int num_cpuirqs, u_int timer_irq, driver_filter_t filter); int bcm_mips_detach(device_t dev); /** * Broadcom MIPS PIC interrupt map data. */ struct bcm_mips_intr_map_data { struct intr_map_data mdata; u_int ivec; /**< bus interrupt vector */ }; /** * Nested MIPS CPU interrupt handler state. */ struct bcm_mips_cpuirq { struct bcm_mips_softc *sc; /**< driver instance state, or NULL if uninitialized. */ u_int mips_irq; /**< mips hardware interrupt number (relative to NSOFT_IRQ) */ int irq_rid; /**< mips IRQ resource id, or -1 if this entry is unavailable */ struct resource *irq_res; /**< mips interrupt resource */ void *irq_cookie; /**< mips interrupt handler cookie, or NULL */ struct bcm_mips_irqsrc *isrc_solo; /**< solo isrc assigned to this interrupt, or NULL */ u_int refs; /**< isrc consumer refcount */ }; /** * Broadcom MIPS PIC interrupt source definition. */ struct bcm_mips_irqsrc { struct intr_irqsrc isrc; u_int ivec; /**< bus interrupt vector */ u_int refs; /**< active reference count */ struct bcm_mips_cpuirq *cpuirq; /**< assigned MIPS HW IRQ, or NULL if no assignment */ }; /** * bcm_mips driver instance state. Must be first member of all subclass * softc structures. */ struct bcm_mips_softc { device_t dev; struct bcm_mips_cpuirq cpuirqs[NREAL_IRQS]; /**< nested CPU IRQ handlers */ u_int num_cpuirqs; /**< number of nested CPU IRQ handlers */ u_int timer_irq; /**< CPU timer IRQ */ struct bcm_mips_irqsrc isrcs[BCM_MIPS_NINTR]; struct mtx mtx; }; #define BCM_MIPS_IVEC_MASK(_isrc) (1 << ((_isrc)->ivec)) #define BCM_MIPS_LOCK_INIT(sc) \ mtx_init(&(sc)->mtx, device_get_nameunit((sc)->dev), \ "bhnd mips driver lock", MTX_DEF) #define BCM_MIPS_LOCK(sc) mtx_lock(&(sc)->mtx) #define BCM_MIPS_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define BCM_MIPS_LOCK_ASSERT(sc, what) mtx_assert(&(sc)->mtx, what) #define BCM_MIPS_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx) #endif /* _MIPS_BROADCOM_BCM_MIPSVAR_H_ */ diff --git a/sys/mips/include/fdt.h b/sys/mips/include/fdt.h index ef92de89d849..8da2a0c392f3 100644 --- a/sys/mips/include/fdt.h +++ b/sys/mips/include/fdt.h @@ -1,48 +1,47 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2010 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_FDT_H_ #define _MACHINE_FDT_H_ #include /* * Bus space tag. XXX endianess info needs to be derived from the blob. */ #if defined(CPU_RMI) || defined(CPU_NLM) #define fdtbus_bs_tag rmi_uart_bus_space #else #define fdtbus_bs_tag mips_bus_space_generic #endif #endif /* _MACHINE_FDT_H_ */ diff --git a/sys/net/ieee_oui.h b/sys/net/ieee_oui.h index 068328d83345..166021d3aa09 100644 --- a/sys/net/ieee_oui.h +++ b/sys/net/ieee_oui.h @@ -1,85 +1,84 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials provided * with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * * Author: George V. Neville-Neil * */ /* Organizationally Unique Identifier assigned by IEEE 14 Nov 2013 */ #define OUI_FREEBSD_BASE 0x589cfc000000 #define OUI_FREEBSD(nic) (OUI_FREEBSD_BASE | (nic)) /* * OUIs are most often used to uniquely identify network interfaces * and occupy the first 3 bytes of both destination and source MAC * addresses. The following allocations exist so that various * software systems associated with FreeBSD can have unique IDs in the * absence of hardware. The use of OUIs for this purpose is not fully * fleshed out but is now in common use in virtualization technology. * * Allocations from this range are expected to be made using COMMON * SENSE by developers. Do NOT take a large range just because * they're currently wide open. Take the smallest useful range for * your system. We have (2^24 - 2) available addresses (see Reserved * Values below) but that is far from infinite. * * In the event of a conflict arbitration of allocation in this file * is subject to core@ approval. * * Applications are differentiated based on the high order bit(s) of * the remaining three bytes. Our first allocation has all 0s, the * next allocation has the highest bit set. Allocating in this way * gives us 254 allocations of 64K addresses. Address blocks can be * concatenated if necessary. * * Reserved Values: 0x000000 and 0xffffff are reserved and MUST NOT BE * allocated for any reason. */ /* Allocate 20 bits to bhyve */ #define OUI_FREEBSD_BHYVE_LOW OUI_FREEBSD(0x000001) #define OUI_FREEBSD_BHYVE_HIGH OUI_FREEBSD(0x0fffff) /* * Allocate 16 bits for a pool to give to various interfaces that need a * generated address, but don't quite need to slice off a whole section of * the OUI (e.g. cloned interfaces, one-off NICs of various vendors). * * ether_gen_addr should be used to generate an address from this pool. */ #define OUI_FREEBSD_GENERATED_MASK 0x10ffff #define OUI_FREEBSD_GENERATED_LOW OUI_FREEBSD(0x100000) #define OUI_FREEBSD_GENERATED_HIGH OUI_FREEBSD(OUI_FREEBSD_GENERATED_MASK) /* Allocate 16 bits for emulated NVMe devices */ #define OUI_FREEBSD_NVME_MASK 0x20ffff #define OUI_FREEBSD_NVME_LOW OUI_FREEBSD(0x200000) #define OUI_FREEBSD_NVME_HIGH OUI_FREEBSD(OUI_FREEBSD_NVME_MASK) diff --git a/sys/net80211/ieee80211_hwmp.c b/sys/net80211/ieee80211_hwmp.c index 4fbcc9051c47..37541b1c51f2 100644 --- a/sys/net80211/ieee80211_hwmp.c +++ b/sys/net80211/ieee80211_hwmp.c @@ -1,2090 +1,2089 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Rui Paulo under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifdef __FreeBSD__ __FBSDID("$FreeBSD$"); #endif /* * IEEE 802.11s Hybrid Wireless Mesh Protocol, HWMP. * * Based on March 2009, D3.0 802.11s draft spec. */ #include "opt_inet.h" #include "opt_wlan.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static void hwmp_vattach(struct ieee80211vap *); static void hwmp_vdetach(struct ieee80211vap *); static int hwmp_newstate(struct ieee80211vap *, enum ieee80211_state, int); static int hwmp_send_action(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], uint8_t *, size_t); static uint8_t * hwmp_add_meshpreq(uint8_t *, const struct ieee80211_meshpreq_ie *); static uint8_t * hwmp_add_meshprep(uint8_t *, const struct ieee80211_meshprep_ie *); static uint8_t * hwmp_add_meshperr(uint8_t *, const struct ieee80211_meshperr_ie *); static uint8_t * hwmp_add_meshrann(uint8_t *, const struct ieee80211_meshrann_ie *); static void hwmp_rootmode_setup(struct ieee80211vap *); static void hwmp_rootmode_cb(void *); static void hwmp_rootmode_rann_cb(void *); static void hwmp_recv_preq(struct ieee80211vap *, struct ieee80211_node *, const struct ieee80211_frame *, const struct ieee80211_meshpreq_ie *); static int hwmp_send_preq(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_meshpreq_ie *, struct timeval *, struct timeval *); static void hwmp_recv_prep(struct ieee80211vap *, struct ieee80211_node *, const struct ieee80211_frame *, const struct ieee80211_meshprep_ie *); static int hwmp_send_prep(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_meshprep_ie *); static void hwmp_recv_perr(struct ieee80211vap *, struct ieee80211_node *, const struct ieee80211_frame *, const struct ieee80211_meshperr_ie *); static int hwmp_send_perr(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_meshperr_ie *); static void hwmp_senderror(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_mesh_route *, int); static void hwmp_recv_rann(struct ieee80211vap *, struct ieee80211_node *, const struct ieee80211_frame *, const struct ieee80211_meshrann_ie *); static int hwmp_send_rann(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_meshrann_ie *); static struct ieee80211_node * hwmp_discover(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct mbuf *); static void hwmp_peerdown(struct ieee80211_node *); static struct timeval ieee80211_hwmp_preqminint = { 0, 100000 }; static struct timeval ieee80211_hwmp_perrminint = { 0, 100000 }; /* NB: the Target Address set in a Proactive PREQ is the broadcast address. */ static const uint8_t broadcastaddr[IEEE80211_ADDR_LEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; typedef uint32_t ieee80211_hwmp_seq; #define HWMP_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0) #define HWMP_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0) #define HWMP_SEQ_EQ(a, b) ((int32_t)((a)-(b)) == 0) #define HWMP_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0) #define HWMP_SEQ_MAX(a, b) (a > b ? a : b) /* * Private extension of ieee80211_mesh_route. */ struct ieee80211_hwmp_route { ieee80211_hwmp_seq hr_seq; /* last HWMP seq seen from dst*/ ieee80211_hwmp_seq hr_preqid; /* last PREQ ID seen from dst */ ieee80211_hwmp_seq hr_origseq; /* seq. no. on our latest PREQ*/ struct timeval hr_lastpreq; /* last time we sent a PREQ */ struct timeval hr_lastrootconf; /* last sent PREQ root conf */ int hr_preqretries; /* number of discoveries */ int hr_lastdiscovery; /* last discovery in ticks */ }; struct ieee80211_hwmp_state { ieee80211_hwmp_seq hs_seq; /* next seq to be used */ ieee80211_hwmp_seq hs_preqid; /* next PREQ ID to be used */ int hs_rootmode; /* proactive HWMP */ struct timeval hs_lastperr; /* last time we sent a PERR */ struct callout hs_roottimer; uint8_t hs_maxhops; /* max hop count */ }; static SYSCTL_NODE(_net_wlan, OID_AUTO, hwmp, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "IEEE 802.11s HWMP parameters"); static int ieee80211_hwmp_targetonly = 0; SYSCTL_INT(_net_wlan_hwmp, OID_AUTO, targetonly, CTLFLAG_RW, &ieee80211_hwmp_targetonly, 0, "Set TO bit on generated PREQs"); static int ieee80211_hwmp_pathtimeout = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, pathlifetime, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_pathtimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "path entry lifetime (ms)"); static int ieee80211_hwmp_maxpreq_retries = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, maxpreq_retries, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_maxpreq_retries, 0, ieee80211_sysctl_msecs_ticks, "I", "maximum number of preq retries"); static int ieee80211_hwmp_net_diameter_traversaltime = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, net_diameter_traversal_time, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_net_diameter_traversaltime, 0, ieee80211_sysctl_msecs_ticks, "I", "estimate travelse time across the MBSS (ms)"); static int ieee80211_hwmp_roottimeout = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, roottimeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_roottimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "root PREQ timeout (ms)"); static int ieee80211_hwmp_rootint = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rootint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_rootint, 0, ieee80211_sysctl_msecs_ticks, "I", "root interval (ms)"); static int ieee80211_hwmp_rannint = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rannint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &ieee80211_hwmp_rannint, 0, ieee80211_sysctl_msecs_ticks, "I", "root announcement interval (ms)"); static struct timeval ieee80211_hwmp_rootconfint = { 0, 0 }; static int ieee80211_hwmp_rootconfint_internal = -1; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, rootconfint, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, &ieee80211_hwmp_rootconfint_internal, 0, ieee80211_sysctl_msecs_ticks, "I", "root confirmation interval (ms) (read-only)"); #define IEEE80211_HWMP_DEFAULT_MAXHOPS 31 static ieee80211_recv_action_func hwmp_recv_action_meshpath; static struct ieee80211_mesh_proto_path mesh_proto_hwmp = { .mpp_descr = "HWMP", .mpp_ie = IEEE80211_MESHCONF_PATH_HWMP, .mpp_discover = hwmp_discover, .mpp_peerdown = hwmp_peerdown, .mpp_senderror = hwmp_senderror, .mpp_vattach = hwmp_vattach, .mpp_vdetach = hwmp_vdetach, .mpp_newstate = hwmp_newstate, .mpp_privlen = sizeof(struct ieee80211_hwmp_route), }; SYSCTL_PROC(_net_wlan_hwmp, OID_AUTO, inact, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &mesh_proto_hwmp.mpp_inact, 0, ieee80211_sysctl_msecs_ticks, "I", "mesh route inactivity timeout (ms)"); static void ieee80211_hwmp_init(void) { /* Default values as per amendment */ ieee80211_hwmp_pathtimeout = msecs_to_ticks(5*1000); ieee80211_hwmp_roottimeout = msecs_to_ticks(5*1000); ieee80211_hwmp_rootint = msecs_to_ticks(2*1000); ieee80211_hwmp_rannint = msecs_to_ticks(1*1000); ieee80211_hwmp_rootconfint_internal = msecs_to_ticks(2*1000); ieee80211_hwmp_maxpreq_retries = 3; /* * (TU): A measurement of time equal to 1024 μs, * 500 TU is 512 ms. */ ieee80211_hwmp_net_diameter_traversaltime = msecs_to_ticks(512); /* * NB: I dont know how to make SYSCTL_PROC that calls ms to ticks * and return a struct timeval... */ ieee80211_hwmp_rootconfint.tv_usec = ieee80211_hwmp_rootconfint_internal * 1000; /* * Register action frame handler. */ ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_HWMP, hwmp_recv_action_meshpath); /* NB: default is 5 secs per spec */ mesh_proto_hwmp.mpp_inact = msecs_to_ticks(5*1000); /* * Register HWMP. */ ieee80211_mesh_register_proto_path(&mesh_proto_hwmp); } SYSINIT(wlan_hwmp, SI_SUB_DRIVERS, SI_ORDER_SECOND, ieee80211_hwmp_init, NULL); static void hwmp_vattach(struct ieee80211vap *vap) { struct ieee80211_hwmp_state *hs; KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a mesh vap, opmode %d", vap->iv_opmode)); hs = IEEE80211_MALLOC(sizeof(struct ieee80211_hwmp_state), M_80211_VAP, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); if (hs == NULL) { printf("%s: couldn't alloc HWMP state\n", __func__); return; } hs->hs_maxhops = IEEE80211_HWMP_DEFAULT_MAXHOPS; callout_init(&hs->hs_roottimer, 1); vap->iv_hwmp = hs; } static void hwmp_vdetach(struct ieee80211vap *vap) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; callout_drain(&hs->hs_roottimer); IEEE80211_FREE(vap->iv_hwmp, M_80211_VAP); vap->iv_hwmp = NULL; } static int hwmp_newstate(struct ieee80211vap *vap, enum ieee80211_state ostate, int arg) { enum ieee80211_state nstate = vap->iv_state; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE, "%s: %s -> %s (%d)\n", __func__, ieee80211_state_name[ostate], ieee80211_state_name[nstate], arg); if (nstate != IEEE80211_S_RUN && ostate == IEEE80211_S_RUN) callout_drain(&hs->hs_roottimer); if (nstate == IEEE80211_S_RUN) hwmp_rootmode_setup(vap); return 0; } /* * Verify the length of an HWMP PREQ and return the number * of destinations >= 1, if verification fails -1 is returned. */ static int verify_mesh_preq_len(struct ieee80211vap *vap, const struct ieee80211_frame *wh, const uint8_t *iefrm) { int alloc_sz = -1; int ndest = -1; if (iefrm[2] & IEEE80211_MESHPREQ_FLAGS_AE) { /* Originator External Address present */ alloc_sz = IEEE80211_MESHPREQ_BASE_SZ_AE; ndest = iefrm[IEEE80211_MESHPREQ_TCNT_OFFSET_AE]; } else { /* w/o Originator External Address */ alloc_sz = IEEE80211_MESHPREQ_BASE_SZ; ndest = iefrm[IEEE80211_MESHPREQ_TCNT_OFFSET]; } alloc_sz += ndest * IEEE80211_MESHPREQ_TRGT_SZ; if(iefrm[1] != (alloc_sz)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP, wh, NULL, "PREQ (AE=%s) with wrong len", iefrm[2] & IEEE80211_MESHPREQ_FLAGS_AE ? "1" : "0"); return (-1); } return ndest; } /* * Verify the length of an HWMP PREP and returns 1 on success, * otherwise -1. */ static int verify_mesh_prep_len(struct ieee80211vap *vap, const struct ieee80211_frame *wh, const uint8_t *iefrm) { int alloc_sz = -1; if (iefrm[2] & IEEE80211_MESHPREP_FLAGS_AE) { if (iefrm[1] == IEEE80211_MESHPREP_BASE_SZ_AE) alloc_sz = IEEE80211_MESHPREP_BASE_SZ_AE; } else if (iefrm[1] == IEEE80211_MESHPREP_BASE_SZ) alloc_sz = IEEE80211_MESHPREP_BASE_SZ; if(alloc_sz < 0) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP, wh, NULL, "PREP (AE=%s) with wrong len", iefrm[2] & IEEE80211_MESHPREP_FLAGS_AE ? "1" : "0"); return (-1); } return (1); } /* * Verify the length of an HWMP PERR and return the number * of destinations >= 1, if verification fails -1 is returned. */ static int verify_mesh_perr_len(struct ieee80211vap *vap, const struct ieee80211_frame *wh, const uint8_t *iefrm) { int alloc_sz = -1; const uint8_t *iefrm_t = iefrm; uint8_t ndest = iefrm_t[IEEE80211_MESHPERR_NDEST_OFFSET]; int i; if(ndest > IEEE80211_MESHPERR_MAXDEST) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP, wh, NULL, "PERR with wrong number of destionat (>19), %u", ndest); return (-1); } iefrm_t += IEEE80211_MESHPERR_NDEST_OFFSET + 1; /* flag is next field */ /* We need to check each destination flag to know size */ for(i = 0; ini_vap; struct ieee80211_meshpreq_ie *preq; struct ieee80211_meshprep_ie *prep; struct ieee80211_meshperr_ie *perr; struct ieee80211_meshrann_ie rann; const uint8_t *iefrm = frm + 2; /* action + code */ const uint8_t *iefrm_t = iefrm; /* temporary pointer */ int ndest = -1; int found = 0; while (efrm - iefrm > 1) { IEEE80211_VERIFY_LENGTH(efrm - iefrm, iefrm[1] + 2, return 0); switch (*iefrm) { case IEEE80211_ELEMID_MESHPREQ: { int i = 0; iefrm_t = iefrm; ndest = verify_mesh_preq_len(vap, wh, iefrm_t); if (ndest < 0) { vap->iv_stats.is_rx_mgtdiscard++; break; } preq = IEEE80211_MALLOC(sizeof(*preq) + (ndest - 1) * sizeof(*preq->preq_targets), M_80211_MESH_PREQ, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); KASSERT(preq != NULL, ("preq == NULL")); preq->preq_ie = *iefrm_t++; preq->preq_len = *iefrm_t++; preq->preq_flags = *iefrm_t++; preq->preq_hopcount = *iefrm_t++; preq->preq_ttl = *iefrm_t++; preq->preq_id = le32dec(iefrm_t); iefrm_t += 4; IEEE80211_ADDR_COPY(preq->preq_origaddr, iefrm_t); iefrm_t += 6; preq->preq_origseq = le32dec(iefrm_t); iefrm_t += 4; /* NB: may have Originator Proxied Address */ if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) { IEEE80211_ADDR_COPY( preq->preq_orig_ext_addr, iefrm_t); iefrm_t += 6; } preq->preq_lifetime = le32dec(iefrm_t); iefrm_t += 4; preq->preq_metric = le32dec(iefrm_t); iefrm_t += 4; preq->preq_tcount = *iefrm_t++; for (i = 0; i < preq->preq_tcount; i++) { preq->preq_targets[i].target_flags = *iefrm_t++; IEEE80211_ADDR_COPY( preq->preq_targets[i].target_addr, iefrm_t); iefrm_t += 6; preq->preq_targets[i].target_seq = le32dec(iefrm_t); iefrm_t += 4; } hwmp_recv_preq(vap, ni, wh, preq); IEEE80211_FREE(preq, M_80211_MESH_PREQ); found++; break; } case IEEE80211_ELEMID_MESHPREP: { iefrm_t = iefrm; ndest = verify_mesh_prep_len(vap, wh, iefrm_t); if (ndest < 0) { vap->iv_stats.is_rx_mgtdiscard++; break; } prep = IEEE80211_MALLOC(sizeof(*prep), M_80211_MESH_PREP, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); KASSERT(prep != NULL, ("prep == NULL")); prep->prep_ie = *iefrm_t++; prep->prep_len = *iefrm_t++; prep->prep_flags = *iefrm_t++; prep->prep_hopcount = *iefrm_t++; prep->prep_ttl = *iefrm_t++; IEEE80211_ADDR_COPY(prep->prep_targetaddr, iefrm_t); iefrm_t += 6; prep->prep_targetseq = le32dec(iefrm_t); iefrm_t += 4; /* NB: May have Target Proxied Address */ if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) { IEEE80211_ADDR_COPY( prep->prep_target_ext_addr, iefrm_t); iefrm_t += 6; } prep->prep_lifetime = le32dec(iefrm_t); iefrm_t += 4; prep->prep_metric = le32dec(iefrm_t); iefrm_t += 4; IEEE80211_ADDR_COPY(prep->prep_origaddr, iefrm_t); iefrm_t += 6; prep->prep_origseq = le32dec(iefrm_t); iefrm_t += 4; hwmp_recv_prep(vap, ni, wh, prep); IEEE80211_FREE(prep, M_80211_MESH_PREP); found++; break; } case IEEE80211_ELEMID_MESHPERR: { int i = 0; iefrm_t = iefrm; ndest = verify_mesh_perr_len(vap, wh, iefrm_t); if (ndest < 0) { vap->iv_stats.is_rx_mgtdiscard++; break; } perr = IEEE80211_MALLOC(sizeof(*perr) + (ndest - 1) * sizeof(*perr->perr_dests), M_80211_MESH_PERR, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); KASSERT(perr != NULL, ("perr == NULL")); perr->perr_ie = *iefrm_t++; perr->perr_len = *iefrm_t++; perr->perr_ttl = *iefrm_t++; perr->perr_ndests = *iefrm_t++; for (i = 0; iperr_ndests; i++) { perr->perr_dests[i].dest_flags = *iefrm_t++; IEEE80211_ADDR_COPY( perr->perr_dests[i].dest_addr, iefrm_t); iefrm_t += 6; perr->perr_dests[i].dest_seq = le32dec(iefrm_t); iefrm_t += 4; /* NB: May have Target Proxied Address */ if (perr->perr_dests[i].dest_flags & IEEE80211_MESHPERR_FLAGS_AE) { IEEE80211_ADDR_COPY( perr->perr_dests[i].dest_ext_addr, iefrm_t); iefrm_t += 6; } perr->perr_dests[i].dest_rcode = le16dec(iefrm_t); iefrm_t += 2; } hwmp_recv_perr(vap, ni, wh, perr); IEEE80211_FREE(perr, M_80211_MESH_PERR); found++; break; } case IEEE80211_ELEMID_MESHRANN: { const struct ieee80211_meshrann_ie *mrann = (const struct ieee80211_meshrann_ie *) iefrm; if (mrann->rann_len != sizeof(struct ieee80211_meshrann_ie) - 2) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP, wh, NULL, "%s", "RAN with wrong len"); vap->iv_stats.is_rx_mgtdiscard++; return 1; } memcpy(&rann, mrann, sizeof(rann)); rann.rann_seq = le32dec(&mrann->rann_seq); rann.rann_interval = le32dec(&mrann->rann_interval); rann.rann_metric = le32dec(&mrann->rann_metric); hwmp_recv_rann(vap, ni, wh, &rann); found++; break; } } iefrm += iefrm[1] + 2; } if (!found) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_HWMP, wh, NULL, "%s", "PATH SEL action without IE"); vap->iv_stats.is_rx_mgtdiscard++; } return 0; } static int hwmp_send_action(struct ieee80211vap *vap, const uint8_t da[IEEE80211_ADDR_LEN], uint8_t *ie, size_t len) { struct ieee80211_node *ni; struct ieee80211com *ic; struct ieee80211_bpf_params params; struct mbuf *m; uint8_t *frm; int ret; if (IEEE80211_IS_MULTICAST(da)) { ni = ieee80211_ref_node(vap->iv_bss); #ifdef IEEE80211_DEBUG_REFCNT IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); #endif ieee80211_ref_node(ni); } else ni = ieee80211_mesh_find_txnode(vap, da); if (vap->iv_state == IEEE80211_S_CAC) { IEEE80211_NOTE(vap, IEEE80211_MSG_OUTPUT, ni, "block %s frame in CAC state", "HWMP action"); vap->iv_stats.is_tx_badstate++; return EIO; /* XXX */ } KASSERT(ni != NULL, ("null node")); ic = ni->ni_ic; m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(struct ieee80211_action) + len ); if (m == NULL) { ieee80211_free_node(ni); vap->iv_stats.is_tx_nobuf++; return ENOMEM; } *frm++ = IEEE80211_ACTION_CAT_MESH; *frm++ = IEEE80211_ACTION_MESH_HWMP; switch (*ie) { case IEEE80211_ELEMID_MESHPREQ: frm = hwmp_add_meshpreq(frm, (struct ieee80211_meshpreq_ie *)ie); break; case IEEE80211_ELEMID_MESHPREP: frm = hwmp_add_meshprep(frm, (struct ieee80211_meshprep_ie *)ie); break; case IEEE80211_ELEMID_MESHPERR: frm = hwmp_add_meshperr(frm, (struct ieee80211_meshperr_ie *)ie); break; case IEEE80211_ELEMID_MESHRANN: frm = hwmp_add_meshrann(frm, (struct ieee80211_meshrann_ie *)ie); break; } m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); M_PREPEND(m, sizeof(struct ieee80211_frame), M_NOWAIT); if (m == NULL) { ieee80211_free_node(ni); vap->iv_stats.is_tx_nobuf++; return ENOMEM; } IEEE80211_TX_LOCK(ic); ieee80211_send_setup(ni, m, IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_ACTION, IEEE80211_NONQOS_TID, vap->iv_myaddr, da, vap->iv_myaddr); m->m_flags |= M_ENCAP; /* mark encapsulated */ IEEE80211_NODE_STAT(ni, tx_mgmt); memset(¶ms, 0, sizeof(params)); params.ibp_pri = WME_AC_VO; params.ibp_rate0 = ni->ni_txparms->mgmtrate; if (IEEE80211_IS_MULTICAST(da)) params.ibp_try0 = 1; else params.ibp_try0 = ni->ni_txparms->maxretry; params.ibp_power = ni->ni_txpower; ret = ieee80211_raw_output(vap, ni, m, ¶ms); IEEE80211_TX_UNLOCK(ic); return (ret); } #define ADDSHORT(frm, v) do { \ le16enc(frm, v); \ frm += 2; \ } while (0) #define ADDWORD(frm, v) do { \ le32enc(frm, v); \ frm += 4; \ } while (0) /* * Add a Mesh Path Request IE to a frame. */ #define PREQ_TFLAGS(n) preq->preq_targets[n].target_flags #define PREQ_TADDR(n) preq->preq_targets[n].target_addr #define PREQ_TSEQ(n) preq->preq_targets[n].target_seq static uint8_t * hwmp_add_meshpreq(uint8_t *frm, const struct ieee80211_meshpreq_ie *preq) { int i; *frm++ = IEEE80211_ELEMID_MESHPREQ; *frm++ = preq->preq_len; /* len already calculated */ *frm++ = preq->preq_flags; *frm++ = preq->preq_hopcount; *frm++ = preq->preq_ttl; ADDWORD(frm, preq->preq_id); IEEE80211_ADDR_COPY(frm, preq->preq_origaddr); frm += 6; ADDWORD(frm, preq->preq_origseq); if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) { IEEE80211_ADDR_COPY(frm, preq->preq_orig_ext_addr); frm += 6; } ADDWORD(frm, preq->preq_lifetime); ADDWORD(frm, preq->preq_metric); *frm++ = preq->preq_tcount; for (i = 0; i < preq->preq_tcount; i++) { *frm++ = PREQ_TFLAGS(i); IEEE80211_ADDR_COPY(frm, PREQ_TADDR(i)); frm += 6; ADDWORD(frm, PREQ_TSEQ(i)); } return frm; } #undef PREQ_TFLAGS #undef PREQ_TADDR #undef PREQ_TSEQ /* * Add a Mesh Path Reply IE to a frame. */ static uint8_t * hwmp_add_meshprep(uint8_t *frm, const struct ieee80211_meshprep_ie *prep) { *frm++ = IEEE80211_ELEMID_MESHPREP; *frm++ = prep->prep_len; /* len already calculated */ *frm++ = prep->prep_flags; *frm++ = prep->prep_hopcount; *frm++ = prep->prep_ttl; IEEE80211_ADDR_COPY(frm, prep->prep_targetaddr); frm += 6; ADDWORD(frm, prep->prep_targetseq); if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) { IEEE80211_ADDR_COPY(frm, prep->prep_target_ext_addr); frm += 6; } ADDWORD(frm, prep->prep_lifetime); ADDWORD(frm, prep->prep_metric); IEEE80211_ADDR_COPY(frm, prep->prep_origaddr); frm += 6; ADDWORD(frm, prep->prep_origseq); return frm; } /* * Add a Mesh Path Error IE to a frame. */ #define PERR_DFLAGS(n) perr->perr_dests[n].dest_flags #define PERR_DADDR(n) perr->perr_dests[n].dest_addr #define PERR_DSEQ(n) perr->perr_dests[n].dest_seq #define PERR_EXTADDR(n) perr->perr_dests[n].dest_ext_addr #define PERR_DRCODE(n) perr->perr_dests[n].dest_rcode static uint8_t * hwmp_add_meshperr(uint8_t *frm, const struct ieee80211_meshperr_ie *perr) { int i; *frm++ = IEEE80211_ELEMID_MESHPERR; *frm++ = perr->perr_len; /* len already calculated */ *frm++ = perr->perr_ttl; *frm++ = perr->perr_ndests; for (i = 0; i < perr->perr_ndests; i++) { *frm++ = PERR_DFLAGS(i); IEEE80211_ADDR_COPY(frm, PERR_DADDR(i)); frm += 6; ADDWORD(frm, PERR_DSEQ(i)); if (PERR_DFLAGS(i) & IEEE80211_MESHPERR_FLAGS_AE) { IEEE80211_ADDR_COPY(frm, PERR_EXTADDR(i)); frm += 6; } ADDSHORT(frm, PERR_DRCODE(i)); } return frm; } #undef PERR_DFLAGS #undef PERR_DADDR #undef PERR_DSEQ #undef PERR_EXTADDR #undef PERR_DRCODE /* * Add a Root Annoucement IE to a frame. */ static uint8_t * hwmp_add_meshrann(uint8_t *frm, const struct ieee80211_meshrann_ie *rann) { *frm++ = IEEE80211_ELEMID_MESHRANN; *frm++ = rann->rann_len; *frm++ = rann->rann_flags; *frm++ = rann->rann_hopcount; *frm++ = rann->rann_ttl; IEEE80211_ADDR_COPY(frm, rann->rann_addr); frm += 6; ADDWORD(frm, rann->rann_seq); ADDWORD(frm, rann->rann_interval); ADDWORD(frm, rann->rann_metric); return frm; } static void hwmp_rootmode_setup(struct ieee80211vap *vap) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_state *ms = vap->iv_mesh; switch (hs->hs_rootmode) { case IEEE80211_HWMP_ROOTMODE_DISABLED: callout_drain(&hs->hs_roottimer); ms->ms_flags &= ~IEEE80211_MESHFLAGS_ROOT; break; case IEEE80211_HWMP_ROOTMODE_NORMAL: case IEEE80211_HWMP_ROOTMODE_PROACTIVE: callout_reset(&hs->hs_roottimer, ieee80211_hwmp_rootint, hwmp_rootmode_cb, vap); ms->ms_flags |= IEEE80211_MESHFLAGS_ROOT; break; case IEEE80211_HWMP_ROOTMODE_RANN: callout_reset(&hs->hs_roottimer, ieee80211_hwmp_rannint, hwmp_rootmode_rann_cb, vap); ms->ms_flags |= IEEE80211_MESHFLAGS_ROOT; break; } } /* * Send a broadcast Path Request to find all nodes on the mesh. We are * called when the vap is configured as a HWMP root node. */ #define PREQ_TFLAGS(n) preq.preq_targets[n].target_flags #define PREQ_TADDR(n) preq.preq_targets[n].target_addr #define PREQ_TSEQ(n) preq.preq_targets[n].target_seq static void hwmp_rootmode_cb(void *arg) { struct ieee80211vap *vap = (struct ieee80211vap *)arg; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_meshpreq_ie preq; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, vap->iv_bss, "%s", "send broadcast PREQ"); preq.preq_flags = 0; if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) preq.preq_flags |= IEEE80211_MESHPREQ_FLAGS_GATE; if (hs->hs_rootmode == IEEE80211_HWMP_ROOTMODE_PROACTIVE) preq.preq_flags |= IEEE80211_MESHPREQ_FLAGS_PP; preq.preq_hopcount = 0; preq.preq_ttl = ms->ms_ttl; preq.preq_id = ++hs->hs_preqid; IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr); preq.preq_origseq = ++hs->hs_seq; preq.preq_lifetime = ticks_to_msecs(ieee80211_hwmp_roottimeout); preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL; preq.preq_tcount = 1; IEEE80211_ADDR_COPY(PREQ_TADDR(0), broadcastaddr); PREQ_TFLAGS(0) = IEEE80211_MESHPREQ_TFLAGS_TO | IEEE80211_MESHPREQ_TFLAGS_USN; PREQ_TSEQ(0) = 0; vap->iv_stats.is_hwmp_rootreqs++; /* NB: we enforce rate check ourself */ hwmp_send_preq(vap, broadcastaddr, &preq, NULL, NULL); hwmp_rootmode_setup(vap); } #undef PREQ_TFLAGS #undef PREQ_TADDR #undef PREQ_TSEQ /* * Send a Root Annoucement (RANN) to find all the nodes on the mesh. We are * called when the vap is configured as a HWMP RANN root node. */ static void hwmp_rootmode_rann_cb(void *arg) { struct ieee80211vap *vap = (struct ieee80211vap *)arg; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_meshrann_ie rann; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, vap->iv_bss, "%s", "send broadcast RANN"); rann.rann_flags = 0; if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) rann.rann_flags |= IEEE80211_MESHFLAGS_GATE; rann.rann_hopcount = 0; rann.rann_ttl = ms->ms_ttl; IEEE80211_ADDR_COPY(rann.rann_addr, vap->iv_myaddr); rann.rann_seq = ++hs->hs_seq; rann.rann_interval = ieee80211_hwmp_rannint; rann.rann_metric = IEEE80211_MESHLMETRIC_INITIALVAL; vap->iv_stats.is_hwmp_rootrann++; hwmp_send_rann(vap, broadcastaddr, &rann); hwmp_rootmode_setup(vap); } /* * Update forwarding information to TA if metric improves. */ static void hwmp_update_transmitter(struct ieee80211vap *vap, struct ieee80211_node *ni, const char *hwmp_frame) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rttran = NULL; /* Transmitter */ int metric = 0; rttran = ieee80211_mesh_rt_find(vap, ni->ni_macaddr); if (rttran == NULL) { rttran = ieee80211_mesh_rt_add(vap, ni->ni_macaddr); if (rttran == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add path to transmitter %6D of %s", ni->ni_macaddr, ":", hwmp_frame); vap->iv_stats.is_mesh_rtaddfailed++; return; } } metric = ms->ms_pmetric->mpm_metric(ni); if (!(rttran->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) || rttran->rt_metric > metric) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "%s path to transmitter %6D of %s, metric %d:%d", rttran->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ? "prefer" : "update", ni->ni_macaddr, ":", hwmp_frame, rttran->rt_metric, metric); IEEE80211_ADDR_COPY(rttran->rt_nexthop, ni->ni_macaddr); rttran->rt_metric = metric; rttran->rt_nhops = 1; ieee80211_mesh_rt_update(rttran, ms->ms_ppath->mpp_inact); rttran->rt_flags = IEEE80211_MESHRT_FLAGS_VALID; } } #define PREQ_TFLAGS(n) preq->preq_targets[n].target_flags #define PREQ_TADDR(n) preq->preq_targets[n].target_addr #define PREQ_TSEQ(n) preq->preq_targets[n].target_seq static void hwmp_recv_preq(struct ieee80211vap *vap, struct ieee80211_node *ni, const struct ieee80211_frame *wh, const struct ieee80211_meshpreq_ie *preq) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rtorig = NULL; struct ieee80211_mesh_route *rtorig_ext = NULL; struct ieee80211_mesh_route *rttarg = NULL; struct ieee80211_hwmp_route *hrorig = NULL; struct ieee80211_hwmp_route *hrtarg = NULL; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; ieee80211_hwmp_seq preqid; /* last seen preqid for orig */ uint32_t metric = 0; /* * Ignore PREQs from us. Could happen because someone forward it * back to us. */ if (IEEE80211_ADDR_EQ(vap->iv_myaddr, preq->preq_origaddr)) return; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "received PREQ, orig %6D, targ(0) %6D", preq->preq_origaddr, ":", PREQ_TADDR(0), ":"); /* * Acceptance criteria: (if the PREQ is not for us or not broadcast, * or an external mac address not proxied by us), * AND forwarding is disabled, discard this PREQ. */ rttarg = ieee80211_mesh_rt_find(vap, PREQ_TADDR(0)); if (!(ms->ms_flags & IEEE80211_MESHFLAGS_FWD) && (!IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0)) || !IEEE80211_IS_MULTICAST(PREQ_TADDR(0)) || (rttarg != NULL && rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY && IEEE80211_ADDR_EQ(vap->iv_myaddr, rttarg->rt_mesh_gate)))) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP, preq->preq_origaddr, NULL, "%s", "not accepting PREQ"); return; } /* * Acceptance criteria: if unicast addressed * AND no valid forwarding for Target of PREQ, discard this PREQ. */ if(rttarg != NULL) hrtarg = IEEE80211_MESH_ROUTE_PRIV(rttarg, struct ieee80211_hwmp_route); /* Address mode: ucast */ if(preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AM && rttarg == NULL && !IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0))) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP, preq->preq_origaddr, NULL, "unicast addressed PREQ of unknown target %6D", PREQ_TADDR(0), ":"); return; } /* PREQ ACCEPTED */ rtorig = ieee80211_mesh_rt_find(vap, preq->preq_origaddr); if (rtorig == NULL) { rtorig = ieee80211_mesh_rt_add(vap, preq->preq_origaddr); if (rtorig == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add orig path to %6D", preq->preq_origaddr, ":"); vap->iv_stats.is_mesh_rtaddfailed++; return; } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "adding originator %6D", preq->preq_origaddr, ":"); } hrorig = IEEE80211_MESH_ROUTE_PRIV(rtorig, struct ieee80211_hwmp_route); /* record last seen preqid */ preqid = hrorig->hr_preqid; hrorig->hr_preqid = HWMP_SEQ_MAX(hrorig->hr_preqid, preq->preq_id); /* Data creation and update of forwarding information * according to Table 11C-8 for originator mesh STA. */ metric = preq->preq_metric + ms->ms_pmetric->mpm_metric(ni); if (HWMP_SEQ_GT(preq->preq_origseq, hrorig->hr_seq) || (HWMP_SEQ_EQ(preq->preq_origseq, hrorig->hr_seq) && metric < rtorig->rt_metric)) { hrorig->hr_seq = preq->preq_origseq; IEEE80211_ADDR_COPY(rtorig->rt_nexthop, wh->i_addr2); rtorig->rt_metric = metric; rtorig->rt_nhops = preq->preq_hopcount + 1; ieee80211_mesh_rt_update(rtorig, preq->preq_lifetime); /* Path to orig is valid now. * NB: we know it can't be Proxy, and if it is GATE * it will be marked below. */ rtorig->rt_flags = IEEE80211_MESHRT_FLAGS_VALID; } else if ((hrtarg != NULL && !HWMP_SEQ_EQ(hrtarg->hr_seq, PREQ_TSEQ(0))) || (rtorig->rt_flags & IEEE80211_MESHRT_FLAGS_VALID && preqid >= preq->preq_id)) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "discard PREQ from %6D, old seqno %u <= %u," " or old preqid %u < %u", preq->preq_origaddr, ":", preq->preq_origseq, hrorig->hr_seq, preq->preq_id, preqid); return; } /* Update forwarding information to TA if metric improves. */ hwmp_update_transmitter(vap, ni, "PREQ"); /* * Check if the PREQ is addressed to us. * or a Proxy currently gated by us. */ if (IEEE80211_ADDR_EQ(vap->iv_myaddr, PREQ_TADDR(0)) || (ms->ms_flags & IEEE80211_MESHFLAGS_GATE && rttarg != NULL && IEEE80211_ADDR_EQ(vap->iv_myaddr, rttarg->rt_mesh_gate) && rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY && rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)) { struct ieee80211_meshprep_ie prep; /* * When we are the target we shall update our own HWMP seq * number with max of (current and preq->seq) + 1 */ hs->hs_seq = HWMP_SEQ_MAX(hs->hs_seq, PREQ_TSEQ(0)) + 1; prep.prep_flags = 0; prep.prep_hopcount = 0; prep.prep_metric = IEEE80211_MESHLMETRIC_INITIALVAL; IEEE80211_ADDR_COPY(prep.prep_targetaddr, vap->iv_myaddr); if (rttarg != NULL && /* if NULL it means we are the target */ rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "reply for proxy %6D", rttarg->rt_dest, ":"); prep.prep_flags |= IEEE80211_MESHPREP_FLAGS_AE; IEEE80211_ADDR_COPY(prep.prep_target_ext_addr, rttarg->rt_dest); /* update proxy seqno to HWMP seqno */ rttarg->rt_ext_seq = hs->hs_seq; prep.prep_hopcount = rttarg->rt_nhops; prep.prep_metric = rttarg->rt_metric; IEEE80211_ADDR_COPY(prep.prep_targetaddr, rttarg->rt_mesh_gate); } /* * Build and send a PREP frame. */ prep.prep_ttl = ms->ms_ttl; prep.prep_targetseq = hs->hs_seq; prep.prep_lifetime = preq->preq_lifetime; IEEE80211_ADDR_COPY(prep.prep_origaddr, preq->preq_origaddr); prep.prep_origseq = preq->preq_origseq; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "reply to %6D", preq->preq_origaddr, ":"); hwmp_send_prep(vap, wh->i_addr2, &prep); return; } /* we may update our proxy information for the orig external */ else if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE) { rtorig_ext = ieee80211_mesh_rt_find(vap, preq->preq_orig_ext_addr); if (rtorig_ext == NULL) { rtorig_ext = ieee80211_mesh_rt_add(vap, preq->preq_orig_ext_addr); if (rtorig_ext == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add orig ext proxy to %6D", preq->preq_orig_ext_addr, ":"); vap->iv_stats.is_mesh_rtaddfailed++; return; } IEEE80211_ADDR_COPY(rtorig_ext->rt_mesh_gate, preq->preq_origaddr); } rtorig_ext->rt_ext_seq = preq->preq_origseq; ieee80211_mesh_rt_update(rtorig_ext, preq->preq_lifetime); } /* * Proactive PREQ: reply with a proactive PREP to the * root STA if requested. */ if (IEEE80211_ADDR_EQ(PREQ_TADDR(0), broadcastaddr) && (PREQ_TFLAGS(0) & IEEE80211_MESHPREQ_TFLAGS_TO)) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "root mesh station @ %6D", preq->preq_origaddr, ":"); /* Check if root is a mesh gate, mark it */ if (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_GATE) { struct ieee80211_mesh_gate_route *gr; rtorig->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE; gr = ieee80211_mesh_mark_gate(vap, preq->preq_origaddr, rtorig); gr->gr_lastseq = 0; /* NOT GANN */ } /* * Reply with a PREP if we don't have a path to the root * or if the root sent us a proactive PREQ. */ if ((rtorig->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0 || (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_PP)) { struct ieee80211_meshprep_ie prep; prep.prep_flags = 0; prep.prep_hopcount = 0; prep.prep_ttl = ms->ms_ttl; IEEE80211_ADDR_COPY(prep.prep_origaddr, preq->preq_origaddr); prep.prep_origseq = preq->preq_origseq; prep.prep_lifetime = preq->preq_lifetime; prep.prep_metric = IEEE80211_MESHLMETRIC_INITIALVAL; IEEE80211_ADDR_COPY(prep.prep_targetaddr, vap->iv_myaddr); prep.prep_targetseq = ++hs->hs_seq; hwmp_send_prep(vap, rtorig->rt_nexthop, &prep); } } /* * Forwarding and Intermediate reply for PREQs with 1 target. */ if ((preq->preq_tcount == 1) && (preq->preq_ttl > 1) && (ms->ms_flags & IEEE80211_MESHFLAGS_FWD)) { struct ieee80211_meshpreq_ie ppreq; /* propagated PREQ */ memcpy(&ppreq, preq, sizeof(ppreq)); /* * We have a valid route to this node. * NB: if target is proxy dont reply. */ if (rttarg != NULL && rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_VALID && !(rttarg->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY)) { /* * Check if we can send an intermediate Path Reply, * i.e., Target Only bit is not set and target is not * the MAC broadcast address. */ if (!(PREQ_TFLAGS(0) & IEEE80211_MESHPREQ_TFLAGS_TO) && !IEEE80211_ADDR_EQ(PREQ_TADDR(0), broadcastaddr)) { struct ieee80211_meshprep_ie prep; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "intermediate reply for PREQ from %6D", preq->preq_origaddr, ":"); prep.prep_flags = 0; prep.prep_hopcount = rttarg->rt_nhops; prep.prep_ttl = ms->ms_ttl; IEEE80211_ADDR_COPY(&prep.prep_targetaddr, PREQ_TADDR(0)); prep.prep_targetseq = hrtarg->hr_seq; prep.prep_lifetime = preq->preq_lifetime; prep.prep_metric =rttarg->rt_metric; IEEE80211_ADDR_COPY(&prep.prep_origaddr, preq->preq_origaddr); prep.prep_origseq = hrorig->hr_seq; hwmp_send_prep(vap, rtorig->rt_nexthop, &prep); /* * Set TO and unset RF bits because we have * sent a PREP. */ ppreq.preq_targets[0].target_flags |= IEEE80211_MESHPREQ_TFLAGS_TO; } } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "forward PREQ from %6D", preq->preq_origaddr, ":"); ppreq.preq_hopcount += 1; ppreq.preq_ttl -= 1; ppreq.preq_metric += ms->ms_pmetric->mpm_metric(ni); /* don't do PREQ ratecheck when we propagate */ hwmp_send_preq(vap, broadcastaddr, &ppreq, NULL, NULL); } } #undef PREQ_TFLAGS #undef PREQ_TADDR #undef PREQ_TSEQ static int hwmp_send_preq(struct ieee80211vap *vap, const uint8_t da[IEEE80211_ADDR_LEN], struct ieee80211_meshpreq_ie *preq, struct timeval *last, struct timeval *minint) { /* * Enforce PREQ interval. * NB: Proactive ROOT PREQs rate is handled by cb task. */ if (last != NULL && minint != NULL) { if (ratecheck(last, minint) == 0) return EALREADY; /* XXX: we should postpone */ getmicrouptime(last); } /* * mesh preq action frame format * [6] da * [6] sa * [6] addr3 = sa * [1] action * [1] category * [tlv] mesh path request */ preq->preq_ie = IEEE80211_ELEMID_MESHPREQ; preq->preq_len = (preq->preq_flags & IEEE80211_MESHPREQ_FLAGS_AE ? IEEE80211_MESHPREQ_BASE_SZ_AE : IEEE80211_MESHPREQ_BASE_SZ) + preq->preq_tcount * IEEE80211_MESHPREQ_TRGT_SZ; return hwmp_send_action(vap, da, (uint8_t *)preq, preq->preq_len+2); } static void hwmp_recv_prep(struct ieee80211vap *vap, struct ieee80211_node *ni, const struct ieee80211_frame *wh, const struct ieee80211_meshprep_ie *prep) { #define IS_PROXY(rt) (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) #define PROXIED_BY_US(rt) \ (IEEE80211_ADDR_EQ(vap->iv_myaddr, rt->rt_mesh_gate)) struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_route *rt = NULL; struct ieee80211_mesh_route *rtorig = NULL; struct ieee80211_mesh_route *rtext = NULL; struct ieee80211_hwmp_route *hr; struct ieee80211com *ic = vap->iv_ic; struct mbuf *m, *next; uint32_t metric = 0; const uint8_t *addr; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "received PREP, orig %6D, targ %6D", prep->prep_origaddr, ":", prep->prep_targetaddr, ":"); /* * Acceptance criteria: (If the corresponding PREP was not generated * by us OR not generated by an external mac that is not proxied by us) * AND forwarding is disabled, discard this PREP. */ rtorig = ieee80211_mesh_rt_find(vap, prep->prep_origaddr); if ((!IEEE80211_ADDR_EQ(vap->iv_myaddr, prep->prep_origaddr) || (rtorig != NULL && IS_PROXY(rtorig) && !PROXIED_BY_US(rtorig))) && !(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)){ IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "discard PREP, orig(%6D) not proxied or generated by us", prep->prep_origaddr, ":"); return; } /* PREP ACCEPTED */ /* * If accepted shall create or update the active forwarding information * it maintains for the target mesh STA of the PREP (according to the * rules defined in 13.10.8.4). If the conditions for creating or * updating the forwarding information have not been met in those * rules, no further steps are applied to the PREP. */ rt = ieee80211_mesh_rt_find(vap, prep->prep_targetaddr); if (rt == NULL) { rt = ieee80211_mesh_rt_add(vap, prep->prep_targetaddr); if (rt == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add PREP path to %6D", prep->prep_targetaddr, ":"); vap->iv_stats.is_mesh_rtaddfailed++; return; } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "adding target %6D", prep->prep_targetaddr, ":"); } hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); /* update path metric */ metric = prep->prep_metric + ms->ms_pmetric->mpm_metric(ni); if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)) { if (HWMP_SEQ_LT(prep->prep_targetseq, hr->hr_seq)) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "discard PREP from %6D, old seq no %u < %u", prep->prep_targetaddr, ":", prep->prep_targetseq, hr->hr_seq); return; } else if (HWMP_SEQ_LEQ(prep->prep_targetseq, hr->hr_seq) && metric > rt->rt_metric) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "discard PREP from %6D, new metric %u > %u", prep->prep_targetaddr, ":", metric, rt->rt_metric); return; } } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "%s path to %6D, hopcount %d:%d metric %d:%d", rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ? "prefer" : "update", prep->prep_targetaddr, ":", rt->rt_nhops, prep->prep_hopcount + 1, rt->rt_metric, metric); hr->hr_seq = prep->prep_targetseq; hr->hr_preqretries = 0; IEEE80211_ADDR_COPY(rt->rt_nexthop, ni->ni_macaddr); rt->rt_metric = metric; rt->rt_nhops = prep->prep_hopcount + 1; ieee80211_mesh_rt_update(rt, prep->prep_lifetime); if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER) { /* discovery complete */ rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_DISCOVER; } rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID; /* mark valid */ /* Update forwarding information to TA if metric improves */ hwmp_update_transmitter(vap, ni, "PREP"); /* * If it's NOT for us, propagate the PREP */ if (!IEEE80211_ADDR_EQ(vap->iv_myaddr, prep->prep_origaddr) && prep->prep_ttl > 1 && prep->prep_hopcount < hs->hs_maxhops) { struct ieee80211_meshprep_ie pprep; /* propagated PREP */ /* * NB: We should already have setup the path to orig * mesh STA when we propagated PREQ to target mesh STA, * no PREP is generated without a corresponding PREQ. * XXX: for now just ignore. */ if (rtorig == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "received PREP for an unknown orig(%6D)", prep->prep_origaddr, ":"); return; } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "propagate PREP from %6D", prep->prep_targetaddr, ":"); memcpy(&pprep, prep, sizeof(pprep)); pprep.prep_hopcount += 1; pprep.prep_ttl -= 1; pprep.prep_metric += ms->ms_pmetric->mpm_metric(ni); hwmp_send_prep(vap, rtorig->rt_nexthop, &pprep); /* precursor list for the Target Mesh STA Address is updated */ } /* * Check if we received a PREP w/ AE and store target external address. * We may store target external address if recevied PREP w/ AE * and we are not final destination */ if (prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE) { rtext = ieee80211_mesh_rt_find(vap, prep->prep_target_ext_addr); if (rtext == NULL) { rtext = ieee80211_mesh_rt_add(vap, prep->prep_target_ext_addr); if (rtext == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add PREP path to proxy %6D", prep->prep_targetaddr, ":"); vap->iv_stats.is_mesh_rtaddfailed++; return; } } IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "%s path to %6D, hopcount %d:%d metric %d:%d", rtext->rt_flags & IEEE80211_MESHRT_FLAGS_VALID ? "prefer" : "update", prep->prep_target_ext_addr, ":", rtext->rt_nhops, prep->prep_hopcount + 1, rtext->rt_metric, metric); rtext->rt_flags = IEEE80211_MESHRT_FLAGS_PROXY | IEEE80211_MESHRT_FLAGS_VALID; IEEE80211_ADDR_COPY(rtext->rt_dest, prep->prep_target_ext_addr); IEEE80211_ADDR_COPY(rtext->rt_mesh_gate, prep->prep_targetaddr); IEEE80211_ADDR_COPY(rtext->rt_nexthop, wh->i_addr2); rtext->rt_metric = metric; rtext->rt_lifetime = prep->prep_lifetime; rtext->rt_nhops = prep->prep_hopcount + 1; rtext->rt_ext_seq = prep->prep_origseq; /* new proxy seq */ /* * XXX: proxy entries have no HWMP priv data, * nullify them to be sure? */ } /* * Check for frames queued awaiting path discovery. * XXX probably can tell exactly and avoid remove call * NB: hash may have false matches, if so they will get * stuck back on the stageq because there won't be * a path. */ addr = prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE ? prep->prep_target_ext_addr : prep->prep_targetaddr; m = ieee80211_ageq_remove(&ic->ic_stageq, (struct ieee80211_node *)(uintptr_t) ieee80211_mac_hash(ic, addr)); /* either dest or ext_dest */ /* * All frames in the stageq here should be non-M_ENCAP; or things * will get very unhappy. */ for (; m != NULL; m = next) { next = m->m_nextpkt; m->m_nextpkt = NULL; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "flush queued frame %p len %d", m, m->m_pkthdr.len); /* * If the mbuf has M_ENCAP set, ensure we free it. * Note that after if_transmit() is called, m is invalid. */ (void) ieee80211_vap_xmitpkt(vap, m); } #undef IS_PROXY #undef PROXIED_BY_US } static int hwmp_send_prep(struct ieee80211vap *vap, const uint8_t da[IEEE80211_ADDR_LEN], struct ieee80211_meshprep_ie *prep) { /* NB: there's no PREP minimum interval. */ /* * mesh prep action frame format * [6] da * [6] sa * [6] addr3 = sa * [1] action * [1] category * [tlv] mesh path reply */ prep->prep_ie = IEEE80211_ELEMID_MESHPREP; prep->prep_len = prep->prep_flags & IEEE80211_MESHPREP_FLAGS_AE ? IEEE80211_MESHPREP_BASE_SZ_AE : IEEE80211_MESHPREP_BASE_SZ; return hwmp_send_action(vap, da, (uint8_t *)prep, prep->prep_len + 2); } #define PERR_DFLAGS(n) perr.perr_dests[n].dest_flags #define PERR_DADDR(n) perr.perr_dests[n].dest_addr #define PERR_DSEQ(n) perr.perr_dests[n].dest_seq #define PERR_DRCODE(n) perr.perr_dests[n].dest_rcode static void hwmp_peerdown(struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_meshperr_ie perr; struct ieee80211_mesh_route *rt; struct ieee80211_hwmp_route *hr; rt = ieee80211_mesh_rt_find(vap, ni->ni_macaddr); if (rt == NULL) return; hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "%s", "delete route entry"); perr.perr_ttl = ms->ms_ttl; perr.perr_ndests = 1; PERR_DFLAGS(0) = 0; if (hr->hr_seq == 0) PERR_DFLAGS(0) |= IEEE80211_MESHPERR_DFLAGS_USN; PERR_DFLAGS(0) |= IEEE80211_MESHPERR_DFLAGS_RC; IEEE80211_ADDR_COPY(PERR_DADDR(0), rt->rt_dest); PERR_DSEQ(0) = ++hr->hr_seq; PERR_DRCODE(0) = IEEE80211_REASON_MESH_PERR_DEST_UNREACH; /* NB: flush everything passing through peer */ ieee80211_mesh_rt_flush_peer(vap, ni->ni_macaddr); hwmp_send_perr(vap, broadcastaddr, &perr); } #undef PERR_DFLAGS #undef PERR_DADDR #undef PERR_DSEQ #undef PERR_DRCODE #define PERR_DFLAGS(n) perr->perr_dests[n].dest_flags #define PERR_DADDR(n) perr->perr_dests[n].dest_addr #define PERR_DSEQ(n) perr->perr_dests[n].dest_seq #define PERR_DEXTADDR(n) perr->perr_dests[n].dest_ext_addr static void hwmp_recv_perr(struct ieee80211vap *vap, struct ieee80211_node *ni, const struct ieee80211_frame *wh, const struct ieee80211_meshperr_ie *perr) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt = NULL; struct ieee80211_mesh_route *rt_ext = NULL; struct ieee80211_hwmp_route *hr; struct ieee80211_meshperr_ie *pperr = NULL; int i, j = 0, forward = 0; IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "received PERR from %6D", wh->i_addr2, ":"); /* * if forwarding is true, prepare pperr */ if (ms->ms_flags & IEEE80211_MESHFLAGS_FWD) { forward = 1; pperr = IEEE80211_MALLOC(sizeof(*perr) + 31*sizeof(*perr->perr_dests), M_80211_MESH_PERR, IEEE80211_M_NOWAIT); /* XXX: magic number, 32 err dests */ } /* * Acceptance criteria: check if we have forwarding information * stored about destination, and that nexthop == TA of this PERR. * NB: we also build a new PERR to propagate in case we should forward. */ for (i = 0; i < perr->perr_ndests; i++) { rt = ieee80211_mesh_rt_find(vap, PERR_DADDR(i)); if (rt == NULL) continue; if (!IEEE80211_ADDR_EQ(rt->rt_nexthop, wh->i_addr2)) continue; /* found and accepted a PERR ndest element, process it... */ if (forward) memcpy(&pperr->perr_dests[j], &perr->perr_dests[i], sizeof(*perr->perr_dests)); hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); switch(PERR_DFLAGS(i)) { case (IEEE80211_REASON_MESH_PERR_NO_FI): if (PERR_DSEQ(i) == 0) { hr->hr_seq++; if (forward) { pperr->perr_dests[j].dest_seq = hr->hr_seq; } } else { hr->hr_seq = PERR_DSEQ(i); } rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID; j++; break; case (IEEE80211_REASON_MESH_PERR_DEST_UNREACH): if(HWMP_SEQ_GT(PERR_DSEQ(i), hr->hr_seq)) { hr->hr_seq = PERR_DSEQ(i); rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID; j++; } break; case (IEEE80211_REASON_MESH_PERR_NO_PROXY): rt_ext = ieee80211_mesh_rt_find(vap, PERR_DEXTADDR(i)); if (rt_ext != NULL) { rt_ext->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID; j++; } break; default: IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "PERR, unknown reason code %u\n", PERR_DFLAGS(i)); goto done; /* XXX: stats?? */ } ieee80211_mesh_rt_flush_peer(vap, PERR_DADDR(i)); KASSERT(j < 32, ("PERR, error ndest >= 32 (%u)", j)); } if (j == 0) { IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "%s", "PERR not accepted"); goto done; /* XXX: stats?? */ } /* * Propagate the PERR if we previously found it on our routing table. */ if (forward && perr->perr_ttl > 1) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "propagate PERR from %6D", wh->i_addr2, ":"); pperr->perr_ndests = j; pperr->perr_ttl--; hwmp_send_perr(vap, broadcastaddr, pperr); } done: if (pperr != NULL) IEEE80211_FREE(pperr, M_80211_MESH_PERR); } #undef PERR_DFLAGS #undef PERR_DADDR #undef PERR_DSEQ #undef PERR_DEXTADDR static int hwmp_send_perr(struct ieee80211vap *vap, const uint8_t da[IEEE80211_ADDR_LEN], struct ieee80211_meshperr_ie *perr) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; int i; uint8_t length = 0; /* * Enforce PERR interval. */ if (ratecheck(&hs->hs_lastperr, &ieee80211_hwmp_perrminint) == 0) return EALREADY; getmicrouptime(&hs->hs_lastperr); /* * mesh perr action frame format * [6] da * [6] sa * [6] addr3 = sa * [1] action * [1] category * [tlv] mesh path error */ perr->perr_ie = IEEE80211_ELEMID_MESHPERR; length = IEEE80211_MESHPERR_BASE_SZ; for (i = 0; iperr_ndests; i++) { if (perr->perr_dests[i].dest_flags & IEEE80211_MESHPERR_FLAGS_AE) { length += IEEE80211_MESHPERR_DEST_SZ_AE; continue ; } length += IEEE80211_MESHPERR_DEST_SZ; } perr->perr_len =length; return hwmp_send_action(vap, da, (uint8_t *)perr, perr->perr_len+2); } /* * Called from the rest of the net80211 code (mesh code for example). * NB: IEEE80211_REASON_MESH_PERR_DEST_UNREACH can be trigger by the fact that * a mesh STA is unable to forward an MSDU/MMPDU to a next-hop mesh STA. */ #define PERR_DFLAGS(n) perr.perr_dests[n].dest_flags #define PERR_DADDR(n) perr.perr_dests[n].dest_addr #define PERR_DSEQ(n) perr.perr_dests[n].dest_seq #define PERR_DEXTADDR(n) perr.perr_dests[n].dest_ext_addr #define PERR_DRCODE(n) perr.perr_dests[n].dest_rcode static void hwmp_senderror(struct ieee80211vap *vap, const uint8_t addr[IEEE80211_ADDR_LEN], struct ieee80211_mesh_route *rt, int rcode) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_hwmp_route *hr = NULL; struct ieee80211_meshperr_ie perr; if (rt != NULL) hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); perr.perr_ndests = 1; perr.perr_ttl = ms->ms_ttl; PERR_DFLAGS(0) = 0; PERR_DRCODE(0) = rcode; switch (rcode) { case IEEE80211_REASON_MESH_PERR_NO_FI: IEEE80211_ADDR_COPY(PERR_DADDR(0), addr); PERR_DSEQ(0) = 0; /* reserved */ break; case IEEE80211_REASON_MESH_PERR_NO_PROXY: KASSERT(rt != NULL, ("no proxy info for sending PERR")); KASSERT(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY, ("route is not marked proxy")); PERR_DFLAGS(0) |= IEEE80211_MESHPERR_FLAGS_AE; IEEE80211_ADDR_COPY(PERR_DADDR(0), vap->iv_myaddr); PERR_DSEQ(0) = rt->rt_ext_seq; IEEE80211_ADDR_COPY(PERR_DEXTADDR(0), addr); break; case IEEE80211_REASON_MESH_PERR_DEST_UNREACH: KASSERT(rt != NULL, ("no route info for sending PERR")); IEEE80211_ADDR_COPY(PERR_DADDR(0), addr); PERR_DSEQ(0) = hr->hr_seq; break; default: KASSERT(0, ("unknown reason code for HWMP PERR (%u)", rcode)); } hwmp_send_perr(vap, broadcastaddr, &perr); } #undef PERR_DFLAGS #undef PEER_DADDR #undef PERR_DSEQ #undef PERR_DEXTADDR #undef PERR_DRCODE static void hwmp_recv_rann(struct ieee80211vap *vap, struct ieee80211_node *ni, const struct ieee80211_frame *wh, const struct ieee80211_meshrann_ie *rann) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_route *rt = NULL; struct ieee80211_hwmp_route *hr; struct ieee80211_meshpreq_ie preq; struct ieee80211_meshrann_ie prann; if (IEEE80211_ADDR_EQ(rann->rann_addr, vap->iv_myaddr)) return; rt = ieee80211_mesh_rt_find(vap, rann->rann_addr); if (rt != NULL && rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) { hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); /* Acceptance criteria: if RANN.seq < stored seq, discard RANN */ if (HWMP_SEQ_LT(rann->rann_seq, hr->hr_seq)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "RANN seq %u < %u", rann->rann_seq, hr->hr_seq); return; } /* Acceptance criteria: if RANN.seq == stored seq AND * RANN.metric > stored metric, discard RANN */ if (HWMP_SEQ_EQ(rann->rann_seq, hr->hr_seq) && rann->rann_metric > rt->rt_metric) { IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "RANN metric %u > %u", rann->rann_metric, rt->rt_metric); return; } } /* RANN ACCEPTED */ ieee80211_hwmp_rannint = rann->rann_interval; /* XXX: mtx lock? */ if (rt == NULL) { rt = ieee80211_mesh_rt_add(vap, rann->rann_addr); if (rt == NULL) { IEEE80211_DISCARD(vap, IEEE80211_MSG_HWMP, wh, NULL, "unable to add mac for RANN root %6D", rann->rann_addr, ":"); vap->iv_stats.is_mesh_rtaddfailed++; return; } } hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); /* Check if root is a mesh gate, mark it */ if (rann->rann_flags & IEEE80211_MESHRANN_FLAGS_GATE) { struct ieee80211_mesh_gate_route *gr; rt->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE; gr = ieee80211_mesh_mark_gate(vap, rann->rann_addr, rt); gr->gr_lastseq = 0; /* NOT GANN */ } /* discovery timeout */ ieee80211_mesh_rt_update(rt, ticks_to_msecs(ieee80211_hwmp_roottimeout)); preq.preq_flags = IEEE80211_MESHPREQ_FLAGS_AM; preq.preq_hopcount = 0; preq.preq_ttl = ms->ms_ttl; preq.preq_id = 0; /* reserved */ IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr); preq.preq_origseq = ++hs->hs_seq; preq.preq_lifetime = ieee80211_hwmp_roottimeout; preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL; preq.preq_tcount = 1; preq.preq_targets[0].target_flags = IEEE80211_MESHPREQ_TFLAGS_TO; /* NB: IEEE80211_MESHPREQ_TFLAGS_USN = 0 implicitly implied */ IEEE80211_ADDR_COPY(preq.preq_targets[0].target_addr, rann->rann_addr); preq.preq_targets[0].target_seq = rann->rann_seq; /* XXX: if rootconfint have not passed, we built this preq in vain */ hwmp_send_preq(vap, wh->i_addr2, &preq, &hr->hr_lastrootconf, &ieee80211_hwmp_rootconfint); /* propagate a RANN */ if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID && rann->rann_ttl > 1 && ms->ms_flags & IEEE80211_MESHFLAGS_FWD) { hr->hr_seq = rann->rann_seq; memcpy(&prann, rann, sizeof(prann)); prann.rann_hopcount += 1; prann.rann_ttl -= 1; prann.rann_metric += ms->ms_pmetric->mpm_metric(ni); hwmp_send_rann(vap, broadcastaddr, &prann); } } static int hwmp_send_rann(struct ieee80211vap *vap, const uint8_t da[IEEE80211_ADDR_LEN], struct ieee80211_meshrann_ie *rann) { /* * mesh rann action frame format * [6] da * [6] sa * [6] addr3 = sa * [1] action * [1] category * [tlv] root annoucement */ rann->rann_ie = IEEE80211_ELEMID_MESHRANN; rann->rann_len = IEEE80211_MESHRANN_BASE_SZ; return hwmp_send_action(vap, da, (uint8_t *)rann, rann->rann_len + 2); } #define PREQ_TFLAGS(n) preq.preq_targets[n].target_flags #define PREQ_TADDR(n) preq.preq_targets[n].target_addr #define PREQ_TSEQ(n) preq.preq_targets[n].target_seq static void hwmp_rediscover_cb(void *arg) { struct ieee80211_mesh_route *rt = arg; struct ieee80211vap *vap = rt->rt_vap; struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_hwmp_route *hr; struct ieee80211_meshpreq_ie preq; /* Optimize: storing first preq? */ if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID)) return ; /* nothing to do */ hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); if (hr->hr_preqretries >= ieee80211_hwmp_maxpreq_retries) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_ANY, rt->rt_dest, "%s", "max number of discovery, send queued frames to GATE"); ieee80211_mesh_forward_to_gates(vap, rt); vap->iv_stats.is_mesh_fwd_nopath++; return ; /* XXX: flush queue? */ } hr->hr_preqretries++; IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, rt->rt_dest, "start path rediscovery , target seq %u", hr->hr_seq); /* * Try to discover the path for this node. * Group addressed PREQ Case A */ preq.preq_flags = 0; preq.preq_hopcount = 0; preq.preq_ttl = ms->ms_ttl; preq.preq_id = ++hs->hs_preqid; IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr); preq.preq_origseq = hr->hr_origseq; preq.preq_lifetime = ticks_to_msecs(ieee80211_hwmp_pathtimeout); preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL; preq.preq_tcount = 1; IEEE80211_ADDR_COPY(PREQ_TADDR(0), rt->rt_dest); PREQ_TFLAGS(0) = 0; if (ieee80211_hwmp_targetonly) PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_TO; PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_USN; PREQ_TSEQ(0) = 0; /* RESERVED when USN flag is set */ /* XXX check return value */ hwmp_send_preq(vap, broadcastaddr, &preq, &hr->hr_lastpreq, &ieee80211_hwmp_preqminint); callout_reset(&rt->rt_discovery, ieee80211_hwmp_net_diameter_traversaltime * 2, hwmp_rediscover_cb, rt); } static struct ieee80211_node * hwmp_discover(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN], struct mbuf *m) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt = NULL; struct ieee80211_hwmp_route *hr; struct ieee80211_meshpreq_ie preq; struct ieee80211_node *ni; int sendpreq = 0; KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a mesh vap, opmode %d", vap->iv_opmode)); KASSERT(!IEEE80211_ADDR_EQ(vap->iv_myaddr, dest), ("%s: discovering self!", __func__)); ni = NULL; if (!IEEE80211_IS_MULTICAST(dest)) { rt = ieee80211_mesh_rt_find(vap, dest); if (rt == NULL) { rt = ieee80211_mesh_rt_add(vap, dest); if (rt == NULL) { IEEE80211_NOTE(vap, IEEE80211_MSG_HWMP, ni, "unable to add discovery path to %6D", dest, ":"); vap->iv_stats.is_mesh_rtaddfailed++; goto done; } } hr = IEEE80211_MESH_ROUTE_PRIV(rt, struct ieee80211_hwmp_route); if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest, "%s", "already discovering queue frame until path found"); sendpreq = 1; goto done; } if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) { if (hr->hr_lastdiscovery != 0 && (ticks - hr->hr_lastdiscovery < (ieee80211_hwmp_net_diameter_traversaltime * 2))) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, dest, NULL, "%s", "too frequent discovery requeust"); sendpreq = 1; goto done; } hr->hr_lastdiscovery = ticks; if (hr->hr_preqretries >= ieee80211_hwmp_maxpreq_retries) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, dest, NULL, "%s", "no valid path , max number of discovery"); vap->iv_stats.is_mesh_fwd_nopath++; goto done; } rt->rt_flags = IEEE80211_MESHRT_FLAGS_DISCOVER; hr->hr_preqretries++; if (hr->hr_origseq == 0) hr->hr_origseq = ++hs->hs_seq; rt->rt_metric = IEEE80211_MESHLMETRIC_INITIALVAL; sendpreq = 1; IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest, "start path discovery (src %s), target seq %u", m == NULL ? "" : ether_sprintf( mtod(m, struct ether_header *)->ether_shost), hr->hr_seq); /* * Try to discover the path for this node. * Group addressed PREQ Case A */ preq.preq_flags = 0; preq.preq_hopcount = 0; preq.preq_ttl = ms->ms_ttl; preq.preq_id = ++hs->hs_preqid; IEEE80211_ADDR_COPY(preq.preq_origaddr, vap->iv_myaddr); preq.preq_origseq = hr->hr_origseq; preq.preq_lifetime = ticks_to_msecs(ieee80211_hwmp_pathtimeout); preq.preq_metric = IEEE80211_MESHLMETRIC_INITIALVAL; preq.preq_tcount = 1; IEEE80211_ADDR_COPY(PREQ_TADDR(0), dest); PREQ_TFLAGS(0) = 0; if (ieee80211_hwmp_targetonly) PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_TO; PREQ_TFLAGS(0) |= IEEE80211_MESHPREQ_TFLAGS_USN; PREQ_TSEQ(0) = 0; /* RESERVED when USN flag is set */ /* XXX check return value */ hwmp_send_preq(vap, broadcastaddr, &preq, &hr->hr_lastpreq, &ieee80211_hwmp_preqminint); callout_reset(&rt->rt_discovery, ieee80211_hwmp_net_diameter_traversaltime * 2, hwmp_rediscover_cb, rt); } if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) ni = ieee80211_find_txnode(vap, rt->rt_nexthop); } else { ni = ieee80211_find_txnode(vap, dest); /* NB: if null then we leak mbuf */ KASSERT(ni != NULL, ("leak mcast frame")); return ni; } done: if (ni == NULL && m != NULL) { if (sendpreq) { struct ieee80211com *ic = vap->iv_ic; /* * Queue packet for transmit when path discovery * completes. If discovery never completes the * frame will be flushed by way of the aging timer. */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, dest, "%s", "queue frame until path found"); MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); m->m_pkthdr.rcvif = (void *)(uintptr_t) ieee80211_mac_hash(ic, dest); /* XXX age chosen randomly */ ieee80211_ageq_append(&ic->ic_stageq, m, IEEE80211_INACT_WAIT); } else { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_HWMP, dest, NULL, "%s", "no valid path to this node"); m_freem(m); } } return ni; } #undef PREQ_TFLAGS #undef PREQ_TADDR #undef PREQ_TSEQ static int hwmp_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; int error; if (vap->iv_opmode != IEEE80211_M_MBSS) return ENOSYS; error = 0; switch (ireq->i_type) { case IEEE80211_IOC_HWMP_ROOTMODE: ireq->i_val = hs->hs_rootmode; break; case IEEE80211_IOC_HWMP_MAXHOPS: ireq->i_val = hs->hs_maxhops; break; default: return ENOSYS; } return error; } IEEE80211_IOCTL_GET(hwmp, hwmp_ioctl_get80211); static int hwmp_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq) { struct ieee80211_hwmp_state *hs = vap->iv_hwmp; int error; if (vap->iv_opmode != IEEE80211_M_MBSS) return ENOSYS; error = 0; switch (ireq->i_type) { case IEEE80211_IOC_HWMP_ROOTMODE: if (ireq->i_val < 0 || ireq->i_val > 3) return EINVAL; hs->hs_rootmode = ireq->i_val; hwmp_rootmode_setup(vap); break; case IEEE80211_IOC_HWMP_MAXHOPS: if (ireq->i_val <= 0 || ireq->i_val > 255) return EINVAL; hs->hs_maxhops = ireq->i_val; break; default: return ENOSYS; } return error; } IEEE80211_IOCTL_SET(hwmp, hwmp_ioctl_set80211); diff --git a/sys/net80211/ieee80211_mesh.c b/sys/net80211/ieee80211_mesh.c index 48a3590d0cf3..7da344707a9b 100644 --- a/sys/net80211/ieee80211_mesh.c +++ b/sys/net80211/ieee80211_mesh.c @@ -1,3617 +1,3616 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Rui Paulo under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifdef __FreeBSD__ __FBSDID("$FreeBSD$"); #endif /* * IEEE 802.11s Mesh Point (MBSS) support. * * Based on March 2009, D3.0 802.11s draft spec. */ #include "opt_inet.h" #include "opt_wlan.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef IEEE80211_SUPPORT_SUPERG #include #endif #include #include static void mesh_rt_flush_invalid(struct ieee80211vap *); static int mesh_select_proto_path(struct ieee80211vap *, const char *); static int mesh_select_proto_metric(struct ieee80211vap *, const char *); static void mesh_vattach(struct ieee80211vap *); static int mesh_newstate(struct ieee80211vap *, enum ieee80211_state, int); static void mesh_rt_cleanup_cb(void *); static void mesh_gatemode_setup(struct ieee80211vap *); static void mesh_gatemode_cb(void *); static void mesh_linkchange(struct ieee80211_node *, enum ieee80211_mesh_mlstate); static void mesh_checkid(void *, struct ieee80211_node *); static uint32_t mesh_generateid(struct ieee80211vap *); static int mesh_checkpseq(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], uint32_t); static void mesh_transmit_to_gate(struct ieee80211vap *, struct mbuf *, struct ieee80211_mesh_route *); static void mesh_forward(struct ieee80211vap *, struct mbuf *, const struct ieee80211_meshcntl *); static int mesh_input(struct ieee80211_node *, struct mbuf *, const struct ieee80211_rx_stats *rxs, int, int); static void mesh_recv_mgmt(struct ieee80211_node *, struct mbuf *, int, const struct ieee80211_rx_stats *rxs, int, int); static void mesh_recv_ctl(struct ieee80211_node *, struct mbuf *, int); static void mesh_peer_timeout_setup(struct ieee80211_node *); static void mesh_peer_timeout_backoff(struct ieee80211_node *); static void mesh_peer_timeout_cb(void *); static __inline void mesh_peer_timeout_stop(struct ieee80211_node *); static int mesh_verify_meshid(struct ieee80211vap *, const uint8_t *); static int mesh_verify_meshconf(struct ieee80211vap *, const uint8_t *); static int mesh_verify_meshpeer(struct ieee80211vap *, uint8_t, const uint8_t *); uint32_t mesh_airtime_calc(struct ieee80211_node *); /* * Timeout values come from the specification and are in milliseconds. */ static SYSCTL_NODE(_net_wlan, OID_AUTO, mesh, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "IEEE 802.11s parameters"); static int ieee80211_mesh_gateint = -1; SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, gateint, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &ieee80211_mesh_gateint, 0, ieee80211_sysctl_msecs_ticks, "I", "mesh gate interval (ms)"); static int ieee80211_mesh_retrytimeout = -1; SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, retrytimeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &ieee80211_mesh_retrytimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "Retry timeout (msec)"); static int ieee80211_mesh_holdingtimeout = -1; SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, holdingtimeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &ieee80211_mesh_holdingtimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "Holding state timeout (msec)"); static int ieee80211_mesh_confirmtimeout = -1; SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, confirmtimeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &ieee80211_mesh_confirmtimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "Confirm state timeout (msec)"); static int ieee80211_mesh_backofftimeout = -1; SYSCTL_PROC(_net_wlan_mesh, OID_AUTO, backofftimeout, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &ieee80211_mesh_backofftimeout, 0, ieee80211_sysctl_msecs_ticks, "I", "Backoff timeout (msec). This is to throutles peering forever when " "not receiving answer or is rejected by a neighbor"); static int ieee80211_mesh_maxretries = 2; SYSCTL_INT(_net_wlan_mesh, OID_AUTO, maxretries, CTLFLAG_RW, &ieee80211_mesh_maxretries, 0, "Maximum retries during peer link establishment"); static int ieee80211_mesh_maxholding = 2; SYSCTL_INT(_net_wlan_mesh, OID_AUTO, maxholding, CTLFLAG_RW, &ieee80211_mesh_maxholding, 0, "Maximum times we are allowed to transition to HOLDING state before " "backinoff during peer link establishment"); static const uint8_t broadcastaddr[IEEE80211_ADDR_LEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; static ieee80211_recv_action_func mesh_recv_action_meshpeering_open; static ieee80211_recv_action_func mesh_recv_action_meshpeering_confirm; static ieee80211_recv_action_func mesh_recv_action_meshpeering_close; static ieee80211_recv_action_func mesh_recv_action_meshlmetric; static ieee80211_recv_action_func mesh_recv_action_meshgate; static ieee80211_send_action_func mesh_send_action_meshpeering_open; static ieee80211_send_action_func mesh_send_action_meshpeering_confirm; static ieee80211_send_action_func mesh_send_action_meshpeering_close; static ieee80211_send_action_func mesh_send_action_meshlmetric; static ieee80211_send_action_func mesh_send_action_meshgate; static const struct ieee80211_mesh_proto_metric mesh_metric_airtime = { .mpm_descr = "AIRTIME", .mpm_ie = IEEE80211_MESHCONF_METRIC_AIRTIME, .mpm_metric = mesh_airtime_calc, }; static struct ieee80211_mesh_proto_path mesh_proto_paths[4]; static struct ieee80211_mesh_proto_metric mesh_proto_metrics[4]; MALLOC_DEFINE(M_80211_MESH_PREQ, "80211preq", "802.11 MESH Path Request frame"); MALLOC_DEFINE(M_80211_MESH_PREP, "80211prep", "802.11 MESH Path Reply frame"); MALLOC_DEFINE(M_80211_MESH_PERR, "80211perr", "802.11 MESH Path Error frame"); /* The longer one of the lifetime should be stored as new lifetime */ #define MESH_ROUTE_LIFETIME_MAX(a, b) (a > b ? a : b) MALLOC_DEFINE(M_80211_MESH_RT, "80211mesh_rt", "802.11s routing table"); MALLOC_DEFINE(M_80211_MESH_GT_RT, "80211mesh_gt", "802.11s known gates table"); /* * Helper functions to manipulate the Mesh routing table. */ static struct ieee80211_mesh_route * mesh_rt_find_locked(struct ieee80211_mesh_state *ms, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_route *rt; MESH_RT_LOCK_ASSERT(ms); TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) { if (IEEE80211_ADDR_EQ(dest, rt->rt_dest)) return rt; } return NULL; } static struct ieee80211_mesh_route * mesh_rt_add_locked(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt; KASSERT(!IEEE80211_ADDR_EQ(broadcastaddr, dest), ("%s: adding broadcast to the routing table", __func__)); MESH_RT_LOCK_ASSERT(ms); rt = IEEE80211_MALLOC(ALIGN(sizeof(struct ieee80211_mesh_route)) + ms->ms_ppath->mpp_privlen, M_80211_MESH_RT, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); if (rt != NULL) { rt->rt_vap = vap; IEEE80211_ADDR_COPY(rt->rt_dest, dest); rt->rt_priv = (void *)ALIGN(&rt[1]); MESH_RT_ENTRY_LOCK_INIT(rt, "MBSS_RT"); callout_init(&rt->rt_discovery, 1); rt->rt_updtime = ticks; /* create time */ TAILQ_INSERT_TAIL(&ms->ms_routes, rt, rt_next); } return rt; } struct ieee80211_mesh_route * ieee80211_mesh_rt_find(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt; MESH_RT_LOCK(ms); rt = mesh_rt_find_locked(ms, dest); MESH_RT_UNLOCK(ms); return rt; } struct ieee80211_mesh_route * ieee80211_mesh_rt_add(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt; KASSERT(ieee80211_mesh_rt_find(vap, dest) == NULL, ("%s: duplicate entry in the routing table", __func__)); KASSERT(!IEEE80211_ADDR_EQ(vap->iv_myaddr, dest), ("%s: adding self to the routing table", __func__)); MESH_RT_LOCK(ms); rt = mesh_rt_add_locked(vap, dest); MESH_RT_UNLOCK(ms); return rt; } /* * Update the route lifetime and returns the updated lifetime. * If new_lifetime is zero and route is timedout it will be invalidated. * new_lifetime is in msec */ int ieee80211_mesh_rt_update(struct ieee80211_mesh_route *rt, int new_lifetime) { int timesince, now; uint32_t lifetime = 0; KASSERT(rt != NULL, ("route is NULL")); now = ticks; MESH_RT_ENTRY_LOCK(rt); /* dont clobber a proxy entry gated by us */ if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY && rt->rt_nhops == 0) { MESH_RT_ENTRY_UNLOCK(rt); return rt->rt_lifetime; } timesince = ticks_to_msecs(now - rt->rt_updtime); rt->rt_updtime = now; if (timesince >= rt->rt_lifetime) { if (new_lifetime != 0) { rt->rt_lifetime = new_lifetime; } else { rt->rt_flags &= ~IEEE80211_MESHRT_FLAGS_VALID; rt->rt_lifetime = 0; } } else { /* update what is left of lifetime */ rt->rt_lifetime = rt->rt_lifetime - timesince; rt->rt_lifetime = MESH_ROUTE_LIFETIME_MAX( new_lifetime, rt->rt_lifetime); } lifetime = rt->rt_lifetime; MESH_RT_ENTRY_UNLOCK(rt); return lifetime; } /* * Add a proxy route (as needed) for the specified destination. */ void ieee80211_mesh_proxy_check(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt; MESH_RT_LOCK(ms); rt = mesh_rt_find_locked(ms, dest); if (rt == NULL) { rt = mesh_rt_add_locked(vap, dest); if (rt == NULL) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest, "%s", "unable to add proxy entry"); vap->iv_stats.is_mesh_rtaddfailed++; } else { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest, "%s", "add proxy entry"); IEEE80211_ADDR_COPY(rt->rt_mesh_gate, vap->iv_myaddr); IEEE80211_ADDR_COPY(rt->rt_nexthop, vap->iv_myaddr); rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID | IEEE80211_MESHRT_FLAGS_PROXY; } } else if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) { KASSERT(rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY, ("no proxy flag for poxy entry")); struct ieee80211com *ic = vap->iv_ic; /* * Fix existing entry created by received frames from * stations that have some memory of dest. We also * flush any frames held on the staging queue; delivering * them is too much trouble right now. */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest, "%s", "fix proxy entry"); IEEE80211_ADDR_COPY(rt->rt_nexthop, vap->iv_myaddr); rt->rt_flags |= IEEE80211_MESHRT_FLAGS_VALID | IEEE80211_MESHRT_FLAGS_PROXY; /* XXX belongs in hwmp */ ieee80211_ageq_drain_node(&ic->ic_stageq, (void *)(uintptr_t) ieee80211_mac_hash(ic, dest)); /* XXX stat? */ } MESH_RT_UNLOCK(ms); } static __inline void mesh_rt_del(struct ieee80211_mesh_state *ms, struct ieee80211_mesh_route *rt) { TAILQ_REMOVE(&ms->ms_routes, rt, rt_next); /* * Grab the lock before destroying it, to be sure no one else * is holding the route. */ MESH_RT_ENTRY_LOCK(rt); callout_drain(&rt->rt_discovery); MESH_RT_ENTRY_LOCK_DESTROY(rt); IEEE80211_FREE(rt, M_80211_MESH_RT); } void ieee80211_mesh_rt_del(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt, *next; MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) { if (IEEE80211_ADDR_EQ(rt->rt_dest, dest)) { if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) { ms->ms_ppath->mpp_senderror(vap, dest, rt, IEEE80211_REASON_MESH_PERR_NO_PROXY); } else { ms->ms_ppath->mpp_senderror(vap, dest, rt, IEEE80211_REASON_MESH_PERR_DEST_UNREACH); } mesh_rt_del(ms, rt); MESH_RT_UNLOCK(ms); return; } } MESH_RT_UNLOCK(ms); } void ieee80211_mesh_rt_flush(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt, *next; if (ms == NULL) return; MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) mesh_rt_del(ms, rt); MESH_RT_UNLOCK(ms); } void ieee80211_mesh_rt_flush_peer(struct ieee80211vap *vap, const uint8_t peer[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt, *next; MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) { if (IEEE80211_ADDR_EQ(rt->rt_nexthop, peer)) mesh_rt_del(ms, rt); } MESH_RT_UNLOCK(ms); } /* * Flush expired routing entries, i.e. those in invalid state for * some time. */ static void mesh_rt_flush_invalid(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt, *next; if (ms == NULL) return; MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(rt, &ms->ms_routes, rt_next, next) { /* Discover paths will be deleted by their own callout */ if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_DISCOVER) continue; ieee80211_mesh_rt_update(rt, 0); if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) mesh_rt_del(ms, rt); } MESH_RT_UNLOCK(ms); } int ieee80211_mesh_register_proto_path(const struct ieee80211_mesh_proto_path *mpp) { int i, firstempty = -1; for (i = 0; i < nitems(mesh_proto_paths); i++) { if (strncmp(mpp->mpp_descr, mesh_proto_paths[i].mpp_descr, IEEE80211_MESH_PROTO_DSZ) == 0) return EEXIST; if (!mesh_proto_paths[i].mpp_active && firstempty == -1) firstempty = i; } if (firstempty < 0) return ENOSPC; memcpy(&mesh_proto_paths[firstempty], mpp, sizeof(*mpp)); mesh_proto_paths[firstempty].mpp_active = 1; return 0; } int ieee80211_mesh_register_proto_metric(const struct ieee80211_mesh_proto_metric *mpm) { int i, firstempty = -1; for (i = 0; i < nitems(mesh_proto_metrics); i++) { if (strncmp(mpm->mpm_descr, mesh_proto_metrics[i].mpm_descr, IEEE80211_MESH_PROTO_DSZ) == 0) return EEXIST; if (!mesh_proto_metrics[i].mpm_active && firstempty == -1) firstempty = i; } if (firstempty < 0) return ENOSPC; memcpy(&mesh_proto_metrics[firstempty], mpm, sizeof(*mpm)); mesh_proto_metrics[firstempty].mpm_active = 1; return 0; } static int mesh_select_proto_path(struct ieee80211vap *vap, const char *name) { struct ieee80211_mesh_state *ms = vap->iv_mesh; int i; for (i = 0; i < nitems(mesh_proto_paths); i++) { if (strcasecmp(mesh_proto_paths[i].mpp_descr, name) == 0) { ms->ms_ppath = &mesh_proto_paths[i]; return 0; } } return ENOENT; } static int mesh_select_proto_metric(struct ieee80211vap *vap, const char *name) { struct ieee80211_mesh_state *ms = vap->iv_mesh; int i; for (i = 0; i < nitems(mesh_proto_metrics); i++) { if (strcasecmp(mesh_proto_metrics[i].mpm_descr, name) == 0) { ms->ms_pmetric = &mesh_proto_metrics[i]; return 0; } } return ENOENT; } static void mesh_gatemode_setup(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; /* * NB: When a mesh gate is running as a ROOT it shall * not send out periodic GANNs but instead mark the * mesh gate flag for the corresponding proactive PREQ * and RANN frames. */ if (ms->ms_flags & IEEE80211_MESHFLAGS_ROOT || (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) == 0) { callout_drain(&ms->ms_gatetimer); return ; } callout_reset(&ms->ms_gatetimer, ieee80211_mesh_gateint, mesh_gatemode_cb, vap); } static void mesh_gatemode_cb(void *arg) { struct ieee80211vap *vap = (struct ieee80211vap *)arg; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_meshgann_ie gann; gann.gann_flags = 0; /* Reserved */ gann.gann_hopcount = 0; gann.gann_ttl = ms->ms_ttl; IEEE80211_ADDR_COPY(gann.gann_addr, vap->iv_myaddr); gann.gann_seq = ms->ms_gateseq++; gann.gann_interval = ieee80211_mesh_gateint; IEEE80211_NOTE(vap, IEEE80211_MSG_MESH, vap->iv_bss, "send broadcast GANN (seq %u)", gann.gann_seq); ieee80211_send_action(vap->iv_bss, IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_GANN, &gann); mesh_gatemode_setup(vap); } static void ieee80211_mesh_init(void) { memset(mesh_proto_paths, 0, sizeof(mesh_proto_paths)); memset(mesh_proto_metrics, 0, sizeof(mesh_proto_metrics)); /* * Setup mesh parameters that depends on the clock frequency. */ ieee80211_mesh_gateint = msecs_to_ticks(10000); ieee80211_mesh_retrytimeout = msecs_to_ticks(40); ieee80211_mesh_holdingtimeout = msecs_to_ticks(40); ieee80211_mesh_confirmtimeout = msecs_to_ticks(40); ieee80211_mesh_backofftimeout = msecs_to_ticks(5000); /* * Register action frame handlers. */ ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_OPEN, mesh_recv_action_meshpeering_open); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, mesh_recv_action_meshpeering_confirm); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, mesh_recv_action_meshpeering_close); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_LMETRIC, mesh_recv_action_meshlmetric); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_GANN, mesh_recv_action_meshgate); ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_OPEN, mesh_send_action_meshpeering_open); ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, mesh_send_action_meshpeering_confirm); ieee80211_send_action_register(IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, mesh_send_action_meshpeering_close); ieee80211_send_action_register(IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_LMETRIC, mesh_send_action_meshlmetric); ieee80211_send_action_register(IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_GANN, mesh_send_action_meshgate); /* * Register Airtime Link Metric. */ ieee80211_mesh_register_proto_metric(&mesh_metric_airtime); } SYSINIT(wlan_mesh, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_mesh_init, NULL); void ieee80211_mesh_attach(struct ieee80211com *ic) { ic->ic_vattach[IEEE80211_M_MBSS] = mesh_vattach; } void ieee80211_mesh_detach(struct ieee80211com *ic) { } static void mesh_vdetach_peers(void *arg, struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; uint16_t args[3]; if (ni->ni_mlstate == IEEE80211_NODE_MESH_ESTABLISHED) { args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); } callout_drain(&ni->ni_mltimer); /* XXX belongs in hwmp */ ieee80211_ageq_drain_node(&ic->ic_stageq, (void *)(uintptr_t) ieee80211_mac_hash(ic, ni->ni_macaddr)); } static void mesh_vdetach(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; callout_drain(&ms->ms_cleantimer); ieee80211_iterate_nodes(&vap->iv_ic->ic_sta, mesh_vdetach_peers, NULL); ieee80211_mesh_rt_flush(vap); MESH_RT_LOCK_DESTROY(ms); ms->ms_ppath->mpp_vdetach(vap); IEEE80211_FREE(vap->iv_mesh, M_80211_VAP); vap->iv_mesh = NULL; } static void mesh_vattach(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms; vap->iv_newstate = mesh_newstate; vap->iv_input = mesh_input; vap->iv_opdetach = mesh_vdetach; vap->iv_recv_mgmt = mesh_recv_mgmt; vap->iv_recv_ctl = mesh_recv_ctl; ms = IEEE80211_MALLOC(sizeof(struct ieee80211_mesh_state), M_80211_VAP, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); if (ms == NULL) { printf("%s: couldn't alloc MBSS state\n", __func__); return; } vap->iv_mesh = ms; ms->ms_seq = 0; ms->ms_flags = (IEEE80211_MESHFLAGS_AP | IEEE80211_MESHFLAGS_FWD); ms->ms_ttl = IEEE80211_MESH_DEFAULT_TTL; TAILQ_INIT(&ms->ms_known_gates); TAILQ_INIT(&ms->ms_routes); MESH_RT_LOCK_INIT(ms, "MBSS"); callout_init(&ms->ms_cleantimer, 1); callout_init(&ms->ms_gatetimer, 1); ms->ms_gateseq = 0; mesh_select_proto_metric(vap, "AIRTIME"); KASSERT(ms->ms_pmetric, ("ms_pmetric == NULL")); mesh_select_proto_path(vap, "HWMP"); KASSERT(ms->ms_ppath, ("ms_ppath == NULL")); ms->ms_ppath->mpp_vattach(vap); } /* * IEEE80211_M_MBSS vap state machine handler. */ static int mesh_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211com *ic = vap->iv_ic; struct ieee80211_node *ni; enum ieee80211_state ostate; IEEE80211_LOCK_ASSERT(ic); ostate = vap->iv_state; IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE, "%s: %s -> %s (%d)\n", __func__, ieee80211_state_name[ostate], ieee80211_state_name[nstate], arg); vap->iv_state = nstate; /* state transition */ if (ostate != IEEE80211_S_SCAN) ieee80211_cancel_scan(vap); /* background scan */ ni = vap->iv_bss; /* NB: no reference held */ if (nstate != IEEE80211_S_RUN && ostate == IEEE80211_S_RUN) { callout_drain(&ms->ms_cleantimer); callout_drain(&ms->ms_gatetimer); } switch (nstate) { case IEEE80211_S_INIT: switch (ostate) { case IEEE80211_S_SCAN: ieee80211_cancel_scan(vap); break; case IEEE80211_S_CAC: ieee80211_dfs_cac_stop(vap); break; case IEEE80211_S_RUN: ieee80211_iterate_nodes(&ic->ic_sta, mesh_vdetach_peers, NULL); break; default: break; } if (ostate != IEEE80211_S_INIT) { /* NB: optimize INIT -> INIT case */ ieee80211_reset_bss(vap); ieee80211_mesh_rt_flush(vap); } break; case IEEE80211_S_SCAN: switch (ostate) { case IEEE80211_S_INIT: if (vap->iv_des_chan != IEEE80211_CHAN_ANYC && !IEEE80211_IS_CHAN_RADAR(vap->iv_des_chan) && ms->ms_idlen != 0) { /* * Already have a channel and a mesh ID; bypass * the scan and startup immediately. */ ieee80211_create_ibss(vap, vap->iv_des_chan); break; } /* * Initiate a scan. We can come here as a result * of an IEEE80211_IOC_SCAN_REQ too in which case * the vap will be marked with IEEE80211_FEXT_SCANREQ * and the scan request parameters will be present * in iv_scanreq. Otherwise we do the default. */ if (vap->iv_flags_ext & IEEE80211_FEXT_SCANREQ) { ieee80211_check_scan(vap, vap->iv_scanreq_flags, vap->iv_scanreq_duration, vap->iv_scanreq_mindwell, vap->iv_scanreq_maxdwell, vap->iv_scanreq_nssid, vap->iv_scanreq_ssid); vap->iv_flags_ext &= ~IEEE80211_FEXT_SCANREQ; } else ieee80211_check_scan_current(vap); break; default: break; } break; case IEEE80211_S_CAC: /* * Start CAC on a DFS channel. We come here when starting * a bss on a DFS channel (see ieee80211_create_ibss). */ ieee80211_dfs_cac_start(vap); break; case IEEE80211_S_RUN: switch (ostate) { case IEEE80211_S_INIT: /* * Already have a channel; bypass the * scan and startup immediately. * Note that ieee80211_create_ibss will call * back to do a RUN->RUN state change. */ ieee80211_create_ibss(vap, ieee80211_ht_adjust_channel(ic, ic->ic_curchan, vap->iv_flags_ht)); /* NB: iv_bss is changed on return */ break; case IEEE80211_S_CAC: /* * NB: This is the normal state change when CAC * expires and no radar was detected; no need to * clear the CAC timer as it's already expired. */ /* fall thru... */ case IEEE80211_S_CSA: #if 0 /* * Shorten inactivity timer of associated stations * to weed out sta's that don't follow a CSA. */ ieee80211_iterate_nodes(&ic->ic_sta, sta_csa, vap); #endif /* * Update bss node channel to reflect where * we landed after CSA. */ ieee80211_node_set_chan(ni, ieee80211_ht_adjust_channel(ic, ic->ic_curchan, ieee80211_htchanflags(ni->ni_chan))); /* XXX bypass debug msgs */ break; case IEEE80211_S_SCAN: case IEEE80211_S_RUN: #ifdef IEEE80211_DEBUG if (ieee80211_msg_debug(vap)) { ieee80211_note(vap, "synchronized with %s meshid ", ether_sprintf(ni->ni_meshid)); ieee80211_print_essid(ni->ni_meshid, ni->ni_meshidlen); /* XXX MCS/HT */ printf(" channel %d\n", ieee80211_chan2ieee(ic, ic->ic_curchan)); } #endif break; default: break; } ieee80211_node_authorize(ni); callout_reset(&ms->ms_cleantimer, ms->ms_ppath->mpp_inact, mesh_rt_cleanup_cb, vap); mesh_gatemode_setup(vap); break; default: break; } /* NB: ostate not nstate */ ms->ms_ppath->mpp_newstate(vap, ostate, arg); return 0; } static void mesh_rt_cleanup_cb(void *arg) { struct ieee80211vap *vap = arg; struct ieee80211_mesh_state *ms = vap->iv_mesh; mesh_rt_flush_invalid(vap); callout_reset(&ms->ms_cleantimer, ms->ms_ppath->mpp_inact, mesh_rt_cleanup_cb, vap); } /* * Mark a mesh STA as gate and return a pointer to it. * If this is first time, we create a new gate route. * Always update the path route to this mesh gate. */ struct ieee80211_mesh_gate_route * ieee80211_mesh_mark_gate(struct ieee80211vap *vap, const uint8_t *addr, struct ieee80211_mesh_route *rt) { struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_gate_route *gr = NULL, *next; int found = 0; MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, next) { if (IEEE80211_ADDR_EQ(gr->gr_addr, addr)) { found = 1; break; } } if (!found) { /* New mesh gate add it to known table. */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, addr, "%s", "stored new gate information from pro-PREQ."); gr = IEEE80211_MALLOC(ALIGN(sizeof(struct ieee80211_mesh_gate_route)), M_80211_MESH_GT_RT, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); IEEE80211_ADDR_COPY(gr->gr_addr, addr); TAILQ_INSERT_TAIL(&ms->ms_known_gates, gr, gr_next); } gr->gr_route = rt; /* TODO: link from path route to gate route */ MESH_RT_UNLOCK(ms); return gr; } /* * Helper function to note the Mesh Peer Link FSM change. */ static void mesh_linkchange(struct ieee80211_node *ni, enum ieee80211_mesh_mlstate state) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; #ifdef IEEE80211_DEBUG static const char *meshlinkstates[] = { [IEEE80211_NODE_MESH_IDLE] = "IDLE", [IEEE80211_NODE_MESH_OPENSNT] = "OPEN SENT", [IEEE80211_NODE_MESH_OPENRCV] = "OPEN RECEIVED", [IEEE80211_NODE_MESH_CONFIRMRCV] = "CONFIRM RECEIVED", [IEEE80211_NODE_MESH_ESTABLISHED] = "ESTABLISHED", [IEEE80211_NODE_MESH_HOLDING] = "HOLDING" }; #endif IEEE80211_NOTE(vap, IEEE80211_MSG_MESH, ni, "peer link: %s -> %s", meshlinkstates[ni->ni_mlstate], meshlinkstates[state]); /* track neighbor count */ if (state == IEEE80211_NODE_MESH_ESTABLISHED && ni->ni_mlstate != IEEE80211_NODE_MESH_ESTABLISHED) { KASSERT(ms->ms_neighbors < 65535, ("neighbor count overflow")); ms->ms_neighbors++; ieee80211_beacon_notify(vap, IEEE80211_BEACON_MESHCONF); } else if (ni->ni_mlstate == IEEE80211_NODE_MESH_ESTABLISHED && state != IEEE80211_NODE_MESH_ESTABLISHED) { KASSERT(ms->ms_neighbors > 0, ("neighbor count 0")); ms->ms_neighbors--; ieee80211_beacon_notify(vap, IEEE80211_BEACON_MESHCONF); } ni->ni_mlstate = state; switch (state) { case IEEE80211_NODE_MESH_HOLDING: ms->ms_ppath->mpp_peerdown(ni); break; case IEEE80211_NODE_MESH_ESTABLISHED: ieee80211_mesh_discover(vap, ni->ni_macaddr, NULL); break; default: break; } } /* * Helper function to generate a unique local ID required for mesh * peer establishment. */ static void mesh_checkid(void *arg, struct ieee80211_node *ni) { uint16_t *r = arg; if (*r == ni->ni_mllid) *(uint16_t *)arg = 0; } static uint32_t mesh_generateid(struct ieee80211vap *vap) { int maxiter = 4; uint16_t r; do { net80211_get_random_bytes(&r, 2); ieee80211_iterate_nodes(&vap->iv_ic->ic_sta, mesh_checkid, &r); maxiter--; } while (r == 0 && maxiter > 0); return r; } /* * Verifies if we already received this packet by checking its * sequence number. * Returns 0 if the frame is to be accepted, 1 otherwise. */ static int mesh_checkpseq(struct ieee80211vap *vap, const uint8_t source[IEEE80211_ADDR_LEN], uint32_t seq) { struct ieee80211_mesh_route *rt; rt = ieee80211_mesh_rt_find(vap, source); if (rt == NULL) { rt = ieee80211_mesh_rt_add(vap, source); if (rt == NULL) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, source, "%s", "add mcast route failed"); vap->iv_stats.is_mesh_rtaddfailed++; return 1; } IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, source, "add mcast route, mesh seqno %d", seq); rt->rt_lastmseq = seq; return 0; } if (IEEE80211_MESH_SEQ_GEQ(rt->rt_lastmseq, seq)) { return 1; } else { rt->rt_lastmseq = seq; return 0; } } /* * Iterate the routing table and locate the next hop. */ struct ieee80211_node * ieee80211_mesh_find_txnode(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN]) { struct ieee80211_mesh_route *rt; rt = ieee80211_mesh_rt_find(vap, dest); if (rt == NULL) return NULL; if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest, "%s: !valid, flags 0x%x", __func__, rt->rt_flags); /* XXX stat */ return NULL; } if (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) { rt = ieee80211_mesh_rt_find(vap, rt->rt_mesh_gate); if (rt == NULL) return NULL; if ((rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, dest, "%s: meshgate !valid, flags 0x%x", __func__, rt->rt_flags); /* XXX stat */ return NULL; } } return ieee80211_find_txnode(vap, rt->rt_nexthop); } static void mesh_transmit_to_gate(struct ieee80211vap *vap, struct mbuf *m, struct ieee80211_mesh_route *rt_gate) { struct ifnet *ifp = vap->iv_ifp; struct ieee80211_node *ni; IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic); ni = ieee80211_mesh_find_txnode(vap, rt_gate->rt_dest); if (ni == NULL) { if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); m_freem(m); return; } /* * Send through the VAP packet transmit path. * This consumes the node ref grabbed above and * the mbuf, regardless of whether there's a problem * or not. */ (void) ieee80211_vap_pkt_send_dest(vap, m, ni); } /* * Forward the queued frames to known valid mesh gates. * Assume destination to be outside the MBSS (i.e. proxy entry), * If no valid mesh gates are known silently discard queued frames. * After transmitting frames to all known valid mesh gates, this route * will be marked invalid, and a new path discovery will happen in the hopes * that (at least) one of the mesh gates have a new proxy entry for us to use. */ void ieee80211_mesh_forward_to_gates(struct ieee80211vap *vap, struct ieee80211_mesh_route *rt_dest) { struct ieee80211com *ic = vap->iv_ic; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt_gate; struct ieee80211_mesh_gate_route *gr = NULL, *gr_next; struct mbuf *m, *mcopy, *next; IEEE80211_TX_UNLOCK_ASSERT(ic); KASSERT( rt_dest->rt_flags == IEEE80211_MESHRT_FLAGS_DISCOVER, ("Route is not marked with IEEE80211_MESHRT_FLAGS_DISCOVER")); /* XXX: send to more than one valid mash gate */ MESH_RT_LOCK(ms); m = ieee80211_ageq_remove(&ic->ic_stageq, (struct ieee80211_node *)(uintptr_t) ieee80211_mac_hash(ic, rt_dest->rt_dest)); TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, gr_next) { rt_gate = gr->gr_route; if (rt_gate == NULL) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, rt_dest->rt_dest, "mesh gate with no path %6D", gr->gr_addr, ":"); continue; } if ((rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) == 0) continue; KASSERT(rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_GATE, ("route not marked as a mesh gate")); KASSERT((rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) == 0, ("found mesh gate that is also marked porxy")); /* * convert route to a proxy route gated by the current * mesh gate, this is needed so encap can built data * frame with correct address. */ rt_dest->rt_flags = IEEE80211_MESHRT_FLAGS_PROXY | IEEE80211_MESHRT_FLAGS_VALID; rt_dest->rt_ext_seq = 1; /* random value */ IEEE80211_ADDR_COPY(rt_dest->rt_mesh_gate, rt_gate->rt_dest); IEEE80211_ADDR_COPY(rt_dest->rt_nexthop, rt_gate->rt_nexthop); rt_dest->rt_metric = rt_gate->rt_metric; rt_dest->rt_nhops = rt_gate->rt_nhops; ieee80211_mesh_rt_update(rt_dest, ms->ms_ppath->mpp_inact); MESH_RT_UNLOCK(ms); /* XXX: lock?? */ mcopy = m_dup(m, M_NOWAIT); for (; mcopy != NULL; mcopy = next) { next = mcopy->m_nextpkt; mcopy->m_nextpkt = NULL; IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_HWMP, rt_dest->rt_dest, "flush queued frame %p len %d", mcopy, mcopy->m_pkthdr.len); mesh_transmit_to_gate(vap, mcopy, rt_gate); } MESH_RT_LOCK(ms); } rt_dest->rt_flags = 0; /* Mark invalid */ m_freem(m); MESH_RT_UNLOCK(ms); } /* * Forward the specified frame. * Decrement the TTL and set TA to our MAC address. */ static void mesh_forward(struct ieee80211vap *vap, struct mbuf *m, const struct ieee80211_meshcntl *mc) { struct ieee80211com *ic = vap->iv_ic; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ifnet *ifp = vap->iv_ifp; const struct ieee80211_frame *wh = mtod(m, const struct ieee80211_frame *); struct mbuf *mcopy; struct ieee80211_meshcntl *mccopy; struct ieee80211_frame *whcopy; struct ieee80211_node *ni; int err; /* This is called from the RX path - don't hold this lock */ IEEE80211_TX_UNLOCK_ASSERT(ic); /* * mesh ttl of 1 means we are the last one receiving it, * according to amendment we decrement and then check if * 0, if so we dont forward. */ if (mc->mc_ttl < 1) { IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh, "%s", "frame not fwd'd, ttl 1"); vap->iv_stats.is_mesh_fwd_ttl++; return; } if (!(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)) { IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh, "%s", "frame not fwd'd, fwding disabled"); vap->iv_stats.is_mesh_fwd_disabled++; return; } mcopy = m_dup(m, M_NOWAIT); if (mcopy == NULL) { IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh, "%s", "frame not fwd'd, cannot dup"); vap->iv_stats.is_mesh_fwd_nobuf++; if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); return; } mcopy = m_pullup(mcopy, ieee80211_hdrspace(ic, wh) + sizeof(struct ieee80211_meshcntl)); if (mcopy == NULL) { IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh, "%s", "frame not fwd'd, too short"); vap->iv_stats.is_mesh_fwd_tooshort++; if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); m_freem(mcopy); return; } whcopy = mtod(mcopy, struct ieee80211_frame *); mccopy = (struct ieee80211_meshcntl *) (mtod(mcopy, uint8_t *) + ieee80211_hdrspace(ic, wh)); /* XXX clear other bits? */ whcopy->i_fc[1] &= ~IEEE80211_FC1_RETRY; IEEE80211_ADDR_COPY(whcopy->i_addr2, vap->iv_myaddr); if (IEEE80211_IS_MULTICAST(wh->i_addr1)) { ni = ieee80211_ref_node(vap->iv_bss); mcopy->m_flags |= M_MCAST; } else { ni = ieee80211_mesh_find_txnode(vap, whcopy->i_addr3); if (ni == NULL) { /* * [Optional] any of the following three actions: * o silently discard * o trigger a path discovery * o inform TA that meshDA is unknown. */ IEEE80211_NOTE_FRAME(vap, IEEE80211_MSG_MESH, wh, "%s", "frame not fwd'd, no path"); ms->ms_ppath->mpp_senderror(vap, whcopy->i_addr3, NULL, IEEE80211_REASON_MESH_PERR_NO_FI); vap->iv_stats.is_mesh_fwd_nopath++; m_freem(mcopy); return; } IEEE80211_ADDR_COPY(whcopy->i_addr1, ni->ni_macaddr); } KASSERT(mccopy->mc_ttl > 0, ("%s called with wrong ttl", __func__)); mccopy->mc_ttl--; /* XXX calculate priority so drivers can find the tx queue */ M_WME_SETAC(mcopy, WME_AC_BE); /* XXX do we know m_nextpkt is NULL? */ MPASS((mcopy->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); mcopy->m_pkthdr.rcvif = (void *) ni; /* * XXX this bypasses all of the VAP TX handling; it passes frames * directly to the parent interface. * * Because of this, there's no TX lock being held as there's no * encaps state being used. * * Doing a direct parent transmit may not be the correct thing * to do here; we'll have to re-think this soon. */ IEEE80211_TX_LOCK(ic); err = ieee80211_parent_xmitpkt(ic, mcopy); IEEE80211_TX_UNLOCK(ic); if (!err) if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } static struct mbuf * mesh_decap(struct ieee80211vap *vap, struct mbuf *m, int hdrlen, int meshdrlen) { #define WHDIR(wh) ((wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK) #define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc) uint8_t b[sizeof(struct ieee80211_qosframe_addr4) + sizeof(struct ieee80211_meshcntl_ae10)]; const struct ieee80211_qosframe_addr4 *wh; const struct ieee80211_meshcntl_ae10 *mc; struct ether_header *eh; struct llc *llc; int ae; if (m->m_len < hdrlen + sizeof(*llc) && (m = m_pullup(m, hdrlen + sizeof(*llc))) == NULL) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_ANY, "discard data frame: %s", "m_pullup failed"); vap->iv_stats.is_rx_tooshort++; return NULL; } memcpy(b, mtod(m, caddr_t), hdrlen); wh = (const struct ieee80211_qosframe_addr4 *)&b[0]; mc = (const struct ieee80211_meshcntl_ae10 *)&b[hdrlen - meshdrlen]; KASSERT(WHDIR(wh) == IEEE80211_FC1_DIR_FROMDS || WHDIR(wh) == IEEE80211_FC1_DIR_DSTODS, ("bogus dir, fc 0x%x:0x%x", wh->i_fc[0], wh->i_fc[1])); llc = (struct llc *)(mtod(m, caddr_t) + hdrlen); if (llc->llc_dsap == LLC_SNAP_LSAP && llc->llc_ssap == LLC_SNAP_LSAP && llc->llc_control == LLC_UI && llc->llc_snap.org_code[0] == 0 && llc->llc_snap.org_code[1] == 0 && llc->llc_snap.org_code[2] == 0 && /* NB: preserve AppleTalk frames that have a native SNAP hdr */ !(llc->llc_snap.ether_type == htons(ETHERTYPE_AARP) || llc->llc_snap.ether_type == htons(ETHERTYPE_IPX))) { m_adj(m, hdrlen + sizeof(struct llc) - sizeof(*eh)); llc = NULL; } else { m_adj(m, hdrlen - sizeof(*eh)); } eh = mtod(m, struct ether_header *); ae = mc->mc_flags & IEEE80211_MESH_AE_MASK; if (WHDIR(wh) == IEEE80211_FC1_DIR_FROMDS) { IEEE80211_ADDR_COPY(eh->ether_dhost, wh->i_addr1); if (ae == IEEE80211_MESH_AE_00) { IEEE80211_ADDR_COPY(eh->ether_shost, wh->i_addr3); } else if (ae == IEEE80211_MESH_AE_01) { IEEE80211_ADDR_COPY(eh->ether_shost, MC01(mc)->mc_addr4); } else { IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY, (const struct ieee80211_frame *)wh, NULL, "bad AE %d", ae); vap->iv_stats.is_mesh_badae++; m_freem(m); return NULL; } } else { if (ae == IEEE80211_MESH_AE_00) { IEEE80211_ADDR_COPY(eh->ether_dhost, wh->i_addr3); IEEE80211_ADDR_COPY(eh->ether_shost, wh->i_addr4); } else if (ae == IEEE80211_MESH_AE_10) { IEEE80211_ADDR_COPY(eh->ether_dhost, mc->mc_addr5); IEEE80211_ADDR_COPY(eh->ether_shost, mc->mc_addr6); } else { IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY, (const struct ieee80211_frame *)wh, NULL, "bad AE %d", ae); vap->iv_stats.is_mesh_badae++; m_freem(m); return NULL; } } #ifndef __NO_STRICT_ALIGNMENT if (!ALIGNED_POINTER(mtod(m, caddr_t) + sizeof(*eh), uint32_t)) { m = ieee80211_realign(vap, m, sizeof(*eh)); if (m == NULL) return NULL; } #endif /* !__NO_STRICT_ALIGNMENT */ if (llc != NULL) { eh = mtod(m, struct ether_header *); eh->ether_type = htons(m->m_pkthdr.len - sizeof(*eh)); } return m; #undef WDIR #undef MC01 } /* * Return non-zero if the unicast mesh data frame should be processed * locally. Frames that are not proxy'd have our address, otherwise * we need to consult the routing table to look for a proxy entry. */ static __inline int mesh_isucastforme(struct ieee80211vap *vap, const struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc) { int ae = mc->mc_flags & 3; KASSERT((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS, ("bad dir 0x%x:0x%x", wh->i_fc[0], wh->i_fc[1])); KASSERT(ae == IEEE80211_MESH_AE_00 || ae == IEEE80211_MESH_AE_10, ("bad AE %d", ae)); if (ae == IEEE80211_MESH_AE_10) { /* ucast w/ proxy */ const struct ieee80211_meshcntl_ae10 *mc10 = (const struct ieee80211_meshcntl_ae10 *) mc; struct ieee80211_mesh_route *rt = ieee80211_mesh_rt_find(vap, mc10->mc_addr5); /* check for proxy route to ourself */ return (rt != NULL && (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY)); } else /* ucast w/o proxy */ return IEEE80211_ADDR_EQ(wh->i_addr3, vap->iv_myaddr); } /* * Verifies transmitter, updates lifetime, precursor list and forwards data. * > 0 means we have forwarded data and no need to process locally * == 0 means we want to process locally (and we may have forwarded data * < 0 means there was an error and data should be discarded */ static int mesh_recv_indiv_data_to_fwrd(struct ieee80211vap *vap, struct mbuf *m, struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc) { struct ieee80211_qosframe_addr4 *qwh; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt_meshda, *rt_meshsa; /* This is called from the RX path - don't hold this lock */ IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic); qwh = (struct ieee80211_qosframe_addr4 *)wh; /* * TODO: * o verify addr2 is a legitimate transmitter * o lifetime of precursor of addr3 (addr2) is max(init, curr) * o lifetime of precursor of addr4 (nexthop) is max(init, curr) */ /* set lifetime of addr3 (meshDA) to initial value */ rt_meshda = ieee80211_mesh_rt_find(vap, qwh->i_addr3); if (rt_meshda == NULL) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, qwh->i_addr2, "no route to meshDA(%6D)", qwh->i_addr3, ":"); /* * [Optional] any of the following three actions: * o silently discard [X] * o trigger a path discovery [ ] * o inform TA that meshDA is unknown. [ ] */ /* XXX: stats */ return (-1); } ieee80211_mesh_rt_update(rt_meshda, ticks_to_msecs( ms->ms_ppath->mpp_inact)); /* set lifetime of addr4 (meshSA) to initial value */ rt_meshsa = ieee80211_mesh_rt_find(vap, qwh->i_addr4); KASSERT(rt_meshsa != NULL, ("no route")); ieee80211_mesh_rt_update(rt_meshsa, ticks_to_msecs( ms->ms_ppath->mpp_inact)); mesh_forward(vap, m, mc); return (1); /* dont process locally */ } /* * Verifies transmitter, updates lifetime, precursor list and process data * locally, if data is proxy with AE = 10 it could mean data should go * on another mesh path or data should be forwarded to the DS. * * > 0 means we have forwarded data and no need to process locally * == 0 means we want to process locally (and we may have forwarded data * < 0 means there was an error and data should be discarded */ static int mesh_recv_indiv_data_to_me(struct ieee80211vap *vap, struct mbuf *m, struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc) { struct ieee80211_qosframe_addr4 *qwh; const struct ieee80211_meshcntl_ae10 *mc10; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_route *rt; int ae; /* This is called from the RX path - don't hold this lock */ IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic); qwh = (struct ieee80211_qosframe_addr4 *)wh; mc10 = (const struct ieee80211_meshcntl_ae10 *)mc; /* * TODO: * o verify addr2 is a legitimate transmitter * o lifetime of precursor entry is max(init, curr) */ /* set lifetime of addr4 (meshSA) to initial value */ rt = ieee80211_mesh_rt_find(vap, qwh->i_addr4); KASSERT(rt != NULL, ("no route")); ieee80211_mesh_rt_update(rt, ticks_to_msecs(ms->ms_ppath->mpp_inact)); rt = NULL; ae = mc10->mc_flags & IEEE80211_MESH_AE_MASK; KASSERT(ae == IEEE80211_MESH_AE_00 || ae == IEEE80211_MESH_AE_10, ("bad AE %d", ae)); if (ae == IEEE80211_MESH_AE_10) { if (IEEE80211_ADDR_EQ(mc10->mc_addr5, qwh->i_addr3)) { return (0); /* process locally */ } rt = ieee80211_mesh_rt_find(vap, mc10->mc_addr5); if (rt != NULL && (rt->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) && (rt->rt_flags & IEEE80211_MESHRT_FLAGS_PROXY) == 0) { /* * Forward on another mesh-path, according to * amendment as specified in 9.32.4.1 */ IEEE80211_ADDR_COPY(qwh->i_addr3, mc10->mc_addr5); mesh_forward(vap, m, (const struct ieee80211_meshcntl *)mc10); return (1); /* dont process locally */ } /* * All other cases: forward of MSDUs from the MBSS to DS indiv. * addressed according to 13.11.3.2. */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_OUTPUT, qwh->i_addr2, "forward frame to DS, SA(%6D) DA(%6D)", mc10->mc_addr6, ":", mc10->mc_addr5, ":"); } return (0); /* process locally */ } /* * Try to forward the group addressed data on to other mesh STAs, and * also to the DS. * * > 0 means we have forwarded data and no need to process locally * == 0 means we want to process locally (and we may have forwarded data * < 0 means there was an error and data should be discarded */ static int mesh_recv_group_data(struct ieee80211vap *vap, struct mbuf *m, struct ieee80211_frame *wh, const struct ieee80211_meshcntl *mc) { #define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc) struct ieee80211_mesh_state *ms = vap->iv_mesh; /* This is called from the RX path - don't hold this lock */ IEEE80211_TX_UNLOCK_ASSERT(vap->iv_ic); mesh_forward(vap, m, mc); if(mc->mc_ttl > 0) { if (mc->mc_flags & IEEE80211_MESH_AE_01) { /* * Forward of MSDUs from the MBSS to DS group addressed * (according to 13.11.3.2) * This happens by delivering the packet, and a bridge * will sent it on another port member. */ if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE && ms->ms_flags & IEEE80211_MESHFLAGS_FWD) { IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, MC01(mc)->mc_addr4, "%s", "forward from MBSS to the DS"); } } } return (0); /* process locally */ #undef MC01 } static int mesh_input(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_rx_stats *rxs, int rssi, int nf) { #define HAS_SEQ(type) ((type & 0x4) == 0) #define MC01(mc) ((const struct ieee80211_meshcntl_ae01 *)mc) struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct ifnet *ifp = vap->iv_ifp; struct ieee80211_frame *wh; const struct ieee80211_meshcntl *mc; int hdrspace, meshdrlen, need_tap, error; uint8_t dir, type, subtype, ae; uint32_t seq; const uint8_t *addr; uint8_t qos[2]; KASSERT(ni != NULL, ("null node")); ni->ni_inact = ni->ni_inact_reload; need_tap = 1; /* mbuf need to be tapped. */ type = -1; /* undefined */ /* This is called from the RX path - don't hold this lock */ IEEE80211_TX_UNLOCK_ASSERT(ic); if (m->m_pkthdr.len < sizeof(struct ieee80211_frame_min)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, NULL, "too short (1): len %u", m->m_pkthdr.len); vap->iv_stats.is_rx_tooshort++; goto out; } /* * Bit of a cheat here, we use a pointer for a 3-address * frame format but don't reference fields past outside * ieee80211_frame_min w/o first validating the data is * present. */ wh = mtod(m, struct ieee80211_frame *); if ((wh->i_fc[0] & IEEE80211_FC0_VERSION_MASK) != IEEE80211_FC0_VERSION_0) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, NULL, "wrong version %x", wh->i_fc[0]); vap->iv_stats.is_rx_badversion++; goto err; } dir = wh->i_fc[1] & IEEE80211_FC1_DIR_MASK; type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; if ((ic->ic_flags & IEEE80211_F_SCAN) == 0) { IEEE80211_RSSI_LPF(ni->ni_avgrssi, rssi); ni->ni_noise = nf; if (HAS_SEQ(type)) { uint8_t tid = ieee80211_gettid(wh); if (IEEE80211_QOS_HAS_SEQ(wh) && TID_TO_WME_AC(tid) >= WME_AC_VI) ic->ic_wme.wme_hipri_traffic++; if (! ieee80211_check_rxseq(ni, wh, wh->i_addr1, rxs)) goto out; } } #ifdef IEEE80211_DEBUG /* * It's easier, but too expensive, to simulate different mesh * topologies by consulting the ACL policy very early, so do this * only under DEBUG. * * NB: this check is also done upon peering link initiation. */ if (vap->iv_acl != NULL && !vap->iv_acl->iac_check(vap, wh)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACL, wh, NULL, "%s", "disallowed by ACL"); vap->iv_stats.is_rx_acl++; goto out; } #endif switch (type) { case IEEE80211_FC0_TYPE_DATA: if (ni == vap->iv_bss) goto out; if (ni->ni_mlstate != IEEE80211_NODE_MESH_ESTABLISHED) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr, NULL, "peer link not yet established (%d)", ni->ni_mlstate); vap->iv_stats.is_mesh_nolink++; goto out; } if (dir != IEEE80211_FC1_DIR_FROMDS && dir != IEEE80211_FC1_DIR_DSTODS) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, "data", "incorrect dir 0x%x", dir); vap->iv_stats.is_rx_wrongdir++; goto err; } /* All Mesh data frames are QoS subtype */ if (!HAS_SEQ(type)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, "data", "incorrect subtype 0x%x", subtype); vap->iv_stats.is_rx_badsubtype++; goto err; } /* * Next up, any fragmentation. * XXX: we defrag before we even try to forward, * Mesh Control field is not present in sub-sequent * fragmented frames. This is in contrast to Draft 4.0. */ hdrspace = ieee80211_hdrspace(ic, wh); if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { m = ieee80211_defrag(ni, m, hdrspace); if (m == NULL) { /* Fragment dropped or frame not complete yet */ goto out; } } wh = mtod(m, struct ieee80211_frame *); /* NB: after defrag */ /* * Now we have a complete Mesh Data frame. */ /* * Only fromDStoDS data frames use 4 address qos frames * as specified in amendment. Otherwise addr4 is located * in the Mesh Control field and a 3 address qos frame * is used. */ *(uint16_t *)qos = *(uint16_t *)ieee80211_getqos(wh); /* * NB: The mesh STA sets the Mesh Control Present * subfield to 1 in the Mesh Data frame containing * an unfragmented MSDU, an A-MSDU, or the first * fragment of an MSDU. * After defrag it should always be present. */ if (!(qos[1] & IEEE80211_QOS_MC)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr, NULL, "%s", "Mesh control field not present"); vap->iv_stats.is_rx_elem_missing++; /* XXX: kinda */ goto err; } /* pull up enough to get to the mesh control */ if (m->m_len < hdrspace + sizeof(struct ieee80211_meshcntl) && (m = m_pullup(m, hdrspace + sizeof(struct ieee80211_meshcntl))) == NULL) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, NULL, "data too short: expecting %u", hdrspace); vap->iv_stats.is_rx_tooshort++; goto out; /* XXX */ } /* * Now calculate the full extent of the headers. Note * mesh_decap will pull up anything we didn't get * above when it strips the 802.11 headers. */ mc = (const struct ieee80211_meshcntl *) (mtod(m, const uint8_t *) + hdrspace); ae = mc->mc_flags & IEEE80211_MESH_AE_MASK; meshdrlen = sizeof(struct ieee80211_meshcntl) + ae * IEEE80211_ADDR_LEN; hdrspace += meshdrlen; /* pull complete hdrspace = ieee80211_hdrspace + meshcontrol */ if ((meshdrlen > sizeof(struct ieee80211_meshcntl)) && (m->m_len < hdrspace) && ((m = m_pullup(m, hdrspace)) == NULL)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, NULL, "data too short: expecting %u", hdrspace); vap->iv_stats.is_rx_tooshort++; goto out; /* XXX */ } /* XXX: are we sure there is no reallocating after m_pullup? */ seq = le32dec(mc->mc_seq); if (IEEE80211_IS_MULTICAST(wh->i_addr1)) addr = wh->i_addr3; else if (ae == IEEE80211_MESH_AE_01) addr = MC01(mc)->mc_addr4; else addr = ((struct ieee80211_qosframe_addr4 *)wh)->i_addr4; if (IEEE80211_ADDR_EQ(vap->iv_myaddr, addr)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT, addr, "data", "%s", "not to me"); vap->iv_stats.is_rx_wrongbss++; /* XXX kinda */ goto out; } if (mesh_checkpseq(vap, addr, seq) != 0) { vap->iv_stats.is_rx_dup++; goto out; } /* This code "routes" the frame to the right control path */ if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { if (IEEE80211_ADDR_EQ(vap->iv_myaddr, wh->i_addr3)) error = mesh_recv_indiv_data_to_me(vap, m, wh, mc); else if (IEEE80211_IS_MULTICAST(wh->i_addr3)) error = mesh_recv_group_data(vap, m, wh, mc); else error = mesh_recv_indiv_data_to_fwrd(vap, m, wh, mc); } else error = mesh_recv_group_data(vap, m, wh, mc); if (error < 0) goto err; else if (error > 0) goto out; if (ieee80211_radiotap_active_vap(vap)) ieee80211_radiotap_rx(vap, m); need_tap = 0; /* * Finally, strip the 802.11 header. */ m = mesh_decap(vap, m, hdrspace, meshdrlen); if (m == NULL) { /* XXX mask bit to check for both */ /* don't count Null data frames as errors */ if (subtype == IEEE80211_FC0_SUBTYPE_NODATA || subtype == IEEE80211_FC0_SUBTYPE_QOS_NULL) goto out; IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT, ni->ni_macaddr, "data", "%s", "decap error"); vap->iv_stats.is_rx_decap++; IEEE80211_NODE_STAT(ni, rx_decap); goto err; } if (qos[0] & IEEE80211_QOS_AMSDU) { m = ieee80211_decap_amsdu(ni, m); if (m == NULL) return IEEE80211_FC0_TYPE_DATA; } ieee80211_deliver_data(vap, ni, m); return type; case IEEE80211_FC0_TYPE_MGT: vap->iv_stats.is_rx_mgmt++; IEEE80211_NODE_STAT(ni, rx_mgmt); if (dir != IEEE80211_FC1_DIR_NODS) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, "mgt", "incorrect dir 0x%x", dir); vap->iv_stats.is_rx_wrongdir++; goto err; } if (m->m_pkthdr.len < sizeof(struct ieee80211_frame)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, "mgt", "too short: len %u", m->m_pkthdr.len); vap->iv_stats.is_rx_tooshort++; goto out; } #ifdef IEEE80211_DEBUG if ((ieee80211_msg_debug(vap) && (vap->iv_ic->ic_flags & IEEE80211_F_SCAN)) || ieee80211_msg_dumppkts(vap)) { if_printf(ifp, "received %s from %s rssi %d\n", ieee80211_mgt_subtype_name(subtype), ether_sprintf(wh->i_addr2), rssi); } #endif if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "%s", "WEP set but not permitted"); vap->iv_stats.is_rx_mgtdiscard++; /* XXX */ goto out; } vap->iv_recv_mgmt(ni, m, subtype, rxs, rssi, nf); goto out; case IEEE80211_FC0_TYPE_CTL: vap->iv_stats.is_rx_ctl++; IEEE80211_NODE_STAT(ni, rx_ctrl); goto out; default: IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY, wh, "bad", "frame type 0x%x", type); /* should not come here */ break; } err: if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); out: if (m != NULL) { if (need_tap && ieee80211_radiotap_active_vap(vap)) ieee80211_radiotap_rx(vap, m); m_freem(m); } return type; #undef HAS_SEQ #undef MC01 } static void mesh_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0, int subtype, const struct ieee80211_rx_stats *rxs, int rssi, int nf) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211com *ic = ni->ni_ic; struct ieee80211_channel *rxchan = ic->ic_curchan; struct ieee80211_frame *wh; struct ieee80211_mesh_route *rt; uint8_t *frm, *efrm; wh = mtod(m0, struct ieee80211_frame *); frm = (uint8_t *)&wh[1]; efrm = mtod(m0, uint8_t *) + m0->m_len; switch (subtype) { case IEEE80211_FC0_SUBTYPE_PROBE_RESP: case IEEE80211_FC0_SUBTYPE_BEACON: { struct ieee80211_scanparams scan; struct ieee80211_channel *c; /* * We process beacon/probe response * frames to discover neighbors. */ if (rxs != NULL) { c = ieee80211_lookup_channel_rxstatus(vap, rxs); if (c != NULL) rxchan = c; } if (ieee80211_parse_beacon(ni, m0, rxchan, &scan) != 0) return; /* * Count frame now that we know it's to be processed. */ if (subtype == IEEE80211_FC0_SUBTYPE_BEACON) { vap->iv_stats.is_rx_beacon++; /* XXX remove */ IEEE80211_NODE_STAT(ni, rx_beacons); } else IEEE80211_NODE_STAT(ni, rx_proberesp); /* * If scanning, just pass information to the scan module. */ if (ic->ic_flags & IEEE80211_F_SCAN) { if (ic->ic_flags_ext & IEEE80211_FEXT_PROBECHAN) { /* * Actively scanning a channel marked passive; * send a probe request now that we know there * is 802.11 traffic present. * * XXX check if the beacon we recv'd gives * us what we need and suppress the probe req */ ieee80211_probe_curchan(vap, 1); ic->ic_flags_ext &= ~IEEE80211_FEXT_PROBECHAN; } ieee80211_add_scan(vap, rxchan, &scan, wh, subtype, rssi, nf); return; } /* The rest of this code assumes we are running */ if (vap->iv_state != IEEE80211_S_RUN) return; /* * Ignore non-mesh STAs. */ if ((scan.capinfo & (IEEE80211_CAPINFO_ESS|IEEE80211_CAPINFO_IBSS)) || scan.meshid == NULL || scan.meshconf == NULL) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, "beacon", "%s", "not a mesh sta"); vap->iv_stats.is_mesh_wrongmesh++; return; } /* * Ignore STAs for other mesh networks. */ if (memcmp(scan.meshid+2, ms->ms_id, ms->ms_idlen) != 0 || mesh_verify_meshconf(vap, scan.meshconf)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, "beacon", "%s", "not for our mesh"); vap->iv_stats.is_mesh_wrongmesh++; return; } /* * Peer only based on the current ACL policy. */ if (vap->iv_acl != NULL && !vap->iv_acl->iac_check(vap, wh)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_ACL, wh, NULL, "%s", "disallowed by ACL"); vap->iv_stats.is_rx_acl++; return; } /* * Do neighbor discovery. */ if (!IEEE80211_ADDR_EQ(wh->i_addr2, ni->ni_macaddr)) { /* * Create a new entry in the neighbor table. */ ni = ieee80211_add_neighbor(vap, wh, &scan); } /* * Automatically peer with discovered nodes if possible. */ if (ni != vap->iv_bss && (ms->ms_flags & IEEE80211_MESHFLAGS_AP)) { switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_IDLE: { uint16_t args[1]; /* Wait for backoff callout to reset counter */ if (ni->ni_mlhcnt >= ieee80211_mesh_maxholding) return; ni->ni_mlpid = mesh_generateid(vap); if (ni->ni_mlpid == 0) return; mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENSNT); args[0] = ni->ni_mlpid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_OPEN, args); ni->ni_mlrcnt = 0; mesh_peer_timeout_setup(ni); break; } case IEEE80211_NODE_MESH_ESTABLISHED: { /* * Valid beacon from a peer mesh STA * bump TA lifetime */ rt = ieee80211_mesh_rt_find(vap, wh->i_addr2); if(rt != NULL) { ieee80211_mesh_rt_update(rt, ticks_to_msecs( ms->ms_ppath->mpp_inact)); } break; } default: break; /* ignore */ } } break; } case IEEE80211_FC0_SUBTYPE_PROBE_REQ: { uint8_t *ssid, *meshid, *rates, *xrates; if (vap->iv_state != IEEE80211_S_RUN) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "wrong state %s", ieee80211_state_name[vap->iv_state]); vap->iv_stats.is_rx_mgtdiscard++; return; } if (IEEE80211_IS_MULTICAST(wh->i_addr2)) { /* frame must be directed */ IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "%s", "not unicast"); vap->iv_stats.is_rx_mgtdiscard++; /* XXX stat */ return; } /* * prreq frame format * [tlv] ssid * [tlv] supported rates * [tlv] extended supported rates * [tlv] mesh id */ ssid = meshid = rates = xrates = NULL; while (efrm - frm > 1) { IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return); switch (*frm) { case IEEE80211_ELEMID_SSID: ssid = frm; break; case IEEE80211_ELEMID_RATES: rates = frm; break; case IEEE80211_ELEMID_XRATES: xrates = frm; break; case IEEE80211_ELEMID_MESHID: meshid = frm; break; } frm += frm[1] + 2; } IEEE80211_VERIFY_ELEMENT(ssid, IEEE80211_NWID_LEN, return); IEEE80211_VERIFY_ELEMENT(rates, IEEE80211_RATE_MAXSIZE, return); if (xrates != NULL) IEEE80211_VERIFY_ELEMENT(xrates, IEEE80211_RATE_MAXSIZE - rates[1], return); if (meshid != NULL) { IEEE80211_VERIFY_ELEMENT(meshid, IEEE80211_MESHID_LEN, return); /* NB: meshid, not ssid */ IEEE80211_VERIFY_SSID(vap->iv_bss, meshid, return); } /* XXX find a better class or define it's own */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_INPUT, wh->i_addr2, "%s", "recv probe req"); /* * Some legacy 11b clients cannot hack a complete * probe response frame. When the request includes * only a bare-bones rate set, communicate this to * the transmit side. */ ieee80211_send_proberesp(vap, wh->i_addr2, 0); break; } case IEEE80211_FC0_SUBTYPE_ACTION: case IEEE80211_FC0_SUBTYPE_ACTION_NOACK: if (ni == vap->iv_bss) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "%s", "unknown node"); vap->iv_stats.is_rx_mgtdiscard++; } else if (!IEEE80211_ADDR_EQ(vap->iv_myaddr, wh->i_addr1) && !IEEE80211_IS_MULTICAST(wh->i_addr1)) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "%s", "not for us"); vap->iv_stats.is_rx_mgtdiscard++; } else if (vap->iv_state != IEEE80211_S_RUN) { IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "wrong state %s", ieee80211_state_name[vap->iv_state]); vap->iv_stats.is_rx_mgtdiscard++; } else { if (ieee80211_parse_action(ni, m0) == 0) (void)ic->ic_recv_action(ni, wh, frm, efrm); } break; case IEEE80211_FC0_SUBTYPE_ASSOC_REQ: case IEEE80211_FC0_SUBTYPE_ASSOC_RESP: case IEEE80211_FC0_SUBTYPE_REASSOC_REQ: case IEEE80211_FC0_SUBTYPE_REASSOC_RESP: case IEEE80211_FC0_SUBTYPE_TIMING_ADV: case IEEE80211_FC0_SUBTYPE_ATIM: case IEEE80211_FC0_SUBTYPE_DISASSOC: case IEEE80211_FC0_SUBTYPE_AUTH: case IEEE80211_FC0_SUBTYPE_DEAUTH: IEEE80211_DISCARD(vap, IEEE80211_MSG_INPUT, wh, NULL, "%s", "not handled"); vap->iv_stats.is_rx_mgtdiscard++; break; default: IEEE80211_DISCARD(vap, IEEE80211_MSG_ANY, wh, "mgt", "subtype 0x%x not handled", subtype); vap->iv_stats.is_rx_badsubtype++; break; } } static void mesh_recv_ctl(struct ieee80211_node *ni, struct mbuf *m, int subtype) { switch (subtype) { case IEEE80211_FC0_SUBTYPE_BAR: ieee80211_recv_bar(ni, m); break; } } /* * Parse meshpeering action ie's for MPM frames */ static const struct ieee80211_meshpeer_ie * mesh_parse_meshpeering_action(struct ieee80211_node *ni, const struct ieee80211_frame *wh, /* XXX for VERIFY_LENGTH */ const uint8_t *frm, const uint8_t *efrm, struct ieee80211_meshpeer_ie *mp, uint8_t subtype) { struct ieee80211vap *vap = ni->ni_vap; const struct ieee80211_meshpeer_ie *mpie; uint16_t args[3]; const uint8_t *meshid, *meshconf; uint8_t sendclose = 0; /* 1 = MPM frame rejected, close will be sent */ meshid = meshconf = NULL; while (efrm - frm > 1) { IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return NULL); switch (*frm) { case IEEE80211_ELEMID_MESHID: meshid = frm; break; case IEEE80211_ELEMID_MESHCONF: meshconf = frm; break; case IEEE80211_ELEMID_MESHPEER: mpie = (const struct ieee80211_meshpeer_ie *) frm; memset(mp, 0, sizeof(*mp)); mp->peer_len = mpie->peer_len; mp->peer_proto = le16dec(&mpie->peer_proto); mp->peer_llinkid = le16dec(&mpie->peer_llinkid); switch (subtype) { case IEEE80211_ACTION_MESHPEERING_CONFIRM: mp->peer_linkid = le16dec(&mpie->peer_linkid); break; case IEEE80211_ACTION_MESHPEERING_CLOSE: /* NB: peer link ID is optional */ if (mpie->peer_len == (IEEE80211_MPM_BASE_SZ + 2)) { mp->peer_linkid = 0; mp->peer_rcode = le16dec(&mpie->peer_linkid); } else { mp->peer_linkid = le16dec(&mpie->peer_linkid); mp->peer_rcode = le16dec(&mpie->peer_rcode); } break; } break; } frm += frm[1] + 2; } /* * Verify the contents of the frame. * If it fails validation, close the peer link. */ if (mesh_verify_meshpeer(vap, subtype, (const uint8_t *)mp)) { sendclose = 1; IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, wh, NULL, "%s", "MPM validation failed"); } /* If meshid is not the same reject any frames type. */ if (sendclose == 0 && mesh_verify_meshid(vap, meshid)) { sendclose = 1; IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, wh, NULL, "%s", "not for our mesh"); if (subtype == IEEE80211_ACTION_MESHPEERING_CLOSE) { /* * Standard not clear about this, if we dont ignore * there will be an endless loop between nodes sending * CLOSE frames between each other with wrong meshid. * Discard and timers will bring FSM to IDLE state. */ return NULL; } } /* * Close frames are accepted if meshid is the same. * Verify the other two types. */ if (sendclose == 0 && subtype != IEEE80211_ACTION_MESHPEERING_CLOSE && mesh_verify_meshconf(vap, meshconf)) { sendclose = 1; IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, wh, NULL, "%s", "configuration missmatch"); } if (sendclose) { vap->iv_stats.is_rx_mgtdiscard++; switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_IDLE: case IEEE80211_NODE_MESH_ESTABLISHED: case IEEE80211_NODE_MESH_HOLDING: /* ignore */ break; case IEEE80211_NODE_MESH_OPENSNT: case IEEE80211_NODE_MESH_OPENRCV: case IEEE80211_NODE_MESH_CONFIRMRCV: args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; /* Reason codes for rejection */ switch (subtype) { case IEEE80211_ACTION_MESHPEERING_OPEN: args[2] = IEEE80211_REASON_MESH_CPVIOLATION; break; case IEEE80211_ACTION_MESHPEERING_CONFIRM: args[2] = IEEE80211_REASON_MESH_INCONS_PARAMS; break; } ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; } return NULL; } return (const struct ieee80211_meshpeer_ie *) mp; } static int mesh_recv_action_meshpeering_open(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_meshpeer_ie ie; const struct ieee80211_meshpeer_ie *meshpeer; uint16_t args[3]; /* +2+2 for action + code + capabilites */ meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2+2, efrm, &ie, IEEE80211_ACTION_MESHPEERING_OPEN); if (meshpeer == NULL) { return 0; } /* XXX move up */ IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "recv PEER OPEN, lid 0x%x", meshpeer->peer_llinkid); switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_IDLE: /* Reject open request if reached our maximum neighbor count */ if (ms->ms_neighbors >= IEEE80211_MESH_MAX_NEIGHBORS) { args[0] = meshpeer->peer_llinkid; args[1] = 0; args[2] = IEEE80211_REASON_MESH_MAX_PEERS; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); /* stay in IDLE state */ return (0); } /* Open frame accepted */ mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENRCV); ni->ni_mllid = meshpeer->peer_llinkid; ni->ni_mlpid = mesh_generateid(vap); if (ni->ni_mlpid == 0) return 0; /* XXX */ args[0] = ni->ni_mlpid; /* Announce we're open too... */ ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_OPEN, args); /* ...and confirm the link. */ args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, args); mesh_peer_timeout_setup(ni); break; case IEEE80211_NODE_MESH_OPENRCV: /* Wrong Link ID */ if (ni->ni_mllid != meshpeer->peer_llinkid) { args[0] = ni->ni_mllid; args[1] = ni->ni_mlpid; args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; } /* Duplicate open, confirm again. */ args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, args); break; case IEEE80211_NODE_MESH_OPENSNT: ni->ni_mllid = meshpeer->peer_llinkid; mesh_linkchange(ni, IEEE80211_NODE_MESH_OPENRCV); args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, args); /* NB: don't setup/clear any timeout */ break; case IEEE80211_NODE_MESH_CONFIRMRCV: if (ni->ni_mlpid != meshpeer->peer_linkid || ni->ni_mllid != meshpeer->peer_llinkid) { args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; } mesh_linkchange(ni, IEEE80211_NODE_MESH_ESTABLISHED); ni->ni_mllid = meshpeer->peer_llinkid; args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, args); mesh_peer_timeout_stop(ni); break; case IEEE80211_NODE_MESH_ESTABLISHED: if (ni->ni_mllid != meshpeer->peer_llinkid) { args[0] = ni->ni_mllid; args[1] = ni->ni_mlpid; args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; } args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CONFIRM, args); break; case IEEE80211_NODE_MESH_HOLDING: args[0] = ni->ni_mlpid; args[1] = meshpeer->peer_llinkid; /* Standard not clear about what the reaason code should be */ args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); break; } return 0; } static int mesh_recv_action_meshpeering_confirm(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_meshpeer_ie ie; const struct ieee80211_meshpeer_ie *meshpeer; uint16_t args[3]; /* +2+2+2+2 for action + code + capabilites + status code + AID */ meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2+2+2+2, efrm, &ie, IEEE80211_ACTION_MESHPEERING_CONFIRM); if (meshpeer == NULL) { return 0; } IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "recv PEER CONFIRM, local id 0x%x, peer id 0x%x", meshpeer->peer_llinkid, meshpeer->peer_linkid); switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_OPENRCV: mesh_linkchange(ni, IEEE80211_NODE_MESH_ESTABLISHED); mesh_peer_timeout_stop(ni); break; case IEEE80211_NODE_MESH_OPENSNT: mesh_linkchange(ni, IEEE80211_NODE_MESH_CONFIRMRCV); mesh_peer_timeout_setup(ni); break; case IEEE80211_NODE_MESH_HOLDING: args[0] = ni->ni_mlpid; args[1] = meshpeer->peer_llinkid; /* Standard not clear about what the reaason code should be */ args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); break; case IEEE80211_NODE_MESH_CONFIRMRCV: if (ni->ni_mllid != meshpeer->peer_llinkid) { args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; args[2] = IEEE80211_REASON_PEER_LINK_CANCELED; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); } break; default: IEEE80211_DISCARD(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, wh, NULL, "received confirm in invalid state %d", ni->ni_mlstate); vap->iv_stats.is_rx_mgtdiscard++; break; } return 0; } static int mesh_recv_action_meshpeering_close(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211_meshpeer_ie ie; const struct ieee80211_meshpeer_ie *meshpeer; uint16_t args[3]; /* +2 for action + code */ meshpeer = mesh_parse_meshpeering_action(ni, wh, frm+2, efrm, &ie, IEEE80211_ACTION_MESHPEERING_CLOSE); if (meshpeer == NULL) { return 0; } /* * XXX: check reason code, for example we could receive * IEEE80211_REASON_MESH_MAX_PEERS then we should not attempt * to peer again. */ IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "%s", "recv PEER CLOSE"); switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_IDLE: /* ignore */ break; case IEEE80211_NODE_MESH_OPENRCV: case IEEE80211_NODE_MESH_OPENSNT: case IEEE80211_NODE_MESH_CONFIRMRCV: case IEEE80211_NODE_MESH_ESTABLISHED: args[0] = ni->ni_mlpid; args[1] = ni->ni_mllid; args[2] = IEEE80211_REASON_MESH_CLOSE_RCVD; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; case IEEE80211_NODE_MESH_HOLDING: mesh_linkchange(ni, IEEE80211_NODE_MESH_IDLE); mesh_peer_timeout_stop(ni); break; } return 0; } /* * Link Metric handling. */ static int mesh_recv_action_meshlmetric(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { const struct ieee80211_meshlmetric_ie *ie = (const struct ieee80211_meshlmetric_ie *) (frm+2); /* action + code */ struct ieee80211_meshlmetric_ie lm_rep; if (ie->lm_flags & IEEE80211_MESH_LMETRIC_FLAGS_REQ) { lm_rep.lm_flags = 0; lm_rep.lm_metric = mesh_airtime_calc(ni); ieee80211_send_action(ni, IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_LMETRIC, &lm_rep); } /* XXX: else do nothing for now */ return 0; } /* * Parse meshgate action ie's for GANN frames. * Returns -1 if parsing fails, otherwise 0. */ static int mesh_parse_meshgate_action(struct ieee80211_node *ni, const struct ieee80211_frame *wh, /* XXX for VERIFY_LENGTH */ struct ieee80211_meshgann_ie *ie, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211vap *vap = ni->ni_vap; const struct ieee80211_meshgann_ie *gannie; while (efrm - frm > 1) { IEEE80211_VERIFY_LENGTH(efrm - frm, frm[1] + 2, return -1); switch (*frm) { case IEEE80211_ELEMID_MESHGANN: gannie = (const struct ieee80211_meshgann_ie *) frm; memset(ie, 0, sizeof(*ie)); ie->gann_ie = gannie->gann_ie; ie->gann_len = gannie->gann_len; ie->gann_flags = gannie->gann_flags; ie->gann_hopcount = gannie->gann_hopcount; ie->gann_ttl = gannie->gann_ttl; IEEE80211_ADDR_COPY(ie->gann_addr, gannie->gann_addr); ie->gann_seq = le32dec(&gannie->gann_seq); ie->gann_interval = le16dec(&gannie->gann_interval); break; } frm += frm[1] + 2; } return 0; } /* * Mesh Gate Announcement handling. */ static int mesh_recv_action_meshgate(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; struct ieee80211_mesh_gate_route *gr, *next; struct ieee80211_mesh_route *rt_gate; struct ieee80211_meshgann_ie pgann; struct ieee80211_meshgann_ie ie; int found = 0; /* +2 for action + code */ if (mesh_parse_meshgate_action(ni, wh, &ie, frm+2, efrm) != 0) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr, NULL, "%s", "GANN parsing failed"); vap->iv_stats.is_rx_mgtdiscard++; return (0); } if (IEEE80211_ADDR_EQ(vap->iv_myaddr, ie.gann_addr)) return 0; IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr, "received GANN, meshgate: %6D (seq %u)", ie.gann_addr, ":", ie.gann_seq); if (ms == NULL) return (0); MESH_RT_LOCK(ms); TAILQ_FOREACH_SAFE(gr, &ms->ms_known_gates, gr_next, next) { if (!IEEE80211_ADDR_EQ(gr->gr_addr, ie.gann_addr)) continue; if (ie.gann_seq <= gr->gr_lastseq) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_MESH, ni->ni_macaddr, NULL, "GANN old seqno %u <= %u", ie.gann_seq, gr->gr_lastseq); MESH_RT_UNLOCK(ms); return (0); } /* corresponding mesh gate found & GANN accepted */ found = 1; break; } if (found == 0) { /* this GANN is from a new mesh Gate add it to known table. */ IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ie.gann_addr, "stored new GANN information, seq %u.", ie.gann_seq); gr = IEEE80211_MALLOC(ALIGN(sizeof(struct ieee80211_mesh_gate_route)), M_80211_MESH_GT_RT, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); IEEE80211_ADDR_COPY(gr->gr_addr, ie.gann_addr); TAILQ_INSERT_TAIL(&ms->ms_known_gates, gr, gr_next); } gr->gr_lastseq = ie.gann_seq; /* check if we have a path to this gate */ rt_gate = mesh_rt_find_locked(ms, gr->gr_addr); if (rt_gate != NULL && rt_gate->rt_flags & IEEE80211_MESHRT_FLAGS_VALID) { gr->gr_route = rt_gate; rt_gate->rt_flags |= IEEE80211_MESHRT_FLAGS_GATE; } MESH_RT_UNLOCK(ms); /* popagate only if decremented ttl >= 1 && forwarding is enabled */ if ((ie.gann_ttl - 1) < 1 && !(ms->ms_flags & IEEE80211_MESHFLAGS_FWD)) return 0; pgann.gann_flags = ie.gann_flags; /* Reserved */ pgann.gann_hopcount = ie.gann_hopcount + 1; pgann.gann_ttl = ie.gann_ttl - 1; IEEE80211_ADDR_COPY(pgann.gann_addr, ie.gann_addr); pgann.gann_seq = ie.gann_seq; pgann.gann_interval = ie.gann_interval; IEEE80211_NOTE_MAC(vap, IEEE80211_MSG_MESH, ie.gann_addr, "%s", "propagate GANN"); ieee80211_send_action(vap->iv_bss, IEEE80211_ACTION_CAT_MESH, IEEE80211_ACTION_MESH_GANN, &pgann); return 0; } static int mesh_send_action(struct ieee80211_node *ni, const uint8_t sa[IEEE80211_ADDR_LEN], const uint8_t da[IEEE80211_ADDR_LEN], struct mbuf *m) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct ieee80211_bpf_params params; int ret; KASSERT(ni != NULL, ("null node")); if (vap->iv_state == IEEE80211_S_CAC) { IEEE80211_NOTE(vap, IEEE80211_MSG_OUTPUT, ni, "block %s frame in CAC state", "Mesh action"); vap->iv_stats.is_tx_badstate++; ieee80211_free_node(ni); m_freem(m); return EIO; /* XXX */ } M_PREPEND(m, sizeof(struct ieee80211_frame), M_NOWAIT); if (m == NULL) { ieee80211_free_node(ni); return ENOMEM; } IEEE80211_TX_LOCK(ic); ieee80211_send_setup(ni, m, IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_ACTION, IEEE80211_NONQOS_TID, sa, da, sa); m->m_flags |= M_ENCAP; /* mark encapsulated */ memset(¶ms, 0, sizeof(params)); params.ibp_pri = WME_AC_VO; params.ibp_rate0 = ni->ni_txparms->mgmtrate; if (IEEE80211_IS_MULTICAST(da)) params.ibp_try0 = 1; else params.ibp_try0 = ni->ni_txparms->maxretry; params.ibp_power = ni->ni_txpower; IEEE80211_NODE_STAT(ni, tx_mgmt); ret = ieee80211_raw_output(vap, ni, m, ¶ms); IEEE80211_TX_UNLOCK(ic); return (ret); } #define ADDSHORT(frm, v) do { \ frm[0] = (v) & 0xff; \ frm[1] = (v) >> 8; \ frm += 2; \ } while (0) #define ADDWORD(frm, v) do { \ frm[0] = (v) & 0xff; \ frm[1] = ((v) >> 8) & 0xff; \ frm[2] = ((v) >> 16) & 0xff; \ frm[3] = ((v) >> 24) & 0xff; \ frm += 4; \ } while (0) static int mesh_send_action_meshpeering_open(struct ieee80211_node *ni, int category, int action, void *args0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; uint16_t *args = args0; const struct ieee80211_rateset *rs; struct mbuf *m; uint8_t *frm; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "send PEER OPEN action: localid 0x%x", args[0]); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ + sizeof(uint16_t) /* capabilites */ + 2 + IEEE80211_RATE_SIZE + 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE) + 2 + IEEE80211_MESHID_LEN + sizeof(struct ieee80211_meshconf_ie) + sizeof(struct ieee80211_meshpeer_ie) ); if (m != NULL) { /* * mesh peer open action frame format: * [1] category * [1] action * [2] capabilities * [tlv] rates * [tlv] xrates * [tlv] mesh id * [tlv] mesh conf * [tlv] mesh peer link mgmt */ *frm++ = category; *frm++ = action; ADDSHORT(frm, ieee80211_getcapinfo(vap, ni->ni_chan)); rs = ieee80211_get_suprates(ic, ic->ic_curchan); frm = ieee80211_add_rates(frm, rs); frm = ieee80211_add_xrates(frm, rs); frm = ieee80211_add_meshid(frm, vap); frm = ieee80211_add_meshconf(frm, vap); frm = ieee80211_add_meshpeer(frm, IEEE80211_ACTION_MESHPEERING_OPEN, args[0], 0, 0); m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int mesh_send_action_meshpeering_confirm(struct ieee80211_node *ni, int category, int action, void *args0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; uint16_t *args = args0; const struct ieee80211_rateset *rs; struct mbuf *m; uint8_t *frm; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "send PEER CONFIRM action: localid 0x%x, peerid 0x%x", args[0], args[1]); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ + sizeof(uint16_t) /* capabilites */ + sizeof(uint16_t) /* status code */ + sizeof(uint16_t) /* AID */ + 2 + IEEE80211_RATE_SIZE + 2 + (IEEE80211_RATE_MAXSIZE - IEEE80211_RATE_SIZE) + 2 + IEEE80211_MESHID_LEN + sizeof(struct ieee80211_meshconf_ie) + sizeof(struct ieee80211_meshpeer_ie) ); if (m != NULL) { /* * mesh peer confirm action frame format: * [1] category * [1] action * [2] capabilities * [2] status code * [2] association id (peer ID) * [tlv] rates * [tlv] xrates * [tlv] mesh id * [tlv] mesh conf * [tlv] mesh peer link mgmt */ *frm++ = category; *frm++ = action; ADDSHORT(frm, ieee80211_getcapinfo(vap, ni->ni_chan)); ADDSHORT(frm, 0); /* status code */ ADDSHORT(frm, args[1]); /* AID */ rs = ieee80211_get_suprates(ic, ic->ic_curchan); frm = ieee80211_add_rates(frm, rs); frm = ieee80211_add_xrates(frm, rs); frm = ieee80211_add_meshid(frm, vap); frm = ieee80211_add_meshconf(frm, vap); frm = ieee80211_add_meshpeer(frm, IEEE80211_ACTION_MESHPEERING_CONFIRM, args[0], args[1], 0); m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int mesh_send_action_meshpeering_close(struct ieee80211_node *ni, int category, int action, void *args0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; uint16_t *args = args0; struct mbuf *m; uint8_t *frm; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "send PEER CLOSE action: localid 0x%x, peerid 0x%x reason %d (%s)", args[0], args[1], args[2], ieee80211_reason_to_string(args[2])); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ + sizeof(uint16_t) /* reason code */ + 2 + IEEE80211_MESHID_LEN + sizeof(struct ieee80211_meshpeer_ie) ); if (m != NULL) { /* * mesh peer close action frame format: * [1] category * [1] action * [tlv] mesh id * [tlv] mesh peer link mgmt */ *frm++ = category; *frm++ = action; frm = ieee80211_add_meshid(frm, vap); frm = ieee80211_add_meshpeer(frm, IEEE80211_ACTION_MESHPEERING_CLOSE, args[0], args[1], args[2]); m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int mesh_send_action_meshlmetric(struct ieee80211_node *ni, int category, int action, void *arg0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct ieee80211_meshlmetric_ie *ie = arg0; struct mbuf *m; uint8_t *frm; if (ie->lm_flags & IEEE80211_MESH_LMETRIC_FLAGS_REQ) { IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "%s", "send LINK METRIC REQUEST action"); } else { IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, ni, "send LINK METRIC REPLY action: metric 0x%x", ie->lm_metric); } IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) + /* action+category */ sizeof(struct ieee80211_meshlmetric_ie) ); if (m != NULL) { /* * mesh link metric * [1] category * [1] action * [tlv] mesh link metric */ *frm++ = category; *frm++ = action; frm = ieee80211_add_meshlmetric(frm, ie->lm_flags, ie->lm_metric); m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return mesh_send_action(ni, vap->iv_myaddr, ni->ni_macaddr, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int mesh_send_action_meshgate(struct ieee80211_node *ni, int category, int action, void *arg0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct ieee80211_meshgann_ie *ie = arg0; struct mbuf *m; uint8_t *frm; IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) + /* action+category */ IEEE80211_MESHGANN_BASE_SZ ); if (m != NULL) { /* * mesh link metric * [1] category * [1] action * [tlv] mesh gate annoucement */ *frm++ = category; *frm++ = action; frm = ieee80211_add_meshgate(frm, ie); m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return mesh_send_action(ni, vap->iv_myaddr, broadcastaddr, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static void mesh_peer_timeout_setup(struct ieee80211_node *ni) { switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_HOLDING: ni->ni_mltval = ieee80211_mesh_holdingtimeout; break; case IEEE80211_NODE_MESH_CONFIRMRCV: ni->ni_mltval = ieee80211_mesh_confirmtimeout; break; case IEEE80211_NODE_MESH_IDLE: ni->ni_mltval = 0; break; default: ni->ni_mltval = ieee80211_mesh_retrytimeout; break; } if (ni->ni_mltval) callout_reset(&ni->ni_mltimer, ni->ni_mltval, mesh_peer_timeout_cb, ni); } /* * Same as above but backoffs timer statisically 50%. */ static void mesh_peer_timeout_backoff(struct ieee80211_node *ni) { uint32_t r; r = arc4random(); ni->ni_mltval += r % ni->ni_mltval; callout_reset(&ni->ni_mltimer, ni->ni_mltval, mesh_peer_timeout_cb, ni); } static __inline void mesh_peer_timeout_stop(struct ieee80211_node *ni) { callout_drain(&ni->ni_mltimer); } static void mesh_peer_backoff_cb(void *arg) { struct ieee80211_node *ni = (struct ieee80211_node *)arg; /* After backoff timeout, try to peer automatically again. */ ni->ni_mlhcnt = 0; } /* * Mesh Peer Link Management FSM timeout handling. */ static void mesh_peer_timeout_cb(void *arg) { struct ieee80211_node *ni = (struct ieee80211_node *)arg; uint16_t args[3]; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_MESH, ni, "mesh link timeout, state %d, retry counter %d", ni->ni_mlstate, ni->ni_mlrcnt); switch (ni->ni_mlstate) { case IEEE80211_NODE_MESH_IDLE: case IEEE80211_NODE_MESH_ESTABLISHED: break; case IEEE80211_NODE_MESH_OPENSNT: case IEEE80211_NODE_MESH_OPENRCV: if (ni->ni_mlrcnt == ieee80211_mesh_maxretries) { args[0] = ni->ni_mlpid; args[2] = IEEE80211_REASON_MESH_MAX_RETRIES; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); ni->ni_mlrcnt = 0; mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); } else { args[0] = ni->ni_mlpid; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_OPEN, args); ni->ni_mlrcnt++; mesh_peer_timeout_backoff(ni); } break; case IEEE80211_NODE_MESH_CONFIRMRCV: args[0] = ni->ni_mlpid; args[2] = IEEE80211_REASON_MESH_CONFIRM_TIMEOUT; ieee80211_send_action(ni, IEEE80211_ACTION_CAT_SELF_PROT, IEEE80211_ACTION_MESHPEERING_CLOSE, args); mesh_linkchange(ni, IEEE80211_NODE_MESH_HOLDING); mesh_peer_timeout_setup(ni); break; case IEEE80211_NODE_MESH_HOLDING: ni->ni_mlhcnt++; if (ni->ni_mlhcnt >= ieee80211_mesh_maxholding) callout_reset(&ni->ni_mlhtimer, ieee80211_mesh_backofftimeout, mesh_peer_backoff_cb, ni); mesh_linkchange(ni, IEEE80211_NODE_MESH_IDLE); break; } } static int mesh_verify_meshid(struct ieee80211vap *vap, const uint8_t *ie) { struct ieee80211_mesh_state *ms = vap->iv_mesh; if (ie == NULL || ie[1] != ms->ms_idlen) return 1; return memcmp(ms->ms_id, ie + 2, ms->ms_idlen); } /* * Check if we are using the same algorithms for this mesh. */ static int mesh_verify_meshconf(struct ieee80211vap *vap, const uint8_t *ie) { const struct ieee80211_meshconf_ie *meshconf = (const struct ieee80211_meshconf_ie *) ie; const struct ieee80211_mesh_state *ms = vap->iv_mesh; if (meshconf == NULL) return 1; if (meshconf->conf_pselid != ms->ms_ppath->mpp_ie) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "unknown path selection algorithm: 0x%x\n", meshconf->conf_pselid); return 1; } if (meshconf->conf_pmetid != ms->ms_pmetric->mpm_ie) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "unknown path metric algorithm: 0x%x\n", meshconf->conf_pmetid); return 1; } if (meshconf->conf_ccid != 0) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "unknown congestion control algorithm: 0x%x\n", meshconf->conf_ccid); return 1; } if (meshconf->conf_syncid != IEEE80211_MESHCONF_SYNC_NEIGHOFF) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "unknown sync algorithm: 0x%x\n", meshconf->conf_syncid); return 1; } if (meshconf->conf_authid != 0) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "unknown auth auth algorithm: 0x%x\n", meshconf->conf_pselid); return 1; } /* Not accepting peers */ if (!(meshconf->conf_cap & IEEE80211_MESHCONF_CAP_AP)) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_MESH, "not accepting peers: 0x%x\n", meshconf->conf_cap); return 1; } return 0; } static int mesh_verify_meshpeer(struct ieee80211vap *vap, uint8_t subtype, const uint8_t *ie) { const struct ieee80211_meshpeer_ie *meshpeer = (const struct ieee80211_meshpeer_ie *) ie; if (meshpeer == NULL || meshpeer->peer_len < IEEE80211_MPM_BASE_SZ || meshpeer->peer_len > IEEE80211_MPM_MAX_SZ) return 1; if (meshpeer->peer_proto != IEEE80211_MPPID_MPM) { IEEE80211_DPRINTF(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_MESH, "Only MPM protocol is supported (proto: 0x%02X)", meshpeer->peer_proto); return 1; } switch (subtype) { case IEEE80211_ACTION_MESHPEERING_OPEN: if (meshpeer->peer_len != IEEE80211_MPM_BASE_SZ) return 1; break; case IEEE80211_ACTION_MESHPEERING_CONFIRM: if (meshpeer->peer_len != IEEE80211_MPM_BASE_SZ + 2) return 1; break; case IEEE80211_ACTION_MESHPEERING_CLOSE: if (meshpeer->peer_len < IEEE80211_MPM_BASE_SZ + 2) return 1; if (meshpeer->peer_len == (IEEE80211_MPM_BASE_SZ + 2) && meshpeer->peer_linkid != 0) return 1; if (meshpeer->peer_rcode == 0) return 1; break; } return 0; } /* * Add a Mesh ID IE to a frame. */ uint8_t * ieee80211_add_meshid(uint8_t *frm, struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a mbss vap")); *frm++ = IEEE80211_ELEMID_MESHID; *frm++ = ms->ms_idlen; memcpy(frm, ms->ms_id, ms->ms_idlen); return frm + ms->ms_idlen; } /* * Add a Mesh Configuration IE to a frame. * For now just use HWMP routing, Airtime link metric, Null Congestion * Signaling, Null Sync Protocol and Null Authentication. */ uint8_t * ieee80211_add_meshconf(uint8_t *frm, struct ieee80211vap *vap) { const struct ieee80211_mesh_state *ms = vap->iv_mesh; uint16_t caps; KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a MBSS vap")); *frm++ = IEEE80211_ELEMID_MESHCONF; *frm++ = IEEE80211_MESH_CONF_SZ; *frm++ = ms->ms_ppath->mpp_ie; /* path selection */ *frm++ = ms->ms_pmetric->mpm_ie; /* link metric */ *frm++ = IEEE80211_MESHCONF_CC_DISABLED; *frm++ = IEEE80211_MESHCONF_SYNC_NEIGHOFF; *frm++ = IEEE80211_MESHCONF_AUTH_DISABLED; /* NB: set the number of neighbors before the rest */ *frm = (ms->ms_neighbors > IEEE80211_MESH_MAX_NEIGHBORS ? IEEE80211_MESH_MAX_NEIGHBORS : ms->ms_neighbors) << 1; if (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) *frm |= IEEE80211_MESHCONF_FORM_GATE; frm += 1; caps = 0; if (ms->ms_flags & IEEE80211_MESHFLAGS_AP) caps |= IEEE80211_MESHCONF_CAP_AP; if (ms->ms_flags & IEEE80211_MESHFLAGS_FWD) caps |= IEEE80211_MESHCONF_CAP_FWRD; *frm++ = caps; return frm; } /* * Add a Mesh Peer Management IE to a frame. */ uint8_t * ieee80211_add_meshpeer(uint8_t *frm, uint8_t subtype, uint16_t localid, uint16_t peerid, uint16_t reason) { KASSERT(localid != 0, ("localid == 0")); *frm++ = IEEE80211_ELEMID_MESHPEER; switch (subtype) { case IEEE80211_ACTION_MESHPEERING_OPEN: *frm++ = IEEE80211_MPM_BASE_SZ; /* length */ ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */ ADDSHORT(frm, localid); /* local ID */ break; case IEEE80211_ACTION_MESHPEERING_CONFIRM: KASSERT(peerid != 0, ("sending peer confirm without peer id")); *frm++ = IEEE80211_MPM_BASE_SZ + 2; /* length */ ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */ ADDSHORT(frm, localid); /* local ID */ ADDSHORT(frm, peerid); /* peer ID */ break; case IEEE80211_ACTION_MESHPEERING_CLOSE: if (peerid) *frm++ = IEEE80211_MPM_MAX_SZ; /* length */ else *frm++ = IEEE80211_MPM_BASE_SZ + 2; /* length */ ADDSHORT(frm, IEEE80211_MPPID_MPM); /* proto */ ADDSHORT(frm, localid); /* local ID */ if (peerid) ADDSHORT(frm, peerid); /* peer ID */ ADDSHORT(frm, reason); break; } return frm; } /* * Compute an Airtime Link Metric for the link with this node. * * Based on Draft 3.0 spec (11B.10, p.149). */ /* * Max 802.11s overhead. */ #define IEEE80211_MESH_MAXOVERHEAD \ (sizeof(struct ieee80211_qosframe_addr4) \ + sizeof(struct ieee80211_meshcntl_ae10) \ + sizeof(struct llc) \ + IEEE80211_ADDR_LEN \ + IEEE80211_WEP_IVLEN \ + IEEE80211_WEP_KIDLEN \ + IEEE80211_WEP_CRCLEN \ + IEEE80211_WEP_MICLEN \ + IEEE80211_CRC_LEN) uint32_t mesh_airtime_calc(struct ieee80211_node *ni) { #define M_BITS 8 #define S_FACTOR (2 * M_BITS) struct ieee80211com *ic = ni->ni_ic; struct ifnet *ifp = ni->ni_vap->iv_ifp; const static int nbits = 8192 << M_BITS; uint32_t overhead, rate, errrate; uint64_t res; /* Time to transmit a frame */ rate = ni->ni_txrate; overhead = ieee80211_compute_duration(ic->ic_rt, ifp->if_mtu + IEEE80211_MESH_MAXOVERHEAD, rate, 0) << M_BITS; /* Error rate in percentage */ /* XXX assuming small failures are ok */ errrate = (((ifp->if_get_counter(ifp, IFCOUNTER_OERRORS) + ifp->if_get_counter(ifp, IFCOUNTER_IERRORS)) / 100) << M_BITS) / 100; res = (overhead + (nbits / rate)) * ((1 << S_FACTOR) / ((1 << M_BITS) - errrate)); return (uint32_t)(res >> S_FACTOR); #undef M_BITS #undef S_FACTOR } /* * Add a Mesh Link Metric report IE to a frame. */ uint8_t * ieee80211_add_meshlmetric(uint8_t *frm, uint8_t flags, uint32_t metric) { *frm++ = IEEE80211_ELEMID_MESHLINK; *frm++ = 5; *frm++ = flags; ADDWORD(frm, metric); return frm; } /* * Add a Mesh Gate Announcement IE to a frame. */ uint8_t * ieee80211_add_meshgate(uint8_t *frm, struct ieee80211_meshgann_ie *ie) { *frm++ = IEEE80211_ELEMID_MESHGANN; /* ie */ *frm++ = IEEE80211_MESHGANN_BASE_SZ; /* len */ *frm++ = ie->gann_flags; *frm++ = ie->gann_hopcount; *frm++ = ie->gann_ttl; IEEE80211_ADDR_COPY(frm, ie->gann_addr); frm += 6; ADDWORD(frm, ie->gann_seq); ADDSHORT(frm, ie->gann_interval); return frm; } #undef ADDSHORT #undef ADDWORD /* * Initialize any mesh-specific node state. */ void ieee80211_mesh_node_init(struct ieee80211vap *vap, struct ieee80211_node *ni) { ni->ni_flags |= IEEE80211_NODE_QOS; callout_init(&ni->ni_mltimer, 1); callout_init(&ni->ni_mlhtimer, 1); } /* * Cleanup any mesh-specific node state. */ void ieee80211_mesh_node_cleanup(struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_mesh_state *ms = vap->iv_mesh; callout_drain(&ni->ni_mltimer); callout_drain(&ni->ni_mlhtimer); /* NB: short-circuit callbacks after mesh_vdetach */ if (vap->iv_mesh != NULL) ms->ms_ppath->mpp_peerdown(ni); } void ieee80211_parse_meshid(struct ieee80211_node *ni, const uint8_t *ie) { ni->ni_meshidlen = ie[1]; memcpy(ni->ni_meshid, ie + 2, ie[1]); } /* * Setup mesh-specific node state on neighbor discovery. */ void ieee80211_mesh_init_neighbor(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const struct ieee80211_scanparams *sp) { ieee80211_parse_meshid(ni, sp->meshid); } void ieee80211_mesh_update_beacon(struct ieee80211vap *vap, struct ieee80211_beacon_offsets *bo) { KASSERT(vap->iv_opmode == IEEE80211_M_MBSS, ("not a MBSS vap")); if (isset(bo->bo_flags, IEEE80211_BEACON_MESHCONF)) { (void)ieee80211_add_meshconf(bo->bo_meshconf, vap); clrbit(bo->bo_flags, IEEE80211_BEACON_MESHCONF); } } static int mesh_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq) { struct ieee80211_mesh_state *ms = vap->iv_mesh; uint8_t tmpmeshid[IEEE80211_NWID_LEN]; struct ieee80211_mesh_route *rt; struct ieee80211req_mesh_route *imr; size_t len, off; uint8_t *p; int error; if (vap->iv_opmode != IEEE80211_M_MBSS) return ENOSYS; error = 0; switch (ireq->i_type) { case IEEE80211_IOC_MESH_ID: ireq->i_len = ms->ms_idlen; memcpy(tmpmeshid, ms->ms_id, ireq->i_len); error = copyout(tmpmeshid, ireq->i_data, ireq->i_len); break; case IEEE80211_IOC_MESH_AP: ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_AP) != 0; break; case IEEE80211_IOC_MESH_FWRD: ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_FWD) != 0; break; case IEEE80211_IOC_MESH_GATE: ireq->i_val = (ms->ms_flags & IEEE80211_MESHFLAGS_GATE) != 0; break; case IEEE80211_IOC_MESH_TTL: ireq->i_val = ms->ms_ttl; break; case IEEE80211_IOC_MESH_RTCMD: switch (ireq->i_val) { case IEEE80211_MESH_RTCMD_LIST: len = 0; MESH_RT_LOCK(ms); TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) { len += sizeof(*imr); } MESH_RT_UNLOCK(ms); if (len > ireq->i_len || ireq->i_len < sizeof(*imr)) { ireq->i_len = len; return ENOMEM; } ireq->i_len = len; /* XXX M_WAIT? */ p = IEEE80211_MALLOC(len, M_TEMP, IEEE80211_M_NOWAIT | IEEE80211_M_ZERO); if (p == NULL) return ENOMEM; off = 0; MESH_RT_LOCK(ms); TAILQ_FOREACH(rt, &ms->ms_routes, rt_next) { if (off >= len) break; imr = (struct ieee80211req_mesh_route *) (p + off); IEEE80211_ADDR_COPY(imr->imr_dest, rt->rt_dest); IEEE80211_ADDR_COPY(imr->imr_nexthop, rt->rt_nexthop); imr->imr_metric = rt->rt_metric; imr->imr_nhops = rt->rt_nhops; imr->imr_lifetime = ieee80211_mesh_rt_update(rt, 0); imr->imr_lastmseq = rt->rt_lastmseq; imr->imr_flags = rt->rt_flags; /* last */ off += sizeof(*imr); } MESH_RT_UNLOCK(ms); error = copyout(p, (uint8_t *)ireq->i_data, ireq->i_len); IEEE80211_FREE(p, M_TEMP); break; case IEEE80211_MESH_RTCMD_FLUSH: case IEEE80211_MESH_RTCMD_ADD: case IEEE80211_MESH_RTCMD_DELETE: return EINVAL; default: return ENOSYS; } break; case IEEE80211_IOC_MESH_PR_METRIC: len = strlen(ms->ms_pmetric->mpm_descr); if (ireq->i_len < len) return EINVAL; ireq->i_len = len; error = copyout(ms->ms_pmetric->mpm_descr, (uint8_t *)ireq->i_data, len); break; case IEEE80211_IOC_MESH_PR_PATH: len = strlen(ms->ms_ppath->mpp_descr); if (ireq->i_len < len) return EINVAL; ireq->i_len = len; error = copyout(ms->ms_ppath->mpp_descr, (uint8_t *)ireq->i_data, len); break; default: return ENOSYS; } return error; } IEEE80211_IOCTL_GET(mesh, mesh_ioctl_get80211); static int mesh_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq) { struct ieee80211_mesh_state *ms = vap->iv_mesh; uint8_t tmpmeshid[IEEE80211_NWID_LEN]; uint8_t tmpaddr[IEEE80211_ADDR_LEN]; char tmpproto[IEEE80211_MESH_PROTO_DSZ]; int error; if (vap->iv_opmode != IEEE80211_M_MBSS) return ENOSYS; error = 0; switch (ireq->i_type) { case IEEE80211_IOC_MESH_ID: if (ireq->i_val != 0 || ireq->i_len > IEEE80211_MESHID_LEN) return EINVAL; error = copyin(ireq->i_data, tmpmeshid, ireq->i_len); if (error != 0) break; memset(ms->ms_id, 0, IEEE80211_NWID_LEN); ms->ms_idlen = ireq->i_len; memcpy(ms->ms_id, tmpmeshid, ireq->i_len); error = ENETRESET; break; case IEEE80211_IOC_MESH_AP: if (ireq->i_val) ms->ms_flags |= IEEE80211_MESHFLAGS_AP; else ms->ms_flags &= ~IEEE80211_MESHFLAGS_AP; error = ENETRESET; break; case IEEE80211_IOC_MESH_FWRD: if (ireq->i_val) ms->ms_flags |= IEEE80211_MESHFLAGS_FWD; else ms->ms_flags &= ~IEEE80211_MESHFLAGS_FWD; mesh_gatemode_setup(vap); break; case IEEE80211_IOC_MESH_GATE: if (ireq->i_val) ms->ms_flags |= IEEE80211_MESHFLAGS_GATE; else ms->ms_flags &= ~IEEE80211_MESHFLAGS_GATE; break; case IEEE80211_IOC_MESH_TTL: ms->ms_ttl = (uint8_t) ireq->i_val; break; case IEEE80211_IOC_MESH_RTCMD: switch (ireq->i_val) { case IEEE80211_MESH_RTCMD_LIST: return EINVAL; case IEEE80211_MESH_RTCMD_FLUSH: ieee80211_mesh_rt_flush(vap); break; case IEEE80211_MESH_RTCMD_ADD: error = copyin(ireq->i_data, tmpaddr, IEEE80211_ADDR_LEN); if (error != 0) break; if (IEEE80211_ADDR_EQ(vap->iv_myaddr, tmpaddr) || IEEE80211_ADDR_EQ(broadcastaddr, tmpaddr)) return EINVAL; ieee80211_mesh_discover(vap, tmpaddr, NULL); break; case IEEE80211_MESH_RTCMD_DELETE: error = copyin(ireq->i_data, tmpaddr, IEEE80211_ADDR_LEN); if (error != 0) break; ieee80211_mesh_rt_del(vap, tmpaddr); break; default: return ENOSYS; } break; case IEEE80211_IOC_MESH_PR_METRIC: error = copyin(ireq->i_data, tmpproto, sizeof(tmpproto)); if (error == 0) { error = mesh_select_proto_metric(vap, tmpproto); if (error == 0) error = ENETRESET; } break; case IEEE80211_IOC_MESH_PR_PATH: error = copyin(ireq->i_data, tmpproto, sizeof(tmpproto)); if (error == 0) { error = mesh_select_proto_path(vap, tmpproto); if (error == 0) error = ENETRESET; } break; default: return ENOSYS; } return error; } IEEE80211_IOCTL_SET(mesh, mesh_ioctl_set80211); diff --git a/sys/net80211/ieee80211_mesh.h b/sys/net80211/ieee80211_mesh.h index 9fff6c3c548c..cff4adb1504d 100644 --- a/sys/net80211/ieee80211_mesh.h +++ b/sys/net80211/ieee80211_mesh.h @@ -1,606 +1,605 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Rui Paulo under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _NET80211_IEEE80211_MESH_H_ #define _NET80211_IEEE80211_MESH_H_ #define IEEE80211_MESH_DEFAULT_TTL 31 #define IEEE80211_MESH_MAX_NEIGHBORS 15 /* * NB: all structures are __packed so sizeof works on arm, et. al. */ /* * 802.11s Information Elements. */ /* Mesh Configuration */ #define IEEE80211_MESH_CONF_SZ (7) struct ieee80211_meshconf_ie { uint8_t conf_ie; /* IEEE80211_ELEMID_MESHCONF */ uint8_t conf_len; uint8_t conf_pselid; /* Active Path Sel. Proto. ID */ uint8_t conf_pmetid; /* Active Metric Identifier */ uint8_t conf_ccid; /* Congestion Control Mode ID */ uint8_t conf_syncid; /* Sync. Protocol ID */ uint8_t conf_authid; /* Auth. Protocol ID */ uint8_t conf_form; /* Formation Information */ uint8_t conf_cap; } __packed; /* Hybrid Wireless Mesh Protocol */ enum { /* 0 reserved */ IEEE80211_MESHCONF_PATH_HWMP = 1, /* 2-254 reserved */ IEEE80211_MESHCONF_PATH_VENDOR = 255, }; /* Airtime Link Metric */ enum { /* 0 reserved */ IEEE80211_MESHCONF_METRIC_AIRTIME = 1, /* 2-254 reserved */ IEEE80211_MESHCONF_METRIC_VENDOR = 255, }; /* Congestion Control */ enum { IEEE80211_MESHCONF_CC_DISABLED = 0, IEEE80211_MESHCONF_CC_SIG = 1, /* 2-254 reserved */ IEEE80211_MESHCONF_CC_VENDOR = 255, }; /* Neighbour Offset */ enum { /* 0 reserved */ IEEE80211_MESHCONF_SYNC_NEIGHOFF = 1, /* 2-254 rserved */ IEEE80211_MESHCONF_SYNC_VENDOR = 255, }; /* Authentication Protocol Identifier */ enum { IEEE80211_MESHCONF_AUTH_DISABLED = 0, /* Simultaneous Authenticaction of Equals */ IEEE80211_MESHCONF_AUTH_SEA = 1, IEEE80211_MESHCONF_AUTH_8021X = 2, /* IEEE 802.1X */ /* 3-254 reserved */ IEEE80211_MESHCONF_AUTH_VENDOR = 255, }; /* Mesh Formation Info */ #define IEEE80211_MESHCONF_FORM_GATE 0x01 /* Connected to Gate */ #define IEEE80211_MESHCONF_FORM_NNEIGH_MASK 0x7E /* Number of Neighbours */ #define IEEE80211_MESHCONF_FORM_SA 0xF0 /* indicating 802.1X auth */ /* Mesh Capability */ #define IEEE80211_MESHCONF_CAP_AP 0x01 /* Accepting Peers */ #define IEEE80211_MESHCONF_CAP_MCCAS 0x02 /* MCCA supported */ #define IEEE80211_MESHCONF_CAP_MCCAE 0x04 /* MCCA enabled */ #define IEEE80211_MESHCONF_CAP_FWRD 0x08 /* forwarding enabled */ #define IEEE80211_MESHCONF_CAP_BTR 0x10 /* Beacon Timing Report Enab */ #define IEEE80211_MESHCONF_CAP_TBTT 0x20 /* TBTT Adjusting */ #define IEEE80211_MESHCONF_CAP_PSL 0x40 /* Power Save Level */ /* 0x80 reserved */ /* Mesh Identifier */ struct ieee80211_meshid_ie { uint8_t id_ie; /* IEEE80211_ELEMID_MESHID */ uint8_t id_len; } __packed; /* Link Metric Report */ struct ieee80211_meshlmetric_ie { uint8_t lm_ie; /* IEEE80211_ACTION_MESH_LMETRIC */ uint8_t lm_len; uint8_t lm_flags; #define IEEE80211_MESH_LMETRIC_FLAGS_REQ 0x01 /* Request */ /* * XXX: this field should be variable in size and depend on * the active active path selection metric identifier */ uint32_t lm_metric; #define IEEE80211_MESHLMETRIC_INITIALVAL 0 } __packed; /* Congestion Notification */ struct ieee80211_meshcngst_ie { uint8_t cngst_ie; /* IEEE80211_ELEMID_MESHCNGST */ uint8_t cngst_len; uint16_t cngst_timer[4]; /* Expiration Timers: AC_BK, AC_BE, AC_VI, AC_VO */ } __packed; /* Peer Link Management */ #define IEEE80211_MPM_BASE_SZ (4) #define IEEE80211_MPM_MAX_SZ (8) struct ieee80211_meshpeer_ie { uint8_t peer_ie; /* IEEE80211_ELEMID_MESHPEER */ uint8_t peer_len; uint16_t peer_proto; /* Peer Management Protocol */ uint16_t peer_llinkid; /* Local Link ID */ uint16_t peer_linkid; /* Peer Link ID */ uint16_t peer_rcode; } __packed; /* Mesh Peering Protocol Identifier field value */ enum { IEEE80211_MPPID_MPM = 0, /* Mesh peering management */ IEEE80211_MPPID_AUTH_MPM = 1, /* Auth. mesh peering exchange */ /* 2-65535 reserved */ }; #ifdef notyet /* Mesh Channel Switch Annoucement */ struct ieee80211_meshcsa_ie { uint8_t csa_ie; /* IEEE80211_ELEMID_MESHCSA */ uint8_t csa_len; uint8_t csa_mode; uint8_t csa_newclass; /* New Regulatory Class */ uint8_t csa_newchan; uint8_t csa_precvalue; /* Precedence Value */ uint8_t csa_count; } __packed; /* Mesh TIM */ /* Equal to the non Mesh version */ /* Mesh Awake Window */ struct ieee80211_meshawakew_ie { uint8_t awakew_ie; /* IEEE80211_ELEMID_MESHAWAKEW */ uint8_t awakew_len; uint8_t awakew_windowlen; /* in TUs */ } __packed; /* Mesh Beacon Timing */ struct ieee80211_meshbeacont_ie { uint8_t beacont_ie; /* IEEE80211_ELEMID_MESHBEACONT */ uint8_t beacont_len; struct { uint8_t mp_aid; /* Least Octet of AID */ uint16_t mp_btime; /* Beacon Time */ uint16_t mp_bint; /* Beacon Interval */ } __packed mp[1]; /* NB: variable size */ } __packed; #endif /* Gate (GANN) Annoucement */ /* * NB: these macros used for the length in the IEs does not include 2 bytes * for _ie and _len fields as is defined by the standard. */ #define IEEE80211_MESHGANN_BASE_SZ (15) struct ieee80211_meshgann_ie { uint8_t gann_ie; /* IEEE80211_ELEMID_MESHGANN */ uint8_t gann_len; uint8_t gann_flags; uint8_t gann_hopcount; uint8_t gann_ttl; uint8_t gann_addr[IEEE80211_ADDR_LEN]; uint32_t gann_seq; /* GANN Sequence Number */ uint16_t gann_interval; /* GANN Interval */ } __packed; /* Root (MP) Annoucement */ #define IEEE80211_MESHRANN_BASE_SZ (21) struct ieee80211_meshrann_ie { uint8_t rann_ie; /* IEEE80211_ELEMID_MESHRANN */ uint8_t rann_len; uint8_t rann_flags; #define IEEE80211_MESHRANN_FLAGS_GATE 0x01 /* Mesh Gate */ uint8_t rann_hopcount; uint8_t rann_ttl; uint8_t rann_addr[IEEE80211_ADDR_LEN]; uint32_t rann_seq; /* HWMP Sequence Number */ uint32_t rann_interval; uint32_t rann_metric; } __packed; /* Mesh Path Request */ #define IEEE80211_MESHPREQ_BASE_SZ (26) #define IEEE80211_MESHPREQ_BASE_SZ_AE (32) #define IEEE80211_MESHPREQ_TRGT_SZ (11) #define IEEE80211_MESHPREQ_TCNT_OFFSET (27) #define IEEE80211_MESHPREQ_TCNT_OFFSET_AE (33) struct ieee80211_meshpreq_ie { uint8_t preq_ie; /* IEEE80211_ELEMID_MESHPREQ */ uint8_t preq_len; uint8_t preq_flags; #define IEEE80211_MESHPREQ_FLAGS_GATE 0x01 /* Mesh Gate */ #define IEEE80211_MESHPREQ_FLAGS_AM 0x02 /* 0 = bcast / 1 = ucast */ #define IEEE80211_MESHPREQ_FLAGS_PP 0x04 /* Proactive PREP */ #define IEEE80211_MESHPREQ_FLAGS_AE 0x40 /* Address Extension */ uint8_t preq_hopcount; uint8_t preq_ttl; uint32_t preq_id; uint8_t preq_origaddr[IEEE80211_ADDR_LEN]; uint32_t preq_origseq; /* HWMP Sequence Number */ /* NB: may have Originator External Address */ uint8_t preq_orig_ext_addr[IEEE80211_ADDR_LEN]; uint32_t preq_lifetime; uint32_t preq_metric; uint8_t preq_tcount; /* target count */ struct { uint8_t target_flags; #define IEEE80211_MESHPREQ_TFLAGS_TO 0x01 /* Target Only */ #define IEEE80211_MESHPREQ_TFLAGS_USN 0x04 /* Unknown HWMP seq number */ uint8_t target_addr[IEEE80211_ADDR_LEN]; uint32_t target_seq; /* HWMP Sequence Number */ } __packed preq_targets[1]; /* NB: variable size */ } __packed; /* Mesh Path Reply */ #define IEEE80211_MESHPREP_BASE_SZ (31) #define IEEE80211_MESHPREP_BASE_SZ_AE (37) struct ieee80211_meshprep_ie { uint8_t prep_ie; /* IEEE80211_ELEMID_MESHPREP */ uint8_t prep_len; uint8_t prep_flags; #define IEEE80211_MESHPREP_FLAGS_AE 0x40 /* Address Extension */ uint8_t prep_hopcount; uint8_t prep_ttl; uint8_t prep_targetaddr[IEEE80211_ADDR_LEN]; uint32_t prep_targetseq; /* NB: May have Target External Address */ uint8_t prep_target_ext_addr[IEEE80211_ADDR_LEN]; uint32_t prep_lifetime; uint32_t prep_metric; uint8_t prep_origaddr[IEEE80211_ADDR_LEN]; uint32_t prep_origseq; /* HWMP Sequence Number */ } __packed; /* Mesh Path Error */ #define IEEE80211_MESHPERR_MAXDEST (19) #define IEEE80211_MESHPERR_NDEST_OFFSET (3) #define IEEE80211_MESHPERR_BASE_SZ (2) #define IEEE80211_MESHPERR_DEST_SZ (13) #define IEEE80211_MESHPERR_DEST_SZ_AE (19) struct ieee80211_meshperr_ie { uint8_t perr_ie; /* IEEE80211_ELEMID_MESHPERR */ uint8_t perr_len; uint8_t perr_ttl; uint8_t perr_ndests; /* Number of Destinations */ struct { uint8_t dest_flags; #define IEEE80211_MESHPERR_DFLAGS_USN 0x01 /* XXX: not part of standard */ #define IEEE80211_MESHPERR_DFLAGS_RC 0x02 /* XXX: not part of standard */ #define IEEE80211_MESHPERR_FLAGS_AE 0x40 /* Address Extension */ uint8_t dest_addr[IEEE80211_ADDR_LEN]; uint32_t dest_seq; /* HWMP Sequence Number */ /* NB: May have Destination External Address */ uint8_t dest_ext_addr[IEEE80211_ADDR_LEN]; uint16_t dest_rcode; } __packed perr_dests[1]; /* NB: variable size */ } __packed; #ifdef notyet /* Mesh Proxy Update */ struct ieee80211_meshpu_ie { uint8_t pu_ie; /* IEEE80211_ELEMID_MESHPU */ uint8_t pu_len; uint8_t pu_flags; #define IEEE80211_MESHPU_FLAGS_MASK 0x1 #define IEEE80211_MESHPU_FLAGS_DEL 0x0 #define IEEE80211_MESHPU_FLAGS_ADD 0x1 uint8_t pu_seq; /* PU Sequence Number */ uint8_t pu_addr[IEEE80211_ADDR_LEN]; uint8_t pu_naddr; /* Number of Proxied Addresses */ /* NB: proxied address follows */ } __packed; /* Mesh Proxy Update Confirmation */ struct ieee80211_meshpuc_ie { uint8_t puc_ie; /* IEEE80211_ELEMID_MESHPUC */ uint8_t puc_len; uint8_t puc_flags; uint8_t puc_seq; /* PU Sequence Number */ uint8_t puc_daddr[IEEE80211_ADDR_LEN]; } __packed; #endif /* * 802.11s Action Frames * XXX: these are wrong, and some of them should be * under MESH category while PROXY is under MULTIHOP category. */ #define IEEE80211_ACTION_CAT_INTERWORK 15 #define IEEE80211_ACTION_CAT_RESOURCE 16 #define IEEE80211_ACTION_CAT_PROXY 17 /* * Mesh Peering Action codes. */ enum { /* 0 reserved */ IEEE80211_ACTION_MESHPEERING_OPEN = 1, IEEE80211_ACTION_MESHPEERING_CONFIRM = 2, IEEE80211_ACTION_MESHPEERING_CLOSE = 3, /* 4-255 reserved */ }; /* * Mesh Action code. */ enum { IEEE80211_ACTION_MESH_LMETRIC = 0, /* Mesh Link Metric Report */ IEEE80211_ACTION_MESH_HWMP = 1, /* HWMP Mesh Path Selection */ IEEE80211_ACTION_MESH_GANN = 2, /* Gate Announcement */ IEEE80211_ACTION_MESH_CC = 3, /* Congestion Control */ IEEE80211_ACTION_MESH_MCCA_SREQ = 4, /* MCCA Setup Request */ IEEE80211_ACTION_MESH_MCCA_SREP = 5, /* MCCA Setup Reply */ IEEE80211_ACTION_MESH_MCCA_AREQ = 6, /* MCCA Advertisement Req. */ IEEE80211_ACTION_MESH_MCCA_ADVER =7, /* MCCA Advertisement */ IEEE80211_ACTION_MESH_MCCA_TRDOWN = 8, /* MCCA Teardown */ IEEE80211_ACTION_MESH_TBTT_REQ = 9, /* TBTT Adjustment Request */ IEEE80211_ACTION_MESH_TBTT_RES = 10, /* TBTT Adjustment Response */ /* 11-255 reserved */ }; /* * Different mesh control structures based on the AE * (Address Extension) bits. */ struct ieee80211_meshcntl { uint8_t mc_flags; /* Address Extension 00 */ uint8_t mc_ttl; /* TTL */ uint8_t mc_seq[4]; /* Sequence No. */ /* NB: more addresses may follow */ } __packed; struct ieee80211_meshcntl_ae01 { uint8_t mc_flags; /* Address Extension 01 */ uint8_t mc_ttl; /* TTL */ uint8_t mc_seq[4]; /* Sequence No. */ uint8_t mc_addr4[IEEE80211_ADDR_LEN]; } __packed; struct ieee80211_meshcntl_ae10 { uint8_t mc_flags; /* Address Extension 10 */ uint8_t mc_ttl; /* TTL */ uint8_t mc_seq[4]; /* Sequence No. */ uint8_t mc_addr5[IEEE80211_ADDR_LEN]; uint8_t mc_addr6[IEEE80211_ADDR_LEN]; } __packed; #define IEEE80211_MESH_AE_MASK 0x03 enum { IEEE80211_MESH_AE_00 = 0, /* MC has no AE subfield */ IEEE80211_MESH_AE_01 = 1, /* MC contain addr4 */ IEEE80211_MESH_AE_10 = 2, /* MC contain addr5 & addr6 */ IEEE80211_MESH_AE_11 = 3, /* RESERVED */ }; #ifdef _KERNEL MALLOC_DECLARE(M_80211_MESH_PREQ); MALLOC_DECLARE(M_80211_MESH_PREP); MALLOC_DECLARE(M_80211_MESH_PERR); MALLOC_DECLARE(M_80211_MESH_RT); MALLOC_DECLARE(M_80211_MESH_GT_RT); /* * Basic forwarding information: * o Destination MAC * o Next-hop MAC * o Precursor list (not implemented yet) * o Path timeout * The rest is part of the active Mesh path selection protocol. * XXX: to be moved out later. */ struct ieee80211_mesh_route { TAILQ_ENTRY(ieee80211_mesh_route) rt_next; struct ieee80211vap *rt_vap; ieee80211_rte_lock_t rt_lock; /* fine grained route lock */ struct callout rt_discovery; /* discovery timeout */ int rt_updtime; /* last update time */ uint8_t rt_dest[IEEE80211_ADDR_LEN]; uint8_t rt_mesh_gate[IEEE80211_ADDR_LEN]; /* meshDA */ uint8_t rt_nexthop[IEEE80211_ADDR_LEN]; uint32_t rt_metric; /* path metric */ uint16_t rt_nhops; /* number of hops */ uint16_t rt_flags; #define IEEE80211_MESHRT_FLAGS_DISCOVER 0x01 /* path discovery */ #define IEEE80211_MESHRT_FLAGS_VALID 0x02 /* path discovery complete */ #define IEEE80211_MESHRT_FLAGS_PROXY 0x04 /* proxy entry */ #define IEEE80211_MESHRT_FLAGS_GATE 0x08 /* mesh gate entry */ uint32_t rt_lifetime; /* route timeout */ uint32_t rt_lastmseq; /* last seq# seen dest */ uint32_t rt_ext_seq; /* proxy seq number */ void *rt_priv; /* private data */ }; #define IEEE80211_MESH_ROUTE_PRIV(rt, cast) ((cast *)rt->rt_priv) /* * Stored information about known mesh gates. */ struct ieee80211_mesh_gate_route { TAILQ_ENTRY(ieee80211_mesh_gate_route) gr_next; uint8_t gr_addr[IEEE80211_ADDR_LEN]; uint32_t gr_lastseq; struct ieee80211_mesh_route *gr_route; }; #define IEEE80211_MESH_PROTO_DSZ 12 /* description size */ /* * Mesh Path Selection Protocol. */ enum ieee80211_state; struct ieee80211_mesh_proto_path { uint8_t mpp_active; char mpp_descr[IEEE80211_MESH_PROTO_DSZ]; uint8_t mpp_ie; struct ieee80211_node * (*mpp_discover)(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct mbuf *); void (*mpp_peerdown)(struct ieee80211_node *); void (*mpp_senderror)(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN], struct ieee80211_mesh_route *, int); void (*mpp_vattach)(struct ieee80211vap *); void (*mpp_vdetach)(struct ieee80211vap *); int (*mpp_newstate)(struct ieee80211vap *, enum ieee80211_state, int); const size_t mpp_privlen; /* size required in the routing table for private data */ int mpp_inact; /* inact. timeout for invalid routes (ticks) */ }; /* * Mesh Link Metric Report Protocol. */ struct ieee80211_mesh_proto_metric { uint8_t mpm_active; char mpm_descr[IEEE80211_MESH_PROTO_DSZ]; uint8_t mpm_ie; uint32_t (*mpm_metric)(struct ieee80211_node *); }; #ifdef notyet /* * Mesh Authentication Protocol. */ struct ieee80211_mesh_proto_auth { uint8_t mpa_ie[4]; }; struct ieee80211_mesh_proto_congestion { }; struct ieee80211_mesh_proto_sync { }; #endif typedef uint32_t ieee80211_mesh_seq; #define IEEE80211_MESH_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0) #define IEEE80211_MESH_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0) struct ieee80211_mesh_state { int ms_idlen; uint8_t ms_id[IEEE80211_MESHID_LEN]; ieee80211_mesh_seq ms_seq; /* seq no for meshcntl */ uint16_t ms_neighbors; uint8_t ms_ttl; /* mesh ttl set in packets */ #define IEEE80211_MESHFLAGS_AP 0x01 /* accept peers */ #define IEEE80211_MESHFLAGS_GATE 0x02 /* mesh gate role */ #define IEEE80211_MESHFLAGS_FWD 0x04 /* forward packets */ #define IEEE80211_MESHFLAGS_ROOT 0x08 /* configured as root */ uint8_t ms_flags; ieee80211_rt_lock_t ms_rt_lock; struct callout ms_cleantimer; struct callout ms_gatetimer; ieee80211_mesh_seq ms_gateseq; TAILQ_HEAD(, ieee80211_mesh_gate_route) ms_known_gates; TAILQ_HEAD(, ieee80211_mesh_route) ms_routes; struct ieee80211_mesh_proto_metric *ms_pmetric; struct ieee80211_mesh_proto_path *ms_ppath; }; void ieee80211_mesh_attach(struct ieee80211com *); void ieee80211_mesh_detach(struct ieee80211com *); struct ieee80211_mesh_route * ieee80211_mesh_rt_find(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); struct ieee80211_mesh_route * ieee80211_mesh_rt_add(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); void ieee80211_mesh_rt_del(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); void ieee80211_mesh_rt_flush(struct ieee80211vap *); void ieee80211_mesh_rt_flush_peer(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); int ieee80211_mesh_rt_update(struct ieee80211_mesh_route *rt, int); void ieee80211_mesh_proxy_check(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); int ieee80211_mesh_register_proto_path(const struct ieee80211_mesh_proto_path *); int ieee80211_mesh_register_proto_metric(const struct ieee80211_mesh_proto_metric *); uint8_t * ieee80211_add_meshid(uint8_t *, struct ieee80211vap *); uint8_t * ieee80211_add_meshconf(uint8_t *, struct ieee80211vap *); uint8_t * ieee80211_add_meshpeer(uint8_t *, uint8_t, uint16_t, uint16_t, uint16_t); uint8_t * ieee80211_add_meshlmetric(uint8_t *, uint8_t, uint32_t); uint8_t * ieee80211_add_meshgate(uint8_t *, struct ieee80211_meshgann_ie *); void ieee80211_mesh_node_init(struct ieee80211vap *, struct ieee80211_node *); void ieee80211_mesh_node_cleanup(struct ieee80211_node *); void ieee80211_parse_meshid(struct ieee80211_node *, const uint8_t *); struct ieee80211_scanparams; void ieee80211_mesh_init_neighbor(struct ieee80211_node *, const struct ieee80211_frame *, const struct ieee80211_scanparams *); void ieee80211_mesh_update_beacon(struct ieee80211vap *, struct ieee80211_beacon_offsets *); struct ieee80211_mesh_gate_route * ieee80211_mesh_mark_gate(struct ieee80211vap *, const uint8_t *, struct ieee80211_mesh_route *); void ieee80211_mesh_forward_to_gates(struct ieee80211vap *, struct ieee80211_mesh_route *); struct ieee80211_node * ieee80211_mesh_find_txnode(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); /* * Return non-zero if proxy operation is enabled. */ static __inline int ieee80211_mesh_isproxyena(struct ieee80211vap *vap) { struct ieee80211_mesh_state *ms = vap->iv_mesh; return (ms->ms_flags & (IEEE80211_MESHFLAGS_AP | IEEE80211_MESHFLAGS_GATE)) != 0; } /* * Process an outbound frame: if a path is known to the * destination then return a reference to the next hop * for immediate transmission. Otherwise initiate path * discovery and, if possible queue the packet to be * sent when path discovery completes. */ static __inline struct ieee80211_node * ieee80211_mesh_discover(struct ieee80211vap *vap, const uint8_t dest[IEEE80211_ADDR_LEN], struct mbuf *m) { struct ieee80211_mesh_state *ms = vap->iv_mesh; return ms->ms_ppath->mpp_discover(vap, dest, m); } #endif /* _KERNEL */ #endif /* !_NET80211_IEEE80211_MESH_H_ */ diff --git a/sys/opencrypto/gfmult.c b/sys/opencrypto/gfmult.c index b34af26790f0..febabfdd4ed5 100644 --- a/sys/opencrypto/gfmult.c +++ b/sys/opencrypto/gfmult.c @@ -1,275 +1,274 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by John-Mark Gurney under * the sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #include "gfmult.h" #define REV_POLY_REDUCT 0xe1 /* 0x87 bit reversed */ /* reverse the bits of a nibble */ static const uint8_t nib_rev[] = { 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf, }; /* calculate v * 2 */ static inline struct gf128 gf128_mulalpha(struct gf128 v) { uint64_t mask; mask = !!(v.v[1] & 1); mask = ~(mask - 1); v.v[1] = (v.v[1] >> 1) | ((v.v[0] & 1) << 63); v.v[0] = (v.v[0] >> 1) ^ ((mask & REV_POLY_REDUCT) << 56); return v; } /* * Generate a table for 0-16 * h. Store the results in the table w/ indexes * bit reversed, and the words striped across the values. */ void gf128_genmultable(struct gf128 h, struct gf128table *t) { struct gf128 tbl[16]; int i; tbl[0] = MAKE_GF128(0, 0); tbl[1] = h; for (i = 2; i < 16; i += 2) { tbl[i] = gf128_mulalpha(tbl[i / 2]); tbl[i + 1] = gf128_add(tbl[i], h); } for (i = 0; i < 16; i++) { t->a[nib_rev[i]] = tbl[i].v[0] >> 32; t->b[nib_rev[i]] = tbl[i].v[0]; t->c[nib_rev[i]] = tbl[i].v[1] >> 32; t->d[nib_rev[i]] = tbl[i].v[1]; } } /* * Generate tables containing h, h^2, h^3 and h^4, starting at 0. */ void gf128_genmultable4(struct gf128 h, struct gf128table4 *t) { struct gf128 h2, h3, h4; gf128_genmultable(h, &t->tbls[0]); h2 = gf128_mul(h, &t->tbls[0]); gf128_genmultable(h2, &t->tbls[1]); h3 = gf128_mul(h, &t->tbls[1]); gf128_genmultable(h3, &t->tbls[2]); h4 = gf128_mul(h2, &t->tbls[1]); gf128_genmultable(h4, &t->tbls[3]); } /* * Read a row from the table. */ static inline struct gf128 readrow(struct gf128table *tbl, unsigned bits) { struct gf128 r; bits = bits % 16; r.v[0] = ((uint64_t)tbl->a[bits] << 32) | tbl->b[bits]; r.v[1] = ((uint64_t)tbl->c[bits] << 32) | tbl->d[bits]; return r; } /* * These are the reduction values. Since we are dealing with bit reversed * version, the values need to be bit reversed, AND the indexes are also * bit reversed to make lookups quicker. */ static uint16_t reduction[] = { 0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0, 0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0, }; /* * Calculate: * (x*2^4 + word[3,0]*h) * * 2^4 + word[7,4]*h) * * ... * 2^4 + word[63,60]*h */ static struct gf128 gfmultword(uint64_t word, struct gf128 x, struct gf128table *tbl) { struct gf128 row; unsigned bits; unsigned redbits; int i; for (i = 0; i < 64; i += 4) { bits = word % 16; /* fetch row */ row = readrow(tbl, bits); /* x * 2^4 */ redbits = x.v[1] % 16; x.v[1] = (x.v[1] >> 4) | (x.v[0] % 16) << 60; x.v[0] >>= 4; x.v[0] ^= (uint64_t)reduction[redbits] << (64 - 16); word >>= 4; x = gf128_add(x, row); } return x; } /* * Calculate * (x*2^4 + worda[3,0]*h^4+wordb[3,0]*h^3+...+wordd[3,0]*h) * * ... * 2^4 + worda[63,60]*h^4+ ... + wordd[63,60]*h * * Passing/returning struct is .5% faster than passing in via pointer on * amd64. */ static struct gf128 gfmultword4(uint64_t worda, uint64_t wordb, uint64_t wordc, uint64_t wordd, struct gf128 x, struct gf128table4 *tbl) { struct gf128 rowa, rowb, rowc, rowd; unsigned bitsa, bitsb, bitsc, bitsd; unsigned redbits; int i; /* * XXX - nibble reverse words to save a shift? probably not as * nibble reverse would take 20 ops (5 * 4) verse 16 */ for (i = 0; i < 64; i += 4) { bitsa = worda % 16; bitsb = wordb % 16; bitsc = wordc % 16; bitsd = wordd % 16; /* fetch row */ rowa = readrow(&tbl->tbls[3], bitsa); rowb = readrow(&tbl->tbls[2], bitsb); rowc = readrow(&tbl->tbls[1], bitsc); rowd = readrow(&tbl->tbls[0], bitsd); /* x * 2^4 */ redbits = x.v[1] % 16; x.v[1] = (x.v[1] >> 4) | (x.v[0] % 16) << 60; x.v[0] >>= 4; x.v[0] ^= (uint64_t)reduction[redbits] << (64 - 16); worda >>= 4; wordb >>= 4; wordc >>= 4; wordd >>= 4; x = gf128_add(x, gf128_add(rowa, gf128_add(rowb, gf128_add(rowc, rowd)))); } return x; } struct gf128 gf128_mul(struct gf128 v, struct gf128table *tbl) { struct gf128 ret; ret = MAKE_GF128(0, 0); ret = gfmultword(v.v[1], ret, tbl); ret = gfmultword(v.v[0], ret, tbl); return ret; } /* * Calculate a*h^4 + b*h^3 + c*h^2 + d*h, or: * (((a*h+b)*h+c)*h+d)*h */ struct gf128 gf128_mul4(struct gf128 a, struct gf128 b, struct gf128 c, struct gf128 d, struct gf128table4 *tbl) { struct gf128 tmp; tmp = MAKE_GF128(0, 0); tmp = gfmultword4(a.v[1], b.v[1], c.v[1], d.v[1], tmp, tbl); tmp = gfmultword4(a.v[0], b.v[0], c.v[0], d.v[0], tmp, tbl); return tmp; } /* * a = data[0..15] + r * b = data[16..31] * c = data[32..47] * d = data[48..63] * * Calculate a*h^4 + b*h^3 + c*h^2 + d*h, or: * (((a*h+b)*h+c)*h+d)*h */ struct gf128 gf128_mul4b(struct gf128 r, const uint8_t *v, struct gf128table4 *tbl) { struct gf128 a, b, c, d; struct gf128 tmp; tmp = MAKE_GF128(0, 0); a = gf128_add(r, gf128_read(&v[0*16])); b = gf128_read(&v[1*16]); c = gf128_read(&v[2*16]); d = gf128_read(&v[3*16]); tmp = gfmultword4(a.v[1], b.v[1], c.v[1], d.v[1], tmp, tbl); tmp = gfmultword4(a.v[0], b.v[0], c.v[0], d.v[0], tmp, tbl); return tmp; } diff --git a/sys/opencrypto/gfmult.h b/sys/opencrypto/gfmult.h index c385618ace37..93bdd092e927 100644 --- a/sys/opencrypto/gfmult.h +++ b/sys/opencrypto/gfmult.h @@ -1,128 +1,127 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by John-Mark Gurney under * the sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef _GFMULT_H_ #define _GFMULT_H_ #ifdef __APPLE__ #define __aligned(x) __attribute__((__aligned__(x))) #define be64dec(buf) __builtin_bswap64(*(uint64_t *)buf) #define be64enc(buf, x) (*(uint64_t *)buf = __builtin_bswap64(x)) #else #include #endif #ifdef _KERNEL #include #else #include #include #endif #define REQ_ALIGN (16 * 4) /* * The rows are striped across cache lines. Note that the indexes * are bit reversed to make accesses quicker. */ struct gf128table { uint32_t a[16] __aligned(REQ_ALIGN); /* bits 0 - 31 */ uint32_t b[16] __aligned(REQ_ALIGN); /* bits 63 - 32 */ uint32_t c[16] __aligned(REQ_ALIGN); /* bits 95 - 64 */ uint32_t d[16] __aligned(REQ_ALIGN); /* bits 127 - 96 */ } __aligned(REQ_ALIGN); /* * A set of tables that contain h, h^2, h^3, h^4. To be used w/ gf128_mul4. */ struct gf128table4 { struct gf128table tbls[4]; }; /* * GCM per spec is bit reversed in memory. So byte 0 is really bit reversed * and contains bits 0-7. We can deal w/ this by using right shifts and * related math instead of having to bit reverse everything. This means that * the low bits are in v[0] (bits 0-63) and reverse order, while the high * bits are in v[1] (bits 64-127) and reverse order. The high bit of v[0] is * bit 0, and the low bit of v[1] is bit 127. */ struct gf128 { uint64_t v[2]; }; /* Note that we don't bit reverse in MAKE_GF128. */ #define MAKE_GF128(a, b) ((struct gf128){.v = { (a), (b) } }) #define GF128_EQ(a, b) ((((a).v[0] ^ (b).v[0]) | \ ((a).v[1] ^ (b).v[1])) == 0) static inline struct gf128 gf128_read(const uint8_t *buf) { struct gf128 r; r.v[0] = be64dec(buf); buf += sizeof(uint64_t); r.v[1] = be64dec(buf); return r; } static inline void gf128_write(struct gf128 v, uint8_t *buf) { uint64_t tmp; be64enc(buf, v.v[0]); buf += sizeof tmp; be64enc(buf, v.v[1]); } static inline struct gf128 __pure /* XXX - __pure2 instead */ gf128_add(struct gf128 a, struct gf128 b) { a.v[0] ^= b.v[0]; a.v[1] ^= b.v[1]; return a; } void gf128_genmultable(struct gf128 h, struct gf128table *t); void gf128_genmultable4(struct gf128 h, struct gf128table4 *t); struct gf128 gf128_mul(struct gf128 v, struct gf128table *tbl); struct gf128 gf128_mul4(struct gf128 a, struct gf128 b, struct gf128 c, struct gf128 d, struct gf128table4 *tbl); struct gf128 gf128_mul4b(struct gf128 r, const uint8_t *v, struct gf128table4 *tbl); #endif /* _GFMULT_H_ */ diff --git a/sys/opencrypto/gmac.c b/sys/opencrypto/gmac.c index 656977acf3e6..07fa6bffb6e7 100644 --- a/sys/opencrypto/gmac.c +++ b/sys/opencrypto/gmac.c @@ -1,131 +1,130 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by John-Mark Gurney under * the sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #include #include #include #include void AES_GMAC_Init(void *ctx) { struct aes_gmac_ctx *agc; agc = ctx; bzero(agc, sizeof *agc); } void AES_GMAC_Setkey(void *ctx, const uint8_t *key, u_int klen) { struct aes_gmac_ctx *agc; const uint8_t zeros[GMAC_BLOCK_LEN] = {}; struct gf128 h; uint8_t hbuf[GMAC_BLOCK_LEN]; agc = ctx; agc->rounds = rijndaelKeySetupEnc(agc->keysched, key, klen * 8); rijndaelEncrypt(agc->keysched, agc->rounds, zeros, hbuf); h = gf128_read(hbuf); gf128_genmultable4(h, &agc->ghashtbl); explicit_bzero(&h, sizeof h); explicit_bzero(hbuf, sizeof hbuf); } void AES_GMAC_Reinit(void *ctx, const uint8_t *iv, u_int ivlen) { struct aes_gmac_ctx *agc; agc = ctx; KASSERT(ivlen <= sizeof agc->counter, ("passed ivlen too large!")); bcopy(iv, agc->counter, ivlen); } int AES_GMAC_Update(void *ctx, const void *vdata, u_int len) { struct aes_gmac_ctx *agc; const uint8_t *data; struct gf128 v; uint8_t buf[GMAC_BLOCK_LEN] = {}; int i; agc = ctx; data = vdata; v = agc->hash; while (len > 0) { if (len >= 4*GMAC_BLOCK_LEN) { i = 4*GMAC_BLOCK_LEN; v = gf128_mul4b(v, data, &agc->ghashtbl); } else if (len >= GMAC_BLOCK_LEN) { i = GMAC_BLOCK_LEN; v = gf128_add(v, gf128_read(data)); v = gf128_mul(v, &agc->ghashtbl.tbls[0]); } else { i = len; bcopy(data, buf, i); v = gf128_add(v, gf128_read(&buf[0])); v = gf128_mul(v, &agc->ghashtbl.tbls[0]); explicit_bzero(buf, sizeof buf); } len -= i; data += i; } agc->hash = v; explicit_bzero(&v, sizeof v); return (0); } void AES_GMAC_Final(uint8_t *digest, void *ctx) { struct aes_gmac_ctx *agc; uint8_t enccntr[GMAC_BLOCK_LEN]; struct gf128 a; /* XXX - zero additional bytes? */ agc = ctx; agc->counter[GMAC_BLOCK_LEN - 1] = 1; rijndaelEncrypt(agc->keysched, agc->rounds, agc->counter, enccntr); a = gf128_add(agc->hash, gf128_read(enccntr)); gf128_write(a, digest); explicit_bzero(enccntr, sizeof enccntr); } diff --git a/sys/opencrypto/gmac.h b/sys/opencrypto/gmac.h index 9de8e381ddb4..48d32e171b8f 100644 --- a/sys/opencrypto/gmac.h +++ b/sys/opencrypto/gmac.h @@ -1,56 +1,55 @@ /*- * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. * * This software was developed by John-Mark Gurney under * the sponsorship of the FreeBSD Foundation and * Rubicon Communications, LLC (Netgate). * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef _GMAC_H_ #define _GMAC_H_ #include "gfmult.h" #include #define GMAC_BLOCK_LEN 16 #define GMAC_DIGEST_LEN 16 struct aes_gmac_ctx { struct gf128table4 ghashtbl; struct gf128 hash; uint32_t keysched[4*(RIJNDAEL_MAXNR + 1)]; uint8_t counter[GMAC_BLOCK_LEN]; int rounds; }; void AES_GMAC_Init(void *); void AES_GMAC_Setkey(void *, const uint8_t *, u_int); void AES_GMAC_Reinit(void *, const uint8_t *, u_int); int AES_GMAC_Update(void *, const void *, u_int); void AES_GMAC_Final(uint8_t *, void *); #endif /* _GMAC_H_ */ diff --git a/sys/riscv/include/bus_dma_impl.h b/sys/riscv/include/bus_dma_impl.h index 64bac5410294..e482fb76a716 100644 --- a/sys/riscv/include/bus_dma_impl.h +++ b/sys/riscv/include/bus_dma_impl.h @@ -1,96 +1,95 @@ /*- * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_BUS_DMA_IMPL_H_ #define _MACHINE_BUS_DMA_IMPL_H_ struct bus_dma_tag_common { struct bus_dma_impl *impl; struct bus_dma_tag_common *parent; bus_size_t alignment; bus_addr_t boundary; bus_addr_t lowaddr; bus_addr_t highaddr; bus_dma_filter_t *filter; void *filterarg; bus_size_t maxsize; u_int nsegments; bus_size_t maxsegsz; int flags; bus_dma_lock_t *lockfunc; void *lockfuncarg; int ref_count; }; struct bus_dma_impl { int (*tag_create)(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); int (*tag_destroy)(bus_dma_tag_t dmat); int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp); void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map); int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp); int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp); int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, int *segp); void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg); bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error); void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); int bus_dma_run_filter(struct bus_dma_tag_common *dmat, bus_addr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat); extern struct bus_dma_impl bus_dma_bounce_impl; #endif diff --git a/sys/riscv/riscv/db_interface.c b/sys/riscv/riscv/db_interface.c index 0f447ea627c6..291fd2c2d8fa 100644 --- a/sys/riscv/riscv/db_interface.c +++ b/sys/riscv/riscv/db_interface.c @@ -1,160 +1,159 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include static int db_frame(struct db_variable *vp, db_expr_t *valuep, int op) { long *reg; if (kdb_frame == NULL) return (0); reg = (long *)((uintptr_t)kdb_frame + (db_expr_t)vp->valuep); if (op == DB_VAR_GET) *valuep = *reg; else *reg = *valuep; return (1); } #define DB_OFFSET(x) (db_expr_t *)offsetof(struct trapframe, x) struct db_variable db_regs[] = { { "ra", DB_OFFSET(tf_ra), db_frame }, { "sp", DB_OFFSET(tf_sp), db_frame }, { "gp", DB_OFFSET(tf_gp), db_frame }, { "tp", DB_OFFSET(tf_tp), db_frame }, { "t0", DB_OFFSET(tf_t[0]), db_frame }, { "t1", DB_OFFSET(tf_t[1]), db_frame }, { "t2", DB_OFFSET(tf_t[2]), db_frame }, { "t3", DB_OFFSET(tf_t[3]), db_frame }, { "t4", DB_OFFSET(tf_t[4]), db_frame }, { "t5", DB_OFFSET(tf_t[5]), db_frame }, { "t6", DB_OFFSET(tf_t[6]), db_frame }, { "s0", DB_OFFSET(tf_s[0]), db_frame }, { "s1", DB_OFFSET(tf_s[1]), db_frame }, { "s2", DB_OFFSET(tf_s[2]), db_frame }, { "s3", DB_OFFSET(tf_s[3]), db_frame }, { "s4", DB_OFFSET(tf_s[4]), db_frame }, { "s5", DB_OFFSET(tf_s[5]), db_frame }, { "s6", DB_OFFSET(tf_s[6]), db_frame }, { "s7", DB_OFFSET(tf_s[7]), db_frame }, { "s8", DB_OFFSET(tf_s[8]), db_frame }, { "s9", DB_OFFSET(tf_s[9]), db_frame }, { "s10", DB_OFFSET(tf_s[10]), db_frame }, { "s11", DB_OFFSET(tf_s[11]), db_frame }, { "a0", DB_OFFSET(tf_a[0]), db_frame }, { "a1", DB_OFFSET(tf_a[1]), db_frame }, { "a2", DB_OFFSET(tf_a[2]), db_frame }, { "a3", DB_OFFSET(tf_a[3]), db_frame }, { "a4", DB_OFFSET(tf_a[4]), db_frame }, { "a5", DB_OFFSET(tf_a[5]), db_frame }, { "a6", DB_OFFSET(tf_a[6]), db_frame }, { "a7", DB_OFFSET(tf_a[7]), db_frame }, { "sepc", DB_OFFSET(tf_sepc), db_frame }, { "sstatus", DB_OFFSET(tf_sstatus), db_frame }, { "stval", DB_OFFSET(tf_stval), db_frame }, { "scause", DB_OFFSET(tf_scause), db_frame }, }; struct db_variable *db_eregs = db_regs + nitems(db_regs); void db_show_mdpcpu(struct pcpu *pc) { } /* * Read bytes from kernel address space for debugger. */ int db_read_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; const char *src; int ret; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { src = (const char *)addr; while (size-- > 0) *data++ = *src++; } (void)kdb_jmpbuf(prev_jb); return (ret); } /* * Write bytes to kernel address space for debugger. */ int db_write_bytes(vm_offset_t addr, size_t size, char *data) { jmp_buf jb; void *prev_jb; char *dst; int ret; prev_jb = kdb_jmpbuf(jb); ret = setjmp(jb); if (ret == 0) { dst = (char *)addr; while (size-- > 0) *dst++ = *data++; /* Invalidate I-cache */ fence_i(); } (void)kdb_jmpbuf(prev_jb); return (ret); } diff --git a/sys/sys/_rangeset.h b/sys/sys/_rangeset.h index f7c72810ead5..17f5e374c91f 100644 --- a/sys/sys/_rangeset.h +++ b/sys/sys/_rangeset.h @@ -1,50 +1,49 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2019 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS__RANGESET_H #define _SYS__RANGESET_H #include typedef void *(*rs_dup_data_t)(void *ctx, void *data); typedef void (*rs_free_data_t)(void *ctx, void *data); struct rangeset { struct pctrie rs_trie; rs_dup_data_t rs_dup_data; rs_free_data_t rs_free_data; void *rs_data_ctx; u_int rs_alloc_flags; }; #endif diff --git a/sys/sys/atomic_common.h b/sys/sys/atomic_common.h index 403b3de8a092..e3969b241a8c 100644 --- a/sys/sys/atomic_common.h +++ b/sys/sys/atomic_common.h @@ -1,83 +1,82 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2017 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_ATOMIC_COMMON_H_ #define _SYS_ATOMIC_COMMON_H_ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif #define atomic_load_char(p) (*(volatile u_char *)(p)) #define atomic_load_short(p) (*(volatile u_short *)(p)) #define atomic_load_int(p) (*(volatile u_int *)(p)) #define atomic_load_long(p) (*(volatile u_long *)(p)) #define atomic_load_ptr(p) (*(volatile __typeof(*p) *)(p)) #define atomic_load_8(p) (*(volatile uint8_t *)(p)) #define atomic_load_16(p) (*(volatile uint16_t *)(p)) #define atomic_load_32(p) (*(volatile uint32_t *)(p)) #ifdef _LP64 #define atomic_load_64(p) (*(volatile uint64_t *)(p)) #endif #define atomic_store_char(p, v) \ (*(volatile u_char *)(p) = (u_char)(v)) #define atomic_store_short(p, v) \ (*(volatile u_short *)(p) = (u_short)(v)) #define atomic_store_int(p, v) \ (*(volatile u_int *)(p) = (u_int)(v)) #define atomic_store_long(p, v) \ (*(volatile u_long *)(p) = (u_long)(v)) #define atomic_store_ptr(p, v) \ (*(volatile __typeof(*p) *)(p) = (v)) #define atomic_store_8(p, v) \ (*(volatile uint8_t *)(p) = (uint8_t)(v)) #define atomic_store_16(p, v) \ (*(volatile uint16_t *)(p) = (uint16_t)(v)) #define atomic_store_32(p, v) \ (*(volatile uint32_t *)(p) = (uint32_t)(v)) #ifdef _LP64 #define atomic_store_64(p, v) \ (*(volatile uint64_t *)(p) = (uint64_t)(v)) #endif /* * Currently all architectures provide acquire and release fences on their own, * but they don't provide consume. Kludge below allows relevant code to stop * openly resorting to the stronger acquire fence, to be sorted out. */ #define atomic_load_consume_ptr(p) \ ((__typeof(*p)) atomic_load_acq_ptr((uintptr_t *)p)) #define atomic_interrupt_fence() __compiler_membar() #endif diff --git a/sys/sys/caprights.h b/sys/sys/caprights.h index 9f396a0df553..f4325ea0ba57 100644 --- a/sys/sys/caprights.h +++ b/sys/sys/caprights.h @@ -1,111 +1,110 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_CAPRIGHTS_H_ #define _SYS_CAPRIGHTS_H_ /* * The top two bits in the first element of the cr_rights[] array contain * total number of elements in the array - 2. This means if those two bits are * equal to 0, we have 2 array elements. * The top two bits in all remaining array elements should be 0. * The next five bits contain array index. Only one bit is used and bit position * in this five-bits range defines array index. This means there can be at most * five array elements. */ #define CAP_RIGHTS_VERSION_00 0 /* #define CAP_RIGHTS_VERSION_01 1 #define CAP_RIGHTS_VERSION_02 2 #define CAP_RIGHTS_VERSION_03 3 */ #define CAP_RIGHTS_VERSION CAP_RIGHTS_VERSION_00 struct cap_rights { uint64_t cr_rights[CAP_RIGHTS_VERSION + 2]; }; #ifndef _CAP_RIGHTS_T_DECLARED #define _CAP_RIGHTS_T_DECLARED typedef struct cap_rights cap_rights_t; #endif #ifdef _KERNEL extern cap_rights_t cap_accept_rights; extern cap_rights_t cap_bind_rights; extern cap_rights_t cap_connect_rights; extern cap_rights_t cap_event_rights; extern cap_rights_t cap_fchdir_rights; extern cap_rights_t cap_fchflags_rights; extern cap_rights_t cap_fchmod_rights; extern cap_rights_t cap_fchown_rights; extern cap_rights_t cap_fcntl_rights; extern cap_rights_t cap_fexecve_rights; extern cap_rights_t cap_flock_rights; extern cap_rights_t cap_fpathconf_rights; extern cap_rights_t cap_fstat_rights; extern cap_rights_t cap_fstatfs_rights; extern cap_rights_t cap_fsync_rights; extern cap_rights_t cap_ftruncate_rights; extern cap_rights_t cap_futimes_rights; extern cap_rights_t cap_getpeername_rights; extern cap_rights_t cap_getsockopt_rights; extern cap_rights_t cap_getsockname_rights; extern cap_rights_t cap_ioctl_rights; extern cap_rights_t cap_linkat_source_rights; extern cap_rights_t cap_linkat_target_rights; extern cap_rights_t cap_listen_rights; extern cap_rights_t cap_mkdirat_rights; extern cap_rights_t cap_mkfifoat_rights; extern cap_rights_t cap_mknodat_rights; extern cap_rights_t cap_mmap_rights; extern cap_rights_t cap_no_rights; extern cap_rights_t cap_pdgetpid_rights; extern cap_rights_t cap_pdkill_rights; extern cap_rights_t cap_pread_rights; extern cap_rights_t cap_pwrite_rights; extern cap_rights_t cap_read_rights; extern cap_rights_t cap_recv_rights; extern cap_rights_t cap_renameat_source_rights; extern cap_rights_t cap_renameat_target_rights; extern cap_rights_t cap_seek_rights; extern cap_rights_t cap_send_rights; extern cap_rights_t cap_send_connect_rights; extern cap_rights_t cap_setsockopt_rights; extern cap_rights_t cap_shutdown_rights; extern cap_rights_t cap_symlinkat_rights; extern cap_rights_t cap_unlinkat_rights; extern cap_rights_t cap_write_rights; #endif #endif /* !_SYS_CAPRIGHTS_H_ */ diff --git a/sys/sys/dnv.h b/sys/sys/dnv.h index 3e0322e8565f..9ddd8720c923 100644 --- a/sys/sys/dnv.h +++ b/sys/sys/dnv.h @@ -1,87 +1,86 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DNV_H_ #define _DNV_H_ #include #ifndef _KERNEL #include #include #include #include #endif #ifndef _NVLIST_T_DECLARED #define _NVLIST_T_DECLARED struct nvlist; typedef struct nvlist nvlist_t; #endif __BEGIN_DECLS /* * The dnvlist_get functions returns value associated with the given name. * If it returns a pointer, the pointer represents internal buffer and should * not be freed by the caller. * If no element of the given name and type exists, the function will return * provided default value. */ bool dnvlist_get_bool(const nvlist_t *nvl, const char *name, bool defval); uint64_t dnvlist_get_number(const nvlist_t *nvl, const char *name, uint64_t defval); const char *dnvlist_get_string(const nvlist_t *nvl, const char *name, const char *defval); const nvlist_t *dnvlist_get_nvlist(const nvlist_t *nvl, const char *name, const nvlist_t *defval); int dnvlist_get_descriptor(const nvlist_t *nvl, const char *name, int defval); const void *dnvlist_get_binary(const nvlist_t *nvl, const char *name, size_t *sizep, const void *defval, size_t defsize); /* * The dnvlist_take functions returns value associated with the given name and * remove corresponding nvpair. * If it returns a pointer, the caller has to free it. * If no element of the given name and type exists, the function will return * provided default value. */ bool dnvlist_take_bool(nvlist_t *nvl, const char *name, bool defval); uint64_t dnvlist_take_number(nvlist_t *nvl, const char *name, uint64_t defval); char *dnvlist_take_string(nvlist_t *nvl, const char *name, char *defval); nvlist_t *dnvlist_take_nvlist(nvlist_t *nvl, const char *name, nvlist_t *defval); int dnvlist_take_descriptor(nvlist_t *nvl, const char *name, int defval); void *dnvlist_take_binary(nvlist_t *nvl, const char *name, size_t *sizep, void *defval, size_t defsize); __END_DECLS #endif /* !_DNV_H_ */ diff --git a/sys/sys/font.h b/sys/sys/font.h index 969a9bce4e6d..04cab126e906 100644 --- a/sys/sys/font.h +++ b/sys/sys/font.h @@ -1,122 +1,121 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009, 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Portions of this software were developed by Oleksandr Rybalko * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_FONT_H_ #define _SYS_FONT_H_ #include /* * Fonts. * * Fonts support normal and bold weights, and single and double width glyphs. * Mapping tables are used to map Unicode points to glyphs. They are sorted by * code point, and vtfont_lookup() uses this to perform a binary search. Each * font has four mapping tables: two weights times two halves (left/single, * right). When a character is not present in a bold map the glyph from the * normal map is used. When no glyph is available, it uses glyph 0, which is * normally equal to U+FFFD. */ enum vfnt_map_type { VFNT_MAP_NORMAL = 0, /* Normal font. */ VFNT_MAP_NORMAL_RIGHT, /* Normal font right hand. */ VFNT_MAP_BOLD, /* Bold font. */ VFNT_MAP_BOLD_RIGHT, /* Bold font right hand. */ VFNT_MAPS /* Number of maps. */ }; struct font_info { int32_t fi_checksum; uint32_t fi_width; uint32_t fi_height; uint32_t fi_bitmap_size; uint32_t fi_map_count[VFNT_MAPS]; }; struct vfnt_map { uint32_t vfm_src; uint16_t vfm_dst; uint16_t vfm_len; } __packed; typedef struct vfnt_map vfnt_map_t; struct vt_font { vfnt_map_t *vf_map[VFNT_MAPS]; uint8_t *vf_bytes; uint32_t vf_height; uint32_t vf_width; uint32_t vf_map_count[VFNT_MAPS]; uint32_t vf_refcount; }; typedef struct vt_font_bitmap_data { uint32_t vfbd_width; uint32_t vfbd_height; uint32_t vfbd_compressed_size; uint32_t vfbd_uncompressed_size; uint8_t *vfbd_compressed_data; struct vt_font *vfbd_font; } vt_font_bitmap_data_t; typedef enum { FONT_AUTO, /* This font is loaded by software */ FONT_MANUAL, /* This font is loaded manually by user */ FONT_BUILTIN, /* This font was built in at compile time */ FONT_RELOAD /* This font is marked to be re-read from file */ } FONT_FLAGS; struct fontlist { char *font_name; FONT_FLAGS font_flags; vt_font_bitmap_data_t *font_data; vt_font_bitmap_data_t *(*font_load)(char *); STAILQ_ENTRY(fontlist) font_next; }; typedef STAILQ_HEAD(font_list, fontlist) font_list_t; #define FONT_HEADER_MAGIC "VFNT0002" struct font_header { uint8_t fh_magic[8]; uint8_t fh_width; uint8_t fh_height; uint16_t fh_pad; uint32_t fh_glyph_count; uint32_t fh_map_count[VFNT_MAPS]; } __packed; #endif /* !_SYS_FONT_H_ */ diff --git a/sys/sys/rangeset.h b/sys/sys/rangeset.h index 99733ecfaf51..58dded5d37e2 100644 --- a/sys/sys/rangeset.h +++ b/sys/sys/rangeset.h @@ -1,88 +1,87 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2019 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_RANGESET_H #define _SYS_RANGESET_H #ifdef _KERNEL #include typedef bool (*rs_pred_t)(void *ctx, void *r); /* * This structure must be embedded at the start of the rangeset element. */ struct rs_el { uint64_t re_start; /* pctrie key */ uint64_t re_end; }; void rangeset_init(struct rangeset *rs, rs_dup_data_t dup_data, rs_free_data_t free_data, void *rs_data_ctx, u_int alloc_flags); void rangeset_fini(struct rangeset *rs); bool rangeset_check_empty(struct rangeset *rs, uint64_t start, uint64_t end); /* * r point to the app data with struct rs_el at the beginning. */ int rangeset_insert(struct rangeset *rs, uint64_t start, uint64_t end, void *r); /* * Guarantees that on error the rangeset is not modified. Remove * might need to split element if its start/end completely cover the * removed range, in which case ENOMEM might be returned. */ void rangeset_remove_all(struct rangeset *rs); int rangeset_remove(struct rangeset *rs, uint64_t start, uint64_t end); int rangeset_remove_pred(struct rangeset *rs, uint64_t start, uint64_t end, rs_pred_t pred); /* * Really returns the pointer to the data with struct rs_el embedded * at the beginning. */ void *rangeset_lookup(struct rangeset *rs, uint64_t place); /* * Copies src_rs entries into dst_rs. dst_rs must be empty. * Leaves dst_rs empty on failure. */ int rangeset_copy(struct rangeset *dst_rs, struct rangeset *src_rs); #endif #endif diff --git a/sys/sys/terminal.h b/sys/sys/terminal.h index dc78cca5394e..a77c985e85ca 100644 --- a/sys/sys/terminal.h +++ b/sys/sys/terminal.h @@ -1,240 +1,239 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2009 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Ed Schouten under sponsorship from the * FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_TERMINAL_H_ #define _SYS_TERMINAL_H_ #include #include #include #include #include #include #include #include "opt_syscons.h" #include "opt_teken.h" struct terminal; struct thread; struct tty; /* * The terminal layer is an abstraction on top of the TTY layer and the * console interface. It can be used by system console drivers to * easily interact with the kernel console and TTYs. * * Terminals contain terminal emulators, which means console drivers * don't need to implement their own terminal emulator. The terminal * emulator deals with UTF-8 exclusively. This means that term_char_t, * the data type used to store input/output characters will always * contain Unicode codepoints. * * To save memory usage, the top bits of term_char_t will contain other * attributes, like colors. Right now term_char_t is composed as * follows: * * Bits Meaning * 0-20: Character value * 21-25: Bold, underline, blink, reverse, right part of CJK fullwidth character * 26-28: Foreground color * 29-31: Background color */ typedef uint32_t term_char_t; #define TCHAR_CHARACTER(c) ((c) & 0x1fffff) #define TCHAR_FORMAT(c) (((c) >> 21) & 0x1f) #define TCHAR_FGCOLOR(c) (((c) >> 26) & 0x7) #define TCHAR_BGCOLOR(c) (((c) >> 29) & 0x7) typedef teken_attr_t term_attr_t; typedef teken_color_t term_color_t; #define TCOLOR_FG(c) (((c) & 0x7) << 26) #define TCOLOR_BG(c) (((c) & 0x7) << 29) #define TCOLOR_LIGHT(c) ((c) | 0x8) #define TCOLOR_DARK(c) ((c) & ~0x8) #define TFORMAT(c) (((c) & 0x1f) << 21) /* syscons(4) compatible color attributes for foreground text */ #define FG_BLACK TCOLOR_FG(TC_BLACK) #define FG_BLUE TCOLOR_FG(TC_BLUE) #define FG_GREEN TCOLOR_FG(TC_GREEN) #define FG_CYAN TCOLOR_FG(TC_CYAN) #define FG_RED TCOLOR_FG(TC_RED) #define FG_MAGENTA TCOLOR_FG(TC_MAGENTA) #define FG_BROWN TCOLOR_FG(TC_BROWN) #define FG_LIGHTGREY TCOLOR_FG(TC_WHITE) #define FG_DARKGREY (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_BLACK)) #define FG_LIGHTBLUE (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_BLUE)) #define FG_LIGHTGREEN (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_GREEN)) #define FG_LIGHTCYAN (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_CYAN)) #define FG_LIGHTRED (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_RED)) #define FG_LIGHTMAGENTA (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_MAGENTA)) #define FG_YELLOW (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_BROWN)) #define FG_WHITE (TFORMAT(TF_BOLD) | TCOLOR_FG(TC_WHITE)) #define FG_BLINK TFORMAT(TF_BLINK) /* syscons(4) compatible color attributes for text background */ #define BG_BLACK TCOLOR_BG(TC_BLACK) #define BG_BLUE TCOLOR_BG(TC_BLUE) #define BG_GREEN TCOLOR_BG(TC_GREEN) #define BG_CYAN TCOLOR_BG(TC_CYAN) #define BG_RED TCOLOR_BG(TC_RED) #define BG_MAGENTA TCOLOR_BG(TC_MAGENTA) #define BG_BROWN TCOLOR_BG(TC_BROWN) #define BG_LIGHTGREY TCOLOR_BG(TC_WHITE) #define BG_DARKGREY (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_BLACK)) #define BG_LIGHTBLUE (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_BLUE)) #define BG_LIGHTGREEN (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_GREEN)) #define BG_LIGHTCYAN (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_CYAN)) #define BG_LIGHTRED (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_RED)) #define BG_LIGHTMAGENTA (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_MAGENTA)) #define BG_YELLOW (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_BROWN)) #define BG_WHITE (TFORMAT(TF_BOLD) | TCOLOR_BG(TC_WHITE)) #ifndef TERMINAL_NORM_ATTR #ifdef SC_NORM_ATTR #define TERMINAL_NORM_ATTR SC_NORM_ATTR #else #define TERMINAL_NORM_ATTR (FG_LIGHTGREY | BG_BLACK) #endif #endif #ifndef TERMINAL_KERN_ATTR #ifdef SC_KERNEL_CONS_ATTR #define TERMINAL_KERN_ATTR SC_KERNEL_CONS_ATTR #else #define TERMINAL_KERN_ATTR (FG_WHITE | BG_BLACK) #endif #endif typedef teken_pos_t term_pos_t; typedef teken_rect_t term_rect_t; typedef void tc_cursor_t(struct terminal *tm, const term_pos_t *p); typedef void tc_putchar_t(struct terminal *tm, const term_pos_t *p, term_char_t c); typedef void tc_fill_t(struct terminal *tm, const term_rect_t *r, term_char_t c); typedef void tc_copy_t(struct terminal *tm, const term_rect_t *r, const term_pos_t *p); typedef void tc_pre_input_t(struct terminal *tm); typedef void tc_post_input_t(struct terminal *tm); typedef void tc_param_t(struct terminal *tm, int cmd, unsigned int arg); typedef void tc_done_t(struct terminal *tm); typedef void tc_cnprobe_t(struct terminal *tm, struct consdev *cd); typedef int tc_cngetc_t(struct terminal *tm); typedef void tc_cngrab_t(struct terminal *tm); typedef void tc_cnungrab_t(struct terminal *tm); typedef void tc_opened_t(struct terminal *tm, int opened); typedef int tc_ioctl_t(struct terminal *tm, u_long cmd, caddr_t data, struct thread *td); typedef int tc_mmap_t(struct terminal *tm, vm_ooffset_t offset, vm_paddr_t * paddr, int nprot, vm_memattr_t *memattr); typedef void tc_bell_t(struct terminal *tm); struct terminal_class { /* Terminal emulator. */ tc_cursor_t *tc_cursor; tc_putchar_t *tc_putchar; tc_fill_t *tc_fill; tc_copy_t *tc_copy; tc_pre_input_t *tc_pre_input; tc_post_input_t *tc_post_input; tc_param_t *tc_param; tc_done_t *tc_done; /* Low-level console interface. */ tc_cnprobe_t *tc_cnprobe; tc_cngetc_t *tc_cngetc; /* DDB & panic handling. */ tc_cngrab_t *tc_cngrab; tc_cnungrab_t *tc_cnungrab; /* Misc. */ tc_opened_t *tc_opened; tc_ioctl_t *tc_ioctl; tc_mmap_t *tc_mmap; tc_bell_t *tc_bell; }; struct terminal { const struct terminal_class *tm_class; void *tm_softc; struct mtx tm_mtx; struct tty *tm_tty; teken_t tm_emulator; struct winsize tm_winsize; unsigned int tm_flags; #define TF_MUTE 0x1 /* Drop incoming data. */ #define TF_BELL 0x2 /* Bell needs to be sent. */ #define TF_CONS 0x4 /* Console device (needs spinlock). */ struct consdev *consdev; }; #ifdef _KERNEL struct terminal *terminal_alloc(const struct terminal_class *tc, void *softc); void terminal_maketty(struct terminal *tm, const char *fmt, ...); void terminal_set_cursor(struct terminal *tm, const term_pos_t *pos); void terminal_set_winsize_blank(struct terminal *tm, const struct winsize *size, int blank, const term_attr_t *attr); void terminal_set_winsize(struct terminal *tm, const struct winsize *size); void terminal_mute(struct terminal *tm, int yes); void terminal_input_char(struct terminal *tm, term_char_t c); void terminal_input_raw(struct terminal *tm, char c); void terminal_input_special(struct terminal *tm, unsigned int k); void termcn_cnregister(struct terminal *tm); /* Kernel console helper interface. */ extern const struct consdev_ops termcn_cnops; #define TERMINAL_DECLARE_EARLY(name, class, softc) \ static struct terminal name = { \ .tm_class = &class, \ .tm_softc = softc, \ .tm_flags = TF_CONS, \ }; \ CONSOLE_DEVICE(name ## _consdev, termcn_cnops, &name) #endif /* _KERNEL */ #endif /* !_SYS_TERMINAL_H_ */ diff --git a/sys/sys/timepps.h b/sys/sys/timepps.h index 578d42704f12..292f7ea1b0f6 100644 --- a/sys/sys/timepps.h +++ b/sys/sys/timepps.h @@ -1,267 +1,266 @@ /*- * SPDX-License-Identifier: Beerware * * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): * wrote this file. As long as you retain this notice you * can do whatever you want with this stuff. If we meet some day, and you think * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * * Copyright (c) 2011 The FreeBSD Foundation - * All rights reserved. * * Portions of this software were developed by Julien Ridoux at the University * of Melbourne under sponsorship from the FreeBSD Foundation. * * $FreeBSD$ * * The is a FreeBSD version of the RFC 2783 API for Pulse Per Second * timing interfaces. */ #ifndef _SYS_TIMEPPS_H_ #define _SYS_TIMEPPS_H_ #include #include #include #define PPS_API_VERS_1 1 typedef int pps_handle_t; typedef unsigned pps_seq_t; typedef struct ntp_fp { unsigned int integral; unsigned int fractional; } ntp_fp_t; typedef union pps_timeu { struct timespec tspec; ntp_fp_t ntpfp; unsigned long longpad[3]; } pps_timeu_t; typedef struct { pps_seq_t assert_sequence; /* assert event seq # */ pps_seq_t clear_sequence; /* clear event seq # */ pps_timeu_t assert_tu; pps_timeu_t clear_tu; int current_mode; /* current mode bits */ } pps_info_t; typedef struct { pps_seq_t assert_sequence; /* assert event seq # */ pps_seq_t clear_sequence; /* clear event seq # */ pps_timeu_t assert_tu; pps_timeu_t clear_tu; ffcounter assert_ffcount; /* ffcounter on assert event */ ffcounter clear_ffcount; /* ffcounter on clear event */ int current_mode; /* current mode bits */ } pps_info_ffc_t; #define assert_timestamp assert_tu.tspec #define clear_timestamp clear_tu.tspec #define assert_timestamp_ntpfp assert_tu.ntpfp #define clear_timestamp_ntpfp clear_tu.ntpfp typedef struct { int api_version; /* API version # */ int mode; /* mode bits */ pps_timeu_t assert_off_tu; pps_timeu_t clear_off_tu; } pps_params_t; #define assert_offset assert_off_tu.tspec #define clear_offset clear_off_tu.tspec #define assert_offset_ntpfp assert_off_tu.ntpfp #define clear_offset_ntpfp clear_off_tu.ntpfp #define PPS_CAPTUREASSERT 0x01 #define PPS_CAPTURECLEAR 0x02 #define PPS_CAPTUREBOTH 0x03 #define PPS_OFFSETASSERT 0x10 #define PPS_OFFSETCLEAR 0x20 #define PPS_ECHOASSERT 0x40 #define PPS_ECHOCLEAR 0x80 #define PPS_CANWAIT 0x100 #define PPS_CANPOLL 0x200 #define PPS_TSFMT_TSPEC 0x1000 #define PPS_TSFMT_NTPFP 0x2000 #define PPS_TSCLK_FBCK 0x10000 #define PPS_TSCLK_FFWD 0x20000 #define PPS_TSCLK_MASK 0x30000 #define PPS_KC_HARDPPS 0 #define PPS_KC_HARDPPS_PLL 1 #define PPS_KC_HARDPPS_FLL 2 struct pps_fetch_args { int tsformat; pps_info_t pps_info_buf; struct timespec timeout; }; struct pps_fetch_ffc_args { int tsformat; pps_info_ffc_t pps_info_buf_ffc; struct timespec timeout; }; struct pps_kcbind_args { int kernel_consumer; int edge; int tsformat; }; #define PPS_IOC_CREATE _IO('1', 1) #define PPS_IOC_DESTROY _IO('1', 2) #define PPS_IOC_SETPARAMS _IOW('1', 3, pps_params_t) #define PPS_IOC_GETPARAMS _IOR('1', 4, pps_params_t) #define PPS_IOC_GETCAP _IOR('1', 5, int) #define PPS_IOC_FETCH _IOWR('1', 6, struct pps_fetch_args) #define PPS_IOC_KCBIND _IOW('1', 7, struct pps_kcbind_args) #define PPS_IOC_FETCH_FFCOUNTER _IOWR('1', 8, struct pps_fetch_ffc_args) #ifdef _KERNEL struct mtx; #define KCMODE_EDGEMASK 0x03 #define KCMODE_ABIFLAG 0x80000000 /* Internal use: abi-aware driver. */ #define PPS_ABI_VERSION 1 #define PPSFLAG_MTX_SPIN 0x01 /* Driver mtx is MTX_SPIN type. */ struct pps_state { /* Capture information. */ struct timehands *capth; struct fftimehands *capffth; unsigned capgen; unsigned capcount; /* State information. */ pps_params_t ppsparam; pps_info_t ppsinfo; pps_info_ffc_t ppsinfo_ffc; int kcmode; int ppscap; struct timecounter *ppstc; unsigned ppscount[3]; /* * The following fields are valid if the driver calls pps_init_abi(). */ uint16_t driver_abi; /* Driver sets before pps_init_abi(). */ uint16_t kernel_abi; /* Kernel sets during pps_init_abi(). */ struct mtx *driver_mtx; /* Optional, valid if non-NULL. */ uint32_t flags; }; void pps_capture(struct pps_state *pps); void pps_event(struct pps_state *pps, int event); void pps_init(struct pps_state *pps); void pps_init_abi(struct pps_state *pps); int pps_ioctl(unsigned long cmd, caddr_t data, struct pps_state *pps); void hardpps(struct timespec *tsp, long nsec); #else /* !_KERNEL */ static __inline int time_pps_create(int filedes, pps_handle_t *handle) { int error; *handle = -1; error = ioctl(filedes, PPS_IOC_CREATE, 0); if (error < 0) return (-1); *handle = filedes; return (0); } static __inline int time_pps_destroy(pps_handle_t handle) { return (ioctl(handle, PPS_IOC_DESTROY, 0)); } static __inline int time_pps_setparams(pps_handle_t handle, const pps_params_t *ppsparams) { return (ioctl(handle, PPS_IOC_SETPARAMS, ppsparams)); } static __inline int time_pps_getparams(pps_handle_t handle, pps_params_t *ppsparams) { return (ioctl(handle, PPS_IOC_GETPARAMS, ppsparams)); } static __inline int time_pps_getcap(pps_handle_t handle, int *mode) { return (ioctl(handle, PPS_IOC_GETCAP, mode)); } static __inline int time_pps_fetch(pps_handle_t handle, const int tsformat, pps_info_t *ppsinfobuf, const struct timespec *timeout) { int error; struct pps_fetch_args arg; arg.tsformat = tsformat; if (timeout == NULL) { arg.timeout.tv_sec = -1; arg.timeout.tv_nsec = -1; } else arg.timeout = *timeout; error = ioctl(handle, PPS_IOC_FETCH, &arg); *ppsinfobuf = arg.pps_info_buf; return (error); } static __inline int time_pps_fetch_ffc(pps_handle_t handle, const int tsformat, pps_info_ffc_t *ppsinfobuf, const struct timespec *timeout) { struct pps_fetch_ffc_args arg; int error; arg.tsformat = tsformat; if (timeout == NULL) { arg.timeout.tv_sec = -1; arg.timeout.tv_nsec = -1; } else { arg.timeout = *timeout; } error = ioctl(handle, PPS_IOC_FETCH_FFCOUNTER, &arg); *ppsinfobuf = arg.pps_info_buf_ffc; return (error); } static __inline int time_pps_kcbind(pps_handle_t handle, const int kernel_consumer, const int edge, const int tsformat) { struct pps_kcbind_args arg; arg.kernel_consumer = kernel_consumer; arg.edge = edge; arg.tsformat = tsformat; return (ioctl(handle, PPS_IOC_KCBIND, &arg)); } #endif /* KERNEL */ #endif /* !_SYS_TIMEPPS_H_ */ diff --git a/sys/tools/fdt/embed_dtb.sh b/sys/tools/fdt/embed_dtb.sh index d5c9c00a9e8a..f6daf799a088 100755 --- a/sys/tools/fdt/embed_dtb.sh +++ b/sys/tools/fdt/embed_dtb.sh @@ -1,46 +1,45 @@ #!/bin/sh # # SPDX-License-Identifier: BSD-2-Clause-FreeBSD # # Copyright (c) 2010 The FreeBSD Foundation -# All rights reserved. # # This software was developed by Semihalf under sponsorship from # the FreeBSD Foundation. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # $1: kernel filename # $2: device tree blob filename dtbsize=`ls -l $2 | awk '{print $5}'` obs=`strings -at d $1 | grep "Device Tree Blob STARTS here" | awk '{print $1}'` eol=`strings -at d $1 | grep "Device Tree Blob STOPS here" | awk '{print $1}'` sz=$((eol - obs)); if [ $sz -lt $dtbsize ] then echo "Selected blob is too big to reembed. Please rebuild the kernel." exit fi dd if=$2 ibs=${dtbsize} of=$1 obs=${obs} oseek=1 conv=notrunc 2> /dev/null diff --git a/sys/x86/include/busdma_impl.h b/sys/x86/include/busdma_impl.h index 97c335f9a79f..c48d98ddfe71 100644 --- a/sys/x86/include/busdma_impl.h +++ b/sys/x86/include/busdma_impl.h @@ -1,101 +1,100 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_BUSDMA_IMPL_H #define __X86_BUSDMA_IMPL_H struct bus_dma_tag_common { struct bus_dma_impl *impl; struct bus_dma_tag_common *parent; bus_size_t alignment; bus_addr_t boundary; bus_addr_t lowaddr; bus_addr_t highaddr; bus_dma_filter_t *filter; void *filterarg; bus_size_t maxsize; u_int nsegments; bus_size_t maxsegsz; int flags; bus_dma_lock_t *lockfunc; void *lockfuncarg; int ref_count; int domain; }; struct bus_dma_impl { int (*tag_create)(bus_dma_tag_t parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat); int (*tag_destroy)(bus_dma_tag_t dmat); int (*tag_set_domain)(bus_dma_tag_t); bool (*id_mapped)(bus_dma_tag_t, vm_paddr_t, bus_size_t); int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, bus_dmamap_t *mapp); void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map); int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, int *segp); int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp); int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, int *segp); void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg); bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int error); void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op); }; void bus_dma_dflt_lock(void *arg, bus_dma_lock_op_t op); int bus_dma_run_filter(struct bus_dma_tag_common *dmat, vm_paddr_t paddr); int common_bus_dma_tag_create(struct bus_dma_tag_common *parent, bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, void *lockfuncarg, size_t sz, void **dmat); extern struct bus_dma_impl bus_dma_bounce_impl; #endif diff --git a/sys/x86/include/ifunc.h b/sys/x86/include/ifunc.h index 690d491d39b9..8756e81eeb9f 100644 --- a/sys/x86/include/ifunc.h +++ b/sys/x86/include/ifunc.h @@ -1,50 +1,49 @@ /*- * Copyright (c) 2015-2018 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_IFUNC_H #define __X86_IFUNC_H #define DEFINE_IFUNC(qual, ret_type, name, args) \ static ret_type (*name##_resolver(void))args __used; \ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ static ret_type (*name##_resolver(void))args #define DEFINE_UIFUNC(qual, ret_type, name, args) \ static ret_type (*name##_resolver(uint32_t, uint32_t, uint32_t, \ uint32_t))args __used; \ qual ret_type name args __attribute__((ifunc(#name "_resolver"))); \ static ret_type (*name##_resolver( \ uint32_t cpu_feature __unused, \ uint32_t cpu_feature2 __unused, \ uint32_t cpu_stdext_feature __unused, \ uint32_t cpu_stdext_feature2 __unused))args #endif diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c index 110740d6ad01..dc8ee4e2f9a2 100644 --- a/sys/x86/iommu/intel_ctx.c +++ b/sys/x86/iommu/intel_ctx.c @@ -1,992 +1,991 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); static void dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain); static void dmar_domain_destroy(struct dmar_domain *domain); static void dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) { struct sf_buf *sf; dmar_root_entry_t *re; vm_page_t ctxm; /* * Allocated context page must be linked. */ ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC); if (ctxm != NULL) return; /* * Page not present, allocate and link. Note that other * thread might execute this sequence in parallel. This * should be safe, because the context entries written by both * threads are equal. */ TD_PREP_PINNED_ASSERT; ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO | IOMMU_PGF_WAITOK); re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf); re += bus; dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & VM_PAGE_TO_PHYS(ctxm))); dmar_flush_root_to_ram(dmar, re); dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; } static dmar_ctx_entry_t * dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) { struct dmar_unit *dmar; dmar_ctx_entry_t *ctxp; dmar = CTX2DMAR(ctx); ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->context.rid), IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp); ctxp += ctx->context.rid & 0xff; return (ctxp); } static void device_tag_init(struct dmar_ctx *ctx, device_t dev) { struct dmar_domain *domain; bus_addr_t maxaddr; domain = CTX2DOM(ctx); maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR); ctx->context.tag->common.ref_count = 1; /* Prevent free */ ctx->context.tag->common.impl = &bus_dma_iommu_impl; ctx->context.tag->common.boundary = 0; ctx->context.tag->common.lowaddr = maxaddr; ctx->context.tag->common.highaddr = maxaddr; ctx->context.tag->common.maxsize = maxaddr; ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; ctx->context.tag->common.maxsegsz = maxaddr; ctx->context.tag->ctx = CTX2IOCTX(ctx); ctx->context.tag->owner = dev; } static void ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, vm_page_t ctx_root) { /* * For update due to move, the store is not atomic. It is * possible that DMAR read upper doubleword, while low * doubleword is not yet updated. The domain id is stored in * the upper doubleword, while the table pointer in the lower. * * There is no good solution, for the same reason it is wrong * to clear P bit in the ctx entry for update. */ dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | domain->awlvl); if (ctx_root == NULL) { dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); } else { dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | DMAR_CTX1_P); } } static void ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, int busno) { struct dmar_unit *unit; struct dmar_domain *domain; vm_page_t ctx_root; int i; domain = CTX2DOM(ctx); unit = DOM2DMAR(domain); KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), pci_get_function(ctx->context.tag->owner), ctxp->ctx1, ctxp->ctx2)); if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 && (unit->hw_ecap & DMAR_ECAP_PT) != 0) { KASSERT(domain->pgtbl_obj == NULL, ("ctx %p non-null pgtbl_obj", ctx)); ctx_root = NULL; } else { ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, IOMMU_PGF_NOALLOC); } if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) { MPASS(!move); for (i = 0; i <= PCI_BUSMAX; i++) { ctx_id_entry_init_one(&ctxp[i], domain, ctx_root); } } else { ctx_id_entry_init_one(ctxp, domain, ctx_root); } dmar_flush_ctx_to_ram(unit, ctxp); } static int dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) { int error; /* * If dmar declares Caching Mode as Set, follow 11.5 "Caching * Mode Consideration" and do the (global) invalidation of the * negative TLB entries. */ if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force) return (0); if (dmar->qi_enabled) { dmar_qi_invalidate_ctx_glob_locked(dmar); if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force) dmar_qi_invalidate_iotlb_glob_locked(dmar); return (0); } error = dmar_inv_ctx_glob(dmar); if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)) error = dmar_inv_iotlb_glob(dmar); return (error); } static int domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, int slot, int func, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len) { struct iommu_map_entries_tailq rmrr_entries; struct iommu_map_entry *entry, *entry1; vm_page_t *ma; iommu_gaddr_t start, end; vm_pindex_t size, i; int error, error1; error = 0; TAILQ_INIT(&rmrr_entries); dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path, dev_path_len, &rmrr_entries); TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { /* * VT-d specification requires that the start of an * RMRR entry is 4k-aligned. Buggy BIOSes put * anything into the start and end fields. Truncate * and round as neccesary. * * We also allow the overlapping RMRR entries, see * iommu_gas_alloc_region(). */ start = entry->start; end = entry->end; if (bootverbose) printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", domain->iodom.iommu->unit, bus, slot, func, (uintmax_t)start, (uintmax_t)end); entry->start = trunc_page(start); entry->end = round_page(end); if (entry->start == entry->end) { /* Workaround for some AMI (?) BIOSes */ if (bootverbose) { if (dev != NULL) device_printf(dev, ""); printf("pci%d:%d:%d ", bus, slot, func); printf("BIOS bug: dmar%d RMRR " "region (%jx, %jx) corrected\n", domain->iodom.iommu->unit, start, end); } entry->end += DMAR_PAGE_SIZE * 0x20; } size = OFF_TO_IDX(entry->end - entry->start); ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); for (i = 0; i < size; i++) { ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, VM_MEMATTR_DEFAULT); } error1 = iommu_gas_map_region(DOM2IODOM(domain), entry, IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); /* * Non-failed RMRR entries are owned by context rb * tree. Get rid of the failed entry, but do not stop * the loop. Rest of the parsed RMRR entries are * loaded and removed on the context destruction. */ if (error1 == 0 && entry->end != entry->start) { IOMMU_LOCK(domain->iodom.iommu); domain->refs++; /* XXXKIB prevent free */ domain->iodom.flags |= IOMMU_DOMAIN_RMRR; IOMMU_UNLOCK(domain->iodom.iommu); } else { if (error1 != 0) { if (dev != NULL) device_printf(dev, ""); printf("pci%d:%d:%d ", bus, slot, func); printf( "dmar%d failed to map RMRR region (%jx, %jx) %d\n", domain->iodom.iommu->unit, start, end, error1); error = error1; } TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); iommu_gas_free_entry(DOM2IODOM(domain), entry); } for (i = 0; i < size; i++) vm_page_putfake(ma[i]); free(ma, M_TEMP); } return (error); } /* * PCI memory address space is shared between memory-mapped devices (MMIO) and * host memory (which may be remapped by an IOMMU). Device accesses to an * address within a memory aperture in a PCIe root port will be treated as * peer-to-peer and not forwarded to an IOMMU. To avoid this, reserve the * address space of the root port's memory apertures in the address space used * by the IOMMU for remapping. */ static int dmar_reserve_pci_regions(struct dmar_domain *domain, device_t dev) { struct iommu_domain *iodom; device_t root; uint32_t val; uint64_t base, limit; int error; iodom = DOM2IODOM(domain); root = pci_find_pcie_root_port(dev); if (root == NULL) return (0); /* Disable downstream memory */ base = PCI_PPBMEMBASE(0, pci_read_config(root, PCIR_MEMBASE_1, 2)); limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_MEMLIMIT_1, 2)); error = iommu_gas_reserve_region_extend(iodom, base, limit + 1); if (bootverbose || error != 0) device_printf(dev, "DMAR reserve [%#jx-%#jx] (error %d)\n", base, limit + 1, error); if (error != 0) return (error); /* Disable downstream prefetchable memory */ val = pci_read_config(root, PCIR_PMBASEL_1, 2); if (val != 0 || pci_read_config(root, PCIR_PMLIMITL_1, 2) != 0) { if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) { base = PCI_PPBMEMBASE( pci_read_config(root, PCIR_PMBASEH_1, 4), val); limit = PCI_PPBMEMLIMIT( pci_read_config(root, PCIR_PMLIMITH_1, 4), pci_read_config(root, PCIR_PMLIMITL_1, 2)); } else { base = PCI_PPBMEMBASE(0, val); limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_PMLIMITL_1, 2)); } error = iommu_gas_reserve_region_extend(iodom, base, limit + 1); if (bootverbose || error != 0) device_printf(dev, "DMAR reserve [%#jx-%#jx] " "(error %d)\n", base, limit + 1, error); if (error != 0) return (error); } return (error); } static struct dmar_domain * dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) { struct iommu_domain *iodom; struct iommu_unit *unit; struct dmar_domain *domain; int error, id, mgaw; id = alloc_unr(dmar->domids); if (id == -1) return (NULL); domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); iodom = DOM2IODOM(domain); unit = DMAR2IOMMU(dmar); domain->domain = id; LIST_INIT(&domain->contexts); iommu_domain_init(unit, iodom, &dmar_domain_map_ops); domain->dmar = dmar; /* * For now, use the maximal usable physical address of the * installed memory to calculate the mgaw on id_mapped domain. * It is useful for the identity mapping, and less so for the * virtualized bus address space. */ domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped); error = domain_set_agaw(domain, mgaw); if (error != 0) goto fail; if (!id_mapped) /* Use all supported address space for remapping. */ domain->iodom.end = 1ULL << (domain->agaw - 1); iommu_gas_init_domain(DOM2IODOM(domain)); if (id_mapped) { if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, domain->iodom.end); } domain->iodom.flags |= IOMMU_DOMAIN_IDMAP; } else { error = domain_alloc_pgtbl(domain); if (error != 0) goto fail; /* Disable local apic region access */ error = iommu_gas_reserve_region(iodom, 0xfee00000, 0xfeefffff + 1, &iodom->msi_entry); if (error != 0) goto fail; } return (domain); fail: dmar_domain_destroy(domain); return (NULL); } static struct dmar_ctx * dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) { struct dmar_ctx *ctx; ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); ctx->context.domain = DOM2IODOM(domain); ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu), M_DMAR_CTX, M_WAITOK | M_ZERO); ctx->context.rid = rid; ctx->refs = 1; return (ctx); } static void dmar_ctx_link(struct dmar_ctx *ctx) { struct dmar_domain *domain; domain = CTX2DOM(ctx); IOMMU_ASSERT_LOCKED(domain->iodom.iommu); KASSERT(domain->refs >= domain->ctx_cnt, ("dom %p ref underflow %d %d", domain, domain->refs, domain->ctx_cnt)); domain->refs++; domain->ctx_cnt++; LIST_INSERT_HEAD(&domain->contexts, ctx, link); } static void dmar_ctx_unlink(struct dmar_ctx *ctx) { struct dmar_domain *domain; domain = CTX2DOM(ctx); IOMMU_ASSERT_LOCKED(domain->iodom.iommu); KASSERT(domain->refs > 0, ("domain %p ctx dtr refs %d", domain, domain->refs)); KASSERT(domain->ctx_cnt >= domain->refs, ("domain %p ctx dtr refs %d ctx_cnt %d", domain, domain->refs, domain->ctx_cnt)); domain->refs--; domain->ctx_cnt--; LIST_REMOVE(ctx, link); } static void dmar_domain_destroy(struct dmar_domain *domain) { struct iommu_domain *iodom; struct dmar_unit *dmar; iodom = DOM2IODOM(domain); KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), ("unfinished unloads %p", domain)); KASSERT(LIST_EMPTY(&domain->contexts), ("destroying dom %p with contexts", domain)); KASSERT(domain->ctx_cnt == 0, ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); KASSERT(domain->refs == 0, ("destroying dom %p with refs %d", domain, domain->refs)); if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) { DMAR_DOMAIN_LOCK(domain); iommu_gas_fini_domain(iodom); DMAR_DOMAIN_UNLOCK(domain); } if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) { if (domain->pgtbl_obj != NULL) DMAR_DOMAIN_PGLOCK(domain); domain_free_pgtbl(domain); } iommu_domain_fini(iodom); dmar = DOM2DMAR(domain); free_unr(dmar->domids, domain->domain); free(domain, M_DMAR_DOMAIN); } static struct dmar_ctx * dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, bool id_mapped, bool rmrr_init) { struct dmar_domain *domain, *domain1; struct dmar_ctx *ctx, *ctx1; struct iommu_unit *unit; dmar_ctx_entry_t *ctxp; struct sf_buf *sf; int bus, slot, func, error; bool enable; if (dev != NULL) { bus = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); } else { bus = PCI_RID2BUS(rid); slot = PCI_RID2SLOT(rid); func = PCI_RID2FUNC(rid); } enable = false; TD_PREP_PINNED_ASSERT; unit = DMAR2IOMMU(dmar); DMAR_LOCK(dmar); KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0), ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus, slot, func)); ctx = dmar_find_ctx_locked(dmar, rid); error = 0; if (ctx == NULL) { /* * Perform the allocations which require sleep or have * higher chance to succeed if the sleep is allowed. */ DMAR_UNLOCK(dmar); dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); domain1 = dmar_domain_alloc(dmar, id_mapped); if (domain1 == NULL) { TD_PINNED_ASSERT; return (NULL); } if (!id_mapped) { error = domain_init_rmrr(domain1, dev, bus, slot, func, dev_domain, dev_busno, dev_path, dev_path_len); if (error == 0) error = dmar_reserve_pci_regions(domain1, dev); if (error != 0) { dmar_domain_destroy(domain1); TD_PINNED_ASSERT; return (NULL); } } ctx1 = dmar_ctx_alloc(domain1, rid); ctxp = dmar_map_ctx_entry(ctx1, &sf); DMAR_LOCK(dmar); /* * Recheck the contexts, other thread might have * already allocated needed one. */ ctx = dmar_find_ctx_locked(dmar, rid); if (ctx == NULL) { domain = domain1; ctx = ctx1; dmar_ctx_link(ctx); ctx->context.tag->owner = dev; device_tag_init(ctx, dev); /* * This is the first activated context for the * DMAR unit. Enable the translation after * everything is set up. */ if (LIST_EMPTY(&dmar->domains)) enable = true; LIST_INSERT_HEAD(&dmar->domains, domain, link); ctx_id_entry_init(ctx, ctxp, false, bus); if (dev != NULL) { device_printf(dev, "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " "agaw %d %s-mapped\n", dmar->iommu.unit, dmar->segment, bus, slot, func, rid, domain->domain, domain->mgaw, domain->agaw, id_mapped ? "id" : "re"); } dmar_unmap_pgtbl(sf); } else { dmar_unmap_pgtbl(sf); dmar_domain_destroy(domain1); /* Nothing needs to be done to destroy ctx1. */ free(ctx1, M_DMAR_CTX); domain = CTX2DOM(ctx); ctx->refs++; /* tag referenced us */ } } else { domain = CTX2DOM(ctx); if (ctx->context.tag->owner == NULL) ctx->context.tag->owner = dev; ctx->refs++; /* tag referenced us */ } error = dmar_flush_for_ctx_entry(dmar, enable); if (error != 0) { dmar_free_ctx_locked(dmar, ctx); TD_PINNED_ASSERT; return (NULL); } /* * The dmar lock was potentially dropped between check for the * empty context list and now. Recheck the state of GCMD_TE * to avoid unneeded command. */ if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { error = dmar_enable_translation(dmar); if (error == 0) { if (bootverbose) { printf("dmar%d: enabled translation\n", dmar->iommu.unit); } } else { printf("dmar%d: enabling translation failed, " "error %d\n", dmar->iommu.unit, error); dmar_free_ctx_locked(dmar, ctx); TD_PINNED_ASSERT; return (NULL); } } DMAR_UNLOCK(dmar); TD_PINNED_ASSERT; return (ctx); } struct dmar_ctx * dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init) { int dev_domain, dev_path_len, dev_busno; dev_domain = pci_get_domain(dev); dev_path_len = dmar_dev_depth(dev); ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno, dev_path, dev_path_len, id_mapped, rmrr_init)); } struct dmar_ctx * dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, bool id_mapped, bool rmrr_init) { return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno, dev_path, dev_path_len, id_mapped, rmrr_init)); } int dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) { struct dmar_unit *dmar; struct dmar_domain *old_domain; dmar_ctx_entry_t *ctxp; struct sf_buf *sf; int error; dmar = domain->dmar; old_domain = CTX2DOM(ctx); if (domain == old_domain) return (0); KASSERT(old_domain->iodom.iommu == domain->iodom.iommu, ("domain %p %u moving between dmars %u %u", domain, domain->domain, old_domain->iodom.iommu->unit, domain->iodom.iommu->unit)); TD_PREP_PINNED_ASSERT; ctxp = dmar_map_ctx_entry(ctx, &sf); DMAR_LOCK(dmar); dmar_ctx_unlink(ctx); ctx->context.domain = &domain->iodom; dmar_ctx_link(ctx); ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); dmar_unmap_pgtbl(sf); error = dmar_flush_for_ctx_entry(dmar, true); /* If flush failed, rolling back would not work as well. */ printf("dmar%d rid %x domain %d->%d %s-mapped\n", dmar->iommu.unit, ctx->context.rid, old_domain->domain, domain->domain, (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? "id" : "re"); dmar_unref_domain_locked(dmar, old_domain); TD_PINNED_ASSERT; return (error); } static void dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) { DMAR_ASSERT_LOCKED(dmar); KASSERT(domain->refs >= 1, ("dmar %d domain %p refs %u", dmar->iommu.unit, domain, domain->refs)); KASSERT(domain->refs > domain->ctx_cnt, ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, domain->refs, domain->ctx_cnt)); if (domain->refs > 1) { domain->refs--; DMAR_UNLOCK(dmar); return; } KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0, ("lost ref on RMRR domain %p", domain)); LIST_REMOVE(domain, link); DMAR_UNLOCK(dmar); taskqueue_drain(dmar->iommu.delayed_taskqueue, &domain->iodom.unload_task); dmar_domain_destroy(domain); } void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) { struct sf_buf *sf; dmar_ctx_entry_t *ctxp; struct dmar_domain *domain; DMAR_ASSERT_LOCKED(dmar); KASSERT(ctx->refs >= 1, ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); /* * If our reference is not last, only the dereference should * be performed. */ if (ctx->refs > 1) { ctx->refs--; DMAR_UNLOCK(dmar); return; } KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, ("lost ref on disabled ctx %p", ctx)); /* * Otherwise, the context entry must be cleared before the * page table is destroyed. The mapping of the context * entries page could require sleep, unlock the dmar. */ DMAR_UNLOCK(dmar); TD_PREP_PINNED_ASSERT; ctxp = dmar_map_ctx_entry(ctx, &sf); DMAR_LOCK(dmar); KASSERT(ctx->refs >= 1, ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); /* * Other thread might have referenced the context, in which * case again only the dereference should be performed. */ if (ctx->refs > 1) { ctx->refs--; DMAR_UNLOCK(dmar); dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; return; } KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, ("lost ref on disabled ctx %p", ctx)); /* * Clear the context pointer and flush the caches. * XXXKIB: cannot do this if any RMRR entries are still present. */ dmar_pte_clear(&ctxp->ctx1); ctxp->ctx2 = 0; dmar_flush_ctx_to_ram(dmar, ctxp); dmar_inv_ctx_glob(dmar); if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { if (dmar->qi_enabled) dmar_qi_invalidate_iotlb_glob_locked(dmar); else dmar_inv_iotlb_glob(dmar); } dmar_unmap_pgtbl(sf); domain = CTX2DOM(ctx); dmar_ctx_unlink(ctx); free(ctx->context.tag, M_DMAR_CTX); free(ctx, M_DMAR_CTX); dmar_unref_domain_locked(dmar, domain); TD_PINNED_ASSERT; } void dmar_free_ctx(struct dmar_ctx *ctx) { struct dmar_unit *dmar; dmar = CTX2DMAR(ctx); DMAR_LOCK(dmar); dmar_free_ctx_locked(dmar, ctx); } /* * Returns with the domain locked. */ struct dmar_ctx * dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) { struct dmar_domain *domain; struct dmar_ctx *ctx; DMAR_ASSERT_LOCKED(dmar); LIST_FOREACH(domain, &dmar->domains, link) { LIST_FOREACH(ctx, &domain->contexts, link) { if (ctx->context.rid == rid) return (ctx); } } return (NULL); } void dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) { struct iommu_domain *domain; domain = entry->domain; IOMMU_DOMAIN_LOCK(domain); if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) iommu_gas_free_region(domain, entry); else iommu_gas_free_space(domain, entry); IOMMU_DOMAIN_UNLOCK(domain); if (free) iommu_gas_free_entry(domain, entry); else entry->flags = 0; } void dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free) { struct dmar_domain *domain; struct dmar_unit *unit; domain = IODOM2DOM(entry->domain); unit = DOM2DMAR(domain); if (unit->qi_enabled) { DMAR_LOCK(unit); dmar_qi_invalidate_locked(IODOM2DOM(entry->domain), entry->start, entry->end - entry->start, &entry->gseq, true); if (!free) entry->flags |= IOMMU_MAP_ENTRY_QI_NF; TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); DMAR_UNLOCK(unit); } else { domain_flush_iotlb_sync(IODOM2DOM(entry->domain), entry->start, entry->end - entry->start); dmar_domain_free_entry(entry, free); } } static bool dmar_domain_unload_emit_wait(struct dmar_domain *domain, struct iommu_map_entry *entry) { if (TAILQ_NEXT(entry, dmamap_link) == NULL) return (true); return (domain->batch_no++ % dmar_batch_coalesce == 0); } void dmar_domain_unload(struct dmar_domain *domain, struct iommu_map_entries_tailq *entries, bool cansleep) { struct dmar_unit *unit; struct iommu_domain *iodom; struct iommu_map_entry *entry, *entry1; int error; iodom = DOM2IODOM(domain); unit = DOM2DMAR(domain); TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, ("not mapped entry %p %p", domain, entry)); error = iodom->ops->unmap(iodom, entry->start, entry->end - entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); KASSERT(error == 0, ("unmap %p error %d", domain, error)); if (!unit->qi_enabled) { domain_flush_iotlb_sync(domain, entry->start, entry->end - entry->start); TAILQ_REMOVE(entries, entry, dmamap_link); dmar_domain_free_entry(entry, true); } } if (TAILQ_EMPTY(entries)) return; KASSERT(unit->qi_enabled, ("loaded entry left")); DMAR_LOCK(unit); TAILQ_FOREACH(entry, entries, dmamap_link) { dmar_qi_invalidate_locked(domain, entry->start, entry->end - entry->start, &entry->gseq, dmar_domain_unload_emit_wait(domain, entry)); } TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link); DMAR_UNLOCK(unit); } struct iommu_ctx * iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init) { struct dmar_unit *dmar; struct dmar_ctx *ret; dmar = IOMMU2DMAR(iommu); ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init); return (CTX2IOCTX(ret)); } void iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context) { struct dmar_unit *dmar; struct dmar_ctx *ctx; dmar = IOMMU2DMAR(iommu); ctx = IOCTX2CTX(context); dmar_free_ctx_locked(dmar, ctx); } void iommu_free_ctx(struct iommu_ctx *context) { struct dmar_ctx *ctx; ctx = IOCTX2CTX(context); dmar_free_ctx(ctx); } void iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free) { dmar_domain_unload_entry(entry, free); } void iommu_domain_unload(struct iommu_domain *iodom, struct iommu_map_entries_tailq *entries, bool cansleep) { struct dmar_domain *domain; domain = IODOM2DOM(iodom); dmar_domain_unload(domain, entries, cansleep); } diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h index 8fe8ad983952..0ad94dbf4123 100644 --- a/sys/x86/iommu/intel_dmar.h +++ b/sys/x86/iommu/intel_dmar.h @@ -1,458 +1,457 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013-2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_IOMMU_INTEL_DMAR_H #define __X86_IOMMU_INTEL_DMAR_H #include struct dmar_unit; /* * Locking annotations: * (u) - Protected by iommu unit lock * (d) - Protected by domain lock * (c) - Immutable after initialization */ /* * The domain abstraction. Most non-constant members of the domain * are protected by owning dmar unit lock, not by the domain lock. * Most important, the dmar lock protects the contexts list. * * The domain lock protects the address map for the domain, and list * of unload entries delayed. * * Page tables pages and pages content is protected by the vm object * lock pgtbl_obj, which contains the page tables pages. */ struct dmar_domain { struct iommu_domain iodom; int domain; /* (c) DID, written in context entry */ int mgaw; /* (c) Real max address width */ int agaw; /* (c) Adjusted guest address width */ int pglvl; /* (c) The pagelevel */ int awlvl; /* (c) The pagelevel as the bitmask, to set in context entry */ u_int ctx_cnt; /* (u) Number of contexts owned */ u_int refs; /* (u) Refs, including ctx */ struct dmar_unit *dmar; /* (c) */ LIST_ENTRY(dmar_domain) link; /* (u) Member in the dmar list */ LIST_HEAD(, dmar_ctx) contexts; /* (u) */ vm_object_t pgtbl_obj; /* (c) Page table pages */ u_int batch_no; }; struct dmar_ctx { struct iommu_ctx context; uint64_t last_fault_rec[2]; /* Last fault reported */ LIST_ENTRY(dmar_ctx) link; /* (u) Member in the domain list */ u_int refs; /* (u) References from tags */ }; #define DMAR_DOMAIN_PGLOCK(dom) VM_OBJECT_WLOCK((dom)->pgtbl_obj) #define DMAR_DOMAIN_PGTRYLOCK(dom) VM_OBJECT_TRYWLOCK((dom)->pgtbl_obj) #define DMAR_DOMAIN_PGUNLOCK(dom) VM_OBJECT_WUNLOCK((dom)->pgtbl_obj) #define DMAR_DOMAIN_ASSERT_PGLOCKED(dom) \ VM_OBJECT_ASSERT_WLOCKED((dom)->pgtbl_obj) #define DMAR_DOMAIN_LOCK(dom) mtx_lock(&(dom)->iodom.lock) #define DMAR_DOMAIN_UNLOCK(dom) mtx_unlock(&(dom)->iodom.lock) #define DMAR_DOMAIN_ASSERT_LOCKED(dom) mtx_assert(&(dom)->iodom.lock, MA_OWNED) #define DMAR2IOMMU(dmar) &((dmar)->iommu) #define IOMMU2DMAR(dmar) \ __containerof((dmar), struct dmar_unit, iommu) #define DOM2IODOM(domain) &((domain)->iodom) #define IODOM2DOM(domain) \ __containerof((domain), struct dmar_domain, iodom) #define CTX2IOCTX(ctx) &((ctx)->context) #define IOCTX2CTX(ctx) \ __containerof((ctx), struct dmar_ctx, context) #define CTX2DOM(ctx) IODOM2DOM((ctx)->context.domain) #define CTX2DMAR(ctx) (CTX2DOM(ctx)->dmar) #define DOM2DMAR(domain) ((domain)->dmar) struct dmar_msi_data { int irq; int irq_rid; struct resource *irq_res; void *intr_handle; int (*handler)(void *); int msi_data_reg; int msi_addr_reg; int msi_uaddr_reg; void (*enable_intr)(struct dmar_unit *); void (*disable_intr)(struct dmar_unit *); const char *name; }; #define DMAR_INTR_FAULT 0 #define DMAR_INTR_QI 1 #define DMAR_INTR_TOTAL 2 struct dmar_unit { struct iommu_unit iommu; device_t dev; uint16_t segment; uint64_t base; /* Resources */ int reg_rid; struct resource *regs; struct dmar_msi_data intrs[DMAR_INTR_TOTAL]; /* Hardware registers cache */ uint32_t hw_ver; uint64_t hw_cap; uint64_t hw_ecap; uint32_t hw_gcmd; /* Data for being a dmar */ LIST_HEAD(, dmar_domain) domains; struct unrhdr *domids; vm_object_t ctx_obj; u_int barrier_flags; /* Fault handler data */ struct mtx fault_lock; uint64_t *fault_log; int fault_log_head; int fault_log_tail; int fault_log_size; struct task fault_task; struct taskqueue *fault_taskqueue; /* QI */ int qi_enabled; vm_offset_t inv_queue; vm_size_t inv_queue_size; uint32_t inv_queue_avail; uint32_t inv_queue_tail; volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait descr completion */ uint64_t inv_waitd_seq_hw_phys; uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */ u_int inv_waitd_gen; /* seq number generation AKA seq overflows */ u_int inv_seq_waiters; /* count of waiters for seq */ u_int inv_queue_full; /* informational counter */ /* IR */ int ir_enabled; vm_paddr_t irt_phys; dmar_irte_t *irt; u_int irte_cnt; vmem_t *irtids; /* Delayed freeing of map entries queue processing */ struct iommu_map_entries_tailq tlb_flush_entries; struct task qi_task; struct taskqueue *qi_taskqueue; }; #define DMAR_LOCK(dmar) mtx_lock(&(dmar)->iommu.lock) #define DMAR_UNLOCK(dmar) mtx_unlock(&(dmar)->iommu.lock) #define DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->iommu.lock, MA_OWNED) #define DMAR_FAULT_LOCK(dmar) mtx_lock_spin(&(dmar)->fault_lock) #define DMAR_FAULT_UNLOCK(dmar) mtx_unlock_spin(&(dmar)->fault_lock) #define DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED) #define DMAR_IS_COHERENT(dmar) (((dmar)->hw_ecap & DMAR_ECAP_C) != 0) #define DMAR_HAS_QI(dmar) (((dmar)->hw_ecap & DMAR_ECAP_QI) != 0) #define DMAR_X2APIC(dmar) \ (x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0) /* Barrier ids */ #define DMAR_BARRIER_RMRR 0 #define DMAR_BARRIER_USEQ 1 struct dmar_unit *dmar_find(device_t dev, bool verbose); struct dmar_unit *dmar_find_hpet(device_t dev, uint16_t *rid); struct dmar_unit *dmar_find_ioapic(u_int apic_id, uint16_t *rid); u_int dmar_nd2mask(u_int nd); bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl); int domain_set_agaw(struct dmar_domain *domain, int mgaw); int dmar_maxaddr2mgaw(struct dmar_unit *unit, iommu_gaddr_t maxaddr, bool allow_less); vm_pindex_t pglvl_max_pages(int pglvl); int domain_is_sp_lvl(struct dmar_domain *domain, int lvl); iommu_gaddr_t pglvl_page_size(int total_pglvl, int lvl); iommu_gaddr_t domain_page_size(struct dmar_domain *domain, int lvl); int calc_am(struct dmar_unit *unit, iommu_gaddr_t base, iommu_gaddr_t size, iommu_gaddr_t *isizep); struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags); void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags); void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, struct sf_buf **sf); void dmar_unmap_pgtbl(struct sf_buf *sf); int dmar_load_root_entry_ptr(struct dmar_unit *unit); int dmar_inv_ctx_glob(struct dmar_unit *unit); int dmar_inv_iotlb_glob(struct dmar_unit *unit); int dmar_flush_write_bufs(struct dmar_unit *unit); void dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst); void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst); void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst); int dmar_enable_translation(struct dmar_unit *unit); int dmar_disable_translation(struct dmar_unit *unit); int dmar_load_irt_ptr(struct dmar_unit *unit); int dmar_enable_ir(struct dmar_unit *unit); int dmar_disable_ir(struct dmar_unit *unit); bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id); void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id); uint64_t dmar_get_timeout(void); void dmar_update_timeout(uint64_t newval); int dmar_fault_intr(void *arg); void dmar_enable_fault_intr(struct dmar_unit *unit); void dmar_disable_fault_intr(struct dmar_unit *unit); int dmar_init_fault_log(struct dmar_unit *unit); void dmar_fini_fault_log(struct dmar_unit *unit); int dmar_qi_intr(void *arg); void dmar_enable_qi_intr(struct dmar_unit *unit); void dmar_disable_qi_intr(struct dmar_unit *unit); int dmar_init_qi(struct dmar_unit *unit); void dmar_fini_qi(struct dmar_unit *unit); void dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t start, iommu_gaddr_t size, struct iommu_qi_genseq *psec, bool emit_wait); void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit); void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit); void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit); void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt); vm_object_t domain_get_idmap_pgtbl(struct dmar_domain *domain, iommu_gaddr_t maxaddr); void put_idmap_pgtbl(vm_object_t obj); void domain_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size); int domain_alloc_pgtbl(struct dmar_domain *domain); void domain_free_pgtbl(struct dmar_domain *domain); extern const struct iommu_domain_map_ops dmar_domain_map_ops; int dmar_dev_depth(device_t child); void dmar_dev_path(device_t child, int *busno, void *path1, int depth); struct dmar_ctx *dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped, bool rmrr_init); struct dmar_ctx *dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, bool id_mapped, bool rmrr_init); int dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx); void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx); void dmar_free_ctx(struct dmar_ctx *ctx); struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid); void dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free); void dmar_domain_unload(struct dmar_domain *domain, struct iommu_map_entries_tailq *entries, bool cansleep); void dmar_domain_free_entry(struct iommu_map_entry *entry, bool free); void dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, struct iommu_map_entries_tailq *rmrr_entries); int dmar_instantiate_rmrr_ctxs(struct iommu_unit *dmar); void dmar_quirks_post_ident(struct dmar_unit *dmar); void dmar_quirks_pre_use(struct iommu_unit *dmar); int dmar_init_irt(struct dmar_unit *unit); void dmar_fini_irt(struct dmar_unit *unit); extern iommu_haddr_t dmar_high; extern int haw; extern int dmar_tbl_pagecnt; extern int dmar_batch_coalesce; static inline uint32_t dmar_read4(const struct dmar_unit *unit, int reg) { return (bus_read_4(unit->regs, reg)); } static inline uint64_t dmar_read8(const struct dmar_unit *unit, int reg) { #ifdef __i386__ uint32_t high, low; low = bus_read_4(unit->regs, reg); high = bus_read_4(unit->regs, reg + 4); return (low | ((uint64_t)high << 32)); #else return (bus_read_8(unit->regs, reg)); #endif } static inline void dmar_write4(const struct dmar_unit *unit, int reg, uint32_t val) { KASSERT(reg != DMAR_GCMD_REG || (val & DMAR_GCMD_TE) == (unit->hw_gcmd & DMAR_GCMD_TE), ("dmar%d clearing TE 0x%08x 0x%08x", unit->iommu.unit, unit->hw_gcmd, val)); bus_write_4(unit->regs, reg, val); } static inline void dmar_write8(const struct dmar_unit *unit, int reg, uint64_t val) { KASSERT(reg != DMAR_GCMD_REG, ("8byte GCMD write")); #ifdef __i386__ uint32_t high, low; low = val; high = val >> 32; bus_write_4(unit->regs, reg, low); bus_write_4(unit->regs, reg + 4, high); #else bus_write_8(unit->regs, reg, val); #endif } /* * dmar_pte_store and dmar_pte_clear ensure that on i386, 32bit writes * are issued in the correct order. For store, the lower word, * containing the P or R and W bits, is set only after the high word * is written. For clear, the P bit is cleared first, then the high * word is cleared. * * dmar_pte_update updates the pte. For amd64, the update is atomic. * For i386, it first disables the entry by clearing the word * containing the P bit, and then defer to dmar_pte_store. The locked * cmpxchg8b is probably available on any machine having DMAR support, * but interrupt translation table may be mapped uncached. */ static inline void dmar_pte_store1(volatile uint64_t *dst, uint64_t val) { #ifdef __i386__ volatile uint32_t *p; uint32_t hi, lo; hi = val >> 32; lo = val; p = (volatile uint32_t *)dst; *(p + 1) = hi; *p = lo; #else *dst = val; #endif } static inline void dmar_pte_store(volatile uint64_t *dst, uint64_t val) { KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx", dst, (uintmax_t)*dst, (uintmax_t)val)); dmar_pte_store1(dst, val); } static inline void dmar_pte_update(volatile uint64_t *dst, uint64_t val) { #ifdef __i386__ volatile uint32_t *p; p = (volatile uint32_t *)dst; *p = 0; #endif dmar_pte_store1(dst, val); } static inline void dmar_pte_clear(volatile uint64_t *dst) { #ifdef __i386__ volatile uint32_t *p; p = (volatile uint32_t *)dst; *p = 0; *(p + 1) = 0; #else *dst = 0; #endif } extern struct timespec dmar_hw_timeout; #define DMAR_WAIT_UNTIL(cond) \ { \ struct timespec last, curr; \ bool forever; \ \ if (dmar_hw_timeout.tv_sec == 0 && \ dmar_hw_timeout.tv_nsec == 0) { \ forever = true; \ } else { \ forever = false; \ nanouptime(&curr); \ timespecadd(&curr, &dmar_hw_timeout, &last); \ } \ for (;;) { \ if (cond) { \ error = 0; \ break; \ } \ nanouptime(&curr); \ if (!forever && timespeccmp(&last, &curr, <)) { \ error = ETIMEDOUT; \ break; \ } \ cpu_spinwait(); \ } \ } #ifdef INVARIANTS #define TD_PREP_PINNED_ASSERT \ int old_td_pinned; \ old_td_pinned = curthread->td_pinned #define TD_PINNED_ASSERT \ KASSERT(curthread->td_pinned == old_td_pinned, \ ("pin count leak: %d %d %s:%d", curthread->td_pinned, \ old_td_pinned, __FILE__, __LINE__)) #else #define TD_PREP_PINNED_ASSERT #define TD_PINNED_ASSERT #endif #endif diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c index 38442795c5e4..a19084eb3e66 100644 --- a/sys/x86/iommu/intel_drv.c +++ b/sys/x86/iommu/intel_drv.c @@ -1,1339 +1,1338 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013-2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_acpi.h" #if defined(__amd64__) #define DEV_APIC #else #include "opt_apic.h" #endif #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEV_APIC #include "pcib_if.h" #include #include #include #endif #define DMAR_FAULT_IRQ_RID 0 #define DMAR_QI_IRQ_RID 1 #define DMAR_REG_RID 2 static devclass_t dmar_devclass; static device_t *dmar_devs; static int dmar_devcnt; typedef int (*dmar_iter_t)(ACPI_DMAR_HEADER *, void *); static void dmar_iterate_tbl(dmar_iter_t iter, void *arg) { ACPI_TABLE_DMAR *dmartbl; ACPI_DMAR_HEADER *dmarh; char *ptr, *ptrend; ACPI_STATUS status; status = AcpiGetTable(ACPI_SIG_DMAR, 1, (ACPI_TABLE_HEADER **)&dmartbl); if (ACPI_FAILURE(status)) return; ptr = (char *)dmartbl + sizeof(*dmartbl); ptrend = (char *)dmartbl + dmartbl->Header.Length; for (;;) { if (ptr >= ptrend) break; dmarh = (ACPI_DMAR_HEADER *)ptr; if (dmarh->Length <= 0) { printf("dmar_identify: corrupted DMAR table, l %d\n", dmarh->Length); break; } ptr += dmarh->Length; if (!iter(dmarh, arg)) break; } AcpiPutTable((ACPI_TABLE_HEADER *)dmartbl); } struct find_iter_args { int i; ACPI_DMAR_HARDWARE_UNIT *res; }; static int dmar_find_iter(ACPI_DMAR_HEADER *dmarh, void *arg) { struct find_iter_args *fia; if (dmarh->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT) return (1); fia = arg; if (fia->i == 0) { fia->res = (ACPI_DMAR_HARDWARE_UNIT *)dmarh; return (0); } fia->i--; return (1); } static ACPI_DMAR_HARDWARE_UNIT * dmar_find_by_index(int idx) { struct find_iter_args fia; fia.i = idx; fia.res = NULL; dmar_iterate_tbl(dmar_find_iter, &fia); return (fia.res); } static int dmar_count_iter(ACPI_DMAR_HEADER *dmarh, void *arg) { if (dmarh->Type == ACPI_DMAR_TYPE_HARDWARE_UNIT) dmar_devcnt++; return (1); } static int dmar_enable = 0; static void dmar_identify(driver_t *driver, device_t parent) { ACPI_TABLE_DMAR *dmartbl; ACPI_DMAR_HARDWARE_UNIT *dmarh; ACPI_STATUS status; int i, error; if (acpi_disabled("dmar")) return; TUNABLE_INT_FETCH("hw.dmar.enable", &dmar_enable); if (!dmar_enable) return; status = AcpiGetTable(ACPI_SIG_DMAR, 1, (ACPI_TABLE_HEADER **)&dmartbl); if (ACPI_FAILURE(status)) return; haw = dmartbl->Width + 1; if ((1ULL << (haw + 1)) > BUS_SPACE_MAXADDR) dmar_high = BUS_SPACE_MAXADDR; else dmar_high = 1ULL << (haw + 1); if (bootverbose) { printf("DMAR HAW=%d flags=<%b>\n", dmartbl->Width, (unsigned)dmartbl->Flags, "\020\001INTR_REMAP\002X2APIC_OPT_OUT"); } AcpiPutTable((ACPI_TABLE_HEADER *)dmartbl); dmar_iterate_tbl(dmar_count_iter, NULL); if (dmar_devcnt == 0) return; dmar_devs = malloc(sizeof(device_t) * dmar_devcnt, M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < dmar_devcnt; i++) { dmarh = dmar_find_by_index(i); if (dmarh == NULL) { printf("dmar_identify: cannot find HWUNIT %d\n", i); continue; } dmar_devs[i] = BUS_ADD_CHILD(parent, 1, "dmar", i); if (dmar_devs[i] == NULL) { printf("dmar_identify: cannot create instance %d\n", i); continue; } error = bus_set_resource(dmar_devs[i], SYS_RES_MEMORY, DMAR_REG_RID, dmarh->Address, PAGE_SIZE); if (error != 0) { printf( "dmar%d: unable to alloc register window at 0x%08jx: error %d\n", i, (uintmax_t)dmarh->Address, error); device_delete_child(parent, dmar_devs[i]); dmar_devs[i] = NULL; } } } static int dmar_probe(device_t dev) { if (acpi_get_handle(dev) != NULL) return (ENXIO); device_set_desc(dev, "DMA remap"); return (BUS_PROBE_NOWILDCARD); } static void dmar_release_intr(device_t dev, struct dmar_unit *unit, int idx) { struct dmar_msi_data *dmd; dmd = &unit->intrs[idx]; if (dmd->irq == -1) return; bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle); bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res); bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid); PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)), dev, dmd->irq); dmd->irq = -1; } static void dmar_release_resources(device_t dev, struct dmar_unit *unit) { int i; iommu_fini_busdma(&unit->iommu); dmar_fini_irt(unit); dmar_fini_qi(unit); dmar_fini_fault_log(unit); for (i = 0; i < DMAR_INTR_TOTAL; i++) dmar_release_intr(dev, unit, i); if (unit->regs != NULL) { bus_deactivate_resource(dev, SYS_RES_MEMORY, unit->reg_rid, unit->regs); bus_release_resource(dev, SYS_RES_MEMORY, unit->reg_rid, unit->regs); unit->regs = NULL; } if (unit->domids != NULL) { delete_unrhdr(unit->domids); unit->domids = NULL; } if (unit->ctx_obj != NULL) { vm_object_deallocate(unit->ctx_obj); unit->ctx_obj = NULL; } } static int dmar_alloc_irq(device_t dev, struct dmar_unit *unit, int idx) { device_t pcib; struct dmar_msi_data *dmd; uint64_t msi_addr; uint32_t msi_data; int error; dmd = &unit->intrs[idx]; pcib = device_get_parent(device_get_parent(dev)); /* Really not pcib */ error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq); if (error != 0) { device_printf(dev, "cannot allocate %s interrupt, %d\n", dmd->name, error); goto err1; } error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq, 1); if (error != 0) { device_printf(dev, "cannot set %s interrupt resource, %d\n", dmd->name, error); goto err2; } dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &dmd->irq_rid, RF_ACTIVE); if (dmd->irq_res == NULL) { device_printf(dev, "cannot allocate resource for %s interrupt\n", dmd->name); error = ENXIO; goto err3; } error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC, dmd->handler, NULL, unit, &dmd->intr_handle); if (error != 0) { device_printf(dev, "cannot setup %s interrupt, %d\n", dmd->name, error); goto err4; } bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, "%s", dmd->name); error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data); if (error != 0) { device_printf(dev, "cannot map %s interrupt, %d\n", dmd->name, error); goto err5; } dmar_write4(unit, dmd->msi_data_reg, msi_data); dmar_write4(unit, dmd->msi_addr_reg, msi_addr); /* Only for xAPIC mode */ dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32); return (0); err5: bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle); err4: bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res); err3: bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid); err2: PCIB_RELEASE_MSIX(pcib, dev, dmd->irq); dmd->irq = -1; err1: return (error); } #ifdef DEV_APIC static int dmar_remap_intr(device_t dev, device_t child, u_int irq) { struct dmar_unit *unit; struct dmar_msi_data *dmd; uint64_t msi_addr; uint32_t msi_data; int i, error; unit = device_get_softc(dev); for (i = 0; i < DMAR_INTR_TOTAL; i++) { dmd = &unit->intrs[i]; if (irq == dmd->irq) { error = PCIB_MAP_MSI(device_get_parent( device_get_parent(dev)), dev, irq, &msi_addr, &msi_data); if (error != 0) return (error); DMAR_LOCK(unit); (dmd->disable_intr)(unit); dmar_write4(unit, dmd->msi_data_reg, msi_data); dmar_write4(unit, dmd->msi_addr_reg, msi_addr); dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32); (dmd->enable_intr)(unit); DMAR_UNLOCK(unit); return (0); } } return (ENOENT); } #endif static void dmar_print_caps(device_t dev, struct dmar_unit *unit, ACPI_DMAR_HARDWARE_UNIT *dmaru) { uint32_t caphi, ecaphi; device_printf(dev, "regs@0x%08jx, ver=%d.%d, seg=%d, flags=<%b>\n", (uintmax_t)dmaru->Address, DMAR_MAJOR_VER(unit->hw_ver), DMAR_MINOR_VER(unit->hw_ver), dmaru->Segment, dmaru->Flags, "\020\001INCLUDE_ALL_PCI"); caphi = unit->hw_cap >> 32; device_printf(dev, "cap=%b,", (u_int)unit->hw_cap, "\020\004AFL\005WBF\006PLMR\007PHMR\010CM\027ZLR\030ISOCH"); printf("%b, ", caphi, "\020\010PSI\027DWD\030DRD\031FL1GP\034PSI"); printf("ndoms=%d, sagaw=%d, mgaw=%d, fro=%d, nfr=%d, superp=%d", DMAR_CAP_ND(unit->hw_cap), DMAR_CAP_SAGAW(unit->hw_cap), DMAR_CAP_MGAW(unit->hw_cap), DMAR_CAP_FRO(unit->hw_cap), DMAR_CAP_NFR(unit->hw_cap), DMAR_CAP_SPS(unit->hw_cap)); if ((unit->hw_cap & DMAR_CAP_PSI) != 0) printf(", mamv=%d", DMAR_CAP_MAMV(unit->hw_cap)); printf("\n"); ecaphi = unit->hw_ecap >> 32; device_printf(dev, "ecap=%b,", (u_int)unit->hw_ecap, "\020\001C\002QI\003DI\004IR\005EIM\007PT\010SC\031ECS\032MTS" "\033NEST\034DIS\035PASID\036PRS\037ERS\040SRS"); printf("%b, ", ecaphi, "\020\002NWFS\003EAFS"); printf("mhmw=%d, iro=%d\n", DMAR_ECAP_MHMV(unit->hw_ecap), DMAR_ECAP_IRO(unit->hw_ecap)); } static int dmar_attach(device_t dev) { struct dmar_unit *unit; ACPI_DMAR_HARDWARE_UNIT *dmaru; uint64_t timeout; int i, error; unit = device_get_softc(dev); unit->dev = dev; unit->iommu.unit = device_get_unit(dev); unit->iommu.dev = dev; dmaru = dmar_find_by_index(unit->iommu.unit); if (dmaru == NULL) return (EINVAL); unit->segment = dmaru->Segment; unit->base = dmaru->Address; unit->reg_rid = DMAR_REG_RID; unit->regs = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &unit->reg_rid, RF_ACTIVE); if (unit->regs == NULL) { device_printf(dev, "cannot allocate register window\n"); return (ENOMEM); } unit->hw_ver = dmar_read4(unit, DMAR_VER_REG); unit->hw_cap = dmar_read8(unit, DMAR_CAP_REG); unit->hw_ecap = dmar_read8(unit, DMAR_ECAP_REG); if (bootverbose) dmar_print_caps(dev, unit, dmaru); dmar_quirks_post_ident(unit); timeout = dmar_get_timeout(); TUNABLE_UINT64_FETCH("hw.dmar.timeout", &timeout); dmar_update_timeout(timeout); for (i = 0; i < DMAR_INTR_TOTAL; i++) unit->intrs[i].irq = -1; unit->intrs[DMAR_INTR_FAULT].name = "fault"; unit->intrs[DMAR_INTR_FAULT].irq_rid = DMAR_FAULT_IRQ_RID; unit->intrs[DMAR_INTR_FAULT].handler = dmar_fault_intr; unit->intrs[DMAR_INTR_FAULT].msi_data_reg = DMAR_FEDATA_REG; unit->intrs[DMAR_INTR_FAULT].msi_addr_reg = DMAR_FEADDR_REG; unit->intrs[DMAR_INTR_FAULT].msi_uaddr_reg = DMAR_FEUADDR_REG; unit->intrs[DMAR_INTR_FAULT].enable_intr = dmar_enable_fault_intr; unit->intrs[DMAR_INTR_FAULT].disable_intr = dmar_disable_fault_intr; error = dmar_alloc_irq(dev, unit, DMAR_INTR_FAULT); if (error != 0) { dmar_release_resources(dev, unit); return (error); } if (DMAR_HAS_QI(unit)) { unit->intrs[DMAR_INTR_QI].name = "qi"; unit->intrs[DMAR_INTR_QI].irq_rid = DMAR_QI_IRQ_RID; unit->intrs[DMAR_INTR_QI].handler = dmar_qi_intr; unit->intrs[DMAR_INTR_QI].msi_data_reg = DMAR_IEDATA_REG; unit->intrs[DMAR_INTR_QI].msi_addr_reg = DMAR_IEADDR_REG; unit->intrs[DMAR_INTR_QI].msi_uaddr_reg = DMAR_IEUADDR_REG; unit->intrs[DMAR_INTR_QI].enable_intr = dmar_enable_qi_intr; unit->intrs[DMAR_INTR_QI].disable_intr = dmar_disable_qi_intr; error = dmar_alloc_irq(dev, unit, DMAR_INTR_QI); if (error != 0) { dmar_release_resources(dev, unit); return (error); } } mtx_init(&unit->iommu.lock, "dmarhw", NULL, MTX_DEF); unit->domids = new_unrhdr(0, dmar_nd2mask(DMAR_CAP_ND(unit->hw_cap)), &unit->iommu.lock); LIST_INIT(&unit->domains); /* * 9.2 "Context Entry": * When Caching Mode (CM) field is reported as Set, the * domain-id value of zero is architecturally reserved. * Software must not use domain-id value of zero * when CM is Set. */ if ((unit->hw_cap & DMAR_CAP_CM) != 0) alloc_unr_specific(unit->domids, 0); unit->ctx_obj = vm_pager_allocate(OBJT_PHYS, NULL, IDX_TO_OFF(1 + DMAR_CTX_CNT), 0, 0, NULL); /* * Allocate and load the root entry table pointer. Enable the * address translation after the required invalidations are * done. */ dmar_pgalloc(unit->ctx_obj, 0, IOMMU_PGF_WAITOK | IOMMU_PGF_ZERO); DMAR_LOCK(unit); error = dmar_load_root_entry_ptr(unit); if (error != 0) { DMAR_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } error = dmar_inv_ctx_glob(unit); if (error != 0) { DMAR_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } if ((unit->hw_ecap & DMAR_ECAP_DI) != 0) { error = dmar_inv_iotlb_glob(unit); if (error != 0) { DMAR_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } } DMAR_UNLOCK(unit); error = dmar_init_fault_log(unit); if (error != 0) { dmar_release_resources(dev, unit); return (error); } error = dmar_init_qi(unit); if (error != 0) { dmar_release_resources(dev, unit); return (error); } error = dmar_init_irt(unit); if (error != 0) { dmar_release_resources(dev, unit); return (error); } error = iommu_init_busdma(&unit->iommu); if (error != 0) { dmar_release_resources(dev, unit); return (error); } #ifdef NOTYET DMAR_LOCK(unit); error = dmar_enable_translation(unit); if (error != 0) { DMAR_UNLOCK(unit); dmar_release_resources(dev, unit); return (error); } DMAR_UNLOCK(unit); #endif return (0); } static int dmar_detach(device_t dev) { return (EBUSY); } static int dmar_suspend(device_t dev) { return (0); } static int dmar_resume(device_t dev) { /* XXXKIB */ return (0); } static device_method_t dmar_methods[] = { DEVMETHOD(device_identify, dmar_identify), DEVMETHOD(device_probe, dmar_probe), DEVMETHOD(device_attach, dmar_attach), DEVMETHOD(device_detach, dmar_detach), DEVMETHOD(device_suspend, dmar_suspend), DEVMETHOD(device_resume, dmar_resume), #ifdef DEV_APIC DEVMETHOD(bus_remap_intr, dmar_remap_intr), #endif DEVMETHOD_END }; static driver_t dmar_driver = { "dmar", dmar_methods, sizeof(struct dmar_unit), }; DRIVER_MODULE(dmar, acpi, dmar_driver, dmar_devclass, 0, 0); MODULE_DEPEND(dmar, acpi, 1, 1, 1); static void dmar_print_path(int busno, int depth, const ACPI_DMAR_PCI_PATH *path) { int i; printf("[%d, ", busno); for (i = 0; i < depth; i++) { if (i != 0) printf(", "); printf("(%d, %d)", path[i].Device, path[i].Function); } printf("]"); } int dmar_dev_depth(device_t child) { devclass_t pci_class; device_t bus, pcib; int depth; pci_class = devclass_find("pci"); for (depth = 1; ; depth++) { bus = device_get_parent(child); pcib = device_get_parent(bus); if (device_get_devclass(device_get_parent(pcib)) != pci_class) return (depth); child = pcib; } } void dmar_dev_path(device_t child, int *busno, void *path1, int depth) { devclass_t pci_class; device_t bus, pcib; ACPI_DMAR_PCI_PATH *path; pci_class = devclass_find("pci"); path = path1; for (depth--; depth != -1; depth--) { path[depth].Device = pci_get_slot(child); path[depth].Function = pci_get_function(child); bus = device_get_parent(child); pcib = device_get_parent(bus); if (device_get_devclass(device_get_parent(pcib)) != pci_class) { /* reached a host bridge */ *busno = pcib_get_bus(bus); return; } child = pcib; } panic("wrong depth"); } static int dmar_match_pathes(int busno1, const ACPI_DMAR_PCI_PATH *path1, int depth1, int busno2, const ACPI_DMAR_PCI_PATH *path2, int depth2, enum AcpiDmarScopeType scope_type) { int i, depth; if (busno1 != busno2) return (0); if (scope_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && depth1 != depth2) return (0); depth = depth1; if (depth2 < depth) depth = depth2; for (i = 0; i < depth; i++) { if (path1[i].Device != path2[i].Device || path1[i].Function != path2[i].Function) return (0); } return (1); } static int dmar_match_devscope(ACPI_DMAR_DEVICE_SCOPE *devscope, int dev_busno, const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len) { ACPI_DMAR_PCI_PATH *path; int path_len; if (devscope->Length < sizeof(*devscope)) { printf("dmar_match_devscope: corrupted DMAR table, dl %d\n", devscope->Length); return (-1); } if (devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_ENDPOINT && devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_BRIDGE) return (0); path_len = devscope->Length - sizeof(*devscope); if (path_len % 2 != 0) { printf("dmar_match_devscope: corrupted DMAR table, dl %d\n", devscope->Length); return (-1); } path_len /= 2; path = (ACPI_DMAR_PCI_PATH *)(devscope + 1); if (path_len == 0) { printf("dmar_match_devscope: corrupted DMAR table, dl %d\n", devscope->Length); return (-1); } return (dmar_match_pathes(devscope->Bus, path, path_len, dev_busno, dev_path, dev_path_len, devscope->EntryType)); } static bool dmar_match_by_path(struct dmar_unit *unit, int dev_domain, int dev_busno, const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len, const char **banner) { ACPI_DMAR_HARDWARE_UNIT *dmarh; ACPI_DMAR_DEVICE_SCOPE *devscope; char *ptr, *ptrend; int match; dmarh = dmar_find_by_index(unit->iommu.unit); if (dmarh == NULL) return (false); if (dmarh->Segment != dev_domain) return (false); if ((dmarh->Flags & ACPI_DMAR_INCLUDE_ALL) != 0) { if (banner != NULL) *banner = "INCLUDE_ALL"; return (true); } ptr = (char *)dmarh + sizeof(*dmarh); ptrend = (char *)dmarh + dmarh->Header.Length; while (ptr < ptrend) { devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr; ptr += devscope->Length; match = dmar_match_devscope(devscope, dev_busno, dev_path, dev_path_len); if (match == -1) return (false); if (match == 1) { if (banner != NULL) *banner = "specific match"; return (true); } } return (false); } static struct dmar_unit * dmar_find_by_scope(int dev_domain, int dev_busno, const ACPI_DMAR_PCI_PATH *dev_path, int dev_path_len) { struct dmar_unit *unit; int i; for (i = 0; i < dmar_devcnt; i++) { if (dmar_devs[i] == NULL) continue; unit = device_get_softc(dmar_devs[i]); if (dmar_match_by_path(unit, dev_domain, dev_busno, dev_path, dev_path_len, NULL)) return (unit); } return (NULL); } struct dmar_unit * dmar_find(device_t dev, bool verbose) { device_t dmar_dev; struct dmar_unit *unit; const char *banner; int i, dev_domain, dev_busno, dev_path_len; /* * This function can only handle PCI(e) devices. */ if (device_get_devclass(device_get_parent(dev)) != devclass_find("pci")) return (NULL); dmar_dev = NULL; dev_domain = pci_get_domain(dev); dev_path_len = dmar_dev_depth(dev); ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); banner = ""; for (i = 0; i < dmar_devcnt; i++) { if (dmar_devs[i] == NULL) continue; unit = device_get_softc(dmar_devs[i]); if (dmar_match_by_path(unit, dev_domain, dev_busno, dev_path, dev_path_len, &banner)) break; } if (i == dmar_devcnt) return (NULL); if (verbose) { device_printf(dev, "pci%d:%d:%d:%d matched dmar%d by %s", dev_domain, pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), unit->iommu.unit, banner); printf(" scope path "); dmar_print_path(dev_busno, dev_path_len, dev_path); printf("\n"); } return (unit); } static struct dmar_unit * dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid) { device_t dmar_dev; struct dmar_unit *unit; ACPI_DMAR_HARDWARE_UNIT *dmarh; ACPI_DMAR_DEVICE_SCOPE *devscope; ACPI_DMAR_PCI_PATH *path; char *ptr, *ptrend; #ifdef DEV_APIC int error; #endif int i; for (i = 0; i < dmar_devcnt; i++) { dmar_dev = dmar_devs[i]; if (dmar_dev == NULL) continue; unit = (struct dmar_unit *)device_get_softc(dmar_dev); dmarh = dmar_find_by_index(i); if (dmarh == NULL) continue; ptr = (char *)dmarh + sizeof(*dmarh); ptrend = (char *)dmarh + dmarh->Header.Length; for (;;) { if (ptr >= ptrend) break; devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr; ptr += devscope->Length; if (devscope->EntryType != entry_type) continue; if (devscope->EnumerationId != id) continue; #ifdef DEV_APIC if (entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) { error = ioapic_get_rid(id, rid); /* * If our IOAPIC has PCI bindings then * use the PCI device rid. */ if (error == 0) return (unit); } #endif if (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE) == 2) { if (rid != NULL) { path = (ACPI_DMAR_PCI_PATH *) (devscope + 1); *rid = PCI_RID(devscope->Bus, path->Device, path->Function); } return (unit); } printf( "dmar_find_nonpci: id %d type %d path length != 2\n", id, entry_type); break; } } return (NULL); } struct dmar_unit * dmar_find_hpet(device_t dev, uint16_t *rid) { return (dmar_find_nonpci(hpet_get_uid(dev), ACPI_DMAR_SCOPE_TYPE_HPET, rid)); } struct dmar_unit * dmar_find_ioapic(u_int apic_id, uint16_t *rid) { return (dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid)); } struct rmrr_iter_args { struct dmar_domain *domain; int dev_domain; int dev_busno; const ACPI_DMAR_PCI_PATH *dev_path; int dev_path_len; struct iommu_map_entries_tailq *rmrr_entries; }; static int dmar_rmrr_iter(ACPI_DMAR_HEADER *dmarh, void *arg) { struct rmrr_iter_args *ria; ACPI_DMAR_RESERVED_MEMORY *resmem; ACPI_DMAR_DEVICE_SCOPE *devscope; struct iommu_map_entry *entry; char *ptr, *ptrend; int match; if (dmarh->Type != ACPI_DMAR_TYPE_RESERVED_MEMORY) return (1); ria = arg; resmem = (ACPI_DMAR_RESERVED_MEMORY *)dmarh; if (resmem->Segment != ria->dev_domain) return (1); ptr = (char *)resmem + sizeof(*resmem); ptrend = (char *)resmem + resmem->Header.Length; for (;;) { if (ptr >= ptrend) break; devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr; ptr += devscope->Length; match = dmar_match_devscope(devscope, ria->dev_busno, ria->dev_path, ria->dev_path_len); if (match == 1) { entry = iommu_gas_alloc_entry(DOM2IODOM(ria->domain), IOMMU_PGF_WAITOK); entry->start = resmem->BaseAddress; /* The RMRR entry end address is inclusive. */ entry->end = resmem->EndAddress; TAILQ_INSERT_TAIL(ria->rmrr_entries, entry, unroll_link); } } return (1); } void dmar_dev_parse_rmrr(struct dmar_domain *domain, int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, struct iommu_map_entries_tailq *rmrr_entries) { struct rmrr_iter_args ria; ria.domain = domain; ria.dev_domain = dev_domain; ria.dev_busno = dev_busno; ria.dev_path = (const ACPI_DMAR_PCI_PATH *)dev_path; ria.dev_path_len = dev_path_len; ria.rmrr_entries = rmrr_entries; dmar_iterate_tbl(dmar_rmrr_iter, &ria); } struct inst_rmrr_iter_args { struct dmar_unit *dmar; }; static device_t dmar_path_dev(int segment, int path_len, int busno, const ACPI_DMAR_PCI_PATH *path, uint16_t *rid) { device_t dev; int i; dev = NULL; for (i = 0; i < path_len; i++) { dev = pci_find_dbsf(segment, busno, path->Device, path->Function); if (i != path_len - 1) { busno = pci_cfgregread(busno, path->Device, path->Function, PCIR_SECBUS_1, 1); path++; } } *rid = PCI_RID(busno, path->Device, path->Function); return (dev); } static int dmar_inst_rmrr_iter(ACPI_DMAR_HEADER *dmarh, void *arg) { const ACPI_DMAR_RESERVED_MEMORY *resmem; const ACPI_DMAR_DEVICE_SCOPE *devscope; struct inst_rmrr_iter_args *iria; const char *ptr, *ptrend; device_t dev; struct dmar_unit *unit; int dev_path_len; uint16_t rid; iria = arg; if (dmarh->Type != ACPI_DMAR_TYPE_RESERVED_MEMORY) return (1); resmem = (ACPI_DMAR_RESERVED_MEMORY *)dmarh; if (resmem->Segment != iria->dmar->segment) return (1); ptr = (const char *)resmem + sizeof(*resmem); ptrend = (const char *)resmem + resmem->Header.Length; for (;;) { if (ptr >= ptrend) break; devscope = (const ACPI_DMAR_DEVICE_SCOPE *)ptr; ptr += devscope->Length; /* XXXKIB bridge */ if (devscope->EntryType != ACPI_DMAR_SCOPE_TYPE_ENDPOINT) continue; rid = 0; dev_path_len = (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE)) / 2; dev = dmar_path_dev(resmem->Segment, dev_path_len, devscope->Bus, (const ACPI_DMAR_PCI_PATH *)(devscope + 1), &rid); if (dev == NULL) { if (bootverbose) { printf("dmar%d no dev found for RMRR " "[%#jx, %#jx] rid %#x scope path ", iria->dmar->iommu.unit, (uintmax_t)resmem->BaseAddress, (uintmax_t)resmem->EndAddress, rid); dmar_print_path(devscope->Bus, dev_path_len, (const ACPI_DMAR_PCI_PATH *)(devscope + 1)); printf("\n"); } unit = dmar_find_by_scope(resmem->Segment, devscope->Bus, (const ACPI_DMAR_PCI_PATH *)(devscope + 1), dev_path_len); if (iria->dmar != unit) continue; dmar_get_ctx_for_devpath(iria->dmar, rid, resmem->Segment, devscope->Bus, (const ACPI_DMAR_PCI_PATH *)(devscope + 1), dev_path_len, false, true); } else { unit = dmar_find(dev, false); if (iria->dmar != unit) continue; iommu_instantiate_ctx(&(iria)->dmar->iommu, dev, true); } } return (1); } /* * Pre-create all contexts for the DMAR which have RMRR entries. */ int dmar_instantiate_rmrr_ctxs(struct iommu_unit *unit) { struct dmar_unit *dmar; struct inst_rmrr_iter_args iria; int error; dmar = IOMMU2DMAR(unit); if (!dmar_barrier_enter(dmar, DMAR_BARRIER_RMRR)) return (0); error = 0; iria.dmar = dmar; dmar_iterate_tbl(dmar_inst_rmrr_iter, &iria); DMAR_LOCK(dmar); if (!LIST_EMPTY(&dmar->domains)) { KASSERT((dmar->hw_gcmd & DMAR_GCMD_TE) == 0, ("dmar%d: RMRR not handled but translation is already enabled", dmar->iommu.unit)); error = dmar_enable_translation(dmar); if (bootverbose) { if (error == 0) { printf("dmar%d: enabled translation\n", dmar->iommu.unit); } else { printf("dmar%d: enabling translation failed, " "error %d\n", dmar->iommu.unit, error); } } } dmar_barrier_exit(dmar, DMAR_BARRIER_RMRR); return (error); } #ifdef DDB #include #include static void dmar_print_domain_entry(const struct iommu_map_entry *entry) { struct iommu_map_entry *l, *r; db_printf( " start %jx end %jx first %jx last %jx free_down %jx flags %x ", entry->start, entry->end, entry->first, entry->last, entry->free_down, entry->flags); db_printf("left "); l = RB_LEFT(entry, rb_entry); if (l == NULL) db_printf("NULL "); else db_printf("%jx ", l->start); db_printf("right "); r = RB_RIGHT(entry, rb_entry); if (r == NULL) db_printf("NULL"); else db_printf("%jx", r->start); db_printf("\n"); } static void dmar_print_ctx(struct dmar_ctx *ctx) { db_printf( " @%p pci%d:%d:%d refs %d flags %x loads %lu unloads %lu\n", ctx, pci_get_bus(ctx->context.tag->owner), pci_get_slot(ctx->context.tag->owner), pci_get_function(ctx->context.tag->owner), ctx->refs, ctx->context.flags, ctx->context.loads, ctx->context.unloads); } static void dmar_print_domain(struct dmar_domain *domain, bool show_mappings) { struct iommu_domain *iodom; struct iommu_map_entry *entry; struct dmar_ctx *ctx; iodom = DOM2IODOM(domain); db_printf( " @%p dom %d mgaw %d agaw %d pglvl %d end %jx refs %d\n" " ctx_cnt %d flags %x pgobj %p map_ents %u\n", domain, domain->domain, domain->mgaw, domain->agaw, domain->pglvl, (uintmax_t)domain->iodom.end, domain->refs, domain->ctx_cnt, domain->iodom.flags, domain->pgtbl_obj, domain->iodom.entries_cnt); if (!LIST_EMPTY(&domain->contexts)) { db_printf(" Contexts:\n"); LIST_FOREACH(ctx, &domain->contexts, link) dmar_print_ctx(ctx); } if (!show_mappings) return; db_printf(" mapped:\n"); RB_FOREACH(entry, iommu_gas_entries_tree, &iodom->rb_root) { dmar_print_domain_entry(entry); if (db_pager_quit) break; } if (db_pager_quit) return; db_printf(" unloading:\n"); TAILQ_FOREACH(entry, &domain->iodom.unload_entries, dmamap_link) { dmar_print_domain_entry(entry); if (db_pager_quit) break; } } DB_FUNC(dmar_domain, db_dmar_print_domain, db_show_table, CS_OWN, NULL) { struct dmar_unit *unit; struct dmar_domain *domain; struct dmar_ctx *ctx; bool show_mappings, valid; int pci_domain, bus, device, function, i, t; db_expr_t radix; valid = false; radix = db_radix; db_radix = 10; t = db_read_token(); if (t == tSLASH) { t = db_read_token(); if (t != tIDENT) { db_printf("Bad modifier\n"); db_radix = radix; db_skip_to_eol(); return; } show_mappings = strchr(db_tok_string, 'm') != NULL; t = db_read_token(); } else { show_mappings = false; } if (t == tNUMBER) { pci_domain = db_tok_number; t = db_read_token(); if (t == tNUMBER) { bus = db_tok_number; t = db_read_token(); if (t == tNUMBER) { device = db_tok_number; t = db_read_token(); if (t == tNUMBER) { function = db_tok_number; valid = true; } } } } db_radix = radix; db_skip_to_eol(); if (!valid) { db_printf("usage: show dmar_domain [/m] " " \n"); return; } for (i = 0; i < dmar_devcnt; i++) { unit = device_get_softc(dmar_devs[i]); LIST_FOREACH(domain, &unit->domains, link) { LIST_FOREACH(ctx, &domain->contexts, link) { if (pci_domain == unit->segment && bus == pci_get_bus(ctx->context.tag->owner) && device == pci_get_slot(ctx->context.tag->owner) && function == pci_get_function(ctx->context.tag->owner)) { dmar_print_domain(domain, show_mappings); goto out; } } } } out:; } static void dmar_print_one(int idx, bool show_domains, bool show_mappings) { struct dmar_unit *unit; struct dmar_domain *domain; int i, frir; unit = device_get_softc(dmar_devs[idx]); db_printf("dmar%d at %p, root at 0x%jx, ver 0x%x\n", unit->iommu.unit, unit, dmar_read8(unit, DMAR_RTADDR_REG), dmar_read4(unit, DMAR_VER_REG)); db_printf("cap 0x%jx ecap 0x%jx gsts 0x%x fsts 0x%x fectl 0x%x\n", (uintmax_t)dmar_read8(unit, DMAR_CAP_REG), (uintmax_t)dmar_read8(unit, DMAR_ECAP_REG), dmar_read4(unit, DMAR_GSTS_REG), dmar_read4(unit, DMAR_FSTS_REG), dmar_read4(unit, DMAR_FECTL_REG)); if (unit->ir_enabled) { db_printf("ir is enabled; IRT @%p phys 0x%jx maxcnt %d\n", unit->irt, (uintmax_t)unit->irt_phys, unit->irte_cnt); } db_printf("fed 0x%x fea 0x%x feua 0x%x\n", dmar_read4(unit, DMAR_FEDATA_REG), dmar_read4(unit, DMAR_FEADDR_REG), dmar_read4(unit, DMAR_FEUADDR_REG)); db_printf("primary fault log:\n"); for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; db_printf(" %d at 0x%x: %jx %jx\n", i, frir, (uintmax_t)dmar_read8(unit, frir), (uintmax_t)dmar_read8(unit, frir + 8)); } if (DMAR_HAS_QI(unit)) { db_printf("ied 0x%x iea 0x%x ieua 0x%x\n", dmar_read4(unit, DMAR_IEDATA_REG), dmar_read4(unit, DMAR_IEADDR_REG), dmar_read4(unit, DMAR_IEUADDR_REG)); if (unit->qi_enabled) { db_printf("qi is enabled: queue @0x%jx (IQA 0x%jx) " "size 0x%jx\n" " head 0x%x tail 0x%x avail 0x%x status 0x%x ctrl 0x%x\n" " hw compl 0x%x@%p/phys@%jx next seq 0x%x gen 0x%x\n", (uintmax_t)unit->inv_queue, (uintmax_t)dmar_read8(unit, DMAR_IQA_REG), (uintmax_t)unit->inv_queue_size, dmar_read4(unit, DMAR_IQH_REG), dmar_read4(unit, DMAR_IQT_REG), unit->inv_queue_avail, dmar_read4(unit, DMAR_ICS_REG), dmar_read4(unit, DMAR_IECTL_REG), unit->inv_waitd_seq_hw, &unit->inv_waitd_seq_hw, (uintmax_t)unit->inv_waitd_seq_hw_phys, unit->inv_waitd_seq, unit->inv_waitd_gen); } else { db_printf("qi is disabled\n"); } } if (show_domains) { db_printf("domains:\n"); LIST_FOREACH(domain, &unit->domains, link) { dmar_print_domain(domain, show_mappings); if (db_pager_quit) break; } } } DB_SHOW_COMMAND(dmar, db_dmar_print) { bool show_domains, show_mappings; show_domains = strchr(modif, 'd') != NULL; show_mappings = strchr(modif, 'm') != NULL; if (!have_addr) { db_printf("usage: show dmar [/d] [/m] index\n"); return; } dmar_print_one((int)addr, show_domains, show_mappings); } DB_SHOW_ALL_COMMAND(dmars, db_show_all_dmars) { int i; bool show_domains, show_mappings; show_domains = strchr(modif, 'd') != NULL; show_mappings = strchr(modif, 'm') != NULL; for (i = 0; i < dmar_devcnt; i++) { dmar_print_one(i, show_domains, show_mappings); if (db_pager_quit) break; } } #endif struct iommu_unit * iommu_find(device_t dev, bool verbose) { struct dmar_unit *dmar; dmar = dmar_find(dev, verbose); return (&dmar->iommu); } diff --git a/sys/x86/iommu/intel_fault.c b/sys/x86/iommu/intel_fault.c index 628bcf2ee763..af8474f41692 100644 --- a/sys/x86/iommu/intel_fault.c +++ b/sys/x86/iommu/intel_fault.c @@ -1,330 +1,329 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_acpi.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Fault interrupt handling for DMARs. If advanced fault logging is * not implemented by hardware, the code emulates it. Fast interrupt * handler flushes the fault registers into circular buffer at * unit->fault_log, and schedules a task. * * The fast handler is used since faults usually come in bursts, and * number of fault log registers is limited, e.g. down to one for 5400 * MCH. We are trying to reduce the latency for clearing the fault * register file. The task is usually long-running, since printf() is * slow, but this is not problematic because bursts are rare. * * For the same reason, each translation unit task is executed in its * own thread. * * XXXKIB It seems there is no hardware available which implements * advanced fault logging, so the code to handle AFL is not written. */ static int dmar_fault_next(struct dmar_unit *unit, int faultp) { faultp += 2; if (faultp == unit->fault_log_size) faultp = 0; return (faultp); } static void dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) { uint32_t clear; clear = 0; if ((fsts & DMAR_FSTS_ITE) != 0) { printf("DMAR%d: Invalidation timed out\n", unit->iommu.unit); clear |= DMAR_FSTS_ITE; } if ((fsts & DMAR_FSTS_ICE) != 0) { printf("DMAR%d: Invalidation completion error\n", unit->iommu.unit); clear |= DMAR_FSTS_ICE; } if ((fsts & DMAR_FSTS_IQE) != 0) { printf("DMAR%d: Invalidation queue error\n", unit->iommu.unit); clear |= DMAR_FSTS_IQE; } if ((fsts & DMAR_FSTS_APF) != 0) { printf("DMAR%d: Advanced pending fault\n", unit->iommu.unit); clear |= DMAR_FSTS_APF; } if ((fsts & DMAR_FSTS_AFO) != 0) { printf("DMAR%d: Advanced fault overflow\n", unit->iommu.unit); clear |= DMAR_FSTS_AFO; } if (clear != 0) dmar_write4(unit, DMAR_FSTS_REG, clear); } int dmar_fault_intr(void *arg) { struct dmar_unit *unit; uint64_t fault_rec[2]; uint32_t fsts; int fri, frir, faultp; bool enqueue; unit = arg; enqueue = false; fsts = dmar_read4(unit, DMAR_FSTS_REG); dmar_fault_intr_clear(unit, fsts); if ((fsts & DMAR_FSTS_PPF) == 0) goto done; fri = DMAR_FSTS_FRI(fsts); for (;;) { frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; fault_rec[1] = dmar_read8(unit, frir + 8); if ((fault_rec[1] & DMAR_FRCD2_F) == 0) break; fault_rec[0] = dmar_read8(unit, frir); dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); DMAR_FAULT_LOCK(unit); faultp = unit->fault_log_head; if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { /* XXXKIB log overflow */ } else { unit->fault_log[faultp] = fault_rec[0]; unit->fault_log[faultp + 1] = fault_rec[1]; unit->fault_log_head = dmar_fault_next(unit, faultp); enqueue = true; } DMAR_FAULT_UNLOCK(unit); fri += 1; if (fri >= DMAR_CAP_NFR(unit->hw_cap)) fri = 0; } done: /* * On SandyBridge, due to errata BJ124, IvyBridge errata * BV100, and Haswell errata HSD40, "Spurious Intel VT-d * Interrupts May Occur When the PFO Bit is Set". Handle the * cases by clearing overflow bit even if no fault is * reported. * * On IvyBridge, errata BV30 states that clearing clear * DMAR_FRCD2_F bit in the fault register causes spurious * interrupt. Do nothing. * */ if ((fsts & DMAR_FSTS_PFO) != 0) { printf("DMAR%d: Fault Overflow\n", unit->iommu.unit); dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); } if (enqueue) { taskqueue_enqueue(unit->fault_taskqueue, &unit->fault_task); } return (FILTER_HANDLED); } static void dmar_fault_task(void *arg, int pending __unused) { struct dmar_unit *unit; struct dmar_ctx *ctx; uint64_t fault_rec[2]; int sid, bus, slot, func, faultp; unit = arg; DMAR_FAULT_LOCK(unit); for (;;) { faultp = unit->fault_log_tail; if (faultp == unit->fault_log_head) break; fault_rec[0] = unit->fault_log[faultp]; fault_rec[1] = unit->fault_log[faultp + 1]; unit->fault_log_tail = dmar_fault_next(unit, faultp); DMAR_FAULT_UNLOCK(unit); sid = DMAR_FRCD2_SID(fault_rec[1]); printf("DMAR%d: ", unit->iommu.unit); DMAR_LOCK(unit); ctx = dmar_find_ctx_locked(unit, sid); if (ctx == NULL) { printf(":"); /* * Note that the slot and function will not be correct * if ARI is in use, but without a ctx entry we have * no way of knowing whether ARI is in use or not. */ bus = PCI_RID2BUS(sid); slot = PCI_RID2SLOT(sid); func = PCI_RID2FUNC(sid); } else { ctx->context.flags |= IOMMU_CTX_FAULTED; ctx->last_fault_rec[0] = fault_rec[0]; ctx->last_fault_rec[1] = fault_rec[1]; device_print_prettyname(ctx->context.tag->owner); bus = pci_get_bus(ctx->context.tag->owner); slot = pci_get_slot(ctx->context.tag->owner); func = pci_get_function(ctx->context.tag->owner); } DMAR_UNLOCK(unit); printf( "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x " "addr %jx\n", bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]), DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), (uintmax_t)fault_rec[0]); DMAR_FAULT_LOCK(unit); } DMAR_FAULT_UNLOCK(unit); } static void dmar_clear_faults(struct dmar_unit *unit) { uint32_t frec, frir, fsts; int i; for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; frec = dmar_read4(unit, frir + 12); if ((frec & DMAR_FRCD2_F32) == 0) continue; dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); } fsts = dmar_read4(unit, DMAR_FSTS_REG); dmar_write4(unit, DMAR_FSTS_REG, fsts); } int dmar_init_fault_log(struct dmar_unit *unit) { mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); unit->fault_log_size = 256; /* 128 fault log entries */ TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); if (unit->fault_log_size % 2 != 0) panic("hw.dmar_fault_log_size must be even"); unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, M_DEVBUF, M_WAITOK | M_ZERO); TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK, taskqueue_thread_enqueue, &unit->fault_taskqueue); taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, "dmar%d fault taskq", unit->iommu.unit); DMAR_LOCK(unit); dmar_disable_fault_intr(unit); dmar_clear_faults(unit); dmar_enable_fault_intr(unit); DMAR_UNLOCK(unit); return (0); } void dmar_fini_fault_log(struct dmar_unit *unit) { if (unit->fault_taskqueue == NULL) return; DMAR_LOCK(unit); dmar_disable_fault_intr(unit); DMAR_UNLOCK(unit); taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); taskqueue_free(unit->fault_taskqueue); unit->fault_taskqueue = NULL; mtx_destroy(&unit->fault_lock); free(unit->fault_log, M_DEVBUF); unit->fault_log = NULL; unit->fault_log_head = unit->fault_log_tail = 0; } void dmar_enable_fault_intr(struct dmar_unit *unit) { uint32_t fectl; DMAR_ASSERT_LOCKED(unit); fectl = dmar_read4(unit, DMAR_FECTL_REG); fectl &= ~DMAR_FECTL_IM; dmar_write4(unit, DMAR_FECTL_REG, fectl); } void dmar_disable_fault_intr(struct dmar_unit *unit) { uint32_t fectl; DMAR_ASSERT_LOCKED(unit); fectl = dmar_read4(unit, DMAR_FECTL_REG); dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); } diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c index 9fd06d49d3b0..2346640ce015 100644 --- a/sys/x86/iommu/intel_idpgtbl.c +++ b/sys/x86/iommu/intel_idpgtbl.c @@ -1,819 +1,818 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int domain_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, int flags); /* * The cache of the identity mapping page tables for the DMARs. Using * the cache saves significant amount of memory for page tables by * reusing the page tables, since usually DMARs are identical and have * the same capabilities. Still, cache records the information needed * to match DMAR capabilities and page table format, to correctly * handle different DMARs. */ struct idpgtbl { iommu_gaddr_t maxaddr; /* Page table covers the guest address range [0..maxaddr) */ int pglvl; /* Total page table levels ignoring superpages */ int leaf; /* The last materialized page table level, it is non-zero if superpages are supported */ vm_object_t pgtbl_obj; /* The page table pages */ LIST_ENTRY(idpgtbl) link; }; static struct sx idpgtbl_lock; SX_SYSINIT(idpgtbl, &idpgtbl_lock, "idpgtbl"); static LIST_HEAD(, idpgtbl) idpgtbls = LIST_HEAD_INITIALIZER(idpgtbls); static MALLOC_DEFINE(M_DMAR_IDPGTBL, "dmar_idpgtbl", "Intel DMAR Identity mappings cache elements"); /* * Build the next level of the page tables for the identity mapping. * - lvl is the level to build; * - idx is the index of the page table page in the pgtbl_obj, which is * being allocated filled now; * - addr is the starting address in the bus address space which is * mapped by the page table page. */ static void domain_idmap_nextlvl(struct idpgtbl *tbl, int lvl, vm_pindex_t idx, iommu_gaddr_t addr) { vm_page_t m1; dmar_pte_t *pte; struct sf_buf *sf; iommu_gaddr_t f, pg_sz; vm_pindex_t base; int i; VM_OBJECT_ASSERT_LOCKED(tbl->pgtbl_obj); if (addr >= tbl->maxaddr) return; (void)dmar_pgalloc(tbl->pgtbl_obj, idx, IOMMU_PGF_OBJL | IOMMU_PGF_WAITOK | IOMMU_PGF_ZERO); base = idx * DMAR_NPTEPG + 1; /* Index of the first child page of idx */ pg_sz = pglvl_page_size(tbl->pglvl, lvl); if (lvl != tbl->leaf) { for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz) domain_idmap_nextlvl(tbl, lvl + 1, base + i, f); } VM_OBJECT_WUNLOCK(tbl->pgtbl_obj); pte = dmar_map_pgtbl(tbl->pgtbl_obj, idx, IOMMU_PGF_WAITOK, &sf); if (lvl == tbl->leaf) { for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz) { if (f >= tbl->maxaddr) break; pte[i].pte = (DMAR_PTE_ADDR_MASK & f) | DMAR_PTE_R | DMAR_PTE_W; } } else { for (i = 0, f = addr; i < DMAR_NPTEPG; i++, f += pg_sz) { if (f >= tbl->maxaddr) break; m1 = dmar_pgalloc(tbl->pgtbl_obj, base + i, IOMMU_PGF_NOALLOC); KASSERT(m1 != NULL, ("lost page table page")); pte[i].pte = (DMAR_PTE_ADDR_MASK & VM_PAGE_TO_PHYS(m1)) | DMAR_PTE_R | DMAR_PTE_W; } } /* domain_get_idmap_pgtbl flushes CPU cache if needed. */ dmar_unmap_pgtbl(sf); VM_OBJECT_WLOCK(tbl->pgtbl_obj); } /* * Find a ready and compatible identity-mapping page table in the * cache. If not found, populate the identity-mapping page table for * the context, up to the maxaddr. The maxaddr byte is allowed to be * not mapped, which is aligned with the definition of Maxmem as the * highest usable physical address + 1. If superpages are used, the * maxaddr is typically mapped. */ vm_object_t domain_get_idmap_pgtbl(struct dmar_domain *domain, iommu_gaddr_t maxaddr) { struct dmar_unit *unit; struct idpgtbl *tbl; vm_object_t res; vm_page_t m; int leaf, i; leaf = 0; /* silence gcc */ /* * First, determine where to stop the paging structures. */ for (i = 0; i < domain->pglvl; i++) { if (i == domain->pglvl - 1 || domain_is_sp_lvl(domain, i)) { leaf = i; break; } } /* * Search the cache for a compatible page table. Qualified * page table must map up to maxaddr, its level must be * supported by the DMAR and leaf should be equal to the * calculated value. The later restriction could be lifted * but I believe it is currently impossible to have any * deviations for existing hardware. */ sx_slock(&idpgtbl_lock); LIST_FOREACH(tbl, &idpgtbls, link) { if (tbl->maxaddr >= maxaddr && dmar_pglvl_supported(domain->dmar, tbl->pglvl) && tbl->leaf == leaf) { res = tbl->pgtbl_obj; vm_object_reference(res); sx_sunlock(&idpgtbl_lock); domain->pglvl = tbl->pglvl; /* XXXKIB ? */ goto end; } } /* * Not found in cache, relock the cache into exclusive mode to * be able to add element, and recheck cache again after the * relock. */ sx_sunlock(&idpgtbl_lock); sx_xlock(&idpgtbl_lock); LIST_FOREACH(tbl, &idpgtbls, link) { if (tbl->maxaddr >= maxaddr && dmar_pglvl_supported(domain->dmar, tbl->pglvl) && tbl->leaf == leaf) { res = tbl->pgtbl_obj; vm_object_reference(res); sx_xunlock(&idpgtbl_lock); domain->pglvl = tbl->pglvl; /* XXXKIB ? */ return (res); } } /* * Still not found, create new page table. */ tbl = malloc(sizeof(*tbl), M_DMAR_IDPGTBL, M_WAITOK); tbl->pglvl = domain->pglvl; tbl->leaf = leaf; tbl->maxaddr = maxaddr; tbl->pgtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL, IDX_TO_OFF(pglvl_max_pages(tbl->pglvl)), 0, 0, NULL); VM_OBJECT_WLOCK(tbl->pgtbl_obj); domain_idmap_nextlvl(tbl, 0, 0, 0); VM_OBJECT_WUNLOCK(tbl->pgtbl_obj); LIST_INSERT_HEAD(&idpgtbls, tbl, link); res = tbl->pgtbl_obj; vm_object_reference(res); sx_xunlock(&idpgtbl_lock); end: /* * Table was found or created. * * If DMAR does not snoop paging structures accesses, flush * CPU cache to memory. Note that dmar_unmap_pgtbl() coherent * argument was possibly invalid at the time of the identity * page table creation, since DMAR which was passed at the * time of creation could be coherent, while current DMAR is * not. * * If DMAR cannot look into the chipset write buffer, flush it * as well. */ unit = domain->dmar; if (!DMAR_IS_COHERENT(unit)) { VM_OBJECT_WLOCK(res); for (m = vm_page_lookup(res, 0); m != NULL; m = vm_page_next(m)) pmap_invalidate_cache_pages(&m, 1); VM_OBJECT_WUNLOCK(res); } if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) { DMAR_LOCK(unit); dmar_flush_write_bufs(unit); DMAR_UNLOCK(unit); } return (res); } /* * Return a reference to the identity mapping page table to the cache. */ void put_idmap_pgtbl(vm_object_t obj) { struct idpgtbl *tbl, *tbl1; vm_object_t rmobj; sx_slock(&idpgtbl_lock); KASSERT(obj->ref_count >= 2, ("lost cache reference")); vm_object_deallocate(obj); /* * Cache always owns one last reference on the page table object. * If there is an additional reference, object must stay. */ if (obj->ref_count > 1) { sx_sunlock(&idpgtbl_lock); return; } /* * Cache reference is the last, remove cache element and free * page table object, returning the page table pages to the * system. */ sx_sunlock(&idpgtbl_lock); sx_xlock(&idpgtbl_lock); LIST_FOREACH_SAFE(tbl, &idpgtbls, link, tbl1) { rmobj = tbl->pgtbl_obj; if (rmobj->ref_count == 1) { LIST_REMOVE(tbl, link); atomic_subtract_int(&dmar_tbl_pagecnt, rmobj->resident_page_count); vm_object_deallocate(rmobj); free(tbl, M_DMAR_IDPGTBL); } } sx_xunlock(&idpgtbl_lock); } /* * The core routines to map and unmap host pages at the given guest * address. Support superpages. */ /* * Index of the pte for the guest address base in the page table at * the level lvl. */ static int domain_pgtbl_pte_off(struct dmar_domain *domain, iommu_gaddr_t base, int lvl) { base >>= DMAR_PAGE_SHIFT + (domain->pglvl - lvl - 1) * DMAR_NPTEPGSHIFT; return (base & DMAR_PTEMASK); } /* * Returns the page index of the page table page in the page table * object, which maps the given address base at the page table level * lvl. */ static vm_pindex_t domain_pgtbl_get_pindex(struct dmar_domain *domain, iommu_gaddr_t base, int lvl) { vm_pindex_t idx, pidx; int i; KASSERT(lvl >= 0 && lvl < domain->pglvl, ("wrong lvl %p %d", domain, lvl)); for (pidx = idx = 0, i = 0; i < lvl; i++, pidx = idx) { idx = domain_pgtbl_pte_off(domain, base, i) + pidx * DMAR_NPTEPG + 1; } return (idx); } static dmar_pte_t * domain_pgtbl_map_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, int flags, vm_pindex_t *idxp, struct sf_buf **sf) { vm_page_t m; struct sf_buf *sfp; dmar_pte_t *pte, *ptep; vm_pindex_t idx, idx1; DMAR_DOMAIN_ASSERT_PGLOCKED(domain); KASSERT((flags & IOMMU_PGF_OBJL) != 0, ("lost PGF_OBJL")); idx = domain_pgtbl_get_pindex(domain, base, lvl); if (*sf != NULL && idx == *idxp) { pte = (dmar_pte_t *)sf_buf_kva(*sf); } else { if (*sf != NULL) dmar_unmap_pgtbl(*sf); *idxp = idx; retry: pte = dmar_map_pgtbl(domain->pgtbl_obj, idx, flags, sf); if (pte == NULL) { KASSERT(lvl > 0, ("lost root page table page %p", domain)); /* * Page table page does not exist, allocate * it and create a pte in the preceeding page level * to reference the allocated page table page. */ m = dmar_pgalloc(domain->pgtbl_obj, idx, flags | IOMMU_PGF_ZERO); if (m == NULL) return (NULL); /* * Prevent potential free while pgtbl_obj is * unlocked in the recursive call to * domain_pgtbl_map_pte(), if other thread did * pte write and clean while the lock is * dropped. */ m->ref_count++; sfp = NULL; ptep = domain_pgtbl_map_pte(domain, base, lvl - 1, flags, &idx1, &sfp); if (ptep == NULL) { KASSERT(m->pindex != 0, ("loosing root page %p", domain)); m->ref_count--; dmar_pgfree(domain->pgtbl_obj, m->pindex, flags); return (NULL); } dmar_pte_store(&ptep->pte, DMAR_PTE_R | DMAR_PTE_W | VM_PAGE_TO_PHYS(m)); dmar_flush_pte_to_ram(domain->dmar, ptep); sf_buf_page(sfp)->ref_count += 1; m->ref_count--; dmar_unmap_pgtbl(sfp); /* Only executed once. */ goto retry; } } pte += domain_pgtbl_pte_off(domain, base, lvl); return (pte); } static int domain_map_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, vm_page_t *ma, uint64_t pflags, int flags) { dmar_pte_t *pte; struct sf_buf *sf; iommu_gaddr_t pg_sz, base1, size1; vm_pindex_t pi, c, idx, run_sz; int lvl; bool superpage; DMAR_DOMAIN_ASSERT_PGLOCKED(domain); base1 = base; size1 = size; flags |= IOMMU_PGF_OBJL; TD_PREP_PINNED_ASSERT; for (sf = NULL, pi = 0; size > 0; base += pg_sz, size -= pg_sz, pi += run_sz) { for (lvl = 0, c = 0, superpage = false;; lvl++) { pg_sz = domain_page_size(domain, lvl); run_sz = pg_sz >> DMAR_PAGE_SHIFT; if (lvl == domain->pglvl - 1) break; /* * Check if the current base suitable for the * superpage mapping. First, verify the level. */ if (!domain_is_sp_lvl(domain, lvl)) continue; /* * Next, look at the size of the mapping and * alignment of both guest and host addresses. */ if (size < pg_sz || (base & (pg_sz - 1)) != 0 || (VM_PAGE_TO_PHYS(ma[pi]) & (pg_sz - 1)) != 0) continue; /* All passed, check host pages contiguouty. */ if (c == 0) { for (c = 1; c < run_sz; c++) { if (VM_PAGE_TO_PHYS(ma[pi + c]) != VM_PAGE_TO_PHYS(ma[pi + c - 1]) + PAGE_SIZE) break; } } if (c >= run_sz) { superpage = true; break; } } KASSERT(size >= pg_sz, ("mapping loop overflow %p %jx %jx %jx", domain, (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); KASSERT(pg_sz > 0, ("pg_sz 0 lvl %d", lvl)); pte = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf); if (pte == NULL) { KASSERT((flags & IOMMU_PGF_WAITOK) == 0, ("failed waitable pte alloc %p", domain)); if (sf != NULL) dmar_unmap_pgtbl(sf); domain_unmap_buf_locked(domain, base1, base - base1, flags); TD_PINNED_ASSERT; return (ENOMEM); } dmar_pte_store(&pte->pte, VM_PAGE_TO_PHYS(ma[pi]) | pflags | (superpage ? DMAR_PTE_SP : 0)); dmar_flush_pte_to_ram(domain->dmar, pte); sf_buf_page(sf)->ref_count += 1; } if (sf != NULL) dmar_unmap_pgtbl(sf); TD_PINNED_ASSERT; return (0); } static int domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags) { struct dmar_domain *domain; struct dmar_unit *unit; uint64_t pflags; int error; pflags = ((eflags & IOMMU_MAP_ENTRY_READ) != 0 ? DMAR_PTE_R : 0) | ((eflags & IOMMU_MAP_ENTRY_WRITE) != 0 ? DMAR_PTE_W : 0) | ((eflags & IOMMU_MAP_ENTRY_SNOOP) != 0 ? DMAR_PTE_SNP : 0) | ((eflags & IOMMU_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0); domain = IODOM2DOM(iodom); unit = domain->dmar; KASSERT((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) == 0, ("modifying idmap pagetable domain %p", domain)); KASSERT((base & DMAR_PAGE_MASK) == 0, ("non-aligned base %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT((size & DMAR_PAGE_MASK) == 0, ("non-aligned size %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT(size > 0, ("zero size %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT(base < (1ULL << domain->agaw), ("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base, (uintmax_t)size, domain->agaw)); KASSERT(base + size < (1ULL << domain->agaw), ("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base, (uintmax_t)size, domain->agaw)); KASSERT(base + size > base, ("size overflow %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT((pflags & (DMAR_PTE_R | DMAR_PTE_W)) != 0, ("neither read nor write %jx", (uintmax_t)pflags)); KASSERT((pflags & ~(DMAR_PTE_R | DMAR_PTE_W | DMAR_PTE_SNP | DMAR_PTE_TM)) == 0, ("invalid pte flags %jx", (uintmax_t)pflags)); KASSERT((pflags & DMAR_PTE_SNP) == 0 || (unit->hw_ecap & DMAR_ECAP_SC) != 0, ("PTE_SNP for dmar without snoop control %p %jx", domain, (uintmax_t)pflags)); KASSERT((pflags & DMAR_PTE_TM) == 0 || (unit->hw_ecap & DMAR_ECAP_DI) != 0, ("PTE_TM for dmar without DIOTLB %p %jx", domain, (uintmax_t)pflags)); KASSERT((flags & ~IOMMU_PGF_WAITOK) == 0, ("invalid flags %x", flags)); DMAR_DOMAIN_PGLOCK(domain); error = domain_map_buf_locked(domain, base, size, ma, pflags, flags); DMAR_DOMAIN_PGUNLOCK(domain); if (error != 0) return (error); if ((unit->hw_cap & DMAR_CAP_CM) != 0) domain_flush_iotlb_sync(domain, base, size); else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) { /* See 11.1 Write Buffer Flushing. */ DMAR_LOCK(unit); dmar_flush_write_bufs(unit); DMAR_UNLOCK(unit); } return (0); } static void domain_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_fs); static void domain_free_pgtbl_pde(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, int flags) { struct sf_buf *sf; dmar_pte_t *pde; vm_pindex_t idx; sf = NULL; pde = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf); domain_unmap_clear_pte(domain, base, lvl, flags, pde, &sf, true); } static void domain_unmap_clear_pte(struct dmar_domain *domain, iommu_gaddr_t base, int lvl, int flags, dmar_pte_t *pte, struct sf_buf **sf, bool free_sf) { vm_page_t m; dmar_pte_clear(&pte->pte); dmar_flush_pte_to_ram(domain->dmar, pte); m = sf_buf_page(*sf); if (free_sf) { dmar_unmap_pgtbl(*sf); *sf = NULL; } m->ref_count--; if (m->ref_count != 0) return; KASSERT(lvl != 0, ("lost reference (lvl) on root pg domain %p base %jx lvl %d", domain, (uintmax_t)base, lvl)); KASSERT(m->pindex != 0, ("lost reference (idx) on root pg domain %p base %jx lvl %d", domain, (uintmax_t)base, lvl)); dmar_pgfree(domain->pgtbl_obj, m->pindex, flags); domain_free_pgtbl_pde(domain, base, lvl - 1, flags); } /* * Assumes that the unmap is never partial. */ static int domain_unmap_buf_locked(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, int flags) { dmar_pte_t *pte; struct sf_buf *sf; vm_pindex_t idx; iommu_gaddr_t pg_sz; int lvl; DMAR_DOMAIN_ASSERT_PGLOCKED(domain); if (size == 0) return (0); KASSERT((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) == 0, ("modifying idmap pagetable domain %p", domain)); KASSERT((base & DMAR_PAGE_MASK) == 0, ("non-aligned base %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT((size & DMAR_PAGE_MASK) == 0, ("non-aligned size %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT(base < (1ULL << domain->agaw), ("base too high %p %jx %jx agaw %d", domain, (uintmax_t)base, (uintmax_t)size, domain->agaw)); KASSERT(base + size < (1ULL << domain->agaw), ("end too high %p %jx %jx agaw %d", domain, (uintmax_t)base, (uintmax_t)size, domain->agaw)); KASSERT(base + size > base, ("size overflow %p %jx %jx", domain, (uintmax_t)base, (uintmax_t)size)); KASSERT((flags & ~IOMMU_PGF_WAITOK) == 0, ("invalid flags %x", flags)); pg_sz = 0; /* silence gcc */ flags |= IOMMU_PGF_OBJL; TD_PREP_PINNED_ASSERT; for (sf = NULL; size > 0; base += pg_sz, size -= pg_sz) { for (lvl = 0; lvl < domain->pglvl; lvl++) { if (lvl != domain->pglvl - 1 && !domain_is_sp_lvl(domain, lvl)) continue; pg_sz = domain_page_size(domain, lvl); if (pg_sz > size) continue; pte = domain_pgtbl_map_pte(domain, base, lvl, flags, &idx, &sf); KASSERT(pte != NULL, ("sleeping or page missed %p %jx %d 0x%x", domain, (uintmax_t)base, lvl, flags)); if ((pte->pte & DMAR_PTE_SP) != 0 || lvl == domain->pglvl - 1) { domain_unmap_clear_pte(domain, base, lvl, flags, pte, &sf, false); break; } } KASSERT(size >= pg_sz, ("unmapping loop overflow %p %jx %jx %jx", domain, (uintmax_t)base, (uintmax_t)size, (uintmax_t)pg_sz)); } if (sf != NULL) dmar_unmap_pgtbl(sf); /* * See 11.1 Write Buffer Flushing for an explanation why RWBF * can be ignored there. */ TD_PINNED_ASSERT; return (0); } static int domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base, iommu_gaddr_t size, int flags) { struct dmar_domain *domain; int error; domain = IODOM2DOM(iodom); DMAR_DOMAIN_PGLOCK(domain); error = domain_unmap_buf_locked(domain, base, size, flags); DMAR_DOMAIN_PGUNLOCK(domain); return (error); } int domain_alloc_pgtbl(struct dmar_domain *domain) { vm_page_t m; KASSERT(domain->pgtbl_obj == NULL, ("already initialized %p", domain)); domain->pgtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL, IDX_TO_OFF(pglvl_max_pages(domain->pglvl)), 0, 0, NULL); DMAR_DOMAIN_PGLOCK(domain); m = dmar_pgalloc(domain->pgtbl_obj, 0, IOMMU_PGF_WAITOK | IOMMU_PGF_ZERO | IOMMU_PGF_OBJL); /* No implicit free of the top level page table page. */ m->ref_count = 1; DMAR_DOMAIN_PGUNLOCK(domain); DMAR_LOCK(domain->dmar); domain->iodom.flags |= IOMMU_DOMAIN_PGTBL_INITED; DMAR_UNLOCK(domain->dmar); return (0); } void domain_free_pgtbl(struct dmar_domain *domain) { vm_object_t obj; vm_page_t m; obj = domain->pgtbl_obj; if (obj == NULL) { KASSERT((domain->dmar->hw_ecap & DMAR_ECAP_PT) != 0 && (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0, ("lost pagetable object domain %p", domain)); return; } DMAR_DOMAIN_ASSERT_PGLOCKED(domain); domain->pgtbl_obj = NULL; if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0) { put_idmap_pgtbl(obj); domain->iodom.flags &= ~IOMMU_DOMAIN_IDMAP; return; } /* Obliterate ref_counts */ VM_OBJECT_ASSERT_WLOCKED(obj); for (m = vm_page_lookup(obj, 0); m != NULL; m = vm_page_next(m)) m->ref_count = 0; VM_OBJECT_WUNLOCK(obj); vm_object_deallocate(obj); } static inline uint64_t domain_wait_iotlb_flush(struct dmar_unit *unit, uint64_t wt, int iro) { uint64_t iotlbr; dmar_write8(unit, iro + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT | DMAR_IOTLB_DR | DMAR_IOTLB_DW | wt); for (;;) { iotlbr = dmar_read8(unit, iro + DMAR_IOTLB_REG_OFF); if ((iotlbr & DMAR_IOTLB_IVT) == 0) break; cpu_spinwait(); } return (iotlbr); } void domain_flush_iotlb_sync(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size) { struct dmar_unit *unit; iommu_gaddr_t isize; uint64_t iotlbr; int am, iro; unit = domain->dmar; KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call", unit->iommu.unit)); iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16; DMAR_LOCK(unit); if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) { iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_DOM | DMAR_IOTLB_DID(domain->domain), iro); KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != DMAR_IOTLB_IAIG_INVLD, ("dmar%d: invalidation failed %jx", unit->iommu.unit, (uintmax_t)iotlbr)); } else { for (; size > 0; base += isize, size -= isize) { am = calc_am(unit, base, size, &isize); dmar_write8(unit, iro, base | am); iotlbr = domain_wait_iotlb_flush(unit, DMAR_IOTLB_IIRG_PAGE | DMAR_IOTLB_DID(domain->domain), iro); KASSERT((iotlbr & DMAR_IOTLB_IAIG_MASK) != DMAR_IOTLB_IAIG_INVLD, ("dmar%d: PSI invalidation failed " "iotlbr 0x%jx base 0x%jx size 0x%jx am %d", unit->iommu.unit, (uintmax_t)iotlbr, (uintmax_t)base, (uintmax_t)size, am)); /* * Any non-page granularity covers whole guest * address space for the domain. */ if ((iotlbr & DMAR_IOTLB_IAIG_MASK) != DMAR_IOTLB_IAIG_PAGE) break; } } DMAR_UNLOCK(unit); } const struct iommu_domain_map_ops dmar_domain_map_ops = { .map = domain_map_buf, .unmap = domain_unmap_buf, }; diff --git a/sys/x86/iommu/intel_intrmap.c b/sys/x86/iommu/intel_intrmap.c index e95d8a8090b3..c926f7a91ec7 100644 --- a/sys/x86/iommu/intel_intrmap.c +++ b/sys/x86/iommu/intel_intrmap.c @@ -1,385 +1,384 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct dmar_unit *dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar); static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low, uint16_t rid); static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie); int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) { struct dmar_unit *unit; vmem_addr_t vmem_res; u_int idx, i; int error; unit = dmar_ir_find(src, NULL, NULL); if (unit == NULL || !unit->ir_enabled) { for (i = 0; i < count; i++) cookies[i] = -1; return (EOPNOTSUPP); } error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT, &vmem_res); if (error != 0) { KASSERT(error != EOPNOTSUPP, ("impossible EOPNOTSUPP from vmem")); return (error); } idx = vmem_res; for (i = 0; i < count; i++) cookies[i] = idx + i; return (0); } int iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, uint64_t *addr, uint32_t *data) { struct dmar_unit *unit; uint64_t low; uint16_t rid; int is_dmar; unit = dmar_ir_find(src, &rid, &is_dmar); if (is_dmar) { KASSERT(unit == NULL, ("DMAR cannot translate itself")); /* * See VT-d specification, 5.1.6 Remapping Hardware - * Interrupt Programming. */ *data = vector; *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); if (x2apic_mode) *addr |= ((uint64_t)cpu & 0xffffff00) << 32; else KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu)); return (0); } if (unit == NULL || !unit->ir_enabled || cookie == -1) return (EOPNOTSUPP); low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : DMAR_IRTE1_DST_xAPIC(cpu)) | DMAR_IRTE1_V(vector) | DMAR_IRTE1_DLM_FM | DMAR_IRTE1_TM_EDGE | DMAR_IRTE1_RH_DIRECT | DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; dmar_ir_program_irte(unit, cookie, low, rid); if (addr != NULL) { /* * See VT-d specification, 5.1.5.2 MSI and MSI-X * Register Programming. */ *addr = MSI_INTEL_ADDR_BASE | ((cookie & 0x7fff) << 5) | ((cookie & 0x8000) << 2) | 0x18; *data = 0; } return (0); } int iommu_unmap_msi_intr(device_t src, u_int cookie) { struct dmar_unit *unit; if (cookie == -1) return (0); unit = dmar_ir_find(src, NULL, NULL); return (dmar_ir_free_irte(unit, cookie)); } int iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo) { struct dmar_unit *unit; vmem_addr_t vmem_res; uint64_t low, iorte; u_int idx; int error; uint16_t rid; unit = dmar_find_ioapic(ioapic_id, &rid); if (unit == NULL || !unit->ir_enabled) { *cookie = -1; return (EOPNOTSUPP); } error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res); if (error != 0) { KASSERT(error != EOPNOTSUPP, ("impossible EOPNOTSUPP from vmem")); return (error); } idx = vmem_res; low = 0; switch (irq) { case IRQ_EXTINT: low |= DMAR_IRTE1_DLM_ExtINT; break; case IRQ_NMI: low |= DMAR_IRTE1_DLM_NMI; break; case IRQ_SMI: low |= DMAR_IRTE1_DLM_SMI; break; default: KASSERT(vector != 0, ("No vector for IRQ %u", irq)); low |= DMAR_IRTE1_DLM_FM | DMAR_IRTE1_V(vector); break; } low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) : DMAR_IRTE1_DST_xAPIC(cpu)) | (edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) | DMAR_IRTE1_RH_DIRECT | DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P; dmar_ir_program_irte(unit, idx, low, rid); if (hi != NULL) { /* * See VT-d specification, 5.1.5.1 I/OxAPIC * Programming. */ iorte = (1ULL << 48) | ((uint64_t)(idx & 0x7fff) << 49) | ((idx & 0x8000) != 0 ? (1 << 11) : 0) | (edge ? IOART_TRGREDG : IOART_TRGRLVL) | (activehi ? IOART_INTAHI : IOART_INTALO) | IOART_DELFIXED | vector; *hi = iorte >> 32; *lo = iorte; } *cookie = idx; return (0); } int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) { struct dmar_unit *unit; u_int idx; idx = *cookie; if (idx == -1) return (0); *cookie = -1; unit = dmar_find_ioapic(ioapic_id, NULL); KASSERT(unit != NULL && unit->ir_enabled, ("unmap: cookie %d unit %p", idx, unit)); return (dmar_ir_free_irte(unit, idx)); } static struct dmar_unit * dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar) { devclass_t src_class; struct dmar_unit *unit; /* * We need to determine if the interrupt source generates FSB * interrupts. If yes, it is either DMAR, in which case * interrupts are not remapped. Or it is HPET, and interrupts * are remapped. For HPET, source id is reported by HPET * record in DMAR ACPI table. */ if (is_dmar != NULL) *is_dmar = FALSE; src_class = device_get_devclass(src); if (src_class == devclass_find("dmar")) { unit = NULL; if (is_dmar != NULL) *is_dmar = TRUE; } else if (src_class == devclass_find("hpet")) { unit = dmar_find_hpet(src, rid); } else { unit = dmar_find(src, bootverbose); if (unit != NULL && rid != NULL) iommu_get_requester(src, rid); } return (unit); } static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low, uint16_t rid) { dmar_irte_t *irte; uint64_t high; KASSERT(idx < unit->irte_cnt, ("bad cookie %d %d", idx, unit->irte_cnt)); irte = &(unit->irt[idx]); high = DMAR_IRTE2_SVT_RID | DMAR_IRTE2_SQ_RID | DMAR_IRTE2_SID_RID(rid); if (bootverbose) { device_printf(unit->dev, "programming irte[%d] rid %#x high %#jx low %#jx\n", idx, rid, (uintmax_t)high, (uintmax_t)low); } DMAR_LOCK(unit); if ((irte->irte1 & DMAR_IRTE1_P) != 0) { /* * The rte is already valid. Assume that the request * is to remap the interrupt for balancing. Only low * word of rte needs to be changed. Assert that the * high word contains expected value. */ KASSERT(irte->irte2 == high, ("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2, (uintmax_t)high)); dmar_pte_update(&irte->irte1, low); } else { dmar_pte_store(&irte->irte2, high); dmar_pte_store(&irte->irte1, low); } dmar_qi_invalidate_iec(unit, idx, 1); DMAR_UNLOCK(unit); } static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie) { dmar_irte_t *irte; KASSERT(unit != NULL && unit->ir_enabled, ("unmap: cookie %d unit %p", cookie, unit)); KASSERT(cookie < unit->irte_cnt, ("bad cookie %u %u", cookie, unit->irte_cnt)); irte = &(unit->irt[cookie]); dmar_pte_clear(&irte->irte1); dmar_pte_clear(&irte->irte2); DMAR_LOCK(unit); dmar_qi_invalidate_iec(unit, cookie, 1); DMAR_UNLOCK(unit); vmem_free(unit->irtids, cookie, 1); return (0); } static u_int clp2(u_int v) { return (powerof2(v) ? v : 1 << fls(v)); } int dmar_init_irt(struct dmar_unit *unit) { if ((unit->hw_ecap & DMAR_ECAP_IR) == 0) return (0); unit->ir_enabled = 1; TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled); if (!unit->ir_enabled) return (0); if (!unit->qi_enabled) { unit->ir_enabled = 0; if (bootverbose) device_printf(unit->dev, "QI disabled, disabling interrupt remapping\n"); return (0); } unit->irte_cnt = clp2(num_io_irqs); unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig( unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO | M_WAITOK, 0, dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ? VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE); if (unit->irt == NULL) return (ENOMEM); unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt); unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0, M_FIRSTFIT | M_NOWAIT); DMAR_LOCK(unit); dmar_load_irt_ptr(unit); dmar_qi_invalidate_iec_glob(unit); DMAR_UNLOCK(unit); /* * Initialize mappings for already configured interrupt pins. * Required, because otherwise the interrupts fault without * irtes. */ intr_reprogram(); DMAR_LOCK(unit); dmar_enable_ir(unit); DMAR_UNLOCK(unit); return (0); } void dmar_fini_irt(struct dmar_unit *unit) { unit->ir_enabled = 0; if (unit->irt != NULL) { dmar_disable_ir(unit); dmar_qi_invalidate_iec_glob(unit); vmem_destroy(unit->irtids); kmem_free((vm_offset_t)unit->irt, unit->irte_cnt * sizeof(dmar_irte_t)); } } diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c index ab2c4d4e80fc..894e3d537ac7 100644 --- a/sys/x86/iommu/intel_qi.c +++ b/sys/x86/iommu/intel_qi.c @@ -1,478 +1,477 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_acpi.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static bool dmar_qi_seq_processed(const struct dmar_unit *unit, const struct iommu_qi_genseq *pseq) { return (pseq->gen < unit->inv_waitd_gen || (pseq->gen == unit->inv_waitd_gen && pseq->seq <= unit->inv_waitd_seq_hw)); } static int dmar_enable_qi(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_QIE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0)); return (error); } static int dmar_disable_qi(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_QIE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) == 0)); return (error); } static void dmar_qi_advance_tail(struct dmar_unit *unit) { DMAR_ASSERT_LOCKED(unit); dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); } static void dmar_qi_ensure(struct dmar_unit *unit, int descr_count) { uint32_t head; int bytes; DMAR_ASSERT_LOCKED(unit); bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; for (;;) { if (bytes <= unit->inv_queue_avail) break; /* refill */ head = dmar_read4(unit, DMAR_IQH_REG); head &= DMAR_IQH_MASK; unit->inv_queue_avail = head - unit->inv_queue_tail - DMAR_IQ_DESCR_SZ; if (head <= unit->inv_queue_tail) unit->inv_queue_avail += unit->inv_queue_size; if (bytes <= unit->inv_queue_avail) break; /* * No space in the queue, do busy wait. Hardware must * make a progress. But first advance the tail to * inform the descriptor streamer about entries we * might have already filled, otherwise they could * clog the whole queue.. */ dmar_qi_advance_tail(unit); unit->inv_queue_full++; cpu_spinwait(); } unit->inv_queue_avail -= bytes; } static void dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) { DMAR_ASSERT_LOCKED(unit); *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, (uintmax_t)unit->inv_queue_size)); unit->inv_queue_tail &= unit->inv_queue_size - 1; *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2; unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, (uintmax_t)unit->inv_queue_size)); unit->inv_queue_tail &= unit->inv_queue_size - 1; } static void dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, bool memw, bool fence) { DMAR_ASSERT_LOCKED(unit); dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), memw ? unit->inv_waitd_seq_hw_phys : 0); } static void dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq, bool emit_wait) { struct iommu_qi_genseq gsec; uint32_t seq; KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); DMAR_ASSERT_LOCKED(unit); if (unit->inv_waitd_seq == 0xffffffff) { gsec.gen = unit->inv_waitd_gen; gsec.seq = unit->inv_waitd_seq; dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); dmar_qi_advance_tail(unit); while (!dmar_qi_seq_processed(unit, &gsec)) cpu_spinwait(); unit->inv_waitd_gen++; unit->inv_waitd_seq = 1; } seq = unit->inv_waitd_seq++; pseq->gen = unit->inv_waitd_gen; pseq->seq = seq; if (emit_wait) { dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_descr(unit, seq, true, true, false); } } static void dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq, bool nowait) { DMAR_ASSERT_LOCKED(unit); unit->inv_seq_waiters++; while (!dmar_qi_seq_processed(unit, gseq)) { if (cold || nowait) { cpu_spinwait(); } else { msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0, "dmarse", hz); } } unit->inv_seq_waiters--; } void dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base, iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) { struct dmar_unit *unit; iommu_gaddr_t isize; int am; unit = domain->dmar; DMAR_ASSERT_LOCKED(unit); for (; size > 0; base += isize, size -= isize) { am = calc_am(unit, base, size, &isize); dmar_qi_ensure(unit, 1); dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR | DMAR_IQ_DESCR_IOTLB_DID(domain->domain), base | am); } dmar_qi_emit_wait_seq(unit, pseq, emit_wait); dmar_qi_advance_tail(unit); } void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) { struct iommu_qi_genseq gseq; DMAR_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) { struct iommu_qi_genseq gseq; DMAR_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) { struct iommu_qi_genseq gseq; DMAR_ASSERT_LOCKED(unit); dmar_qi_ensure(unit, 2); dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); } void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) { struct iommu_qi_genseq gseq; u_int c, l; DMAR_ASSERT_LOCKED(unit); KASSERT(start < unit->irte_cnt && start < start + cnt && start + cnt <= unit->irte_cnt, ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); for (; cnt > 0; cnt -= c, start += c) { l = ffs(start | cnt) - 1; c = 1 << l; dmar_qi_ensure(unit, 1); dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | DMAR_IQ_DESCR_IEC_IM(l), 0); } dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); /* * The caller of the function, in particular, * dmar_ir_program_irte(), may be called from the context * where the sleeping is forbidden (in fact, the * intr_table_lock mutex may be held, locked from * intr_shuffle_irqs()). Wait for the invalidation completion * using the busy wait. * * The impact on the interrupt input setup code is small, the * expected overhead is comparable with the chipset register * read. It is more harmful for the parallel DMA operations, * since we own the dmar unit lock until whole invalidation * queue is processed, which includes requests possibly issued * before our request. */ dmar_qi_wait_for_seq(unit, &gseq, true); } int dmar_qi_intr(void *arg) { struct dmar_unit *unit; unit = arg; KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->iommu.unit)); taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); return (FILTER_HANDLED); } static void dmar_qi_task(void *arg, int pending __unused) { struct dmar_unit *unit; struct iommu_map_entry *entry; uint32_t ics; unit = arg; DMAR_LOCK(unit); for (;;) { entry = TAILQ_FIRST(&unit->tlb_flush_entries); if (entry == NULL) break; if (!dmar_qi_seq_processed(unit, &entry->gseq)) break; TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); DMAR_UNLOCK(unit); dmar_domain_free_entry(entry, (entry->flags & IOMMU_MAP_ENTRY_QI_NF) == 0); DMAR_LOCK(unit); } ics = dmar_read4(unit, DMAR_ICS_REG); if ((ics & DMAR_ICS_IWC) != 0) { ics = DMAR_ICS_IWC; dmar_write4(unit, DMAR_ICS_REG, ics); } if (unit->inv_seq_waiters > 0) wakeup(&unit->inv_seq_waiters); DMAR_UNLOCK(unit); } int dmar_init_qi(struct dmar_unit *unit) { uint64_t iqa; uint32_t ics; int qi_sz; if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) return (0); unit->qi_enabled = 1; TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); if (!unit->qi_enabled) return (0); TAILQ_INIT(&unit->tlb_flush_entries); TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, taskqueue_thread_enqueue, &unit->qi_taskqueue); taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, "dmar%d qi taskq", unit->iommu.unit); unit->inv_waitd_gen = 0; unit->inv_waitd_seq = 1; qi_sz = DMAR_IQA_QS_DEF; TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); if (qi_sz > DMAR_IQA_QS_MAX) qi_sz = DMAR_IQA_QS_MAX; unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; /* Reserve one descriptor to prevent wraparound. */ unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ; /* The invalidation queue reads by DMARs are always coherent. */ unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); unit->inv_waitd_seq_hw_phys = pmap_kextract( (vm_offset_t)&unit->inv_waitd_seq_hw); DMAR_LOCK(unit); dmar_write8(unit, DMAR_IQT_REG, 0); iqa = pmap_kextract(unit->inv_queue); iqa |= qi_sz; dmar_write8(unit, DMAR_IQA_REG, iqa); dmar_enable_qi(unit); ics = dmar_read4(unit, DMAR_ICS_REG); if ((ics & DMAR_ICS_IWC) != 0) { ics = DMAR_ICS_IWC; dmar_write4(unit, DMAR_ICS_REG, ics); } dmar_enable_qi_intr(unit); DMAR_UNLOCK(unit); return (0); } void dmar_fini_qi(struct dmar_unit *unit) { struct iommu_qi_genseq gseq; if (!unit->qi_enabled) return; taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); taskqueue_free(unit->qi_taskqueue); unit->qi_taskqueue = NULL; DMAR_LOCK(unit); /* quisce */ dmar_qi_ensure(unit, 1); dmar_qi_emit_wait_seq(unit, &gseq, true); dmar_qi_advance_tail(unit); dmar_qi_wait_for_seq(unit, &gseq, false); /* only after the quisce, disable queue */ dmar_disable_qi_intr(unit); dmar_disable_qi(unit); KASSERT(unit->inv_seq_waiters == 0, ("dmar%d: waiters on disabled queue", unit->iommu.unit)); DMAR_UNLOCK(unit); kmem_free(unit->inv_queue, unit->inv_queue_size); unit->inv_queue = 0; unit->inv_queue_size = 0; unit->qi_enabled = 0; } void dmar_enable_qi_intr(struct dmar_unit *unit) { uint32_t iectl; DMAR_ASSERT_LOCKED(unit); KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->iommu.unit)); iectl = dmar_read4(unit, DMAR_IECTL_REG); iectl &= ~DMAR_IECTL_IM; dmar_write4(unit, DMAR_IECTL_REG, iectl); } void dmar_disable_qi_intr(struct dmar_unit *unit) { uint32_t iectl; DMAR_ASSERT_LOCKED(unit); KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->iommu.unit)); iectl = dmar_read4(unit, DMAR_IECTL_REG); dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); } diff --git a/sys/x86/iommu/intel_quirks.c b/sys/x86/iommu/intel_quirks.c index 84626936eb81..f75857911416 100644 --- a/sys/x86/iommu/intel_quirks.c +++ b/sys/x86/iommu/intel_quirks.c @@ -1,245 +1,244 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013, 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef void (*dmar_quirk_cpu_fun)(struct dmar_unit *); struct intel_dmar_quirk_cpu { u_int ext_family; u_int ext_model; u_int family_code; u_int model; u_int stepping; dmar_quirk_cpu_fun quirk; const char *descr; }; typedef void (*dmar_quirk_nb_fun)(struct dmar_unit *, device_t nb); struct intel_dmar_quirk_nb { u_int dev_id; u_int rev_no; dmar_quirk_nb_fun quirk; const char *descr; }; #define QUIRK_NB_ALL_REV 0xffffffff static void dmar_match_quirks(struct dmar_unit *dmar, const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len, const struct intel_dmar_quirk_cpu *cpu_quirks, int cpu_quirks_len) { device_t nb; const struct intel_dmar_quirk_nb *nb_quirk; const struct intel_dmar_quirk_cpu *cpu_quirk; u_int p[4]; u_int dev_id, rev_no; u_int ext_family, ext_model, family_code, model, stepping; int i; if (nb_quirks != NULL) { nb = pci_find_bsf(0, 0, 0); if (nb != NULL) { dev_id = pci_get_device(nb); rev_no = pci_get_revid(nb); for (i = 0; i < nb_quirks_len; i++) { nb_quirk = &nb_quirks[i]; if (nb_quirk->dev_id == dev_id && (nb_quirk->rev_no == rev_no || nb_quirk->rev_no == QUIRK_NB_ALL_REV)) { if (bootverbose) { device_printf(dmar->dev, "NB IOMMU quirk %s\n", nb_quirk->descr); } nb_quirk->quirk(dmar, nb); } } } else { device_printf(dmar->dev, "cannot find northbridge\n"); } } if (cpu_quirks != NULL) { do_cpuid(1, p); ext_family = (p[0] & CPUID_EXT_FAMILY) >> 20; ext_model = (p[0] & CPUID_EXT_MODEL) >> 16; family_code = (p[0] & CPUID_FAMILY) >> 8; model = (p[0] & CPUID_MODEL) >> 4; stepping = p[0] & CPUID_STEPPING; for (i = 0; i < cpu_quirks_len; i++) { cpu_quirk = &cpu_quirks[i]; if (cpu_quirk->ext_family == ext_family && cpu_quirk->ext_model == ext_model && cpu_quirk->family_code == family_code && cpu_quirk->model == model && (cpu_quirk->stepping == -1 || cpu_quirk->stepping == stepping)) { if (bootverbose) { device_printf(dmar->dev, "CPU IOMMU quirk %s\n", cpu_quirk->descr); } cpu_quirk->quirk(dmar); } } } } static void nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused) { unit->hw_cap &= ~(DMAR_CAP_PHMR | DMAR_CAP_PLMR); } static void nb_no_ir(struct dmar_unit *unit, device_t nb __unused) { unit->hw_ecap &= ~(DMAR_ECAP_IR | DMAR_ECAP_EIM); } static void nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb) { u_int rev_no; rev_no = pci_get_revid(nb); if (rev_no <= 0x13) nb_no_ir(unit, nb); } static const struct intel_dmar_quirk_nb pre_use_nb[] = { { .dev_id = 0x4001, .rev_no = 0x20, .quirk = nb_5400_no_low_high_prot_mem, .descr = "5400 E23" /* no low/high protected memory */ }, { .dev_id = 0x4003, .rev_no = 0x20, .quirk = nb_5400_no_low_high_prot_mem, .descr = "5400 E23" /* no low/high protected memory */ }, { .dev_id = 0x3403, .rev_no = QUIRK_NB_ALL_REV, .quirk = nb_5500_no_ir_rev13, .descr = "5500 E47, E53" /* interrupt remapping does not work */ }, { .dev_id = 0x3405, .rev_no = QUIRK_NB_ALL_REV, .quirk = nb_5500_no_ir_rev13, .descr = "5500 E47, E53" /* interrupt remapping does not work */ }, { .dev_id = 0x3405, .rev_no = 0x22, .quirk = nb_no_ir, .descr = "5500 E47, E53" /* interrupt remapping does not work */ }, { .dev_id = 0x3406, .rev_no = QUIRK_NB_ALL_REV, .quirk = nb_5500_no_ir_rev13, .descr = "5500 E47, E53" /* interrupt remapping does not work */ }, }; static void cpu_e5_am9(struct dmar_unit *unit) { unit->hw_cap &= ~(0x3fULL << 48); unit->hw_cap |= (9ULL << 48); } static const struct intel_dmar_quirk_cpu post_ident_cpu[] = { { .ext_family = 0, .ext_model = 2, .family_code = 6, .model = 13, .stepping = 6, .quirk = cpu_e5_am9, .descr = "E5 BT176" /* AM should be at most 9 */ }, }; void dmar_quirks_pre_use(struct iommu_unit *unit) { struct dmar_unit *dmar; dmar = IOMMU2DMAR(unit); if (!dmar_barrier_enter(dmar, DMAR_BARRIER_USEQ)) return; DMAR_LOCK(dmar); dmar_match_quirks(dmar, pre_use_nb, nitems(pre_use_nb), NULL, 0); dmar_barrier_exit(dmar, DMAR_BARRIER_USEQ); } void dmar_quirks_post_ident(struct dmar_unit *dmar) { dmar_match_quirks(dmar, NULL, 0, post_ident_cpu, nitems(post_ident_cpu)); } diff --git a/sys/x86/iommu/intel_reg.h b/sys/x86/iommu/intel_reg.h index e7ab4c7ccd1b..d8e1bbc82db3 100644 --- a/sys/x86/iommu/intel_reg.h +++ b/sys/x86/iommu/intel_reg.h @@ -1,416 +1,415 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013-2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_IOMMU_INTEL_REG_H #define __X86_IOMMU_INTEL_REG_H #define DMAR_PAGE_SIZE PAGE_SIZE #define DMAR_PAGE_MASK (DMAR_PAGE_SIZE - 1) #define DMAR_PAGE_SHIFT PAGE_SHIFT #define DMAR_NPTEPG (DMAR_PAGE_SIZE / sizeof(dmar_pte_t)) #define DMAR_NPTEPGSHIFT 9 #define DMAR_PTEMASK (DMAR_NPTEPG - 1) #define IOMMU_PAGE_SIZE DMAR_PAGE_SIZE #define IOMMU_PAGE_MASK DMAR_PAGE_MASK typedef struct dmar_root_entry { uint64_t r1; uint64_t r2; } dmar_root_entry_t; #define DMAR_ROOT_R1_P 1 /* Present */ #define DMAR_ROOT_R1_CTP_MASK 0xfffffffffffff000 /* Mask for Context-Entry Table Pointer */ #define DMAR_CTX_CNT (DMAR_PAGE_SIZE / sizeof(dmar_root_entry_t)) typedef struct dmar_ctx_entry { uint64_t ctx1; uint64_t ctx2; } dmar_ctx_entry_t; #define DMAR_CTX1_P 1 /* Present */ #define DMAR_CTX1_FPD 2 /* Fault Processing Disable */ /* Translation Type: */ #define DMAR_CTX1_T_UNTR 0 /* only Untranslated */ #define DMAR_CTX1_T_TR 4 /* both Untranslated and Translated */ #define DMAR_CTX1_T_PASS 8 /* Pass-Through */ #define DMAR_CTX1_ASR_MASK 0xfffffffffffff000 /* Mask for the Address Space Root */ #define DMAR_CTX2_AW_2LVL 0 /* 2-level page tables */ #define DMAR_CTX2_AW_3LVL 1 /* 3-level page tables */ #define DMAR_CTX2_AW_4LVL 2 /* 4-level page tables */ #define DMAR_CTX2_AW_5LVL 3 /* 5-level page tables */ #define DMAR_CTX2_AW_6LVL 4 /* 6-level page tables */ #define DMAR_CTX2_DID_MASK 0xffff0 #define DMAR_CTX2_DID(x) ((x) << 8) /* Domain Identifier */ #define DMAR_CTX2_GET_DID(ctx2) (((ctx2) & DMAR_CTX2_DID_MASK) >> 8) typedef struct dmar_pte { uint64_t pte; } dmar_pte_t; #define DMAR_PTE_R 1 /* Read */ #define DMAR_PTE_W (1 << 1) /* Write */ #define DMAR_PTE_SP (1 << 7) /* Super Page */ #define DMAR_PTE_SNP (1 << 11) /* Snoop Behaviour */ #define DMAR_PTE_ADDR_MASK 0xffffffffff000 /* Address Mask */ #define DMAR_PTE_TM (1ULL << 62) /* Transient Mapping */ typedef struct dmar_irte { uint64_t irte1; uint64_t irte2; } dmar_irte_t; /* Source Validation Type */ #define DMAR_IRTE2_SVT_NONE (0ULL << (82 - 64)) #define DMAR_IRTE2_SVT_RID (1ULL << (82 - 64)) #define DMAR_IRTE2_SVT_BUS (2ULL << (82 - 64)) /* Source-id Qualifier */ #define DMAR_IRTE2_SQ_RID (0ULL << (80 - 64)) #define DMAR_IRTE2_SQ_RID_N2 (1ULL << (80 - 64)) #define DMAR_IRTE2_SQ_RID_N21 (2ULL << (80 - 64)) #define DMAR_IRTE2_SQ_RID_N210 (3ULL << (80 - 64)) /* Source Identifier */ #define DMAR_IRTE2_SID_RID(x) ((uint64_t)(x)) #define DMAR_IRTE2_SID_BUS(start, end) ((((uint64_t)(start)) << 8) | (end)) /* Destination Id */ #define DMAR_IRTE1_DST_xAPIC(x) (((uint64_t)(x)) << 40) #define DMAR_IRTE1_DST_x2APIC(x) (((uint64_t)(x)) << 32) /* Vector */ #define DMAR_IRTE1_V(x) (((uint64_t)x) << 16) #define DMAR_IRTE1_IM_POSTED (1ULL << 15) /* Posted */ /* Delivery Mode */ #define DMAR_IRTE1_DLM_FM (0ULL << 5) #define DMAR_IRTE1_DLM_LP (1ULL << 5) #define DMAR_IRTE1_DLM_SMI (2ULL << 5) #define DMAR_IRTE1_DLM_NMI (4ULL << 5) #define DMAR_IRTE1_DLM_INIT (5ULL << 5) #define DMAR_IRTE1_DLM_ExtINT (7ULL << 5) /* Trigger Mode */ #define DMAR_IRTE1_TM_EDGE (0ULL << 4) #define DMAR_IRTE1_TM_LEVEL (1ULL << 4) /* Redirection Hint */ #define DMAR_IRTE1_RH_DIRECT (0ULL << 3) #define DMAR_IRTE1_RH_SELECT (1ULL << 3) /* Destination Mode */ #define DMAR_IRTE1_DM_PHYSICAL (0ULL << 2) #define DMAR_IRTE1_DM_LOGICAL (1ULL << 2) #define DMAR_IRTE1_FPD (1ULL << 1) /* Fault Processing Disable */ #define DMAR_IRTE1_P (1ULL) /* Present */ /* Version register */ #define DMAR_VER_REG 0 #define DMAR_MAJOR_VER(x) (((x) >> 4) & 0xf) #define DMAR_MINOR_VER(x) ((x) & 0xf) /* Capabilities register */ #define DMAR_CAP_REG 0x8 #define DMAR_CAP_PI (1ULL << 59) /* Posted Interrupts */ #define DMAR_CAP_FL1GP (1ULL << 56) /* First Level 1GByte Page */ #define DMAR_CAP_DRD (1ULL << 55) /* DMA Read Draining */ #define DMAR_CAP_DWD (1ULL << 54) /* DMA Write Draining */ #define DMAR_CAP_MAMV(x) ((u_int)(((x) >> 48) & 0x3f)) /* Maximum Address Mask */ #define DMAR_CAP_NFR(x) ((u_int)(((x) >> 40) & 0xff) + 1) /* Num of Fault-recording regs */ #define DMAR_CAP_PSI (1ULL << 39) /* Page Selective Invalidation */ #define DMAR_CAP_SPS(x) ((u_int)(((x) >> 34) & 0xf)) /* Super-Page Support */ #define DMAR_CAP_SPS_2M 0x1 #define DMAR_CAP_SPS_1G 0x2 #define DMAR_CAP_SPS_512G 0x4 #define DMAR_CAP_SPS_1T 0x8 #define DMAR_CAP_FRO(x) ((u_int)(((x) >> 24) & 0x1ff)) /* Fault-recording reg offset */ #define DMAR_CAP_ISOCH (1 << 23) /* Isochrony */ #define DMAR_CAP_ZLR (1 << 22) /* Zero-length reads */ #define DMAR_CAP_MGAW(x) ((u_int)(((x) >> 16) & 0x3f)) /* Max Guest Address Width */ #define DMAR_CAP_SAGAW(x) ((u_int)(((x) >> 8) & 0x1f)) /* Adjusted Guest Address Width */ #define DMAR_CAP_SAGAW_2LVL 0x01 #define DMAR_CAP_SAGAW_3LVL 0x02 #define DMAR_CAP_SAGAW_4LVL 0x04 #define DMAR_CAP_SAGAW_5LVL 0x08 #define DMAR_CAP_SAGAW_6LVL 0x10 #define DMAR_CAP_CM (1 << 7) /* Caching mode */ #define DMAR_CAP_PHMR (1 << 6) /* Protected High-mem Region */ #define DMAR_CAP_PLMR (1 << 5) /* Protected Low-mem Region */ #define DMAR_CAP_RWBF (1 << 4) /* Required Write-Buffer Flushing */ #define DMAR_CAP_AFL (1 << 3) /* Advanced Fault Logging */ #define DMAR_CAP_ND(x) ((u_int)((x) & 0x3)) /* Number of domains */ /* Extended Capabilities register */ #define DMAR_ECAP_REG 0x10 #define DMAR_ECAP_PSS(x) (((x) >> 35) & 0xf) /* PASID Size Supported */ #define DMAR_ECAP_EAFS (1ULL << 34) /* Extended Accessed Flag */ #define DMAR_ECAP_NWFS (1ULL << 33) /* No Write Flag */ #define DMAR_ECAP_SRS (1ULL << 31) /* Supervisor Request */ #define DMAR_ECAP_ERS (1ULL << 30) /* Execute Request */ #define DMAR_ECAP_PRS (1ULL << 29) /* Page Request */ #define DMAR_ECAP_PASID (1ULL << 28) /* Process Address Space Id */ #define DMAR_ECAP_DIS (1ULL << 27) /* Deferred Invalidate */ #define DMAR_ECAP_NEST (1ULL << 26) /* Nested Translation */ #define DMAR_ECAP_MTS (1ULL << 25) /* Memory Type */ #define DMAR_ECAP_ECS (1ULL << 24) /* Extended Context */ #define DMAR_ECAP_MHMV(x) ((u_int)(((x) >> 20) & 0xf)) /* Maximum Handle Mask Value */ #define DMAR_ECAP_IRO(x) ((u_int)(((x) >> 8) & 0x3ff)) /* IOTLB Register Offset */ #define DMAR_ECAP_SC (1 << 7) /* Snoop Control */ #define DMAR_ECAP_PT (1 << 6) /* Pass Through */ #define DMAR_ECAP_EIM (1 << 4) /* Extended Interrupt Mode (x2APIC) */ #define DMAR_ECAP_IR (1 << 3) /* Interrupt Remapping */ #define DMAR_ECAP_DI (1 << 2) /* Device IOTLB */ #define DMAR_ECAP_QI (1 << 1) /* Queued Invalidation */ #define DMAR_ECAP_C (1 << 0) /* Coherency */ /* Global Command register */ #define DMAR_GCMD_REG 0x18 #define DMAR_GCMD_TE (1U << 31) /* Translation Enable */ #define DMAR_GCMD_SRTP (1 << 30) /* Set Root Table Pointer */ #define DMAR_GCMD_SFL (1 << 29) /* Set Fault Log */ #define DMAR_GCMD_EAFL (1 << 28) /* Enable Advanced Fault Logging */ #define DMAR_GCMD_WBF (1 << 27) /* Write Buffer Flush */ #define DMAR_GCMD_QIE (1 << 26) /* Queued Invalidation Enable */ #define DMAR_GCMD_IRE (1 << 25) /* Interrupt Remapping Enable */ #define DMAR_GCMD_SIRTP (1 << 24) /* Set Interrupt Remap Table Pointer */ #define DMAR_GCMD_CFI (1 << 23) /* Compatibility Format Interrupt */ /* Global Status register */ #define DMAR_GSTS_REG 0x1c #define DMAR_GSTS_TES (1U << 31) /* Translation Enable Status */ #define DMAR_GSTS_RTPS (1 << 30) /* Root Table Pointer Status */ #define DMAR_GSTS_FLS (1 << 29) /* Fault Log Status */ #define DMAR_GSTS_AFLS (1 << 28) /* Advanced Fault Logging Status */ #define DMAR_GSTS_WBFS (1 << 27) /* Write Buffer Flush Status */ #define DMAR_GSTS_QIES (1 << 26) /* Queued Invalidation Enable Status */ #define DMAR_GSTS_IRES (1 << 25) /* Interrupt Remapping Enable Status */ #define DMAR_GSTS_IRTPS (1 << 24) /* Interrupt Remapping Table Pointer Status */ #define DMAR_GSTS_CFIS (1 << 23) /* Compatibility Format Interrupt Status */ /* Root-Entry Table Address register */ #define DMAR_RTADDR_REG 0x20 #define DMAR_RTADDR_RTT (1 << 11) /* Root Table Type */ #define DMAR_RTADDR_RTA_MASK 0xfffffffffffff000 /* Context Command register */ #define DMAR_CCMD_REG 0x28 #define DMAR_CCMD_ICC (1ULL << 63) /* Invalidate Context-Cache */ #define DMAR_CCMD_ICC32 (1U << 31) #define DMAR_CCMD_CIRG_MASK (0x3ULL << 61) /* Context Invalidation Request Granularity */ #define DMAR_CCMD_CIRG_GLOB (0x1ULL << 61) /* Global */ #define DMAR_CCMD_CIRG_DOM (0x2ULL << 61) /* Domain */ #define DMAR_CCMD_CIRG_DEV (0x3ULL << 61) /* Device */ #define DMAR_CCMD_CAIG(x) (((x) >> 59) & 0x3) /* Context Actual Invalidation Granularity */ #define DMAR_CCMD_CAIG_GLOB 0x1 /* Global */ #define DMAR_CCMD_CAIG_DOM 0x2 /* Domain */ #define DMAR_CCMD_CAIG_DEV 0x3 /* Device */ #define DMAR_CCMD_FM (0x3UUL << 32) /* Function Mask */ #define DMAR_CCMD_SID(x) (((x) & 0xffff) << 16) /* Source-ID */ #define DMAR_CCMD_DID(x) ((x) & 0xffff) /* Domain-ID */ /* Invalidate Address register */ #define DMAR_IVA_REG_OFF 0 #define DMAR_IVA_IH (1 << 6) /* Invalidation Hint */ #define DMAR_IVA_AM(x) ((x) & 0x1f) /* Address Mask */ #define DMAR_IVA_ADDR(x) ((x) & ~0xfffULL) /* Address */ /* IOTLB Invalidate register */ #define DMAR_IOTLB_REG_OFF 0x8 #define DMAR_IOTLB_IVT (1ULL << 63) /* Invalidate IOTLB */ #define DMAR_IOTLB_IVT32 (1U << 31) #define DMAR_IOTLB_IIRG_MASK (0x3ULL << 60) /* Invalidation Request Granularity */ #define DMAR_IOTLB_IIRG_GLB (0x1ULL << 60) /* Global */ #define DMAR_IOTLB_IIRG_DOM (0x2ULL << 60) /* Domain-selective */ #define DMAR_IOTLB_IIRG_PAGE (0x3ULL << 60) /* Page-selective */ #define DMAR_IOTLB_IAIG_MASK (0x3ULL << 57) /* Actual Invalidation Granularity */ #define DMAR_IOTLB_IAIG_INVLD 0 /* Hw detected error */ #define DMAR_IOTLB_IAIG_GLB (0x1ULL << 57) /* Global */ #define DMAR_IOTLB_IAIG_DOM (0x2ULL << 57) /* Domain-selective */ #define DMAR_IOTLB_IAIG_PAGE (0x3ULL << 57) /* Page-selective */ #define DMAR_IOTLB_DR (0x1ULL << 49) /* Drain Reads */ #define DMAR_IOTLB_DW (0x1ULL << 48) /* Drain Writes */ #define DMAR_IOTLB_DID(x) (((uint64_t)(x) & 0xffff) << 32) /* Domain Id */ /* Fault Status register */ #define DMAR_FSTS_REG 0x34 #define DMAR_FSTS_FRI(x) (((x) >> 8) & 0xff) /* Fault Record Index */ #define DMAR_FSTS_ITE (1 << 6) /* Invalidation Time-out */ #define DMAR_FSTS_ICE (1 << 5) /* Invalidation Completion */ #define DMAR_FSTS_IQE (1 << 4) /* Invalidation Queue */ #define DMAR_FSTS_APF (1 << 3) /* Advanced Pending Fault */ #define DMAR_FSTS_AFO (1 << 2) /* Advanced Fault Overflow */ #define DMAR_FSTS_PPF (1 << 1) /* Primary Pending Fault */ #define DMAR_FSTS_PFO 1 /* Fault Overflow */ /* Fault Event Control register */ #define DMAR_FECTL_REG 0x38 #define DMAR_FECTL_IM (1U << 31) /* Interrupt Mask */ #define DMAR_FECTL_IP (1 << 30) /* Interrupt Pending */ /* Fault Event Data register */ #define DMAR_FEDATA_REG 0x3c /* Fault Event Address register */ #define DMAR_FEADDR_REG 0x40 /* Fault Event Upper Address register */ #define DMAR_FEUADDR_REG 0x44 /* Advanced Fault Log register */ #define DMAR_AFLOG_REG 0x58 /* Fault Recording Register, also usable for Advanced Fault Log records */ #define DMAR_FRCD2_F (1ULL << 63) /* Fault */ #define DMAR_FRCD2_F32 (1U << 31) #define DMAR_FRCD2_T(x) ((int)((x >> 62) & 1)) /* Type */ #define DMAR_FRCD2_T_W 0 /* Write request */ #define DMAR_FRCD2_T_R 1 /* Read or AtomicOp */ #define DMAR_FRCD2_AT(x) ((int)((x >> 60) & 0x3)) /* Address Type */ #define DMAR_FRCD2_FR(x) ((int)((x >> 32) & 0xff)) /* Fault Reason */ #define DMAR_FRCD2_SID(x) ((int)(x & 0xffff)) /* Source Identifier */ #define DMAR_FRCS1_FI_MASK 0xffffffffff000 /* Fault Info, Address Mask */ /* Protected Memory Enable register */ #define DMAR_PMEN_REG 0x64 #define DMAR_PMEN_EPM (1U << 31) /* Enable Protected Memory */ #define DMAR_PMEN_PRS 1 /* Protected Region Status */ /* Protected Low-Memory Base register */ #define DMAR_PLMBASE_REG 0x68 /* Protected Low-Memory Limit register */ #define DMAR_PLMLIMIT_REG 0x6c /* Protected High-Memory Base register */ #define DMAR_PHMBASE_REG 0x70 /* Protected High-Memory Limit register */ #define DMAR_PHMLIMIT_REG 0x78 /* Queued Invalidation Descriptors */ #define DMAR_IQ_DESCR_SZ_SHIFT 4 /* Shift for descriptor count to ring offset */ #define DMAR_IQ_DESCR_SZ (1 << DMAR_IQ_DESCR_SZ_SHIFT) /* Descriptor size */ /* Context-cache Invalidate Descriptor */ #define DMAR_IQ_DESCR_CTX_INV 0x1 #define DMAR_IQ_DESCR_CTX_GLOB (0x1 << 4) /* Granularity: Global */ #define DMAR_IQ_DESCR_CTX_DOM (0x2 << 4) /* Granularity: Domain */ #define DMAR_IQ_DESCR_CTX_DEV (0x3 << 4) /* Granularity: Device */ #define DMAR_IQ_DESCR_CTX_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */ #define DMAR_IQ_DESCR_CTX_SRC(x) (((uint64_t)(x)) << 32) /* Source Id */ #define DMAR_IQ_DESCR_CTX_FM(x) (((uint64_t)(x)) << 48) /* Function Mask */ /* IOTLB Invalidate Descriptor */ #define DMAR_IQ_DESCR_IOTLB_INV 0x2 #define DMAR_IQ_DESCR_IOTLB_GLOB (0x1 << 4) /* Granularity: Global */ #define DMAR_IQ_DESCR_IOTLB_DOM (0x2 << 4) /* Granularity: Domain */ #define DMAR_IQ_DESCR_IOTLB_PAGE (0x3 << 4) /* Granularity: Page */ #define DMAR_IQ_DESCR_IOTLB_DW (1 << 6) /* Drain Writes */ #define DMAR_IQ_DESCR_IOTLB_DR (1 << 7) /* Drain Reads */ #define DMAR_IQ_DESCR_IOTLB_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */ /* Device-TLB Invalidate Descriptor */ #define DMAR_IQ_DESCR_DTLB_INV 0x3 /* Invalidate Interrupt Entry Cache */ #define DMAR_IQ_DESCR_IEC_INV 0x4 #define DMAR_IQ_DESCR_IEC_IDX (1 << 4) /* Index-Selective Invalidation */ #define DMAR_IQ_DESCR_IEC_IIDX(x) (((uint64_t)x) << 32) /* Interrupt Index */ #define DMAR_IQ_DESCR_IEC_IM(x) ((x) << 27) /* Index Mask */ /* Invalidation Wait Descriptor */ #define DMAR_IQ_DESCR_WAIT_ID 0x5 #define DMAR_IQ_DESCR_WAIT_IF (1 << 4) /* Interrupt Flag */ #define DMAR_IQ_DESCR_WAIT_SW (1 << 5) /* Status Write */ #define DMAR_IQ_DESCR_WAIT_FN (1 << 6) /* Fence */ #define DMAR_IQ_DESCR_WAIT_SD(x) (((uint64_t)(x)) << 32) /* Status Data */ /* Extended IOTLB Invalidate Descriptor */ #define DMAR_IQ_DESCR_EIOTLB_INV 0x6 /* PASID-Cache Invalidate Descriptor */ #define DMAR_IQ_DESCR_PASIDC_INV 0x7 /* Extended Device-TLB Invalidate Descriptor */ #define DMAR_IQ_DESCR_EDTLB_INV 0x8 /* Invalidation Queue Head register */ #define DMAR_IQH_REG 0x80 #define DMAR_IQH_MASK 0x7fff0 /* Next cmd index mask */ /* Invalidation Queue Tail register */ #define DMAR_IQT_REG 0x88 #define DMAR_IQT_MASK 0x7fff0 /* Invalidation Queue Address register */ #define DMAR_IQA_REG 0x90 #define DMAR_IQA_IQA_MASK 0xfffffffffffff000 /* Invalidation Queue Base Address mask */ #define DMAR_IQA_QS_MASK 0x7 /* Queue Size in pages */ #define DMAR_IQA_QS_MAX 0x7 /* Max Queue size */ #define DMAR_IQA_QS_DEF 3 /* Invalidation Completion Status register */ #define DMAR_ICS_REG 0x9c #define DMAR_ICS_IWC 1 /* Invalidation Wait Descriptor Complete */ /* Invalidation Event Control register */ #define DMAR_IECTL_REG 0xa0 #define DMAR_IECTL_IM (1U << 31) /* Interrupt Mask */ #define DMAR_IECTL_IP (1 << 30) /* Interrupt Pending */ /* Invalidation Event Data register */ #define DMAR_IEDATA_REG 0xa4 /* Invalidation Event Address register */ #define DMAR_IEADDR_REG 0xa8 /* Invalidation Event Upper Address register */ #define DMAR_IEUADDR_REG 0xac /* Interrupt Remapping Table Address register */ #define DMAR_IRTA_REG 0xb8 #define DMAR_IRTA_EIME (1 << 11) /* Extended Interrupt Mode Enable */ #define DMAR_IRTA_S_MASK 0xf /* Size Mask */ #endif diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c index 43a00428c46c..152c7cac3a7d 100644 --- a/sys/x86/iommu/intel_utils.c +++ b/sys/x86/iommu/intel_utils.c @@ -1,669 +1,668 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2013 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include u_int dmar_nd2mask(u_int nd) { static const u_int masks[] = { 0x000f, /* nd == 0 */ 0x002f, /* nd == 1 */ 0x00ff, /* nd == 2 */ 0x02ff, /* nd == 3 */ 0x0fff, /* nd == 4 */ 0x2fff, /* nd == 5 */ 0xffff, /* nd == 6 */ 0x0000, /* nd == 7 reserved */ }; KASSERT(nd <= 6, ("number of domains %d", nd)); return (masks[nd]); } static const struct sagaw_bits_tag { int agaw; int cap; int awlvl; int pglvl; } sagaw_bits[] = { {.agaw = 30, .cap = DMAR_CAP_SAGAW_2LVL, .awlvl = DMAR_CTX2_AW_2LVL, .pglvl = 2}, {.agaw = 39, .cap = DMAR_CAP_SAGAW_3LVL, .awlvl = DMAR_CTX2_AW_3LVL, .pglvl = 3}, {.agaw = 48, .cap = DMAR_CAP_SAGAW_4LVL, .awlvl = DMAR_CTX2_AW_4LVL, .pglvl = 4}, {.agaw = 57, .cap = DMAR_CAP_SAGAW_5LVL, .awlvl = DMAR_CTX2_AW_5LVL, .pglvl = 5}, {.agaw = 64, .cap = DMAR_CAP_SAGAW_6LVL, .awlvl = DMAR_CTX2_AW_6LVL, .pglvl = 6} }; bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl) { int i; for (i = 0; i < nitems(sagaw_bits); i++) { if (sagaw_bits[i].pglvl != pglvl) continue; if ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) return (true); } return (false); } int domain_set_agaw(struct dmar_domain *domain, int mgaw) { int sagaw, i; domain->mgaw = mgaw; sagaw = DMAR_CAP_SAGAW(domain->dmar->hw_cap); for (i = 0; i < nitems(sagaw_bits); i++) { if (sagaw_bits[i].agaw >= mgaw) { domain->agaw = sagaw_bits[i].agaw; domain->pglvl = sagaw_bits[i].pglvl; domain->awlvl = sagaw_bits[i].awlvl; return (0); } } device_printf(domain->dmar->dev, "context request mgaw %d: no agaw found, sagaw %x\n", mgaw, sagaw); return (EINVAL); } /* * Find a best fit mgaw for the given maxaddr: * - if allow_less is false, must find sagaw which maps all requested * addresses (used by identity mappings); * - if allow_less is true, and no supported sagaw can map all requested * address space, accept the biggest sagaw, whatever is it. */ int dmar_maxaddr2mgaw(struct dmar_unit *unit, iommu_gaddr_t maxaddr, bool allow_less) { int i; for (i = 0; i < nitems(sagaw_bits); i++) { if ((1ULL << sagaw_bits[i].agaw) >= maxaddr && (DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) != 0) break; } if (allow_less && i == nitems(sagaw_bits)) { do { i--; } while ((DMAR_CAP_SAGAW(unit->hw_cap) & sagaw_bits[i].cap) == 0); } if (i < nitems(sagaw_bits)) return (sagaw_bits[i].agaw); KASSERT(0, ("no mgaw for maxaddr %jx allow_less %d", (uintmax_t) maxaddr, allow_less)); return (-1); } /* * Calculate the total amount of page table pages needed to map the * whole bus address space on the context with the selected agaw. */ vm_pindex_t pglvl_max_pages(int pglvl) { vm_pindex_t res; int i; for (res = 0, i = pglvl; i > 0; i--) { res *= DMAR_NPTEPG; res++; } return (res); } /* * Return true if the page table level lvl supports the superpage for * the context ctx. */ int domain_is_sp_lvl(struct dmar_domain *domain, int lvl) { int alvl, cap_sps; static const int sagaw_sp[] = { DMAR_CAP_SPS_2M, DMAR_CAP_SPS_1G, DMAR_CAP_SPS_512G, DMAR_CAP_SPS_1T }; alvl = domain->pglvl - lvl - 1; cap_sps = DMAR_CAP_SPS(domain->dmar->hw_cap); return (alvl < nitems(sagaw_sp) && (sagaw_sp[alvl] & cap_sps) != 0); } iommu_gaddr_t pglvl_page_size(int total_pglvl, int lvl) { int rlvl; static const iommu_gaddr_t pg_sz[] = { (iommu_gaddr_t)DMAR_PAGE_SIZE, (iommu_gaddr_t)DMAR_PAGE_SIZE << DMAR_NPTEPGSHIFT, (iommu_gaddr_t)DMAR_PAGE_SIZE << (2 * DMAR_NPTEPGSHIFT), (iommu_gaddr_t)DMAR_PAGE_SIZE << (3 * DMAR_NPTEPGSHIFT), (iommu_gaddr_t)DMAR_PAGE_SIZE << (4 * DMAR_NPTEPGSHIFT), (iommu_gaddr_t)DMAR_PAGE_SIZE << (5 * DMAR_NPTEPGSHIFT) }; KASSERT(lvl >= 0 && lvl < total_pglvl, ("total %d lvl %d", total_pglvl, lvl)); rlvl = total_pglvl - lvl - 1; KASSERT(rlvl < nitems(pg_sz), ("sizeof pg_sz lvl %d", lvl)); return (pg_sz[rlvl]); } iommu_gaddr_t domain_page_size(struct dmar_domain *domain, int lvl) { return (pglvl_page_size(domain->pglvl, lvl)); } int calc_am(struct dmar_unit *unit, iommu_gaddr_t base, iommu_gaddr_t size, iommu_gaddr_t *isizep) { iommu_gaddr_t isize; int am; for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) { isize = 1ULL << (am + DMAR_PAGE_SHIFT); if ((base & (isize - 1)) == 0 && size >= isize) break; if (am == 0) break; } *isizep = isize; return (am); } iommu_haddr_t dmar_high; int haw; int dmar_tbl_pagecnt; vm_page_t dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags) { vm_page_t m; int zeroed, aflags; zeroed = (flags & IOMMU_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0; aflags = zeroed | VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP | ((flags & IOMMU_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL : VM_ALLOC_NOWAIT); for (;;) { if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); m = vm_page_lookup(obj, idx); if ((flags & IOMMU_PGF_NOALLOC) != 0 || m != NULL) { if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); break; } m = vm_page_alloc_contig(obj, idx, aflags, 1, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); if (m != NULL) { if (zeroed && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); atomic_add_int(&dmar_tbl_pagecnt, 1); break; } if ((flags & IOMMU_PGF_WAITOK) == 0) break; } return (m); } void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags) { vm_page_t m; if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); m = vm_page_grab(obj, idx, VM_ALLOC_NOCREAT); if (m != NULL) { vm_page_free(m); atomic_subtract_int(&dmar_tbl_pagecnt, 1); } if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); } void * dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, struct sf_buf **sf) { vm_page_t m; bool allocated; if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WLOCK(obj); m = vm_page_lookup(obj, idx); if (m == NULL && (flags & IOMMU_PGF_ALLOC) != 0) { m = dmar_pgalloc(obj, idx, flags | IOMMU_PGF_OBJL); allocated = true; } else allocated = false; if (m == NULL) { if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); return (NULL); } /* Sleepable allocations cannot fail. */ if ((flags & IOMMU_PGF_WAITOK) != 0) VM_OBJECT_WUNLOCK(obj); sched_pin(); *sf = sf_buf_alloc(m, SFB_CPUPRIVATE | ((flags & IOMMU_PGF_WAITOK) == 0 ? SFB_NOWAIT : 0)); if (*sf == NULL) { sched_unpin(); if (allocated) { VM_OBJECT_ASSERT_WLOCKED(obj); dmar_pgfree(obj, m->pindex, flags | IOMMU_PGF_OBJL); } if ((flags & IOMMU_PGF_OBJL) == 0) VM_OBJECT_WUNLOCK(obj); return (NULL); } if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) == (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) VM_OBJECT_WLOCK(obj); else if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) == 0) VM_OBJECT_WUNLOCK(obj); return ((void *)sf_buf_kva(*sf)); } void dmar_unmap_pgtbl(struct sf_buf *sf) { sf_buf_free(sf); sched_unpin(); } static void dmar_flush_transl_to_ram(struct dmar_unit *unit, void *dst, size_t sz) { if (DMAR_IS_COHERENT(unit)) return; /* * If DMAR does not snoop paging structures accesses, flush * CPU cache to memory. */ pmap_force_invalidate_cache_range((uintptr_t)dst, (uintptr_t)dst + sz); } void dmar_flush_pte_to_ram(struct dmar_unit *unit, dmar_pte_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst) { dmar_flush_transl_to_ram(unit, dst, sizeof(*dst)); } /* * Load the root entry pointer into the hardware, busily waiting for * the completion. */ int dmar_load_root_entry_ptr(struct dmar_unit *unit) { vm_page_t root_entry; int error; /* * Access to the GCMD register must be serialized while the * command is submitted. */ DMAR_ASSERT_LOCKED(unit); VM_OBJECT_RLOCK(unit->ctx_obj); root_entry = vm_page_lookup(unit->ctx_obj, 0); VM_OBJECT_RUNLOCK(unit->ctx_obj); dmar_write8(unit, DMAR_RTADDR_REG, VM_PAGE_TO_PHYS(root_entry)); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SRTP); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_RTPS) != 0)); return (error); } /* * Globally invalidate the context entries cache, busily waiting for * the completion. */ int dmar_inv_ctx_glob(struct dmar_unit *unit) { int error; /* * Access to the CCMD register must be serialized while the * command is submitted. */ DMAR_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); /* * The DMAR_CCMD_ICC bit in the upper dword should be written * after the low dword write is completed. Amd64 * dmar_write8() does not have this issue, i386 dmar_write8() * writes the upper dword last. */ dmar_write8(unit, DMAR_CCMD_REG, DMAR_CCMD_ICC | DMAR_CCMD_CIRG_GLOB); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_CCMD_REG + 4) & DMAR_CCMD_ICC32) == 0)); return (error); } /* * Globally invalidate the IOTLB, busily waiting for the completion. */ int dmar_inv_iotlb_glob(struct dmar_unit *unit) { int error, reg; DMAR_ASSERT_LOCKED(unit); KASSERT(!unit->qi_enabled, ("QI enabled")); reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap); /* See a comment about DMAR_CCMD_ICC in dmar_inv_ctx_glob. */ dmar_write8(unit, reg + DMAR_IOTLB_REG_OFF, DMAR_IOTLB_IVT | DMAR_IOTLB_IIRG_GLB | DMAR_IOTLB_DR | DMAR_IOTLB_DW); DMAR_WAIT_UNTIL(((dmar_read4(unit, reg + DMAR_IOTLB_REG_OFF + 4) & DMAR_IOTLB_IVT32) == 0)); return (error); } /* * Flush the chipset write buffers. See 11.1 "Write Buffer Flushing" * in the architecture specification. */ int dmar_flush_write_bufs(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); /* * DMAR_GCMD_WBF is only valid when CAP_RWBF is reported. */ KASSERT((unit->hw_cap & DMAR_CAP_RWBF) != 0, ("dmar%d: no RWBF", unit->iommu.unit)); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_WBF); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_WBFS) != 0)); return (error); } int dmar_enable_translation(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) != 0)); return (error); } int dmar_disable_translation(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_TE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_TES) == 0)); return (error); } int dmar_load_irt_ptr(struct dmar_unit *unit) { uint64_t irta, s; int error; DMAR_ASSERT_LOCKED(unit); irta = unit->irt_phys; if (DMAR_X2APIC(unit)) irta |= DMAR_IRTA_EIME; s = fls(unit->irte_cnt) - 2; KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK && powerof2(unit->irte_cnt), ("IRTA_REG_S overflow %x", unit->irte_cnt)); irta |= s; dmar_write8(unit, DMAR_IRTA_REG, irta); dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SIRTP); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS) != 0)); return (error); } int dmar_enable_ir(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd |= DMAR_GCMD_IRE; unit->hw_gcmd &= ~DMAR_GCMD_CFI; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) != 0)); return (error); } int dmar_disable_ir(struct dmar_unit *unit) { int error; DMAR_ASSERT_LOCKED(unit); unit->hw_gcmd &= ~DMAR_GCMD_IRE; dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) == 0)); return (error); } #define BARRIER_F \ u_int f_done, f_inproc, f_wakeup; \ \ f_done = 1 << (barrier_id * 3); \ f_inproc = 1 << (barrier_id * 3 + 1); \ f_wakeup = 1 << (barrier_id * 3 + 2) bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id) { BARRIER_F; DMAR_LOCK(dmar); if ((dmar->barrier_flags & f_done) != 0) { DMAR_UNLOCK(dmar); return (false); } if ((dmar->barrier_flags & f_inproc) != 0) { while ((dmar->barrier_flags & f_inproc) != 0) { dmar->barrier_flags |= f_wakeup; msleep(&dmar->barrier_flags, &dmar->iommu.lock, 0, "dmarb", 0); } KASSERT((dmar->barrier_flags & f_done) != 0, ("dmar%d barrier %d missing done", dmar->iommu.unit, barrier_id)); DMAR_UNLOCK(dmar); return (false); } dmar->barrier_flags |= f_inproc; DMAR_UNLOCK(dmar); return (true); } void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id) { BARRIER_F; DMAR_ASSERT_LOCKED(dmar); KASSERT((dmar->barrier_flags & (f_done | f_inproc)) == f_inproc, ("dmar%d barrier %d missed entry", dmar->iommu.unit, barrier_id)); dmar->barrier_flags |= f_done; if ((dmar->barrier_flags & f_wakeup) != 0) wakeup(&dmar->barrier_flags); dmar->barrier_flags &= ~(f_inproc | f_wakeup); DMAR_UNLOCK(dmar); } int dmar_batch_coalesce = 100; struct timespec dmar_hw_timeout = { .tv_sec = 0, .tv_nsec = 1000000 }; static const uint64_t d = 1000000000; void dmar_update_timeout(uint64_t newval) { /* XXXKIB not atomic */ dmar_hw_timeout.tv_sec = newval / d; dmar_hw_timeout.tv_nsec = newval % d; } uint64_t dmar_get_timeout(void) { return ((uint64_t)dmar_hw_timeout.tv_sec * d + dmar_hw_timeout.tv_nsec); } static int dmar_timeout_sysctl(SYSCTL_HANDLER_ARGS) { uint64_t val; int error; val = dmar_get_timeout(); error = sysctl_handle_long(oidp, &val, 0, req); if (error != 0 || req->newptr == NULL) return (error); dmar_update_timeout(val); return (error); } static SYSCTL_NODE(_hw_iommu, OID_AUTO, dmar, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD, &dmar_tbl_pagecnt, 0, "Count of pages used for DMAR pagetables"); SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, batch_coalesce, CTLFLAG_RWTUN, &dmar_batch_coalesce, 0, "Number of qi batches between interrupt"); SYSCTL_PROC(_hw_iommu_dmar, OID_AUTO, timeout, CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0, dmar_timeout_sysctl, "QU", "Timeout for command wait, in nanoseconds"); diff --git a/sys/x86/iommu/iommu_intrmap.h b/sys/x86/iommu/iommu_intrmap.h index 92cdf5a76a2f..713bdb5d7ef4 100644 --- a/sys/x86/iommu/iommu_intrmap.h +++ b/sys/x86/iommu/iommu_intrmap.h @@ -1,43 +1,42 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef __X86_IOMMU_IOMMU_INTRMAP_H #define __X86_IOMMU_IOMMU_INTRMAP_H int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count); int iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie, uint64_t *addr, uint32_t *data); int iommu_unmap_msi_intr(device_t src, u_int cookie); int iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo); int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie); #endif