diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -169,7 +169,7 @@ .macro do_ast mrs x19, daif /* Make sure the IRQs are enabled before calling ast() */ - bic x19, x19, #PSR_I + bic x19, x19, #(PSR_I | PSR_F) 1: /* * Mask interrupts while checking the ast pending flag @@ -212,6 +212,14 @@ ERET END(handle_el1h_irq) +ENTRY(handle_el1h_fiq) + save_registers 1 + mov x0, sp + bl intr_fiq_handler + restore_registers 1 + ERET +END(handle_el1h_irq) + ENTRY(handle_el0_sync) save_registers 0 ldr x0, [x18, #PC_CURTHREAD] @@ -232,6 +240,15 @@ ERET END(handle_el0_irq) +ENTRY(handle_el0_fiq) + save_registers 0 + mov x0, sp + bl intr_fiq_handler + do_ast + restore_registers 0 + ERET +END(handle_el0_fiq) + ENTRY(handle_serror) save_registers 0 mov x0, sp @@ -270,16 +287,16 @@ vector el1h_sync 1 /* Synchronous EL1h */ vector el1h_irq 1 /* IRQ EL1h */ - vempty 1 /* FIQ EL1h */ + vector el1h_fiq 1 /* FIQ EL1h */ vector serror 1 /* Error EL1h */ vector el0_sync 0 /* Synchronous 64-bit EL0 */ vector el0_irq 0 /* IRQ 64-bit EL0 */ - vempty 0 /* FIQ 64-bit EL0 */ + vector el0_fiq 0 /* FIQ 64-bit EL0 */ vector serror 0 /* Error 64-bit EL0 */ vector el0_sync 0 /* Synchronous 32-bit EL0 */ vector el0_irq 0 /* IRQ 32-bit EL0 */ - vempty 0 /* FIQ 32-bit EL0 */ + vector el0_fiq 0 /* FIQ 32-bit EL0 */ vector serror 0 /* Error 32-bit EL0 */ diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h --- a/sys/arm64/include/armreg.h +++ b/sys/arm64/include/armreg.h @@ -201,7 +201,7 @@ #define DAIF_I (1 << 1) #define DAIF_F (1 << 0) #define DAIF_ALL (DAIF_D | DAIF_A | DAIF_I | DAIF_F) -#define DAIF_INTR (DAIF_I) /* All exceptions that pass */ +#define DAIF_INTR (DAIF_I | DAIF_F) /* All exceptions that pass */ /* through the intr framework */ /* DBGBCR_EL1 - Debug Breakpoint Control Registers */ @@ -1796,7 +1796,7 @@ #define PSR_D 0x00000200UL #define PSR_DAIF (PSR_D | PSR_A | PSR_I | PSR_F) /* The default DAIF mask. These bits are valid in spsr_el1 and daif */ -#define PSR_DAIF_DEFAULT (PSR_F) +#define PSR_DAIF_DEFAULT (0) #define PSR_IL 0x00100000UL #define PSR_SS 0x00200000UL #define PSR_V 0x10000000UL diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -97,11 +97,18 @@ /* Main interrupt handler called from assembler -> 'hidden' for C code. */ void intr_irq_handler(struct trapframe *tf); +#if defined(__aarch64__) +void intr_fiq_handler(struct trapframe *tf); +#endif /* Root interrupt controller stuff. */ device_t intr_irq_root_dev; static intr_irq_filter_t *irq_root_filter; static void *irq_root_arg; +#if defined(__aarch64__) +static intr_irq_filter_t *fiq_root_filter; +static void *fiq_root_arg; +#endif static u_int irq_root_ipicount; struct intr_pic_child { @@ -363,6 +370,31 @@ #endif } +#if defined(__aarch64__) +void +intr_fiq_handler(struct trapframe *tf) +{ + struct trapframe * oldframe; + struct thread * td; + + KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); + + VM_CNT_INC(v_intr); + critical_enter(); + td = curthread; + oldframe = td->td_intr_frame; + td->td_intr_frame = tf; + fiq_root_filter(fiq_root_arg); + td->td_intr_frame = oldframe; + critical_exit(); +#ifdef HWPMC_HOOKS + if (pmc_hook && TRAPF_USERMODE(tf) && + (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) + pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); +#endif +} +#endif + int intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) { @@ -924,6 +956,34 @@ return (0); } +#if defined(__aarch64__) +int +intr_pic_claim_fiq(device_t dev, intr_irq_filter_t *filter, void *arg) +{ + KASSERT(dev != NULL, ("%s: NULL device", __func__)); + + if (dev != intr_irq_root_dev) { + device_printf(dev, "FIQ device not root IRQ device\n"); + return (EINVAL); + } + + if (filter == NULL) { + device_printf(dev, "FIQ filter missing\n"); + return (EINVAL); + } + + if (fiq_root_filter != NULL) { + device_printf(dev, "another FIQ handler already set\n"); + return (EBUSY); + } + + fiq_root_filter = filter; + fiq_root_arg = arg; + + return (0); +} +#endif + /* * Add a handler to manage a sub range of a parents interrupts. */ diff --git a/sys/sys/intr.h b/sys/sys/intr.h --- a/sys/sys/intr.h +++ b/sys/sys/intr.h @@ -113,6 +113,9 @@ struct intr_pic *intr_pic_register(device_t, intptr_t); int intr_pic_deregister(device_t, intptr_t); int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *, u_int); +#if defined(__aarch64__) +int intr_pic_claim_fiq(device_t, intr_irq_filter_t *, void *); +#endif int intr_pic_add_handler(device_t, struct intr_pic *, intr_child_irq_filter_t *, void *, uintptr_t, uintptr_t); bool intr_is_per_cpu(struct resource *);