Index: head/sys/arm64/arm64/exception.S
===================================================================
--- head/sys/arm64/arm64/exception.S	(revision 355906)
+++ head/sys/arm64/arm64/exception.S	(revision 355907)
@@ -1,255 +1,255 @@
 /*-
  * Copyright (c) 2014 Andrew Turner
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  */
 
 #include <machine/asm.h>
 #include <machine/armreg.h>
 __FBSDID("$FreeBSD$");
 
 #include "assym.inc"
 
 	.text
 
 .macro	save_registers el
 .if \el == 1
 	mov	x18, sp
 	sub	sp, sp, #128
 .endif
 	sub	sp, sp, #(TF_SIZE + 16)
 	stp	x29, x30, [sp, #(TF_SIZE)]
 	stp	x28, x29, [sp, #(TF_X + 28 * 8)]
 	stp	x26, x27, [sp, #(TF_X + 26 * 8)]
 	stp	x24, x25, [sp, #(TF_X + 24 * 8)]
 	stp	x22, x23, [sp, #(TF_X + 22 * 8)]
 	stp	x20, x21, [sp, #(TF_X + 20 * 8)]
 	stp	x18, x19, [sp, #(TF_X + 18 * 8)]
 	stp	x16, x17, [sp, #(TF_X + 16 * 8)]
 	stp	x14, x15, [sp, #(TF_X + 14 * 8)]
 	stp	x12, x13, [sp, #(TF_X + 12 * 8)]
 	stp	x10, x11, [sp, #(TF_X + 10 * 8)]
 	stp	x8,  x9,  [sp, #(TF_X + 8  * 8)]
 	stp	x6,  x7,  [sp, #(TF_X + 6  * 8)]
 	stp	x4,  x5,  [sp, #(TF_X + 4  * 8)]
 	stp	x2,  x3,  [sp, #(TF_X + 2  * 8)]
 	stp	x0,  x1,  [sp, #(TF_X + 0  * 8)]
 	mrs	x10, elr_el1
 	mrs	x11, spsr_el1
 	mrs	x12, esr_el1
 .if \el == 0
 	mrs	x18, sp_el0
 .endif
 	str	x10, [sp, #(TF_ELR)]
 	stp	w11, w12, [sp, #(TF_SPSR)]
 	stp	x18,  lr, [sp, #(TF_SP)]
 	mrs	x18, tpidr_el1
 	add	x29, sp, #(TF_SIZE)
 .if \el == 0
 	/* Apply the SSBD (CVE-2018-3639) workaround if needed */
 	ldr	x1, [x18, #PC_SSBD]
 	cbz	x1, 1f
 	mov	w0, #1
 	blr	x1
 1:
 
 	ldr	x0, [x18, #(PC_CURTHREAD)]
 	bl	dbg_monitor_enter
 .endif
 	msr	daifclr, #8	/* Enable the debug exception */
 .endm
 
 .macro	restore_registers el
 .if \el == 1
 	/*
 	 * Disable interrupts and debug exceptions, x18 may change in the
 	 * interrupt exception handler.  For EL0 exceptions, do_ast already
 	 * did this.
 	 */
 	msr	daifset, #10
 .endif
 .if \el == 0
 	ldr	x0, [x18, #PC_CURTHREAD]
 	mov	x1, sp
 	bl	dbg_monitor_exit
 
 	/* Remove the SSBD (CVE-2018-3639) workaround if needed */
 	ldr	x1, [x18, #PC_SSBD]
 	cbz	x1, 1f
 	mov	w0, #0
 	blr	x1
 1:
 .endif
 	ldp	x18,  lr, [sp, #(TF_SP)]
 	ldp	x10, x11, [sp, #(TF_ELR)]
 .if \el == 0
 	msr	sp_el0, x18
 .endif
 	msr	spsr_el1, x11
 	msr	elr_el1, x10
 	ldp	x0,  x1,  [sp, #(TF_X + 0  * 8)]
 	ldp	x2,  x3,  [sp, #(TF_X + 2  * 8)]
 	ldp	x4,  x5,  [sp, #(TF_X + 4  * 8)]
 	ldp	x6,  x7,  [sp, #(TF_X + 6  * 8)]
 	ldp	x8,  x9,  [sp, #(TF_X + 8  * 8)]
 	ldp	x10, x11, [sp, #(TF_X + 10 * 8)]
 	ldp	x12, x13, [sp, #(TF_X + 12 * 8)]
 	ldp	x14, x15, [sp, #(TF_X + 14 * 8)]
 	ldp	x16, x17, [sp, #(TF_X + 16 * 8)]
 .if \el == 0
 	/*
 	 * We only restore the callee saved registers when returning to
 	 * userland as they may have been updated by a system call or signal.
 	 */
 	ldp	x18, x19, [sp, #(TF_X + 18 * 8)]
 	ldp	x20, x21, [sp, #(TF_X + 20 * 8)]
 	ldp	x22, x23, [sp, #(TF_X + 22 * 8)]
 	ldp	x24, x25, [sp, #(TF_X + 24 * 8)]
 	ldp	x26, x27, [sp, #(TF_X + 26 * 8)]
 	ldp	x28, x29, [sp, #(TF_X + 28 * 8)]
 .else
 	ldr	     x29, [sp, #(TF_X + 29 * 8)]
 .endif
 .if \el == 0
 	add	sp, sp, #(TF_SIZE + 16)
 .else
 	mov	sp, x18
 	mrs	x18, tpidr_el1
 .endif
 .endm
 
 .macro	do_ast
 	mrs	x19, daif
 	/* Make sure the IRQs are enabled before calling ast() */
 	bic	x19, x19, #PSR_I
 1:
 	/* Disable interrupts */
 	msr	daifset, #10
 
 	/* Read the current thread flags */
 	ldr	x1, [x18, #PC_CURTHREAD]	/* Load curthread */
 	ldr	x2, [x1, #TD_FLAGS]
 
 	/* Check if we have either bits set */
 	mov	x3, #((TDF_ASTPENDING|TDF_NEEDRESCHED) >> 8)
 	lsl	x3, x3, #8
 	and	x2, x2, x3
 	cbz	x2, 2f
 
 	/* Restore interrupts */
 	msr	daif, x19
 
 	/* handle the ast */
 	mov	x0, sp
 	bl	_C_LABEL(ast)
 
 	/* Re-check for new ast scheduled */
 	b	1b
 2:
 .endm
 
 ENTRY(handle_el1h_sync)
 	save_registers 1
 	ldr	x0, [x18, #PC_CURTHREAD]
 	mov	x1, sp
 	bl	do_el1h_sync
 	restore_registers 1
-	eret
+	ERET
 END(handle_el1h_sync)
 
 ENTRY(handle_el1h_irq)
 	save_registers 1
 	mov	x0, sp
 	bl	intr_irq_handler
 	restore_registers 1
-	eret
+	ERET
 END(handle_el1h_irq)
 
 ENTRY(handle_el0_sync)
 	save_registers 0
 	ldr	x0, [x18, #PC_CURTHREAD]
 	mov	x1, sp
 	str	x1, [x0, #TD_FRAME]
 	bl	do_el0_sync
 	do_ast
 	restore_registers 0
-	eret
+	ERET
 END(handle_el0_sync)
 
 ENTRY(handle_el0_irq)
 	save_registers 0
 	mov	x0, sp
 	bl	intr_irq_handler
 	do_ast
 	restore_registers 0
-	eret
+	ERET
 END(handle_el0_irq)
 
 ENTRY(handle_serror)
 	save_registers 0
 	mov	x0, sp
 1:	bl	do_serror
 	b	1b
 END(handle_serror)
 
 ENTRY(handle_empty_exception)
 	save_registers 0
 	mov	x0, sp
 1:	bl	unhandled_exception
 	b	1b
 END(handle_unhandled_exception)
 
 .macro	vempty
 	.align 7
 	b	handle_empty_exception
 .endm
 
 .macro	vector	name
 	.align 7
 	b	handle_\name
 .endm
 
 	.align 11
 	.globl exception_vectors
 exception_vectors:
 	vempty			/* Synchronous EL1t */
 	vempty			/* IRQ EL1t */
 	vempty			/* FIQ EL1t */
 	vempty			/* Error EL1t */
 
 	vector el1h_sync	/* Synchronous EL1h */
 	vector el1h_irq		/* IRQ EL1h */
 	vempty			/* FIQ EL1h */
 	vector serror		/* Error EL1h */
 
 	vector el0_sync		/* Synchronous 64-bit EL0 */
 	vector el0_irq		/* IRQ 64-bit EL0 */
 	vempty			/* FIQ 64-bit EL0 */
 	vector serror		/* Error 64-bit EL0 */
 
 	vector el0_sync		/* Synchronous 32-bit EL0 */
 	vector el0_irq		/* IRQ 32-bit EL0 */
 	vempty			/* FIQ 32-bit EL0 */
 	vector serror		/* Error 32-bit EL0 */
 
Index: head/sys/arm64/arm64/swtch.S
===================================================================
--- head/sys/arm64/arm64/swtch.S	(revision 355906)
+++ head/sys/arm64/arm64/swtch.S	(revision 355907)
@@ -1,292 +1,292 @@
 /*-
  * Copyright (c) 2014 Andrew Turner
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * This software was developed by Andrew Turner under sponsorship from
  * the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  */
 
 #include "assym.inc"
 #include "opt_kstack_pages.h"
 #include "opt_sched.h"
 
 #include <machine/asm.h>
 
 __FBSDID("$FreeBSD$");
 
 .macro clear_step_flag pcbflags, tmp
 	tbz	\pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
 	mrs	\tmp, mdscr_el1
 	bic	\tmp, \tmp, #1
 	msr	mdscr_el1, \tmp
 	isb
 999:
 .endm
 
 .macro set_step_flag pcbflags, tmp
 	tbz	\pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
 	mrs	\tmp, mdscr_el1
 	orr	\tmp, \tmp, #1
 	msr	mdscr_el1, \tmp
 	isb
 999:
 .endm
 
 /*
  * void cpu_throw(struct thread *old, struct thread *new)
  */
 ENTRY(cpu_throw)
 	/* Of old == NULL skip disabling stepping */
 	cbz	x0, 1f
 
 	/* If we were single stepping, disable it */
 	ldr	x4, [x0, #TD_PCB]
 	ldr	w5, [x4, #PCB_FLAGS]
 	clear_step_flag w5, x6
 1:
 
 #ifdef VFP
 	/* Backup the new thread pointer around a call to C code */
 	mov	x19, x0
 	mov	x20, x1
 	bl	vfp_discard
 	mov	x1, x20
 	mov	x0, x19
 #endif
 
 	bl	pmap_switch
 	mov	x4, x0
 
 	/* If we are single stepping, enable it */
 	ldr	w5, [x4, #PCB_FLAGS]
 	set_step_flag w5, x6
 
 	/* Restore the registers */
 	ldp	x5, x6, [x4, #PCB_SP]
 	mov	sp, x5
 	msr	tpidr_el0, x6
 	ldr	x6, [x4, #PCB_TPIDRRO]
 	msr	tpidrro_el0, x6
 	ldp	x8, x9, [x4, #PCB_REGS + 8 * 8]
 	ldp	x10, x11, [x4, #PCB_REGS + 10 * 8]
 	ldp	x12, x13, [x4, #PCB_REGS + 12 * 8]
 	ldp	x14, x15, [x4, #PCB_REGS + 14 * 8]
 	ldp	x16, x17, [x4, #PCB_REGS + 16 * 8]
 	ldr	     x19, [x4, #PCB_REGS + 19 * 8]
 	ldp	x20, x21, [x4, #PCB_REGS + 20 * 8]
 	ldp	x22, x23, [x4, #PCB_REGS + 22 * 8]
 	ldp	x24, x25, [x4, #PCB_REGS + 24 * 8]
 	ldp	x26, x27, [x4, #PCB_REGS + 26 * 8]
 	ldp	x28, x29, [x4, #PCB_REGS + 28 * 8]
 	ldr	x30, [x4, #PCB_REGS + 30 * 8]
 
 	ret
 END(cpu_throw)
 
 /*
  * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
  *
  * x0 = old
  * x1 = new
  * x2 = mtx
  * x3 to x7, x16 and x17 are caller saved
  */
 ENTRY(cpu_switch)
 	/*
 	 * Save the old context.
 	 */
 	ldr	x4, [x0, #TD_PCB]
 
 	/* Store the callee-saved registers */
 	stp	x8, x9, [x4, #PCB_REGS + 8 * 8]
 	stp	x10, x11, [x4, #PCB_REGS + 10 * 8]
 	stp	x12, x13, [x4, #PCB_REGS + 12 * 8]
 	stp	x14, x15, [x4, #PCB_REGS + 14 * 8]
 	stp	x16, x17, [x4, #PCB_REGS + 16 * 8]
 	stp	x18, x19, [x4, #PCB_REGS + 18 * 8]
 	stp	x20, x21, [x4, #PCB_REGS + 20 * 8]
 	stp	x22, x23, [x4, #PCB_REGS + 22 * 8]
 	stp	x24, x25, [x4, #PCB_REGS + 24 * 8]
 	stp	x26, x27, [x4, #PCB_REGS + 26 * 8]
 	stp	x28, x29, [x4, #PCB_REGS + 28 * 8]
 	str	x30, [x4, #PCB_REGS + 30 * 8]
 	/* And the old stack pointer */
 	mov	x5, sp
 	mrs	x6, tpidrro_el0
 	str	x6, [x4, #PCB_TPIDRRO]
 	mrs	x6, tpidr_el0
 	stp	x5, x6, [x4, #PCB_SP]
 
 	/* If we were single stepping, disable it */
 	ldr	w5, [x4, #PCB_FLAGS]
 	clear_step_flag w5, x6
 
 	mov	x19, x0
 	mov	x20, x1
 	mov	x21, x2
 
 #ifdef VFP
 	/* Load the pcb address */
 	mov	x1, x4
 	bl	vfp_save_state
 	mov	x1, x20
 	mov	x0, x19
 #endif
 
 	bl	pmap_switch
 	/* Move the new pcb out of the way */
 	mov	x4, x0
 
 	mov	x2, x21
 	mov	x1, x20
 	mov	x0, x19
 
 	/*
 	 * Release the old thread.
 	 */
 	stlr	x2, [x0, #TD_LOCK]
 #if defined(SCHED_ULE) && defined(SMP)
 	/* Spin if TD_LOCK points to a blocked_lock */
 	ldr	x2, =_C_LABEL(blocked_lock)
 1:
 	ldar	x3, [x1, #TD_LOCK]
 	cmp	x3, x2
 	b.eq	1b
 #endif
 
 	/* If we are single stepping, enable it */
 	ldr	w5, [x4, #PCB_FLAGS]
 	set_step_flag w5, x6
 
 	/* Restore the registers */
 	ldp	x5, x6, [x4, #PCB_SP]
 	mov	sp, x5
 	msr	tpidr_el0, x6
 	ldr	x6, [x4, #PCB_TPIDRRO]
 	msr	tpidrro_el0, x6
 	ldp	x8, x9, [x4, #PCB_REGS + 8 * 8]
 	ldp	x10, x11, [x4, #PCB_REGS + 10 * 8]
 	ldp	x12, x13, [x4, #PCB_REGS + 12 * 8]
 	ldp	x14, x15, [x4, #PCB_REGS + 14 * 8]
 	ldp	x16, x17, [x4, #PCB_REGS + 16 * 8]
 	ldr	     x19, [x4, #PCB_REGS + 19 * 8]
 	ldp	x20, x21, [x4, #PCB_REGS + 20 * 8]
 	ldp	x22, x23, [x4, #PCB_REGS + 22 * 8]
 	ldp	x24, x25, [x4, #PCB_REGS + 24 * 8]
 	ldp	x26, x27, [x4, #PCB_REGS + 26 * 8]
 	ldp	x28, x29, [x4, #PCB_REGS + 28 * 8]
 	ldr	x30, [x4, #PCB_REGS + 30 * 8]
 
 	str	xzr, [x4, #PCB_REGS + 18 * 8]
 	ret
 .Lcpu_switch_panic_str:
 	.asciz "cpu_switch: %p\0"
 END(cpu_switch)
 
 ENTRY(fork_trampoline)
 	mov	x0, x8
 	mov	x1, x9
 	mov	x2, sp
 	mov	fp, #0	/* Stack traceback stops here. */
 	bl	_C_LABEL(fork_exit)
 
 	/* Restore the registers other than x0 and x1 */
 	ldp	x2, x3, [sp, #TF_X + 2 * 8]
 	ldp	x4, x5, [sp, #TF_X + 4 * 8]
 	ldp	x6, x7, [sp, #TF_X + 6 * 8]
 	ldp	x8, x9, [sp, #TF_X + 8 * 8]
 	ldp	x10, x11, [sp, #TF_X + 10 * 8]
 	ldp	x12, x13, [sp, #TF_X + 12 * 8]
 	ldp	x14, x15, [sp, #TF_X + 14 * 8]
 	ldp	x16, x17, [sp, #TF_X + 16 * 8]
 	ldr	     x19, [sp, #TF_X + 19 * 8]
 	ldp	x20, x21, [sp, #TF_X + 20 * 8]
 	ldp	x22, x23, [sp, #TF_X + 22 * 8]
 	ldp	x24, x25, [sp, #TF_X + 24 * 8]
 	ldp	x26, x27, [sp, #TF_X + 26 * 8]
 	ldp	x28, x29, [sp, #TF_X + 28 * 8]
 
 	/*
 	 * Disable interrupts to avoid
 	 * overwriting spsr_el1 and sp_el0 by an IRQ exception.
 	 */
 	msr	daifset, #2
 
 	/* Restore sp and lr */
 	ldp	x0, x1, [sp, #TF_SP]
 	msr	sp_el0, x0
 	mov	lr, x1
 
 	/* Restore elr and spsr */
 	ldp	x0, x1, [sp, #TF_ELR]
 	msr	elr_el1, x0
 	msr	spsr_el1, x1
 
 	/* Finally x0 and x1 */
 	ldp	x0, x1, [sp, #TF_X + 0 * 8]
 	ldr	x18, [sp, #TF_X + 18 * 8]
 
 	/*
 	 * No need for interrupts reenabling since PSR
 	 * will be set to the desired value anyway.
 	 */
-	eret
+	ERET
 	
 END(fork_trampoline)
 
 ENTRY(savectx)
 	/* Store the callee-saved registers */
 	stp	x8,  x9,  [x0, #PCB_REGS + 8 * 8]
 	stp	x10, x11, [x0, #PCB_REGS + 10 * 8]
 	stp	x12, x13, [x0, #PCB_REGS + 12 * 8]
 	stp	x14, x15, [x0, #PCB_REGS + 14 * 8]
 	stp	x16, x17, [x0, #PCB_REGS + 16 * 8]
 	stp	x18, x19, [x0, #PCB_REGS + 18 * 8]
 	stp	x20, x21, [x0, #PCB_REGS + 20 * 8]
 	stp	x22, x23, [x0, #PCB_REGS + 22 * 8]
 	stp	x24, x25, [x0, #PCB_REGS + 24 * 8]
 	stp	x26, x27, [x0, #PCB_REGS + 26 * 8]
 	stp	x28, x29, [x0, #PCB_REGS + 28 * 8]
 	str	x30, [x0, #PCB_REGS + 30 * 8]
 	/* And the old stack pointer */
 	mov	x5, sp
 	mrs	x6, tpidrro_el0
 	str	x6, [x0, #PCB_TPIDRRO]
 	mrs	x6, tpidr_el0
 	stp	x5, x6, [x0, #PCB_SP]
 
 	/* Store the VFP registers */
 #ifdef VFP
 	mov	x28, lr
 	mov	x1, x0			/* move pcb to the correct register */
 	mov	x0, xzr			/* td = NULL */
 	bl	vfp_save_state
 	mov	lr, x28
 #endif
 
 	ret
 END(savectx)
 
Index: head/sys/arm64/include/asm.h
===================================================================
--- head/sys/arm64/include/asm.h	(revision 355906)
+++ head/sys/arm64/include/asm.h	(revision 355907)
@@ -1,93 +1,105 @@
 /*-
  * Copyright (c) 2014 Andrew Turner
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _MACHINE_ASM_H_
 #define	_MACHINE_ASM_H_
 
 #undef __FBSDID
 #if !defined(lint) && !defined(STRIP_FBSDID)
 #define	__FBSDID(s)     .ident s
 #else
 #define	__FBSDID(s)     /* nothing */
 #endif
 
 #define	_C_LABEL(x)	x
 
 #define	ENTRY(sym)						\
 	.text; .globl sym; .align 2; .type sym,#function; sym:
 #define	EENTRY(sym)						\
 	.globl	sym; sym:
 #define	END(sym) .size sym, . - sym
 #define	EEND(sym)
 
 #define	WEAK_REFERENCE(sym, alias)				\
 	.weak alias;						\
 	.set alias,sym
 
 #define	UINT64_C(x)	(x)
 
 #if defined(PIC)
 #define	PIC_SYM(x,y)	x ## @ ## y
 #else
 #define	PIC_SYM(x,y)	x
 #endif
 
 /* Alias for link register x30 */
 #define	lr		x30
 
 /*
  * Sets the trap fault handler. The exception handler will return to the
  * address in the handler register on a data abort or the xzr register to
  * clear the handler. The tmp parameter should be a register able to hold
  * the temporary data.
  */
 #define	SET_FAULT_HANDLER(handler, tmp)					\
 	ldr	tmp, [x18, #PC_CURTHREAD];	/* Load curthread */	\
 	ldr	tmp, [tmp, #TD_PCB];		/* Load the pcb */	\
 	str	handler, [tmp, #PCB_ONFAULT]	/* Set the handler */
 
 #define	ENTER_USER_ACCESS(reg, tmp)					\
 	ldr	tmp, =has_pan;			/* Get the addr of has_pan */ \
 	ldr	reg, [tmp];			/* Read it */		\
 	cbz	reg, 997f;			/* If no PAN skip */	\
 	.inst	0xd500409f | (0 << 8);		/* Clear PAN */		\
 	997:
 
 #define	EXIT_USER_ACCESS(reg)						\
 	cbz	reg, 998f;			/* If no PAN skip */	\
 	.inst	0xd500409f | (1 << 8);		/* Set PAN */		\
 	998:
 
 #define	EXIT_USER_ACCESS_CHECK(reg, tmp)				\
 	ldr	tmp, =has_pan;			/* Get the addr of has_pan */ \
 	ldr	reg, [tmp];			/* Read it */		\
 	cbz	reg, 999f;			/* If no PAN skip */	\
 	.inst	0xd500409f | (1 << 8);		/* Set PAN */		\
 	999:
 
+/*
+ * Some AArch64 CPUs speculate past an eret instruction. As the user may
+ * control the registers at this point add a speculation barrier usable on
+ * all AArch64 CPUs after the eret instruction.
+ * TODO: ARMv8.5 adds a specific instruction for this, we could use that
+ * if we know we are running on something that supports it.
+ */
+#define	ERET								\
+	eret;								\
+	dsb	sy;							\
+	isb
+
 #endif /* _MACHINE_ASM_H_ */