Differential D14633 Diff 41168 sys/i386/i386/exception.s

Changeset View

Standalone View

sys/i386/i386/exception.s

/*-		/*-
* Copyright (c) 1989, 1990 William F. Jolitz.		* Copyright (c) 1989, 1990 William F. Jolitz.
* Copyright (c) 1990 The Regents of the University of California.		* Copyright (c) 1990 The Regents of the University of California.
* Copyright (c) 2007 The FreeBSD Foundation		* Copyright (c) 2007, 2018 The FreeBSD Foundation
* All rights reserved.		* All rights reserved.
*		*
* Portions of this software were developed by A. Joseph Koshy under		* Portions of this software were developed by A. Joseph Koshy under
* sponsorship from the FreeBSD Foundation and Google, Inc.		* sponsorship from the FreeBSD Foundation and Google, Inc.
		* Portions of this software were developed by Konstantin Belousov
		* <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
*		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
* 1. Redistributions of source code must retain the above copyright		* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.		* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright		* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the		* notice, this list of conditions and the following disclaimer in the
Show All 16 Lines
*		*
* $FreeBSD$		* $FreeBSD$
*/		*/

#include "opt_apic.h"		#include "opt_apic.h"
#include "opt_atpic.h"		#include "opt_atpic.h"
#include "opt_hwpmc_hooks.h"		#include "opt_hwpmc_hooks.h"

#include <machine/asmacros.h>		#include "assym.inc"

#include <machine/psl.h>		#include <machine/psl.h>
		#include <machine/asmacros.h>
#include <machine/trap.h>		#include <machine/trap.h>

#include "assym.inc"

#define SEL_RPL_MASK 0x0003
#define GSEL_KPL 0x0020 /* GSEL(GCODE_SEL, SEL_KPL) */

#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
.bss		.bss
.globl dtrace_invop_jump_addr		.globl dtrace_invop_jump_addr
.align 4		.align 4
.type dtrace_invop_jump_addr, @object		.type dtrace_invop_jump_addr, @object
.size dtrace_invop_jump_addr, 4		.size dtrace_invop_jump_addr, 4
dtrace_invop_jump_addr:		dtrace_invop_jump_addr:
.zero 4		.zero 4
.globl dtrace_invop_calltrap_addr		.globl dtrace_invop_calltrap_addr
.align 4		.align 4
.type dtrace_invop_calltrap_addr, @object		.type dtrace_invop_calltrap_addr, @object
.size dtrace_invop_calltrap_addr, 4		.size dtrace_invop_calltrap_addr, 4
dtrace_invop_calltrap_addr:		dtrace_invop_calltrap_addr:
.zero 8		.zero 8
#endif		#endif
.text		.text
#ifdef HWPMC_HOOKS
ENTRY(start_exceptions)		ENTRY(start_exceptions)
#endif		.globl tramp_idleptd
		tramp_idleptd: .long 0

/*****************************************************************************/		/*****************************************************************************/
/* Trap handling */		/* Trap handling */
/*****************************************************************************/		/*****************************************************************************/
/*		/*
* Trap and fault vector routines.		* Trap and fault vector routines.
*		*
* Most traps are 'trap gates', SDT_SYS386TGT. A trap gate pushes state on		* All traps are 'interrupt gates', SDT_SYS386IGT. Interrupts are disabled
* the stack that mostly looks like an interrupt, but does not disable		* by hardware to not allow interrupts until code switched to the kernel
* interrupts. A few of the traps we are use are interrupt gates,		* address space and the kernel thread stack.
* SDT_SYS386IGT, which are nearly the same thing except interrupts are
* disabled on entry.
*		*
* The cpu will push a certain amount of state onto the kernel stack for		* The cpu will push a certain amount of state onto the kernel stack for
* the current process. The amount of state depends on the type of trap		* the current process. The amount of state depends on the type of trap
* and whether the trap crossed rings or not. See i386/include/frame.h.		* and whether the trap crossed rings or not. See i386/include/frame.h.
* At the very least the current EFLAGS (status register, which includes		* At the very least the current EFLAGS (status register, which includes
* the interrupt disable state prior to the trap), the code segment register,		* the interrupt disable state prior to the trap), the code segment register,
* and the return instruction pointer are pushed by the cpu. The cpu		* and the return instruction pointer are pushed by the cpu. The cpu
* will also push an 'error' code for certain traps. We push a dummy		* will also push an 'error' code for certain traps. We push a dummy
* error code for those traps where the cpu doesn't in order to maintain		* error code for those traps where the cpu doesn't in order to maintain
* a consistent frame. We also push a contrived 'trap number'.		* a consistent frame. We also push a contrived 'trap number'.
*		*
* The cpu does not push the general registers, we must do that, and we		* The cpu does not push the general registers, we must do that, and we
* must restore them prior to calling 'iret'. The cpu adjusts the %cs and		* must restore them prior to calling 'iret'. The cpu adjusts the %cs and
* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we		* %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we
* must load them with appropriate values for supervisor mode operation.		* must load them with appropriate values for supervisor mode operation.
		*
		* This code is not executed at the linked address, it is copied to the
		emasteUnsubmitted Done Inline Actions "address of linking" seems a bit nonstandard - I'd probably use "the linked address" or "the address at which it is linked" emaste: "address of linking" seems a bit nonstandard - I'd probably use "the linked address" or "the…
		* trampoline area. As the consequence, all code there and in included files
		* must be PIC.
*/		*/

MCOUNT_LABEL(user)		MCOUNT_LABEL(user)
MCOUNT_LABEL(btrap)		MCOUNT_LABEL(btrap)

#define TRAP(a) pushl $(a) ; jmp alltraps		#define TRAP(a) pushl $(a) ; jmp alltraps

IDTVEC(div)		IDTVEC(div)
pushl $0; TRAP(T_DIVIDE)		pushl $0; TRAP(T_DIVIDE)
IDTVEC(dbg)		IDTVEC(dbg)
pushl $0; TRAP(T_TRCTRAP)		pushl $0; TRAP(T_TRCTRAP)
IDTVEC(nmi)
pushl $0; TRAP(T_NMI)
IDTVEC(bpt)		IDTVEC(bpt)
pushl $0; TRAP(T_BPTFLT)		pushl $0; TRAP(T_BPTFLT)
IDTVEC(dtrace_ret)		IDTVEC(dtrace_ret)
pushl $0; TRAP(T_DTRACE_RET)		pushl $0; TRAP(T_DTRACE_RET)
IDTVEC(ofl)		IDTVEC(ofl)
pushl $0; TRAP(T_OFLOW)		pushl $0; TRAP(T_OFLOW)
IDTVEC(bnd)		IDTVEC(bnd)
pushl $0; TRAP(T_BOUND)		pushl $0; TRAP(T_BOUND)
#ifndef KDTRACE_HOOKS		#ifndef KDTRACE_HOOKS
IDTVEC(ill)		IDTVEC(ill)
pushl $0; TRAP(T_PRIVINFLT)		pushl $0; TRAP(T_PRIVINFLT)
#endif		#endif
IDTVEC(dna)		IDTVEC(dna)
pushl $0; TRAP(T_DNA)		pushl $0; TRAP(T_DNA)
IDTVEC(fpusegm)		IDTVEC(fpusegm)
pushl $0; TRAP(T_FPOPFLT)		pushl $0; TRAP(T_FPOPFLT)
IDTVEC(tss)		IDTVEC(tss)
TRAP(T_TSSFLT)		TRAP(T_TSSFLT)
IDTVEC(missing)		IDTVEC(missing)
TRAP(T_SEGNPFLT)		pushl $T_SEGNPFLT
		jmp irettraps
IDTVEC(stk)		IDTVEC(stk)
TRAP(T_STKFLT)		pushl $T_STKFLT
		jmp irettraps
IDTVEC(prot)		IDTVEC(prot)
TRAP(T_PROTFLT)		pushl $T_PROTFLT
		jmp irettraps
IDTVEC(page)		IDTVEC(page)
TRAP(T_PAGEFLT)		cmpl $PMAP_TRM_MIN_ADDRESS, TF_EIP-TF_ERR(%esp)
IDTVEC(mchk)		jb 1f
pushl $0; TRAP(T_MCHK)		movl %ebx, %cr3
		movl %edx, TF_EIP-TF_ERR(%esp)
		addl $4, %esp
		iret
		1: pushl $T_PAGEFLT
		jmp alltraps
IDTVEC(rsvd_pti)		IDTVEC(rsvd_pti)
IDTVEC(rsvd)		IDTVEC(rsvd)
pushl $0; TRAP(T_RESERVED)		pushl $0; TRAP(T_RESERVED)
IDTVEC(fpu)		IDTVEC(fpu)
pushl $0; TRAP(T_ARITHTRAP)		pushl $0; TRAP(T_ARITHTRAP)
IDTVEC(align)		IDTVEC(align)
TRAP(T_ALIGNFLT)		TRAP(T_ALIGNFLT)
IDTVEC(xmm)		IDTVEC(xmm)
pushl $0; TRAP(T_XMMFLT)		pushl $0; TRAP(T_XMMFLT)

/*		/*
* All traps except ones for syscalls jump to alltraps. If		* All traps except ones for syscalls or invalid segment,
		* jump to alltraps. If
* interrupts were enabled when the trap occurred, then interrupts		* interrupts were enabled when the trap occurred, then interrupts
* are enabled now if the trap was through a trap gate, else		* are enabled now if the trap was through a trap gate, else
* disabled if the trap was through an interrupt gate. Note that		* disabled if the trap was through an interrupt gate. Note that
* int0x80_syscall is a trap gate. Interrupt gates are used by		* int0x80_syscall is a trap gate. Interrupt gates are used by
* page faults, non-maskable interrupts, debug and breakpoint		* page faults, non-maskable interrupts, debug and breakpoint
* exceptions.		* exceptions.
*/		*/
SUPERALIGN_TEXT		SUPERALIGN_TEXT
.globl alltraps		.globl alltraps
.type alltraps,@function		.type alltraps,@function
alltraps:		alltraps:
pushal		PUSH_FRAME2
pushl $0
movw %ds,(%esp)
pushl $0
movw %es,(%esp)
pushl $0
movw %fs,(%esp)
alltraps_with_regs_pushed:		alltraps_with_regs_pushed:
SET_KERNEL_SREGS		SET_KERNEL_SREGS
cld		cld
		KENTER
FAKE_MCOUNT(TF_EIP(%esp))		FAKE_MCOUNT(TF_EIP(%esp))
calltrap:		calltrap:
pushl %esp		pushl %esp
call trap		movl $trap,%eax
		call *%eax
add $4, %esp		add $4, %esp

/*		/*
* Return via doreti to handle ASTs.		* Return via doreti to handle ASTs.
*/		*/
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

		.globl irettraps
		.type irettraps,@function
		irettraps:
		testl $PSL_VM, TF_EFLAGS-TF_TRAPNO(%esp)
		jnz alltraps
		testb $SEL_RPL_MASK, TF_CS-TF_TRAPNO(%esp)
		jnz alltraps

/*		/*
		* Kernel mode.
		* The special case there is the kernel mode with user %cr3 and
		* trampoline stack. We need to copy both current frame and the
		* hardware portion of the frame we tried to return to, to the
		* normal stack. This logic must follow the stack unwind order
		* in doreti.
		*/
		PUSH_FRAME2
		SET_KERNEL_SREGS
		cld
		call 1f
		1: popl %ebx
		leal (doreti_iret - 1b)(%ebx), %edx
		cmpl %edx, TF_EIP(%esp)
		jne 2f
		movl $(2 * TF_SZ - TF_EIP), %ecx
		jmp 6f
		2: leal (doreti_popl_ds - 1b)(%ebx), %edx
		cmpl %edx, TF_EIP(%esp)
		jne 3f
		movl $(2 * TF_SZ - TF_DS), %ecx
		jmp 6f
		3: leal (doreti_popl_es - 1b)(%ebx), %edx
		cmpl %edx, TF_EIP(%esp)
		jne 4f
		movl $(2 * TF_SZ - TF_ES), %ecx
		jmp 6f
		4: leal (doreti_popl_fs - 1b)(%ebx), %edx
		cmpl %edx, TF_EIP(%esp)
		jne 5f
		movl $(2 * TF_SZ - TF_FS), %ecx
		jmp 6f
		/* kernel mode, normal */
		5: FAKE_MCOUNT(TF_EIP(%esp))
		jmp calltrap
		6: cmpl $PMAP_TRM_MIN_ADDRESS, %esp /* trampoline stack ? */
		jb 5b /* if not, no need to change stacks */
		movl (tramp_idleptd - 1b)(%ebx), %eax
		movl %eax, %cr3
		movl PCPU(KESP0), %edx
		subl %ecx, %edx
		movl %edx, %edi
		movl %esp, %esi
		rep; movsb
		movl %edx, %esp
		FAKE_MCOUNT(TF_EIP(%esp))
		jmp calltrap

		/*
* Privileged instruction fault.		* Privileged instruction fault.
*/		*/
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
SUPERALIGN_TEXT		SUPERALIGN_TEXT
IDTVEC(ill)		IDTVEC(ill)
/*		/*
* Check if a DTrace hook is registered. The default (data) segment
* cannot be used for this since %ds is not known good until we
* verify that the entry was from kernel mode.
*/
cmpl $0,%ss:dtrace_invop_jump_addr
je norm_ill

/*
* Check if this is a user fault. If so, just handle it as a normal		* Check if this is a user fault. If so, just handle it as a normal
* trap.		* trap.
*/		*/
cmpl $GSEL_KPL, 4(%esp) /* Check the code segment */
jne norm_ill
testl $PSL_VM, 8(%esp) /* and vm86 mode. */		testl $PSL_VM, 8(%esp) /* and vm86 mode. */
jnz norm_ill		jnz norm_ill
		cmpl $GSEL_KPL, 4(%esp) /* Check the code segment */
		jne norm_ill

/*		/*
		* Check if a DTrace hook is registered. The trampoline cannot
		* be instrumented.
		*/
		cmpl $0, dtrace_invop_jump_addr
		je norm_ill

		/*
* This is a kernel instruction fault that might have been caused		* This is a kernel instruction fault that might have been caused
* by a DTrace provider.		* by a DTrace provider.
*/		*/
pushal		pushal
cld		cld

/*		/*
* Set our jump address for the jump back in the event that		* Set our jump address for the jump back in the event that
* the exception wasn't caused by DTrace at all.		* the exception wasn't caused by DTrace at all.
*/		*/
movl $norm_ill, dtrace_invop_calltrap_addr		movl $norm_ill, dtrace_invop_calltrap_addr

/* Jump to the code hooked in by DTrace. */		/* Jump to the code hooked in by DTrace. */
jmpl *dtrace_invop_jump_addr		jmpl *dtrace_invop_jump_addr

/*		/*
* Process the instruction fault in the normal way.		* Process the instruction fault in the normal way.
*/		*/
norm_ill:		norm_ill:
pushl $0		pushl $0
TRAP(T_PRIVINFLT)		pushl $T_PRIVINFLT
		jmp alltraps
#endif		#endif

/*		IDTVEC(mchk)
* Call gate entry for syscalls (lcall 7,0).
* This is used by FreeBSD 1.x a.out executables and "old" NetBSD executables.
*
* The intersegment call has been set up to specify one dummy parameter.
* This leaves a place to put eflags so that the call frame can be
* converted to a trap frame. Note that the eflags is (semi-)bogusly
* pushed into (what will be) tf_err and then copied later into the
* final spot. It has to be done this way because esp can't be just
* temporarily altered for the pushfl - an interrupt might come in
* and clobber the saved cs/eip.
*/
SUPERALIGN_TEXT
IDTVEC(lcall_syscall)
pushfl /* save eflags */
popl 8(%esp) /* shuffle into tf_eflags */
pushl $7 /* sizeof "lcall 7,0" */
pushl $0 /* tf_trapno */
pushal
pushl $0		pushl $0
movw %ds,(%esp)		pushl $T_MCHK
		jmp nmi_mchk_common

		IDTVEC(nmi)
pushl $0		pushl $0
movw %es,(%esp)		pushl $T_NMI
pushl $0		nmi_mchk_common:
movw %fs,(%esp)		PUSH_FRAME2
SET_KERNEL_SREGS		SET_KERNEL_SREGS
cld		cld
		/*
		* Save %cr3 into tf_err. There is no good place to put it.
		* Always reload %cr3, since we might have interrupted the
		* kernel entry or exit.
		* Do not switch to the thread kernel stack, otherwise we might
		* obliterate the previous context partially copied from the
		* trampoline stack.
		*/
		movl %cr3, %eax
		movl %eax, TF_ERR(%esp)
		call 1f
		1: popl %eax
		movl (tramp_idleptd - 1b)(%eax), %eax
		movl %eax, %cr3
FAKE_MCOUNT(TF_EIP(%esp))		FAKE_MCOUNT(TF_EIP(%esp))
pushl %esp		jmp calltrap
call syscall
add $4, %esp
MEXITCOUNT
jmp doreti

/*		/*
* Trap gate entry for syscalls (int 0x80).		* Trap gate entry for syscalls (int 0x80).
* This is used by FreeBSD ELF executables, "new" NetBSD executables, and all		* This is used by FreeBSD ELF executables, "new" a.out executables, and all
		emasteUnsubmitted Done Inline Actions Aside, `"new" NetBSD executables` should no longer be here. emaste: Aside, `"new" NetBSD executables` should no longer be here.
		kibAuthorUnsubmitted Not Done Inline Actions I changed it to '"new" a.out executables'. kib: I changed it to '"new" a.out executables'.
* Linux executables.		* Linux executables.
*		*
* Even though the name says 'int0x80', this is actually a trap gate, not an		* Even though the name says 'int0x80', this is actually a trap gate, not an
* interrupt gate. Thus interrupts are enabled on entry just as they are for		* interrupt gate. Thus interrupts are enabled on entry just as they are for
* a normal syscall.		* a normal syscall.
*/		*/
SUPERALIGN_TEXT		SUPERALIGN_TEXT
IDTVEC(int0x80_syscall)		IDTVEC(int0x80_syscall)
pushl $2 /* sizeof "int 0x80" */		pushl $2 /* sizeof "int 0x80" */
pushl $0 /* tf_trapno */		pushl $0 /* tf_trapno */
pushal		PUSH_FRAME2
pushl $0
movw %ds,(%esp)
pushl $0
movw %es,(%esp)
pushl $0
movw %fs,(%esp)
SET_KERNEL_SREGS		SET_KERNEL_SREGS
cld		cld
		MOVE_STACKS
		sti
FAKE_MCOUNT(TF_EIP(%esp))		FAKE_MCOUNT(TF_EIP(%esp))
pushl %esp		pushl %esp
call syscall		movl $syscall, %eax
		call *%eax
add $4, %esp		add $4, %esp
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

ENTRY(fork_trampoline)		ENTRY(fork_trampoline)
pushl %esp /* trapframe pointer */		pushl %esp /* trapframe pointer */
pushl %ebx /* arg1 */		pushl %ebx /* arg1 */
pushl %esi /* function */		pushl %esi /* function */
call fork_exit		movl $fork_exit, %eax
		call *%eax
addl $12,%esp		addl $12,%esp
/* cut from syscall */		/* cut from syscall */

/*		/*
* Return via doreti to handle ASTs.		* Return via doreti to handle ASTs.
*/		*/
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti
Show All 34 Lines	#endif
.p2align 4		.p2align 4
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
#include <i386/i386/vm86bios.s>		#include <i386/i386/vm86bios.s>

.text		.text
MCOUNT_LABEL(eintr)		MCOUNT_LABEL(eintr)

		#include <i386/i386/copyout_fast.s>

/*		/*
* void doreti(struct trapframe)		* void doreti(struct trapframe)
*		*
* Handle return from interrupts, traps and syscalls.		* Handle return from interrupts, traps and syscalls.
*/		*/
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
.type doreti,@function		.type doreti,@function
Show All 16 Lines	#endif
* have an RPL in non-VM86 mode.		* have an RPL in non-VM86 mode.
* ASTs can not be handled now if we are in a vm86 call.		* ASTs can not be handled now if we are in a vm86 call.
*/		*/
testl $PSL_VM,TF_EFLAGS(%esp)		testl $PSL_VM,TF_EFLAGS(%esp)
jz doreti_notvm86		jz doreti_notvm86
movl PCPU(CURPCB),%ecx		movl PCPU(CURPCB),%ecx
testl $PCB_VM86CALL,PCB_FLAGS(%ecx)		testl $PCB_VM86CALL,PCB_FLAGS(%ecx)
jz doreti_ast		jz doreti_ast
jmp doreti_exit		jmp doreti_popl_fs

doreti_notvm86:		doreti_notvm86:
testb $SEL_RPL_MASK,TF_CS(%esp) /* are we returning to user mode? */		testb $SEL_RPL_MASK,TF_CS(%esp) /* are we returning to user mode? */
jz doreti_exit /* can't handle ASTs now if not */		jz doreti_exit /* can't handle ASTs now if not */

doreti_ast:		doreti_ast:
/*		/*
* Check for ASTs atomically with returning. Disabling CPU		* Check for ASTs atomically with returning. Disabling CPU
* interrupts provides sufficient locking even in the SMP case,		* interrupts provides sufficient locking even in the SMP case,
* since we will be informed of any new ASTs by an IPI.		* since we will be informed of any new ASTs by an IPI.
*/		*/
cli		cli
movl PCPU(CURTHREAD),%eax		movl PCPU(CURTHREAD),%eax
testl $TDF_ASTPENDING \| TDF_NEEDRESCHED,TD_FLAGS(%eax)		testl $TDF_ASTPENDING \| TDF_NEEDRESCHED,TD_FLAGS(%eax)
je doreti_exit		je doreti_exit
sti		sti
pushl %esp /* pass a pointer to the trapframe */		pushl %esp /* pass a pointer to the trapframe */
call ast		movl $ast, %eax
		call *%eax
add $4,%esp		add $4,%esp
jmp doreti_ast		jmp doreti_ast

/*		/*
* doreti_exit: pop registers, iret.		* doreti_exit: pop registers, iret.
*		*
* The segment register pop is a special case, since it may		* The segment register pop is a special case, since it may
* fault if (for example) a sigreturn specifies bad segment		* fault if (for example) a sigreturn specifies bad segment
* registers. The fault is handled in trap.c.		* registers. The fault is handled in trap.c.
*/		*/
doreti_exit:		doreti_exit:
MEXITCOUNT		MEXITCOUNT

		cmpl $T_NMI, TF_TRAPNO(%esp)
		je doreti_iret_nmi
		cmpl $T_MCHK, TF_TRAPNO(%esp)
		je doreti_iret_nmi
		testl $SEL_RPL_MASK, TF_CS(%esp)
		jz doreti_popl_fs
		movl %esp, %esi
		movl PCPU(TRAMPSTK), %edx
		movl $TF_SZ, %ecx
		subl %ecx, %edx
		movl %edx, %edi
		rep; movsb
		movl %edx, %esp
		movl PCPU(CURPCB),%eax
		movl PCB_CR3(%eax), %eax
		movl %eax, %cr3

.globl doreti_popl_fs		.globl doreti_popl_fs
doreti_popl_fs:		doreti_popl_fs:
popl %fs		popl %fs
.globl doreti_popl_es		.globl doreti_popl_es
doreti_popl_es:		doreti_popl_es:
popl %es		popl %es
.globl doreti_popl_ds		.globl doreti_popl_ds
doreti_popl_ds:		doreti_popl_ds:
popl %ds		popl %ds
popal		popal
addl $8,%esp		addl $8,%esp
.globl doreti_iret		.globl doreti_iret
doreti_iret:		doreti_iret:
iret		iret

		doreti_iret_nmi:
		movl TF_ERR(%esp), %eax
		movl %eax, %cr3
		jmp doreti_popl_fs

/*		/*
* doreti_iret_fault and friends. Alternative return code for		* doreti_iret_fault and friends. Alternative return code for
* the case where we get a fault in the doreti_exit code		* the case where we get a fault in the doreti_exit code
* above. trap() (i386/i386/trap.c) catches this specific		* above. trap() (i386/i386/trap.c) catches this specific
* case, and continues in the corresponding place in the code		* case, and continues in the corresponding place in the code
* below.		* below.
*		*
* If the fault occured during return to usermode, we recreate		* If the fault occured during return to usermode, we recreate
* the trap frame and call trap() to send a signal. Otherwise		* the trap frame and call trap() to send a signal. Otherwise
* the kernel was tricked into fault by attempt to restore invalid		* the kernel was tricked into fault by attempt to restore invalid
* usermode segment selectors on return from nested fault or		* usermode segment selectors on return from nested fault or
* interrupt, where interrupted kernel entry code not yet loaded		* interrupt, where interrupted kernel entry code not yet loaded
* kernel selectors. In the latter case, emulate iret and zero		* kernel selectors. In the latter case, emulate iret and zero
* the invalid selector.		* the invalid selector.
*/		*/
ALIGN_TEXT		ALIGN_TEXT
.globl doreti_iret_fault		.globl doreti_iret_fault
doreti_iret_fault:		doreti_iret_fault:
subl $8,%esp		pushl $0 /* tf_err */
		pushl $0 /* tf_trapno XXXKIB: provide more useful value ? */
pushal		pushal
pushl $0		pushl $0
movw %ds,(%esp)		movw %ds,(%esp)
.globl doreti_popl_ds_fault		.globl doreti_popl_ds_fault
doreti_popl_ds_fault:		doreti_popl_ds_fault:
testb $SEL_RPL_MASK,TF_CS-TF_DS(%esp)		testb $SEL_RPL_MASK,TF_CS-TF_DS(%esp)
jz doreti_popl_ds_kfault		jz doreti_popl_ds_kfault
pushl $0		pushl $0
movw %es,(%esp)		movw %es,(%esp)
.globl doreti_popl_es_fault		.globl doreti_popl_es_fault
doreti_popl_es_fault:		doreti_popl_es_fault:
testb $SEL_RPL_MASK,TF_CS-TF_ES(%esp)		testb $SEL_RPL_MASK,TF_CS-TF_ES(%esp)
jz doreti_popl_es_kfault		jz doreti_popl_es_kfault
pushl $0		pushl $0
movw %fs,(%esp)		movw %fs,(%esp)
.globl doreti_popl_fs_fault		.globl doreti_popl_fs_fault
doreti_popl_fs_fault:		doreti_popl_fs_fault:
testb $SEL_RPL_MASK,TF_CS-TF_FS(%esp)		testb $SEL_RPL_MASK,TF_CS-TF_FS(%esp)
jz doreti_popl_fs_kfault		jz doreti_popl_fs_kfault
sti
movl $0,TF_ERR(%esp) /* XXX should be the error code */		movl $0,TF_ERR(%esp) /* XXX should be the error code */
movl $T_PROTFLT,TF_TRAPNO(%esp)		movl $T_PROTFLT,TF_TRAPNO(%esp)
jmp alltraps_with_regs_pushed		SET_KERNEL_SREGS
		jmp calltrap

doreti_popl_ds_kfault:		doreti_popl_ds_kfault:
movl $0,(%esp)		movl $0,(%esp)
jmp doreti_popl_ds		jmp doreti_popl_ds
doreti_popl_es_kfault:		doreti_popl_es_kfault:
movl $0,(%esp)		movl $0,(%esp)
jmp doreti_popl_es		jmp doreti_popl_es
doreti_popl_fs_kfault:		doreti_popl_fs_kfault:
movl $0,(%esp)		movl $0,(%esp)
jmp doreti_popl_fs		jmp doreti_popl_fs

#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
doreti_nmi:		doreti_nmi:
/*		/*
* Since we are returning from an NMI, check if the current trap		* Since we are returning from an NMI, check if the current trap
* was from user mode and if so whether the current thread		* was from user mode and if so whether the current thread
* needs a user call chain capture.		* needs a user call chain capture.
*/		*/
		testl $PSL_VM, TF_EFLAGS(%esp)
		jnz doreti_exit
testb $SEL_RPL_MASK,TF_CS(%esp)		testb $SEL_RPL_MASK,TF_CS(%esp)
jz doreti_exit		jz doreti_exit
movl PCPU(CURTHREAD),%eax /* curthread present? */		movl PCPU(CURTHREAD),%eax /* curthread present? */
orl %eax,%eax		orl %eax,%eax
jz doreti_exit		jz doreti_exit
testl $TDP_CALLCHAIN,TD_PFLAGS(%eax) /* flagged for capture? */		testl $TDP_CALLCHAIN,TD_PFLAGS(%eax) /* flagged for capture? */
jz doreti_exit		jz doreti_exit
/*		/*
		* Switch to thread stack. Reset tf_trapno to not indicate NMI,
		* to cause normal userspace exit.
		*/
		movl $T_RESERVED, TF_TRAPNO(%esp)
		NMOVE_STACKS
		/*
* Take the processor out of NMI mode by executing a fake "iret".		* Take the processor out of NMI mode by executing a fake "iret".
*/		*/
pushfl		pushfl
pushl %cs		pushl %cs
pushl $outofnmi		call 1f
		1: popl %eax
		leal (outofnmi-1b)(%eax),%eax
		pushl %eax
iret		iret
outofnmi:		outofnmi:
/*		/*
* Call the callchain capture hook after turning interrupts back on.		* Call the callchain capture hook after turning interrupts back on.
*/		*/
movl pmc_hook,%ecx		movl pmc_hook,%ecx
orl %ecx,%ecx		orl %ecx,%ecx
jz doreti_exit		jz doreti_exit
pushl %esp /* frame pointer */		pushl %esp /* frame pointer */
pushl $PMC_FN_USER_CALLCHAIN /* command */		pushl $PMC_FN_USER_CALLCHAIN /* command */
movl PCPU(CURTHREAD),%eax		movl PCPU(CURTHREAD),%eax
pushl %eax /* curthread */		pushl %eax /* curthread */
sti		sti
call *%ecx		call *%ecx
addl $12,%esp		addl $12,%esp
jmp doreti_ast		jmp doreti_ast
ENTRY(end_exceptions)
#endif		#endif

		ENTRY(end_exceptions)