Differential D21566 Diff 62659 head/sys/amd64/amd64/trap.c

Changeset View

Standalone View

head/sys/amd64/amd64/trap.c

Show First 20 Lines • Show All 105 Lines • ▼ Show 20 Lines
extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),		extern inthand_t IDTVEC(bpt), IDTVEC(bpt_pti), IDTVEC(dbg),
IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),		IDTVEC(fast_syscall), IDTVEC(fast_syscall_pti), IDTVEC(fast_syscall32),
IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);		IDTVEC(int0x80_syscall_pti), IDTVEC(int0x80_syscall);

void __noinline trap(struct trapframe *frame);		void __noinline trap(struct trapframe *frame);
void trap_check(struct trapframe *frame);		void trap_check(struct trapframe *frame);
void dblfault_handler(struct trapframe *frame);		void dblfault_handler(struct trapframe *frame);

static int trap_pfault(struct trapframe *, int);		static int trap_pfault(struct trapframe , bool, int , int *);
static void trap_fatal(struct trapframe *, vm_offset_t);		static void trap_fatal(struct trapframe *, vm_offset_t);
#ifdef KDTRACE_HOOKS		#ifdef KDTRACE_HOOKS
static bool trap_user_dtrace(struct trapframe *,		static bool trap_user_dtrace(struct trapframe *,
int (*hook)(struct trapframe ));		int (*hook)(struct trapframe ));
#endif		#endif

static const char UNKNOWN[] = "unknown";		static const char UNKNOWN[] = "unknown";
static const char *const trap_msg[] = {		static const char *const trap_msg[] = {
Show All 27 Lines	static const char *const trap_msg[] = {
[T_STKFLT] = "stack fault",		[T_STKFLT] = "stack fault",
[T_MCHK] = "machine check trap",		[T_MCHK] = "machine check trap",
[T_XMMFLT] = "SIMD floating-point exception",		[T_XMMFLT] = "SIMD floating-point exception",
[T_RESERVED] = "reserved (unknown) fault",		[T_RESERVED] = "reserved (unknown) fault",
[31] = UNKNOWN, /* reserved */		[31] = UNKNOWN, /* reserved */
[T_DTRACE_RET] = "DTrace pid return trap",		[T_DTRACE_RET] = "DTrace pid return trap",
};		};

static int prot_fault_translation;
SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RWTUN,
&prot_fault_translation, 0,
"Select signal to deliver on protection fault");
static int uprintf_signal;		static int uprintf_signal;
SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,		SYSCTL_INT(_machdep, OID_AUTO, uprintf_signal, CTLFLAG_RWTUN,
&uprintf_signal, 0,		&uprintf_signal, 0,
"Print debugging information on trap signal to ctty");		"Print debugging information on trap signal to ctty");

/*		/*
* Control L1D flush on return from NMI.		* Control L1D flush on return from NMI.
*		*
Show All 17 Lines

void		void
trap(struct trapframe *frame)		trap(struct trapframe *frame)
{		{
ksiginfo_t ksi;		ksiginfo_t ksi;
struct thread *td;		struct thread *td;
struct proc *p;		struct proc *p;
register_t addr, dr6;		register_t addr, dr6;
int signo, ucode;		int pf, signo, ucode;
u_int type;		u_int type;

td = curthread;		td = curthread;
p = td->td_proc;		p = td->td_proc;
signo = 0;
ucode = 0;
addr = 0;
dr6 = 0;		dr6 = 0;

VM_CNT_INC(v_trap);		VM_CNT_INC(v_trap);
type = frame->tf_trapno;		type = frame->tf_trapno;

#ifdef SMP		#ifdef SMP
/* Handler for NMI IPIs used for stopping CPUs. */		/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI && ipi_nmi_handler() == 0)		if (type == T_NMI && ipi_nmi_handler() == 0)
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines	#endif
case T_DOUBLEFLT: /* double fault */		case T_DOUBLEFLT: /* double fault */
default:		default:
signo = SIGBUS;		signo = SIGBUS;
ucode = BUS_OBJERR;		ucode = BUS_OBJERR;
break;		break;

case T_PAGEFLT: /* page fault */		case T_PAGEFLT: /* page fault */
/*		/*
* Emulator can take care about this trap?		* Can emulator handle this trap?
*/		*/
if (*p->p_sysent->sv_trap != NULL &&		if (*p->p_sysent->sv_trap != NULL &&
(*p->p_sysent->sv_trap)(td) == 0)		(*p->p_sysent->sv_trap)(td) == 0)
return;		return;

addr = frame->tf_addr;		pf = trap_pfault(frame, true, &signo, &ucode);
signo = trap_pfault(frame, TRUE);		if (pf == -1)
if (signo == -1)
return;		return;
if (signo == 0)		if (pf == 0)
goto userret;		goto userret;
if (signo == SIGSEGV) {		addr = frame->tf_addr;
ucode = SEGV_MAPERR;
} else if (prot_fault_translation == 0) {
/*
* Autodetect. This check also covers
* the images without the ABI-tag ELF
* note.
*/
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD &&
p->p_osrel >= P_OSREL_SIGSEGV) {
signo = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
signo = SIGBUS;
ucode = T_PAGEFLT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
signo = SIGBUS;
ucode = T_PAGEFLT;
} else {
/*
* Always SIGSEGV mode.
*/
signo = SIGSEGV;
ucode = SEGV_ACCERR;
}
break;		break;

case T_DIVIDE: /* integer divide fault */		case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;		ucode = FPE_INTDIV;
signo = SIGFPE;		signo = SIGFPE;
break;		break;

#ifdef DEV_ISA		#ifdef DEV_ISA
Show All 38 Lines	#endif
}		}
} else {		} else {
/* kernel trap */		/* kernel trap */

KASSERT(cold \|\| td->td_ucred != NULL,		KASSERT(cold \|\| td->td_ucred != NULL,
("kernel trap doesn't have ucred"));		("kernel trap doesn't have ucred"));
switch (type) {		switch (type) {
case T_PAGEFLT: /* page fault */		case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE);		(void)trap_pfault(frame, false, NULL, NULL);
return;		return;

case T_DNA:		case T_DNA:
if (PCB_USER_FPU(td->td_pcb))		if (PCB_USER_FPU(td->td_pcb))
panic("Unregistered use of FPU in kernel");		panic("Unregistered use of FPU in kernel");
fpudna();		fpudna();
return;		return;

▲ Show 20 Lines • Show All 255 Lines • ▼ Show 20 Lines	trap_is_pti(struct trapframe *frame)

return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&		return (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 &&
pg_nx != 0 && (frame->tf_err & (PGEX_P \| PGEX_W \|		pg_nx != 0 && (frame->tf_err & (PGEX_P \| PGEX_W \|
PGEX_U \| PGEX_I)) == (PGEX_P \| PGEX_U \| PGEX_I) &&		PGEX_U \| PGEX_I)) == (PGEX_P \| PGEX_U \| PGEX_I) &&
(curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==		(curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK) ==
(PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));		(PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK));
}		}

		/*
		* Handle all details of a page fault.
		* Returns:
		* -1 if this fault was fatal, typically from kernel mode
		* (cannot happen, but we need to return something).
		* 0 if this fault was handled by updating either the user or kernel
		* page table, execution can continue.
		* 1 if this fault was from usermode and it was not handled, a synchronous
		* signal should be delivered to the thread. *signo returns the signal
		* number, *ucode gives si_code.
		*/
static int		static int
trap_pfault(struct trapframe *frame, int usermode)		trap_pfault(struct trapframe frame, bool usermode, int signo, int *ucode)
{		{
struct thread *td;		struct thread *td;
struct proc *p;		struct proc *p;
vm_map_t map;		vm_map_t map;
vm_offset_t va;		vm_offset_t eva;
int rv;		int rv;
vm_prot_t ftype;		vm_prot_t ftype;
vm_offset_t eva;

		MPASS(!usermode \|\| (signo != NULL && ucode != NULL));

td = curthread;		td = curthread;
p = td->td_proc;		p = td->td_proc;
eva = frame->tf_addr;		eva = frame->tf_addr;

if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {		if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
/*		/*
* Due to both processor errata and lazy TLB invalidation when		* Due to both processor errata and lazy TLB invalidation when
* access restrictions are removed from virtual pages, memory		* access restrictions are removed from virtual pages, memory
Show All 32 Lines	if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
*/		*/
if (td->td_critnest != 0 \|\|		if (td->td_critnest != 0 \|\|
WITNESS_CHECK(WARN_SLEEPOK \| WARN_GIANTOK, NULL,		WITNESS_CHECK(WARN_SLEEPOK \| WARN_GIANTOK, NULL,
"Kernel page fault") != 0) {		"Kernel page fault") != 0) {
trap_fatal(frame, eva);		trap_fatal(frame, eva);
return (-1);		return (-1);
}		}
}		}
va = trunc_page(eva);		if (eva >= VM_MIN_KERNEL_ADDRESS) {
if (va >= VM_MIN_KERNEL_ADDRESS) {
/*		/*
* Don't allow user-mode faults in kernel address space.		* Don't allow user-mode faults in kernel address space.
*/		*/
if (usermode)		if (usermode) {
return (SIGSEGV);		*signo = SIGSEGV;
		*ucode = SEGV_MAPERR;
		return (1);
		}

map = kernel_map;		map = kernel_map;
} else {		} else {
map = &p->p_vmspace->vm_map;		map = &p->p_vmspace->vm_map;

/*		/*
* When accessing a usermode address, kernel must be		* When accessing a usermode address, kernel must be
* ready to accept the page fault, and provide a		* ready to accept the page fault, and provide a
Show All 25 Lines	trap_pfault(struct trapframe frame, bool usermode, int signo, int *ucode)
* either from usermode or from kernel if copyin accessed		* either from usermode or from kernel if copyin accessed
* key-protected mapping.		* key-protected mapping.
*/		*/
if ((frame->tf_err & PGEX_PK) != 0) {		if ((frame->tf_err & PGEX_PK) != 0) {
if (eva > VM_MAXUSER_ADDRESS) {		if (eva > VM_MAXUSER_ADDRESS) {
trap_fatal(frame, eva);		trap_fatal(frame, eva);
return (-1);		return (-1);
}		}
rv = KERN_PROTECTION_FAILURE;		if (usermode) {
		*signo = SIGSEGV;
		*ucode = SEGV_PKUERR;
		return (1);
		}
goto after_vmfault;		goto after_vmfault;
}		}

/*		/*
* If nx protection of the usermode portion of kernel page		* If nx protection of the usermode portion of kernel page
* tables caused trap, panic.		* tables caused trap, panic.
*/		*/
if (usermode && trap_is_pti(frame))		if (usermode && trap_is_pti(frame))
panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,		panic("PTI: pid %d comm %s tf_err %#lx", p->p_pid,
p->p_comm, frame->tf_err);		p->p_comm, frame->tf_err);

/*		/*
* PGEX_I is defined only if the execute disable bit capability is		* PGEX_I is defined only if the execute disable bit capability is
* supported and enabled.		* supported and enabled.
*/		*/
if (frame->tf_err & PGEX_W)		if (frame->tf_err & PGEX_W)
ftype = VM_PROT_WRITE;		ftype = VM_PROT_WRITE;
else if ((frame->tf_err & PGEX_I) && pg_nx != 0)		else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
ftype = VM_PROT_EXECUTE;		ftype = VM_PROT_EXECUTE;
else		else
ftype = VM_PROT_READ;		ftype = VM_PROT_READ;

/* Fault in the page. */		/* Fault in the page. */
rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);		rv = vm_fault_trap(map, eva, ftype, VM_FAULT_NORMAL, signo, ucode);
if (rv == KERN_SUCCESS) {		if (rv == KERN_SUCCESS) {
#ifdef HWPMC_HOOKS		#ifdef HWPMC_HOOKS
if (ftype == VM_PROT_READ \|\| ftype == VM_PROT_WRITE) {		if (ftype == VM_PROT_READ \|\| ftype == VM_PROT_WRITE) {
PMC_SOFT_CALL_TF( , , page_fault, all, frame);		PMC_SOFT_CALL_TF( , , page_fault, all, frame);
if (ftype == VM_PROT_READ)		if (ftype == VM_PROT_READ)
PMC_SOFT_CALL_TF( , , page_fault, read,		PMC_SOFT_CALL_TF( , , page_fault, read,
frame);		frame);
else		else
PMC_SOFT_CALL_TF( , , page_fault, write,		PMC_SOFT_CALL_TF( , , page_fault, write,
frame);		frame);
}		}
#endif		#endif
return (0);		return (0);
}		}

		if (usermode)
		return (1);
after_vmfault:		after_vmfault:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&		if (td->td_intr_nesting_level == 0 &&
curpcb->pcb_onfault != NULL) {		curpcb->pcb_onfault != NULL) {
frame->tf_rip = (long)curpcb->pcb_onfault;		frame->tf_rip = (long)curpcb->pcb_onfault;
return (0);		return (0);
}		}
trap_fatal(frame, eva);		trap_fatal(frame, eva);
return (-1);		return (-1);
}
return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV);
}		}

static void		static void
trap_fatal(frame, eva)		trap_fatal(frame, eva)
struct trapframe *frame;		struct trapframe *frame;
vm_offset_t eva;		vm_offset_t eva;
{		{
int code, ss;		int code, ss;
▲ Show 20 Lines • Show All 340 Lines • Show Last 20 Lines