Differential D13797 Diff 38054 sys/amd64/amd64/apic_vector.S

Changeset View

Standalone View

sys/amd64/amd64/apic_vector.S

/*-		/*-
* Copyright (c) 1989, 1990 William F. Jolitz.		* Copyright (c) 1989, 1990 William F. Jolitz.
* Copyright (c) 1990 The Regents of the University of California.		* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.		* All rights reserved.
		* Copyright (c) 2014-2018 The FreeBSD Foundation
		* All rights reserved.
*		*
		* Portions of this software were developed by
		* Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
		* the FreeBSD Foundation.
		*
* Redistribution and use in source and binary forms, with or without		* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions		* modification, are permitted provided that the following conditions
* are met:		* are met:
* 1. Redistributions of source code must retain the above copyright		* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.		* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright		* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the		* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.		* documentation and/or other materials provided with the distribution.
Show All 19 Lines

/*		/*
* Interrupt entry points for external interrupts triggered by I/O APICs		* Interrupt entry points for external interrupts triggered by I/O APICs
* as well as IPI handlers.		* as well as IPI handlers.
*/		*/

#include "opt_smp.h"		#include "opt_smp.h"

		#include "assym.s"

#include <machine/asmacros.h>		#include <machine/asmacros.h>
#include <machine/specialreg.h>		#include <machine/specialreg.h>
#include <x86/apicreg.h>		#include <x86/apicreg.h>

#include "assym.s"

#ifdef SMP		#ifdef SMP
#define LK lock ;		#define LK lock ;
#else		#else
#define LK		#define LK
#endif		#endif

.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
Show All 13 Lines

/*		/*
* I/O Interrupt Entry Point. Rather than having one entry point for		* I/O Interrupt Entry Point. Rather than having one entry point for
* each interrupt source, we use one entry point for each 32-bit word		* each interrupt source, we use one entry point for each 32-bit word
* in the ISR. The handler determines the highest bit set in the ISR,		* in the ISR. The handler determines the highest bit set in the ISR,
* translates that into a vector, and passes the vector to the		* translates that into a vector, and passes the vector to the
* lapic_handle_intr() function.		* lapic_handle_intr() function.
*/		*/
#define ISR_VEC(index, vec_name) \		.macro ISR_VEC index, vec_name
.text ; \		INTR_HANDLER \vec_name
SUPERALIGN_TEXT ; \		FAKE_MCOUNT(TF_RIP(%rsp))
IDTVEC(vec_name) ; \		cmpl $0,x2apic_mode
PUSH_FRAME ; \		je 1f
FAKE_MCOUNT(TF_RIP(%rsp)) ; \		movl $(MSR_APIC_ISR0 + \index),%ecx
cmpl $0,x2apic_mode ; \		rdmsr
je 1f ; \		jmp 2f
movl $(MSR_APIC_ISR0 + index),%ecx ; \		1:
rdmsr ; \		movq lapic_map, %rdx /* pointer to local APIC */
jmp 2f ; \		movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */
1: ; \		2:
movq lapic_map, %rdx ; /* pointer to local APIC */ \		bsrl %eax, %eax /* index of highest set bit in ISR */
movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \		jz 3f
2: ; \		addl $(32 * \index),%eax
bsrl %eax, %eax ; /* index of highest set bit in ISR */ \		movq %rsp, %rsi
jz 3f ; \		movl %eax, %edi /* pass the IRQ */
addl $(32 * index),%eax ; \		call lapic_handle_intr
movq %rsp, %rsi ; \		3:
movl %eax, %edi ; /* pass the IRQ */ \		MEXITCOUNT
call lapic_handle_intr ; \
3: ; \
MEXITCOUNT ; \
jmp doreti		jmp doreti
		.endm

/*		/*
* Handle "spurious INTerrupts".		* Handle "spurious INTerrupts".
* Notes:		* Notes:
* This is different than the "spurious INTerrupt" generated by an		* This is different than the "spurious INTerrupt" generated by an
* 8259 PIC for missing INTs. See the APIC documentation for details.		* 8259 PIC for missing INTs. See the APIC documentation for details.
* This routine should NOT do an 'EOI' cycle.		* This routine should NOT do an 'EOI' cycle.
*/		*/
.text		.text
SUPERALIGN_TEXT		SUPERALIGN_TEXT
IDTVEC(spuriousint)		IDTVEC(spuriousint)

/* No EOI cycle used here */		/* No EOI cycle used here */

jmp doreti_iret		jmp doreti_iret

ISR_VEC(1, apic_isr1)		ISR_VEC 1, apic_isr1
ISR_VEC(2, apic_isr2)		ISR_VEC 2, apic_isr2
ISR_VEC(3, apic_isr3)		ISR_VEC 3, apic_isr3
ISR_VEC(4, apic_isr4)		ISR_VEC 4, apic_isr4
ISR_VEC(5, apic_isr5)		ISR_VEC 5, apic_isr5
ISR_VEC(6, apic_isr6)		ISR_VEC 6, apic_isr6
ISR_VEC(7, apic_isr7)		ISR_VEC 7, apic_isr7

/*		/*
* Local APIC periodic timer handler.		* Local APIC periodic timer handler.
*/		*/
.text		INTR_HANDLER timerint
SUPERALIGN_TEXT
jeffUnsubmitted Done Inline Actions Are we not losing this alignment in your new version? jeff: Are we not losing this alignment in your new version?
emasteUnsubmitted Not Done Inline Actions INTR_HANDLER is .macro INTR_HANDLER vec_name .text PTI_INTRENTRY \vec_name INTR_PUSH_FRAME \vec_name .endm and INTR_PUSH_FRAME is: .macro INTR_PUSH_FRAME vec_name SUPERALIGN_TEXT ... emaste: INTR_HANDLER is ``` .macro INTR_HANDLER vec_name .text …
IDTVEC(timerint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))		FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp, %rdi		movq %rsp, %rdi
call lapic_handle_timer		call lapic_handle_timer
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

/*		/*
* Local APIC CMCI handler.		* Local APIC CMCI handler.
*/		*/
.text		INTR_HANDLER cmcint
SUPERALIGN_TEXT
IDTVEC(cmcint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))		FAKE_MCOUNT(TF_RIP(%rsp))
call lapic_handle_cmc		call lapic_handle_cmc
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

/*		/*
* Local APIC error interrupt handler.		* Local APIC error interrupt handler.
*/		*/
.text		INTR_HANDLER errorint
SUPERALIGN_TEXT
IDTVEC(errorint)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))		FAKE_MCOUNT(TF_RIP(%rsp))
call lapic_handle_error		call lapic_handle_error
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

#ifdef XENHVM		#ifdef XENHVM
/*		/*
* Xen event channel upcall interrupt handler.		* Xen event channel upcall interrupt handler.
* Only used when the hypervisor supports direct vector callbacks.		* Only used when the hypervisor supports direct vector callbacks.
*/		*/
.text		INTR_HANDLER xen_intr_upcall
SUPERALIGN_TEXT
IDTVEC(xen_intr_upcall)
PUSH_FRAME
FAKE_MCOUNT(TF_RIP(%rsp))		FAKE_MCOUNT(TF_RIP(%rsp))
movq %rsp, %rdi		movq %rsp, %rdi
call xen_intr_handle_upcall		call xen_intr_handle_upcall
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti
#endif		#endif

#ifdef SMP		#ifdef SMP
/*		/*
* Global address space TLB shootdown.		* Global address space TLB shootdown.
*/		*/
.text		.text

SUPERALIGN_TEXT		SUPERALIGN_TEXT
invltlb_ret:		invltlb_ret:
call as_lapic_eoi		call as_lapic_eoi
POP_FRAME		jmp ld_regs
jmp doreti_iret

SUPERALIGN_TEXT		SUPERALIGN_TEXT
IDTVEC(invltlb)		INTR_HANDLER invltlb
PUSH_FRAME

call invltlb_handler		call invltlb_handler
jmp invltlb_ret		jmp invltlb_ret

IDTVEC(invltlb_pcid)		INTR_HANDLER invltlb_pcid
PUSH_FRAME

call invltlb_pcid_handler		call invltlb_pcid_handler
jmp invltlb_ret		jmp invltlb_ret

IDTVEC(invltlb_invpcid)		INTR_HANDLER invltlb_invpcid
PUSH_FRAME

call invltlb_invpcid_handler		call invltlb_invpcid_handler
jmp invltlb_ret		jmp invltlb_ret

/*		/*
* Single page TLB shootdown		* Single page TLB shootdown
*/		*/
.text		INTR_HANDLER invlpg

SUPERALIGN_TEXT
IDTVEC(invlpg)
PUSH_FRAME

call invlpg_handler		call invlpg_handler
jmp invltlb_ret		jmp invltlb_ret

/*		/*
* Page range TLB shootdown.		* Page range TLB shootdown.
*/		*/
.text		INTR_HANDLER invlrng
SUPERALIGN_TEXT
IDTVEC(invlrng)
PUSH_FRAME

call invlrng_handler		call invlrng_handler
jmp invltlb_ret		jmp invltlb_ret

/*		/*
* Invalidate cache.		* Invalidate cache.
*/		*/
.text		INTR_HANDLER invlcache
SUPERALIGN_TEXT
IDTVEC(invlcache)
PUSH_FRAME

call invlcache_handler		call invlcache_handler
jmp invltlb_ret		jmp invltlb_ret

/*		/*
* Handler for IPIs sent via the per-cpu IPI bitmap.		* Handler for IPIs sent via the per-cpu IPI bitmap.
*/		*/
.text		INTR_HANDLER ipi_intr_bitmap_handler
SUPERALIGN_TEXT
IDTVEC(ipi_intr_bitmap_handler)
PUSH_FRAME

call as_lapic_eoi		call as_lapic_eoi

FAKE_MCOUNT(TF_RIP(%rsp))		FAKE_MCOUNT(TF_RIP(%rsp))

call ipi_bitmap_handler		call ipi_bitmap_handler
MEXITCOUNT		MEXITCOUNT
jmp doreti		jmp doreti

/*		/*
* Executed by a CPU when it receives an IPI_STOP from another CPU.		* Executed by a CPU when it receives an IPI_STOP from another CPU.
*/		*/
.text		INTR_HANDLER cpustop
SUPERALIGN_TEXT
IDTVEC(cpustop)
PUSH_FRAME

call as_lapic_eoi		call as_lapic_eoi

call cpustop_handler		call cpustop_handler
jmp doreti		jmp doreti

/*		/*
* Executed by a CPU when it receives an IPI_SUSPEND from another CPU.		* Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
*/		*/
.text		INTR_HANDLER cpususpend
SUPERALIGN_TEXT
IDTVEC(cpususpend)
PUSH_FRAME

call cpususpend_handler		call cpususpend_handler
call as_lapic_eoi		call as_lapic_eoi
jmp doreti		jmp doreti

/*		/*
* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.		* Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
*		*
* - Calls the generic rendezvous action function.		* - Calls the generic rendezvous action function.
*/		*/
.text		INTR_HANDLER rendezvous
SUPERALIGN_TEXT
IDTVEC(rendezvous)
PUSH_FRAME
#ifdef COUNT_IPIS		#ifdef COUNT_IPIS
movl PCPU(CPUID), %eax		movl PCPU(CPUID), %eax
movq ipi_rendezvous_counts(,%rax,8), %rax		movq ipi_rendezvous_counts(,%rax,8), %rax
incq (%rax)		incq (%rax)
#endif		#endif
call smp_rendezvous_action		call smp_rendezvous_action
call as_lapic_eoi		call as_lapic_eoi
jmp doreti		jmp doreti
Show All 22 Lines	IDTVEC(justreturn)
pushq %rax		pushq %rax
pushq %rcx		pushq %rcx
pushq %rdx		pushq %rdx
call as_lapic_eoi		call as_lapic_eoi
popq %rdx		popq %rdx
popq %rcx		popq %rcx
popq %rax		popq %rax
jmp doreti_iret		jmp doreti_iret

		INTR_HANDLER justreturn1
		call as_lapic_eoi
		jmp doreti

#endif /* SMP */		#endif /* SMP */