Page MenuHomeFreeBSD
Files F135928887

D5738.id14595.diff
No OneTemporary
Actions

D5738.id14595.diff
View Options

Index: sys/x86/include/apicreg.h
===================================================================
--- sys/x86/include/apicreg.h
+++ sys/x86/include/apicreg.h
@@ -399,10 +399,11 @@
#define APIC_LVTT_VECTOR 0x000000ff
#define APIC_LVTT_DS 0x00001000
#define APIC_LVTT_M 0x00010000
-#define APIC_LVTT_TM 0x00020000
+#define APIC_LVTT_TM 0x00060000
# define APIC_LVTT_TM_ONE_SHOT 0x00000000
# define APIC_LVTT_TM_PERIODIC 0x00020000
-
+# define APIC_LVTT_TM_TSCDLT 0x00040000
+# define APIC_LVTT_TM_RSRV 0x00060000
/* APIC timer current count */
#define APIC_TIMER_MAX_COUNT 0xffffffff
Index: sys/x86/include/specialreg.h
===================================================================
--- sys/x86/include/specialreg.h
+++ sys/x86/include/specialreg.h
@@ -457,6 +457,7 @@
#define MSR_DRAM_ENERGY_STATUS 0x619
#define MSR_PP0_ENERGY_STATUS 0x639
#define MSR_PP1_ENERGY_STATUS 0x641
+#define MSR_TSC_DEADLINE 0x6e0 /* Writes are not serializing */
/*
* VMX MSRs
@@ -478,7 +479,8 @@
#define MSR_VMX_TRUE_ENTRY_CTLS 0x490
/*
- * X2APIC MSRs
+ * X2APIC MSRs.
+ * Writes are not serializing.
*/
#define MSR_APIC_000 0x800
#define MSR_APIC_ID 0x802
Index: sys/x86/x86/local_apic.c
===================================================================
--- sys/x86/x86/local_apic.c
+++ sys/x86/x86/local_apic.c
@@ -56,6 +56,7 @@
#include <vm/pmap.h>
#include <x86/apicreg.h>
+#include <machine/clock.h>
#include <machine/cpufunc.h>
#include <machine/cputypes.h>
#include <machine/frame.h>
@@ -94,6 +95,13 @@
#define IRQ_DTRACE_RET (NUM_IO_INTS + 3)
#define IRQ_EVTCHN (NUM_IO_INTS + 4)
+enum lat_timer_mode {
+ LAT_MODE_UNDEF = 0,
+ LAT_MODE_PERIODIC = 1,
+ LAT_MODE_ONESHOT = 2,
+ LAT_MODE_DEADLINE = 3,
+};
+
/*
* Support for local APICs. Local APICs manage interrupts on each
* individual processor as opposed to I/O APICs which receive interrupts
@@ -119,9 +127,10 @@
u_int la_cluster_id:2;
u_int la_present:1;
u_long *la_timer_count;
- u_long la_timer_period;
- u_int la_timer_mode;
- uint32_t lvt_timer_cache;
+ uint64_t la_timer_period;
+ enum lat_timer_mode la_timer_mode;
+ uint32_t lvt_timer_base;
+ uint32_t lvt_timer_last;
/* Include IDT_SYSCALL to make indexing easier. */
int la_ioint_irqs[APIC_NUM_IOINTS + 1];
} static lapics[MAX_APIC_ID + 1];
@@ -160,6 +169,7 @@
vm_paddr_t lapic_paddr;
int x2apic_mode;
int lapic_eoi_suppression;
+static int lapic_timer_tsc_deadline;
static u_long lapic_timer_divisor;
static struct eventtimer lapic_et;
@@ -167,6 +177,8 @@
SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, "");
SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD,
&lapic_eoi_suppression, 0, "");
+SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD,
+ &lapic_timer_tsc_deadline, 0, "");
static uint32_t
lapic_read32(enum LAPIC_REGISTERS reg)
@@ -260,6 +272,8 @@
u_int count, int enable_int);
static void lapic_timer_periodic(struct lapic *,
u_int count, int enable_int);
+static void lapic_timer_deadline(struct lapic *la, uint64_t when,
+ int enable_int);
static void lapic_timer_stop(struct lapic *);
static void lapic_timer_set_divisor(u_int divisor);
static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value);
@@ -450,7 +464,14 @@
if (!arat) {
lapic_et.et_flags |= ET_FLAGS_C3STOP;
lapic_et.et_quality -= 200;
+ } else if ((cpu_feature & CPUID_TSC) != 0 &&
+ (cpu_feature2 & CPUID2_TSCDLT) != 0 &&
+ tsc_is_invariant && tsc_freq != 0) {
+ lapic_timer_tsc_deadline = 1;
+ TUNABLE_INT_FETCH("hw.lapic_tsc_deadline",
+ &lapic_timer_tsc_deadline);
}
+
lapic_et.et_frequency = 0;
/* We don't know frequency yet, so trying to guess. */
lapic_et.et_min_period = 0x00001000LL;
@@ -604,23 +625,34 @@
}
/* Program timer LVT and setup handler. */
- la->lvt_timer_cache = lvt_mode(la, APIC_LVT_TIMER,
+ la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER,
lapic_read32(LAPIC_LVT_TIMER));
- lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_cache);
+ la->lvt_timer_last = la->lvt_timer_base;
+ lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base);
if (boot) {
snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
intrcnt_add(buf, &la->la_timer_count);
}
/* Setup the timer if configured. */
- if (la->la_timer_mode != 0) {
+ if (la->la_timer_mode != LAT_MODE_UNDEF) {
KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor",
lapic_id()));
lapic_timer_set_divisor(lapic_timer_divisor);
- if (la->la_timer_mode == 1)
+ switch (la->la_timer_mode) {
+ case LAT_MODE_PERIODIC:
lapic_timer_periodic(la, la->la_timer_period, 1);
- else
+ break;
+ case LAT_MODE_ONESHOT:
lapic_timer_oneshot(la, la->la_timer_period, 1);
+ break;
+ case LAT_MODE_DEADLINE:
+ lapic_timer_deadline(la, la->la_timer_period, 1);
+ break;
+ default:
+ panic("corrupted la_timer_mode %p %d", la,
+ la->la_timer_mode);
+ }
}
/* Program error LVT and clear any existing errors. */
@@ -722,45 +754,76 @@
#endif
}
+static void
+lapic_calibrate_initcount(struct eventtimer *et, struct lapic *la)
+{
+ u_long value;
+
+ /* Start off with a divisor of 2 (power on reset default). */
+ lapic_timer_divisor = 2;
+ /* Try to calibrate the local APIC timer. */
+ do {
+ lapic_timer_set_divisor(lapic_timer_divisor);
+ lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0);
+ DELAY(1000000);
+ value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER);
+ if (value != APIC_TIMER_MAX_COUNT)
+ break;
+ lapic_timer_divisor <<= 1;
+ } while (lapic_timer_divisor <= 128);
+ if (lapic_timer_divisor > 128)
+ panic("lapic: Divisor too big");
+ if (bootverbose) {
+ printf("lapic: Divisor %lu, Frequency %lu Hz\n",
+ lapic_timer_divisor, value);
+ }
+ et->et_frequency = value;
+}
+
+static void
+lapic_calibrate_deadline(struct eventtimer *et, struct lapic *la __unused)
+{
+ uint64_t value;
+
+ value = rdtsc();
+ DELAY(1000000);
+ value = rdtsc() - value;
+ if (bootverbose) {
+ printf("lapic: deadline tsc mode, Frequency %ju Hz\n",
+ (uintmax_t)value);
+ }
+ et->et_frequency = value;
+}
+
static int
lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
{
struct lapic *la;
- u_long value;
la = &lapics[PCPU_GET(apic_id)];
if (et->et_frequency == 0) {
- /* Start off with a divisor of 2 (power on reset default). */
- lapic_timer_divisor = 2;
- /* Try to calibrate the local APIC timer. */
- do {
- lapic_timer_set_divisor(lapic_timer_divisor);
- lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0);
- DELAY(1000000);
- value = APIC_TIMER_MAX_COUNT -
- lapic_read32(LAPIC_CCR_TIMER);
- if (value != APIC_TIMER_MAX_COUNT)
- break;
- lapic_timer_divisor <<= 1;
- } while (lapic_timer_divisor <= 128);
- if (lapic_timer_divisor > 128)
- panic("lapic: Divisor too big");
- if (bootverbose)
- printf("lapic: Divisor %lu, Frequency %lu Hz\n",
- lapic_timer_divisor, value);
- et->et_frequency = value;
+ if (lapic_timer_tsc_deadline)
+ lapic_calibrate_deadline(et, la);
+ else
+ lapic_calibrate_initcount(et, la);
et->et_min_period = (0x00000002LLU << 32) / et->et_frequency;
et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency;
}
- if (la->la_timer_mode == 0)
+ if (la->la_timer_mode == LAT_MODE_UNDEF)
lapic_timer_set_divisor(lapic_timer_divisor);
if (period != 0) {
- la->la_timer_mode = 1;
- la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 32;
+ la->la_timer_mode = LAT_MODE_PERIODIC;
+ la->la_timer_period = ((uint32_t)et->et_frequency * period) >>
+ 32;
lapic_timer_periodic(la, la->la_timer_period, 1);
+ } else if (lapic_timer_tsc_deadline) {
+ la->la_timer_mode = LAT_MODE_DEADLINE;
+ la->la_timer_period = (et->et_frequency * first) >> 32;
+ lapic_timer_deadline(la, la->la_timer_period, 1);
} else {
- la->la_timer_mode = 2;
- la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 32;
+ la->la_timer_mode = LAT_MODE_ONESHOT;
+ la->la_timer_period = ((uint32_t)et->et_frequency * first) >>
+ 32;
lapic_timer_oneshot(la, la->la_timer_period, 1);
}
return (0);
@@ -769,10 +832,11 @@
static int
lapic_et_stop(struct eventtimer *et)
{
- struct lapic *la = &lapics[PCPU_GET(apic_id)];
+ struct lapic *la;
- la->la_timer_mode = 0;
+ la = &lapics[PCPU_GET(apic_id)];
lapic_timer_stop(la);
+ la->la_timer_mode = LAT_MODE_UNDEF;
return (0);
}
@@ -1075,11 +1139,12 @@
{
uint32_t value;
- value = la->lvt_timer_cache;
+ value = la->lvt_timer_base;
value &= ~APIC_LVTT_TM;
value |= APIC_LVTT_TM_ONE_SHOT;
if (enable_int)
value &= ~APIC_LVT_M;
+ la->lvt_timer_last = value;
lapic_write32(LAPIC_LVT_TIMER, value);
lapic_write32(LAPIC_ICR_TIMER, count);
}
@@ -1089,23 +1154,48 @@
{
uint32_t value;
- value = la->lvt_timer_cache;
+ value = la->lvt_timer_base;
value &= ~APIC_LVTT_TM;
value |= APIC_LVTT_TM_PERIODIC;
if (enable_int)
value &= ~APIC_LVT_M;
+ la->lvt_timer_last = value;
lapic_write32(LAPIC_LVT_TIMER, value);
lapic_write32(LAPIC_ICR_TIMER, count);
}
static void
+lapic_timer_deadline(struct lapic *la, uint64_t when, int enable_int)
+{
+ uint32_t value;
+
+ value = la->lvt_timer_base;
+ value &= ~APIC_LVTT_TM;
+ value |= APIC_LVTT_TM_TSCDLT;
+ if (enable_int)
+ value &= ~APIC_LVT_M;
+ if (value != la->lvt_timer_last) {
+ la->lvt_timer_last = value;
+ lapic_write32(LAPIC_LVT_TIMER, value);
+ if (!x2apic_mode)
+ mfence();
+ }
+ wrmsr(MSR_TSC_DEADLINE, when + rdtsc());
+}
+
+static void
lapic_timer_stop(struct lapic *la)
{
uint32_t value;
- value = la->lvt_timer_cache;
+ if (la->la_timer_mode == LAT_MODE_DEADLINE) {
+ wrmsr(MSR_TSC_DEADLINE, 0);
+ mfence();
+ }
+ value = la->lvt_timer_base;
value &= ~APIC_LVTT_TM;
value |= APIC_LVT_M;
+ la->lvt_timer_last = value;
lapic_write32(LAPIC_LVT_TIMER, value);
}

File Metadata

Mime Type
text/plain
Expires
Sat, Nov 15, 6:19 AM (5 h, 46 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
25318294
Default Alt Text
D5738.id14595.diff (9 KB)

Event Timeline