Microoptimize LAPIC timer routines to avoid reading from hardware during

programming using earlier cached values. This makes respective routines to
disappear from PMC top and reduces total number of active CPU cycles on idle
24-core system by 10%.

1 files changed, 24 insertions, 19 deletions

diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
index 99a681b01044..e9941729463a 100644
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -119,6 +119,7 @@ struct lapic {
 	u_long *la_timer_count;
 	u_long la_timer_period;
 	u_int la_timer_mode;
+	uint32_t lvt_timer_cache;
 	/* Include IDT_SYSCALL to make indexing easier. */
 	int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 } static lapics[MAX_APIC_ID + 1];
@@ -160,9 +161,11 @@ static struct eventtimer lapic_et;
 
 static void	lapic_enable(void);
 static void	lapic_resume(struct pic *pic);
-static void	lapic_timer_oneshot(u_int count, int enable_int);
-static void	lapic_timer_periodic(u_int count, int enable_int);
-static void	lapic_timer_stop(void);
+static void	lapic_timer_oneshot(struct lapic *,
+		    u_int count, int enable_int);
+static void	lapic_timer_periodic(struct lapic *,
+		    u_int count, int enable_int);
+static void	lapic_timer_stop(struct lapic *);
 static void	lapic_timer_set_divisor(u_int divisor);
 static uint32_t	lvt_mode(struct lapic *la, u_int pin, uint32_t value);
 static int	lapic_et_start(struct eventtimer *et,
@@ -370,7 +373,8 @@ lapic_setup(int boot)
 		lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint);
 
 	/* Program timer LVT and setup handler. */
-	lapic->lvt_timer = lvt_mode(la, LVT_TIMER, lapic->lvt_timer);
+	la->lvt_timer_cache = lapic->lvt_timer =
+	    lvt_mode(la, LVT_TIMER, lapic->lvt_timer);
 	if (boot) {
 		snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid));
 		intrcnt_add(buf, &la->la_timer_count);
@@ -382,9 +386,9 @@ lapic_setup(int boot)
 		    lapic_id()));
 		lapic_timer_set_divisor(lapic_timer_divisor);
 		if (la->la_timer_mode == 1)
-			lapic_timer_periodic(la->la_timer_period, 1);
+			lapic_timer_periodic(la, la->la_timer_period, 1);
 		else
-			lapic_timer_oneshot(la->la_timer_period, 1);
+			lapic_timer_oneshot(la, la->la_timer_period, 1);
 	}
 
 	/* Program error LVT and clear any existing errors. */
@@ -489,13 +493,14 @@ lapic_et_start(struct eventtimer *et,
 	struct lapic *la;
 	u_long value;
 
+	la = &lapics[PCPU_GET(apic_id)];
 	if (et->et_frequency == 0) {
 		/* Start off with a divisor of 2 (power on reset default). */
 		lapic_timer_divisor = 2;
 		/* Try to calibrate the local APIC timer. */
 		do {
 			lapic_timer_set_divisor(lapic_timer_divisor);
-			lapic_timer_oneshot(APIC_TIMER_MAX_COUNT, 0);
+			lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0);
 			DELAY(1000000);
 			value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer;
 			if (value != APIC_TIMER_MAX_COUNT)
@@ -515,22 +520,22 @@ lapic_et_start(struct eventtimer *et,
 		et->et_max_period.frac =
 		    ((0xfffffffeLLU << 32) / et->et_frequency) << 32;
 	}
-	lapic_timer_set_divisor(lapic_timer_divisor);
-	la = &lapics[lapic_id()];
+	if (la->la_timer_mode == 0)
+		lapic_timer_set_divisor(lapic_timer_divisor);
 	if (period != NULL) {
 		la->la_timer_mode = 1;
 		la->la_timer_period =
 		    (et->et_frequency * (period->frac >> 32)) >> 32;
 		if (period->sec != 0)
 			la->la_timer_period += et->et_frequency * period->sec;
-		lapic_timer_periodic(la->la_timer_period, 1);
+		lapic_timer_periodic(la, la->la_timer_period, 1);
 	} else {
 		la->la_timer_mode = 2;
 		la->la_timer_period =
 		    (et->et_frequency * (first->frac >> 32)) >> 32;
 		if (first->sec != 0)
 			la->la_timer_period += et->et_frequency * first->sec;
-		lapic_timer_oneshot(la->la_timer_period, 1);
+		lapic_timer_oneshot(la, la->la_timer_period, 1);
 	}
 	return (0);
 }
@@ -538,10 +543,10 @@ lapic_et_start(struct eventtimer *et,
 static int
 lapic_et_stop(struct eventtimer *et)
 {
-	struct lapic *la = &lapics[lapic_id()];
+	struct lapic *la = &lapics[PCPU_GET(apic_id)];
 
 	la->la_timer_mode = 0;
-	lapic_timer_stop();
+	lapic_timer_stop(la);
 	return (0);
 }
 
@@ -835,11 +840,11 @@ lapic_timer_set_divisor(u_int divisor)
 }
 
 static void
-lapic_timer_oneshot(u_int count, int enable_int)
+lapic_timer_oneshot(struct lapic *la, u_int count, int enable_int)
 {
 	u_int32_t value;
 
-	value = lapic->lvt_timer;
+	value = la->lvt_timer_cache;
 	value &= ~APIC_LVTT_TM;
 	value |= APIC_LVTT_TM_ONE_SHOT;
 	if (enable_int)
@@ -849,11 +854,11 @@ lapic_timer_oneshot(u_int count, int enable_int)
 }
 
 static void
-lapic_timer_periodic(u_int count, int enable_int)
+lapic_timer_periodic(struct lapic *la, u_int count, int enable_int)
 {
 	u_int32_t value;
 
-	value = lapic->lvt_timer;
+	value = la->lvt_timer_cache;
 	value &= ~APIC_LVTT_TM;
 	value |= APIC_LVTT_TM_PERIODIC;
 	if (enable_int)
@@ -863,11 +868,11 @@ lapic_timer_periodic(u_int count, int enable_int)
 }
 
 static void
-lapic_timer_stop(void)
+lapic_timer_stop(struct lapic *la)
 {
 	u_int32_t value;
 
-	value = lapic->lvt_timer;
+	value = la->lvt_timer_cache;
 	value &= ~APIC_LVTT_TM;
 	value |= APIC_LVT_M;
 	lapic->lvt_timer = value;