For SMP, use prv_PPAGE1/prv_PMAP1 instead of PADDR1/PMAP1.

get_ptbase and pmap_pte_quick no longer generates IPIs.
This should reduce the number of IPIs during heavy paging.

18 files changed, 133 insertions, 35 deletions

diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index ddfc7551119f..89117f4ca709 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)genassym.c	5.11 (Berkeley) 5/10/91
- *	$Id: genassym.c,v 1.53 1998/04/06 15:40:10 peter Exp $
+ *	$Id: genassym.c,v 1.54 1998/04/06 18:59:14 peter Exp $
  */
 
 #include "opt_vm86.h"
@@ -219,6 +219,7 @@ main()
 	printf("#define\tGD_PRV_CMAP1 %d\n", &globaldata->prv_CMAP1);
 	printf("#define\tGD_PRV_CMAP2 %d\n", &globaldata->prv_CMAP2);
 	printf("#define\tGD_PRV_CMAP3 %d\n", &globaldata->prv_CMAP3);
+	printf("#define\tGD_PRV_PMAP1 %d\n", &globaldata->prv_PMAP1);
 	printf("#define\tGD_INSIDE_INTR %d\n", &globaldata->inside_intr);
 	printf("#define\tPS_GLOBALDATA 0x%x\n", &privatespace->globaldata);
 	printf("#define\tPS_PRVPT 0x%x\n", &privatespace->prvpt);
@@ -228,6 +229,7 @@ main()
 	printf("#define\tPS_CPAGE1 0x%x\n", &privatespace->CPAGE1);
 	printf("#define\tPS_CPAGE2 0x%x\n", &privatespace->CPAGE2);
 	printf("#define\tPS_CPAGE3 0x%x\n", &privatespace->CPAGE3);
+	printf("#define\tPS_PPAGE1 0x%x\n", &privatespace->PPAGE1);
 	printf("#define\tPS_IOAPICS 0x%x\n", &privatespace->ioapics);
 #endif
 
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index deec139f3579..dbd22bb33366 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.197 1998/05/15 07:25:25 dyson Exp $
+ *	$Id: pmap.c,v 1.198 1998/05/17 17:43:13 tegge Exp $
  */
 
 /*
@@ -190,8 +190,13 @@ extern pd_entry_t *IdlePTDS[];
 extern pt_entry_t SMP_prvpt[];
 #endif
 
+#ifdef SMP
+extern unsigned int prv_PPAGE1[];
+extern pt_entry_t *prv_PMAP1;
+#else
 static pt_entry_t *PMAP1 = 0;
 static unsigned *PADDR1 = 0;
+#endif
 
 static PMAP_INLINE void	free_pv_entry __P((pv_entry_t pv));
 static unsigned * get_ptbase __P((pmap_t pmap));
@@ -347,10 +352,12 @@ pmap_bootstrap(firstaddr, loadaddr)
 	SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
 	       atop(round_page(sizeof(struct msgbuf))))
 
+#if !defined(SMP)
 	/*
 	 * ptemap is used for pmap_pte_quick
 	 */
 	SYSMAP(unsigned *, PMAP1, PADDR1, 1);
+#endif
 
 	virtual_avail = va;
 
@@ -438,6 +445,7 @@ pmap_bootstrap(firstaddr, loadaddr)
 	prv_CMAP1 = &SMP_prvpt[3 + UPAGES];
 	prv_CMAP2 = &SMP_prvpt[4 + UPAGES];
 	prv_CMAP3 = &SMP_prvpt[5 + UPAGES];
+	prv_PMAP1 = &SMP_prvpt[6 + UPAGES];
 #endif
 
 	invltlb();
@@ -735,7 +743,12 @@ get_ptbase(pmap)
 	/* otherwise, we are alternate address space */
 	if (frame != (((unsigned) APTDpde) & PG_FRAME)) {
 		APTDpde = (pd_entry_t) (frame | PG_RW | PG_V);
+#if defined(SMP)
+		/* The page directory is not shared between CPUs */
+		cpu_invltlb();
+#else
 		invltlb();
+#endif
 	}
 	return (unsigned *) APTmap;
 }
@@ -763,11 +776,19 @@ pmap_pte_quick(pmap, va)
 			return (unsigned *) PTmap + index;
 		}
 		newpf = pde & PG_FRAME;
+#ifdef SMP
+		if ( ((* (unsigned *) prv_PMAP1) & PG_FRAME) != newpf) {
+			* (unsigned *) prv_PMAP1 = newpf | PG_RW | PG_V;
+			cpu_invlpg(&prv_PPAGE1);
+		}
+		return prv_PPAGE1 + ((unsigned) index & (NPTEPG - 1));
+#else
 		if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) {
 			* (unsigned *) PMAP1 = newpf | PG_RW | PG_V;
 			invltlb_1pg((vm_offset_t) PADDR1);
 		}
 		return PADDR1 + ((unsigned) index & (NPTEPG - 1));
+#endif
 	}
 	return (0);
 }
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index 1a3ed2bc1029..f4cf0c5737df 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -30,7 +30,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: cpufunc.h,v 1.77 1998/05/11 02:13:47 dyson Exp $
+ *	$Id: cpufunc.h,v 1.78 1998/05/12 18:28:05 dyson Exp $
  */
 
 /*
@@ -225,6 +225,20 @@ cpu_invlpg(void *addr)
 	__asm   __volatile("invlpg %0"::"m"(*(char *)addr):"memory");
 }
 
+static __inline void
+cpu_invltlb(void)
+{
+	u_long	temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3()
+	 * is inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
+			 : : "memory");
+#if defined(SWTCH_OPTIM_STATS)
+	++tlb_flush_count;
+#endif
+}
 #else  /* !SMP */
 
 static __inline void
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index e6b1dc80695f..efaa10a5c88d 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: globaldata.h,v 1.1 1998/04/06 15:37:21 peter Exp $
+ * $Id: globaldata.h,v 1.2 1998/04/06 18:59:15 peter Exp $
  */
 
 /*
@@ -57,6 +57,7 @@ struct globaldata {
 	pt_entry_t	*prv_CMAP1;
 	pt_entry_t	*prv_CMAP2;
 	pt_entry_t	*prv_CMAP3;
+	pt_entry_t	*prv_PMAP1;
 	int		inside_intr;
 #endif
 };
@@ -79,16 +80,17 @@ struct privatespace {
 	lapic_t		lapic;
 	char		__filler1[PAGE_SIZE - sizeof(lapic_t)];
 
-	/* page 3,4 - idle stack (2 pages) */
-	char		idlestack[2 * PAGE_SIZE];
+	/* page 3..2+UPAGES - idle stack (UPAGES pages) */
+	char		idlestack[UPAGES * PAGE_SIZE];
 
-	/* page 5,6,7 - CPAGE1,CPAGE2,CPAGE3 */
+	/* page 3+UPAGES..6+UPAGES - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */
 	char		CPAGE1[PAGE_SIZE];
 	char		CPAGE2[PAGE_SIZE];
 	char		CPAGE3[PAGE_SIZE];
+	char		PPAGE1[PAGE_SIZE];
 
-	/* page 8-15 - spare, unmapped */
-	char		__filler2[8 * PAGE_SIZE];
+	/* page 7+UPAGES..15 - spare, unmapped */
+	char		__filler2[(9-UPAGES) * PAGE_SIZE];
 
 	/* page 16-31 - space for IO apics */
 	char		ioapics[16 * PAGE_SIZE];
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c
index ddfc7551119f..89117f4ca709 100644
--- a/sys/i386/i386/genassym.c
+++ b/sys/i386/i386/genassym.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)genassym.c	5.11 (Berkeley) 5/10/91
- *	$Id: genassym.c,v 1.53 1998/04/06 15:40:10 peter Exp $
+ *	$Id: genassym.c,v 1.54 1998/04/06 18:59:14 peter Exp $
  */
 
 #include "opt_vm86.h"
@@ -219,6 +219,7 @@ main()
 	printf("#define\tGD_PRV_CMAP1 %d\n", &globaldata->prv_CMAP1);
 	printf("#define\tGD_PRV_CMAP2 %d\n", &globaldata->prv_CMAP2);
 	printf("#define\tGD_PRV_CMAP3 %d\n", &globaldata->prv_CMAP3);
+	printf("#define\tGD_PRV_PMAP1 %d\n", &globaldata->prv_PMAP1);
 	printf("#define\tGD_INSIDE_INTR %d\n", &globaldata->inside_intr);
 	printf("#define\tPS_GLOBALDATA 0x%x\n", &privatespace->globaldata);
 	printf("#define\tPS_PRVPT 0x%x\n", &privatespace->prvpt);
@@ -228,6 +229,7 @@ main()
 	printf("#define\tPS_CPAGE1 0x%x\n", &privatespace->CPAGE1);
 	printf("#define\tPS_CPAGE2 0x%x\n", &privatespace->CPAGE2);
 	printf("#define\tPS_CPAGE3 0x%x\n", &privatespace->CPAGE3);
+	printf("#define\tPS_PPAGE1 0x%x\n", &privatespace->PPAGE1);
 	printf("#define\tPS_IOAPICS 0x%x\n", &privatespace->ioapics);
 #endif
 
diff --git a/sys/i386/i386/globals.s b/sys/i386/i386/globals.s
index 0a297a63e58d..dee81e6698e0 100644
--- a/sys/i386/i386/globals.s
+++ b/sys/i386/i386/globals.s
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: globals.s,v 1.1 1998/04/06 15:39:09 peter Exp $
+ * $Id: globals.s,v 1.2 1998/04/06 18:59:14 peter Exp $
  */
 
 #include "opt_vm86.h"
@@ -43,7 +43,7 @@
 	.set	_SMP_prvstart,(MPPTDI << PDRSHIFT)
 
 	.globl	globaldata,_SMP_prvpt,_lapic,_SMP_ioapic
-	.globl	_prv_CPAGE1,_prv_CPAGE2,_prv_CPAGE3
+	.globl	_prv_CPAGE1,_prv_CPAGE2,_prv_CPAGE3,_prv_PPAGE1
 	.globl	_idlestack,_idlestack_top
 
 	.set	globaldata,_SMP_prvstart + PS_GLOBALDATA
@@ -54,6 +54,7 @@
 	.set	_prv_CPAGE1,_SMP_prvstart + PS_CPAGE1
 	.set	_prv_CPAGE2,_SMP_prvstart + PS_CPAGE2
 	.set	_prv_CPAGE3,_SMP_prvstart + PS_CPAGE3
+	.set	_prv_PPAGE1,_SMP_prvstart + PS_PPAGE1
 	.set	_SMP_ioapic,_SMP_prvstart + PS_IOAPICS
 #endif
 
@@ -86,7 +87,7 @@ globaldata:
 	 * the AP versions are setup in mp_machdep.c.
 	 */
 	.globl	_cpuid,_cpu_lockid,_other_cpus,_my_idlePTD,_ss_tpr
-	.globl	_prv_CMAP1,_prv_CMAP2,_prv_CMAP3
+	.globl	_prv_CMAP1,_prv_CMAP2,_prv_CMAP3,_prv_PMAP1
 	.globl	_inside_intr
 
 	.set	_cpuid,globaldata + GD_CPUID
@@ -97,6 +98,7 @@ globaldata:
 	.set	_prv_CMAP1,globaldata + GD_PRV_CMAP1
 	.set	_prv_CMAP2,globaldata + GD_PRV_CMAP2
 	.set	_prv_CMAP3,globaldata + GD_PRV_CMAP3
+	.set	_prv_PMAP1,globaldata + GD_PRV_PMAP1
 	.set	_inside_intr,globaldata + GD_INSIDE_INTR
 #endif
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index deec139f3579..dbd22bb33366 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.197 1998/05/15 07:25:25 dyson Exp $
+ *	$Id: pmap.c,v 1.198 1998/05/17 17:43:13 tegge Exp $
  */
 
 /*
@@ -190,8 +190,13 @@ extern pd_entry_t *IdlePTDS[];
 extern pt_entry_t SMP_prvpt[];
 #endif
 
+#ifdef SMP
+extern unsigned int prv_PPAGE1[];
+extern pt_entry_t *prv_PMAP1;
+#else
 static pt_entry_t *PMAP1 = 0;
 static unsigned *PADDR1 = 0;
+#endif
 
 static PMAP_INLINE void	free_pv_entry __P((pv_entry_t pv));
 static unsigned * get_ptbase __P((pmap_t pmap));
@@ -347,10 +352,12 @@ pmap_bootstrap(firstaddr, loadaddr)
 	SYSMAP(struct msgbuf *, msgbufmap, msgbufp,
 	       atop(round_page(sizeof(struct msgbuf))))
 
+#if !defined(SMP)
 	/*
 	 * ptemap is used for pmap_pte_quick
 	 */
 	SYSMAP(unsigned *, PMAP1, PADDR1, 1);
+#endif
 
 	virtual_avail = va;
 
@@ -438,6 +445,7 @@ pmap_bootstrap(firstaddr, loadaddr)
 	prv_CMAP1 = &SMP_prvpt[3 + UPAGES];
 	prv_CMAP2 = &SMP_prvpt[4 + UPAGES];
 	prv_CMAP3 = &SMP_prvpt[5 + UPAGES];
+	prv_PMAP1 = &SMP_prvpt[6 + UPAGES];
 #endif
 
 	invltlb();
@@ -735,7 +743,12 @@ get_ptbase(pmap)
 	/* otherwise, we are alternate address space */
 	if (frame != (((unsigned) APTDpde) & PG_FRAME)) {
 		APTDpde = (pd_entry_t) (frame | PG_RW | PG_V);
+#if defined(SMP)
+		/* The page directory is not shared between CPUs */
+		cpu_invltlb();
+#else
 		invltlb();
+#endif
 	}
 	return (unsigned *) APTmap;
 }
@@ -763,11 +776,19 @@ pmap_pte_quick(pmap, va)
 			return (unsigned *) PTmap + index;
 		}
 		newpf = pde & PG_FRAME;
+#ifdef SMP
+		if ( ((* (unsigned *) prv_PMAP1) & PG_FRAME) != newpf) {
+			* (unsigned *) prv_PMAP1 = newpf | PG_RW | PG_V;
+			cpu_invlpg(&prv_PPAGE1);
+		}
+		return prv_PPAGE1 + ((unsigned) index & (NPTEPG - 1));
+#else
 		if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) {
 			* (unsigned *) PMAP1 = newpf | PG_RW | PG_V;
 			invltlb_1pg((vm_offset_t) PADDR1);
 		}
 		return PADDR1 + ((unsigned) index & (NPTEPG - 1));
+#endif
 	}
 	return (0);
 }
diff --git a/sys/i386/include/asnames.h b/sys/i386/include/asnames.h
index 5f7c206789fb..37cd14bd4048 100644
--- a/sys/i386/include/asnames.h
+++ b/sys/i386/include/asnames.h
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: asnames.h,v 1.20 1998/04/22 09:46:16 peter Exp $
+ * $Id: asnames.h,v 1.21 1998/05/17 11:52:06 phk Exp $
  */
 
 #ifndef _MACHINE_ASNAMES_H_
@@ -306,9 +306,11 @@
 #define _prv_CMAP1			prv_CMAP1
 #define _prv_CMAP2			prv_CMAP2
 #define _prv_CMAP3			prv_CMAP3
+#define _prv_PMAP1			prv_PMAP1
 #define _prv_CPAGE1			prv_CPAGE1
 #define _prv_CPAGE2			prv_CPAGE2
 #define _prv_CPAGE3			prv_CPAGE3
+#define _prv_PPAGE1			prv_PPAGE1
 #define _qs				qs
 #define _rcpoll				rcpoll
 #define _real_2op_NaN			real_2op_NaN
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
index 1a3ed2bc1029..f4cf0c5737df 100644
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -30,7 +30,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: cpufunc.h,v 1.77 1998/05/11 02:13:47 dyson Exp $
+ *	$Id: cpufunc.h,v 1.78 1998/05/12 18:28:05 dyson Exp $
  */
 
 /*
@@ -225,6 +225,20 @@ cpu_invlpg(void *addr)
 	__asm   __volatile("invlpg %0"::"m"(*(char *)addr):"memory");
 }
 
+static __inline void
+cpu_invltlb(void)
+{
+	u_long	temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3()
+	 * is inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
+			 : : "memory");
+#if defined(SWTCH_OPTIM_STATS)
+	++tlb_flush_count;
+#endif
+}
 #else  /* !SMP */
 
 static __inline void
diff --git a/sys/i386/include/globaldata.h b/sys/i386/include/globaldata.h
index e6b1dc80695f..efaa10a5c88d 100644
--- a/sys/i386/include/globaldata.h
+++ b/sys/i386/include/globaldata.h
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: globaldata.h,v 1.1 1998/04/06 15:37:21 peter Exp $
+ * $Id: globaldata.h,v 1.2 1998/04/06 18:59:15 peter Exp $
  */
 
 /*
@@ -57,6 +57,7 @@ struct globaldata {
 	pt_entry_t	*prv_CMAP1;
 	pt_entry_t	*prv_CMAP2;
 	pt_entry_t	*prv_CMAP3;
+	pt_entry_t	*prv_PMAP1;
 	int		inside_intr;
 #endif
 };
@@ -79,16 +80,17 @@ struct privatespace {
 	lapic_t		lapic;
 	char		__filler1[PAGE_SIZE - sizeof(lapic_t)];
 
-	/* page 3,4 - idle stack (2 pages) */
-	char		idlestack[2 * PAGE_SIZE];
+	/* page 3..2+UPAGES - idle stack (UPAGES pages) */
+	char		idlestack[UPAGES * PAGE_SIZE];
 
-	/* page 5,6,7 - CPAGE1,CPAGE2,CPAGE3 */
+	/* page 3+UPAGES..6+UPAGES - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */
 	char		CPAGE1[PAGE_SIZE];
 	char		CPAGE2[PAGE_SIZE];
 	char		CPAGE3[PAGE_SIZE];
+	char		PPAGE1[PAGE_SIZE];
 
-	/* page 8-15 - spare, unmapped */
-	char		__filler2[8 * PAGE_SIZE];
+	/* page 7+UPAGES..15 - spare, unmapped */
+	char		__filler2[(9-UPAGES) * PAGE_SIZE];
 
 	/* page 16-31 - space for IO apics */
 	char		ioapics[16 * PAGE_SIZE];
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h
index e6b1dc80695f..efaa10a5c88d 100644
--- a/sys/i386/include/pcpu.h
+++ b/sys/i386/include/pcpu.h
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: globaldata.h,v 1.1 1998/04/06 15:37:21 peter Exp $
+ * $Id: globaldata.h,v 1.2 1998/04/06 18:59:15 peter Exp $
  */
 
 /*
@@ -57,6 +57,7 @@ struct globaldata {
 	pt_entry_t	*prv_CMAP1;
 	pt_entry_t	*prv_CMAP2;
 	pt_entry_t	*prv_CMAP3;
+	pt_entry_t	*prv_PMAP1;
 	int		inside_intr;
 #endif
 };
@@ -79,16 +80,17 @@ struct privatespace {
 	lapic_t		lapic;
 	char		__filler1[PAGE_SIZE - sizeof(lapic_t)];
 
-	/* page 3,4 - idle stack (2 pages) */
-	char		idlestack[2 * PAGE_SIZE];
+	/* page 3..2+UPAGES - idle stack (UPAGES pages) */
+	char		idlestack[UPAGES * PAGE_SIZE];
 
-	/* page 5,6,7 - CPAGE1,CPAGE2,CPAGE3 */
+	/* page 3+UPAGES..6+UPAGES - CPAGE1,CPAGE2,CPAGE3,PPAGE1 */
 	char		CPAGE1[PAGE_SIZE];
 	char		CPAGE2[PAGE_SIZE];
 	char		CPAGE3[PAGE_SIZE];
+	char		PPAGE1[PAGE_SIZE];
 
-	/* page 8-15 - spare, unmapped */
-	char		__filler2[8 * PAGE_SIZE];
+	/* page 7+UPAGES..15 - spare, unmapped */
+	char		__filler2[(9-UPAGES) * PAGE_SIZE];
 
 	/* page 16-31 - space for IO apics */
 	char		ioapics[16 * PAGE_SIZE];
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 8d8873bb3a06..7b58b87d385b 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.73 1998/04/06 15:48:30 peter Exp $
+ *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
  */
 
 #include "opt_smp.h"
@@ -1717,6 +1717,7 @@ start_all_aps(u_int boot_addr)
 		newpt[3 + UPAGES] = 0;		/* *prv_CMAP1 */
 		newpt[4 + UPAGES] = 0;		/* *prv_CMAP2 */
 		newpt[5 + UPAGES] = 0;		/* *prv_CMAP3 */
+		newpt[6 + UPAGES] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
 		gd->cpuid = x;
@@ -1725,6 +1726,7 @@ start_all_aps(u_int boot_addr)
 		gd->prv_CMAP1 = &newpt[3 + UPAGES];
 		gd->prv_CMAP2 = &newpt[4 + UPAGES];
 		gd->prv_CMAP3 = &newpt[5 + UPAGES];
+		gd->prv_PMAP1 = &newpt[6 + UPAGES];
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;