Man the liferafts! Here comes the long awaited SMP -> -current merge!

There are various options documented in i386/conf/LINT, there is more to
come over the next few days.

The kernel should run pretty much "as before" without the options to
activate SMP mode.

There are a handful of known "loose ends" that need to be fixed, but
have been put off since the SMP kernel is in a moderately good condition
at the moment.

This commit is the result of the tinkering and testing over the last 14
months by many people.  A special thanks to Steve Passe for implementing
the APIC code!

35 files changed, 7537 insertions, 153 deletions

diff --git a/sys/amd64/amd64/autoconf.c b/sys/amd64/amd64/autoconf.c
index 5d297bfa953f..70e0b137dfee 100644
--- a/sys/amd64/amd64/autoconf.c
+++ b/sys/amd64/amd64/autoconf.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)autoconf.c	7.1 (Berkeley) 5/9/91
- *	$Id$
+ *	$Id: autoconf.c,v 1.64 1997/02/22 09:32:08 peter Exp $
  */
 
 /*
@@ -45,6 +45,8 @@
  * devices are determined (from possibilities mentioned in ioconf.c),
  * and the drivers are initialized.
  */
+#include "opt_smp.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/buf.h>
@@ -61,6 +63,10 @@
 #include <machine/md_var.h>
 #include <i386/isa/icu.h> /* For interrupts */
 
+#if defined(APIC_IO)
+#include <machine/smp.h>
+#endif /* APIC_IO */
+
 #include "isa.h"
 #if NISA > 0
 #include <i386/isa/isa_device.h>
@@ -158,8 +164,18 @@ configure(dummy)
 	configure_start();
 
 	/* Allow all routines to decide for themselves if they want intrs */
+#if defined(APIC_IO)
+	configure_local_apic();
+        enable_intr();
+#else
         enable_intr();
         INTREN(IRQ_SLAVE);
+#endif /* APIC_IO */
+
+#if NCRD > 0
+	/* Before isa_configure to avoid ISA drivers finding our cards */
+	pccard_configure();
+#endif
 
 #if NEISA > 0
 	eisa_configure();
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index a74cd09b5a6c..014ddd2f2c57 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -33,16 +33,24 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.46 1997/04/20 06:41:26 phk Exp $
+ *	$Id: swtch.s,v 1.47 1997/04/22 06:55:29 jdp Exp $
  */
 
 #include "npx.h"
 #include "opt_user_ldt.h"
+#include "opt_smp.h"
+#include "opt_smp_privpages.h"
 
 #include <sys/rtprio.h>
 
 #include <machine/asmacros.h>
 #include <machine/spl.h>
+#include <machine/smpasm.h>
+#include <machine/smptests.h>		/** TEST_LOPRIO */
+
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#include <machine/pmap.h>
+#endif
 
 #include "assym.s"
 
@@ -61,8 +69,12 @@
  * queues.
  */
 	.data
+#ifndef SMP
 	.globl	_curpcb
 _curpcb:	.long	0			/* pointer to curproc's PCB area */
+#endif
+	.globl	_whichqs, _whichrtqs, _whichidqs
+
 _whichqs:	.long	0			/* which run queues have data */
 _whichrtqs:	.long	0			/* which realtime run queues have data */
 _whichidqs:	.long	0			/* which idletime run queues have data */
@@ -229,16 +241,30 @@ rem3id:	.asciz	"remrq.id"
 /*
  * When no processes are on the runq, cpu_switch() branches to _idle
  * to wait for something to come ready.
+ *
+ * NOTE: on an SMP system this routine is a startup-only code path.
+ * once initialization is over, meaning the idle procs have been
+ * created, we should NEVER branch here.
  */
 	ALIGN_TEXT
 _idle:
+#ifdef SMP
+	movl	_smp_active, %eax
+	cmpl	$0, %eax
+	jnz	badsw
+#endif /* SMP */
 	xorl	%ebp,%ebp
 	movl	$HIDENAME(tmpstk),%esp
 	movl	_IdlePTD,%ecx
 	movl	%ecx,%cr3
 
 	/* update common_tss.tss_esp0 pointer */
+#ifdef SMP
+	GETCPUID(%eax)
+	movl	_SMPcommon_tss_ptr(,%eax,4), %eax
+#else
 	movl	$_common_tss, %eax
+#endif
 	movl	%esp, TSS_ESP0(%eax)
 
 #ifdef TSS_IS_CACHED				/* example only */
@@ -288,13 +314,20 @@ ENTRY(default_halt)
  * cpu_switch()
  */
 ENTRY(cpu_switch)
+	
 	/* switch to new process. first, save context as needed */
-	movl	_curproc,%ecx
+	GETCURPROC(%ecx)
 
 	/* if no process to save, don't bother */
 	testl	%ecx,%ecx
 	je	sw1
 
+#ifdef SMP
+	movb	P_ONCPU(%ecx), %al		/* save "last" cpu */
+	movb	%al, P_LASTCPU(%ecx)
+	movb	$0xff, P_ONCPU(%ecx)		/* "leave" the cpu */
+#endif
+
 	movl	P_ADDR(%ecx),%ecx
 
 	movl	(%esp),%eax			/* Hardware registers */
@@ -305,10 +338,19 @@ ENTRY(cpu_switch)
 	movl	%esi,PCB_ESI(%ecx)
 	movl	%edi,PCB_EDI(%ecx)
 
+#ifdef SMP
+	movl	_mp_lock, %eax
+	cmpl	$0xffffffff, %eax		/* is it free? */
+	je	badsw				/* yes, bad medicine! */
+	andl	$0x00ffffff, %eax		/* clear CPU portion */
+	movl	%eax,PCB_MPNEST(%ecx)		/* store it */
+#endif /* SMP */
+
 #if NNPX > 0
 	/* have we used fp, and need a save? */
-	mov	_curproc,%eax
-	cmp	%eax,_npxproc
+	GETCURPROC(%eax)
+	GETNPXPROC(%ebx)
+	cmp	%eax,%ebx
 	jne	1f
 	addl	$PCB_SAVEFPU,%ecx		/* h/w bugs make saving complicated */
 	pushl	%ecx
@@ -319,7 +361,7 @@ ENTRY(cpu_switch)
 
 	movb	$1,_intr_nesting_level		/* charge Intr, not Sys/Idle */
 
-	movl	$0,_curproc			/* out of process */
+	SETCURPROC($0, %edi)
 
 	/* save is done, now choose a new process or idle */
 sw1:
@@ -417,9 +459,30 @@ swtch_com:
 	movl	P_ADDR(%ecx),%edx
 	movl	PCB_CR3(%edx),%ebx
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+	/* Grab the private PT pointer from the outgoing process's PTD */
+	movl	$_PTD,%esi
+	movl	4*MPPTDI(%esi), %eax		/* fetch cpu's prv pt */
+#endif
+
 	/* switch address space */
 	movl	%ebx,%cr3
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+	/* Copy the private PT to the new process's PTD */
+	/* XXX yuck, the _PTD changes when we switch, so we have to
+	 * reload %cr3 after changing the address space.
+	 * We need to fix this by storing a pointer to the virtual
+	 * location of the per-process PTD in the PCB or something quick.
+	 * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm.
+	 */
+	movl	$_PTD,%esi
+	movl	%eax, 4*MPPTDI(%esi)		/* restore cpu's prv page */
+
+	/* XXX: we have just changed the page tables.. reload.. */
+	movl	%ebx,%cr3
+#endif
+
 #ifdef HOW_TO_SWITCH_TSS			/* example only */
 	/* Fix up tss pointer to floating pcb/stack structure */
 	/* XXX probably lots faster to store the 64 bits of tss entry
@@ -435,7 +498,12 @@ swtch_com:
 #endif
 
 	/* update common_tss.tss_esp0 pointer */
+#ifdef SMP
+	GETCPUID(%eax)
+	movl	_SMPcommon_tss_ptr(,%eax,4), %eax
+#else
 	movl	$_common_tss, %eax
+#endif
 	movl	%edx, %ebx			/* pcb */
 	addl	$(UPAGES * PAGE_SIZE), %ebx
 	movl	%ebx, TSS_ESP0(%eax)
@@ -457,9 +525,32 @@ swtch_com:
 	movl	PCB_EIP(%edx),%eax
 	movl	%eax,(%esp)
 
-	movl	%edx,_curpcb
-	movl	%ecx,_curproc			/* into next process */
+#ifdef SMP
+	GETCPUID(%eax)
+	movb	%al, P_ONCPU(%ecx)
+#endif
+	SETCURPCB(%edx, %eax)
+	SETCURPROC(%ecx, %eax)
+
 	movb	$0,_intr_nesting_level
+#ifdef SMP
+	movl	_apic_base, %eax		/* base addr of LOCAL APIC */
+#if defined(TEST_LOPRIO)
+	/**
+	 * FIXME: the belief here is that we ALWAYS leave here
+	 *         holding a the lock, is this TRUE???
+	 */
+	pushl	%edx
+	movl	APIC_TPR(%eax), %edx		/* get TPR register contents */
+	andl	$~0xff, %edx			/* clear the prio field */
+	movl	%edx, APIC_TPR(%eax)		/* now hold loprio for INTs */
+	popl	%edx
+#endif /* TEST_LOPRIO */
+	movl	APIC_ID(%eax), %eax		/* APIC ID register */
+	andl	$APIC_ID_MASK, %eax		/* extract ID portion */
+	orl	PCB_MPNEST(%edx), %eax		/* add count from PROC */
+	movl	%eax, _mp_lock			/* load the mp_lock */
+#endif /* SMP */
 
 #ifdef	USER_LDT
 	cmpl	$0, PCB_USERLDT(%edx)
@@ -520,7 +611,7 @@ ENTRY(savectx)
 	 * have to handle h/w bugs for reloading.  We used to lose the
 	 * parent's npx state for forks by forgetting to reload.
 	 */
-	mov	_npxproc,%eax
+	GETNPXPROC(%eax)
 	testl	%eax,%eax
 	je	1f
 
diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c
index c286027d53bd..adee9ff7fe47 100644
--- a/sys/amd64/amd64/db_interface.c
+++ b/sys/amd64/amd64/db_interface.c
@@ -23,7 +23,7 @@
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  *
- *	$Id: db_interface.c,v 1.26 1997/02/22 09:32:14 peter Exp $
+ *	$Id: db_interface.c,v 1.27 1997/03/28 12:56:07 bde Exp $
  */
 
 /*
@@ -265,6 +265,8 @@ Debugger(msg)
 
 	if (!in_Debugger) {
 	    in_Debugger = 1;
+	    printf("CR0=%08x",rcr0());
+	    printf("CR3=%08x",rcr3());
 	    db_printf("Debugger(\"%s\")\n", msg);
 	    breakpoint();
 	    in_Debugger = 0;
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
index 546df763b208..f8a934cf05ed 100644
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -23,7 +23,7 @@
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  *
- *	$Id: db_trace.c,v 1.19 1997/02/28 12:36:18 bde Exp $
+ *	$Id: db_trace.c,v 1.20 1997/03/01 05:44:09 ache Exp $
  */
 
 #include <sys/param.h>
@@ -199,6 +199,8 @@ db_nextframe(fp, ip)
 			db_printf(
 		    "--- trap %#n, eip = %#n, esp = %#n, ebp = %#n ---\n",
 			    tf->tf_trapno, eip, esp, ebp);
+			db_printf("--- curproc = 0x%x, pid = %d ---\n",
+			    curproc, curproc ? curproc->p_pid : 0);
 		}
 		break;
 	case SYSCALL:
@@ -208,6 +210,8 @@ db_nextframe(fp, ip)
 			db_printf(
 		    "--- syscall %#n, eip = %#n, esp = %#n, ebp = %#n ---\n",
 			    tf->tf_eax, eip, esp, ebp);
+			db_printf("--- curproc = 0x%x, pid = %d ---\n",
+			    curproc, curproc ? curproc->p_pid : 0);
 		}
 		break;
 	case INTERRUPT:
@@ -218,6 +222,8 @@ db_nextframe(fp, ip)
 			db_printf(
 		    "--- interrupt, eip = %#n, esp = %#n, ebp = %#n ---\n",
 			    eip, esp, ebp);
+			db_printf("--- curproc = 0x%x, pid = %d ---\n",
+			    curproc, curproc ? curproc->p_pid : 0);
 		}
 		break;
 	default:
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 0ab3b33952e2..2705c08efcfe 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -30,9 +30,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: exception.s,v 1.23 1997/04/07 11:42:09 peter Exp $
+ *	$Id: exception.s,v 1.24 1997/04/13 16:58:08 bde Exp $
  */
 
+#include "opt_smp.h"
 #include "npx.h"				/* NNPX */
 #include "assym.s"				/* system defines */
 #include <sys/errno.h>				/* error return codes */
@@ -42,6 +43,7 @@
 #include <sys/syscall.h>			/* syscall numbers */
 #include <machine/asmacros.h>			/* miscellaneous macros */
 #include <sys/cdefs.h>				/* CPP macros */
+#include <machine/smpasm.h>
 
 #define	KDSEL		0x10			/* kernel data selector */
 #define	SEL_RPL_MASK	0x0003
@@ -138,6 +140,9 @@ IDTVEC(fpu)
 	movl	_cpl,%eax
 	pushl	%eax
 	pushl	$0				/* dummy unit to finish building intr frame */
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_TRAP
 	orl	$SWI_AST_MASK,%eax
 	movl	%eax,_cpl
@@ -163,6 +168,9 @@ alltraps_with_regs_pushed:
 	movl	%ax,%es
 	FAKE_MCOUNT(12*4(%esp))
 calltrap:
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	FAKE_MCOUNT(_btrap)			/* init "from" _btrap -> calltrap */
 	incl	_cnt+V_TRAP
 	orl	$SWI_AST_MASK,_cpl
@@ -212,6 +220,9 @@ IDTVEC(syscall)
 	movl	%eax,TF_EFLAGS(%esp)
 	movl	$7,TF_ERR(%esp) 		/* sizeof "lcall 7,0" */
 	FAKE_MCOUNT(12*4(%esp))
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_SYSCALL
 	movl	$SWI_AST_MASK,_cpl
 	call	_syscall
@@ -238,6 +249,9 @@ IDTVEC(int0x80_syscall)
 	movl	%ax,%es
 	movl	$2,TF_ERR(%esp)			/* sizeof "int 0x80" */
 	FAKE_MCOUNT(12*4(%esp))
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_SYSCALL
 	movl	$SWI_AST_MASK,_cpl
 	call	_syscall
@@ -254,7 +268,13 @@ ENTRY(fork_trampoline)
 	movl	$SWI_AST_MASK,_cpl
 	call	_splz
 
+#if defined(SMP)
+	GETCPUID(%eax)
+	leal	_SMPruntime(,%eax,8), %eax
+	pushl	%eax
+#else
 	pushl	$_runtime
+#endif
 	call	_microtime
 	popl	%eax
 
diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s
index 0ab3b33952e2..2705c08efcfe 100644
--- a/sys/amd64/amd64/exception.s
+++ b/sys/amd64/amd64/exception.s
@@ -30,9 +30,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: exception.s,v 1.23 1997/04/07 11:42:09 peter Exp $
+ *	$Id: exception.s,v 1.24 1997/04/13 16:58:08 bde Exp $
  */
 
+#include "opt_smp.h"
 #include "npx.h"				/* NNPX */
 #include "assym.s"				/* system defines */
 #include <sys/errno.h>				/* error return codes */
@@ -42,6 +43,7 @@
 #include <sys/syscall.h>			/* syscall numbers */
 #include <machine/asmacros.h>			/* miscellaneous macros */
 #include <sys/cdefs.h>				/* CPP macros */
+#include <machine/smpasm.h>
 
 #define	KDSEL		0x10			/* kernel data selector */
 #define	SEL_RPL_MASK	0x0003
@@ -138,6 +140,9 @@ IDTVEC(fpu)
 	movl	_cpl,%eax
 	pushl	%eax
 	pushl	$0				/* dummy unit to finish building intr frame */
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_TRAP
 	orl	$SWI_AST_MASK,%eax
 	movl	%eax,_cpl
@@ -163,6 +168,9 @@ alltraps_with_regs_pushed:
 	movl	%ax,%es
 	FAKE_MCOUNT(12*4(%esp))
 calltrap:
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	FAKE_MCOUNT(_btrap)			/* init "from" _btrap -> calltrap */
 	incl	_cnt+V_TRAP
 	orl	$SWI_AST_MASK,_cpl
@@ -212,6 +220,9 @@ IDTVEC(syscall)
 	movl	%eax,TF_EFLAGS(%esp)
 	movl	$7,TF_ERR(%esp) 		/* sizeof "lcall 7,0" */
 	FAKE_MCOUNT(12*4(%esp))
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_SYSCALL
 	movl	$SWI_AST_MASK,_cpl
 	call	_syscall
@@ -238,6 +249,9 @@ IDTVEC(int0x80_syscall)
 	movl	%ax,%es
 	movl	$2,TF_ERR(%esp)			/* sizeof "int 0x80" */
 	FAKE_MCOUNT(12*4(%esp))
+#ifdef SMP
+	call	_get_mplock
+#endif /* SMP */
 	incl	_cnt+V_SYSCALL
 	movl	$SWI_AST_MASK,_cpl
 	call	_syscall
@@ -254,7 +268,13 @@ ENTRY(fork_trampoline)
 	movl	$SWI_AST_MASK,_cpl
 	call	_splz
 
+#if defined(SMP)
+	GETCPUID(%eax)
+	leal	_SMPruntime(,%eax,8), %eax
+	pushl	%eax
+#else
 	pushl	$_runtime
+#endif
 	call	_microtime
 	popl	%eax
 
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 6e3f5c77e077..402b1cebb76f 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -32,7 +32,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)npx.c	7.2 (Berkeley) 5/12/91
- *	$Id: npx.c,v 1.40 1997/03/24 11:23:58 bde Exp $
+ *	$Id: npx.c,v 1.41 1997/04/22 06:55:38 jdp Exp $
  */
 
 #include "npx.h"
@@ -40,6 +40,7 @@
 
 #include "opt_cpu.h"
 #include "opt_math_emulate.h"
+#include "opt_smp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -60,6 +61,10 @@
 #include <machine/trap.h>
 #include <machine/clock.h>
 #include <machine/specialreg.h>
+#if defined(APIC_IO)
+#include <machine/apic.h>
+#include <machine/mpapic.h>
+#endif /* APIC_IO */
 
 #include <i386/isa/icu.h>
 #include <i386/isa/isa_device.h>
@@ -133,7 +138,12 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint,
 	"Floatingpoint instructions executed in hardware");
 
 static u_int	npx0_imask = SWI_CLOCK_MASK;
+#ifdef SMP
+#define npxproc	(SMPnpxproc[cpunumber()])
+struct proc	*SMPnpxproc[NCPU];
+#else
 struct proc	*npxproc;
+#endif
 
 static	bool_t			npx_ex16;
 static	bool_t			npx_exists;
@@ -148,8 +158,28 @@ static	volatile u_int		npx_traps_while_probing;
  * interrupts.  We'll still need a special exception 16 handler.  The busy
  * latch stuff in probeintr() can be moved to npxprobe().
  */
-
 inthand_t probeintr;
+
+#if defined(APIC_IO)
+
+asm
+("
+	.text
+	.p2align 2,0x90
+" __XSTRING(CNAME(probeintr)) ":
+	ss
+	incl	" __XSTRING(CNAME(npx_intrs_while_probing)) "
+	pushl	%eax
+	movl	" __XSTRING(CNAME(apic_base)) ",%eax	# EOI to local APIC
+	movl	$0,0xb0(,%eax,1)	# movl $0, APIC_EOI(%eax)
+	movb	$0,%al
+	outb	%al,$0xf0		# clear BUSY# latch
+	popl	%eax
+	iret
+");
+
+#else
+
 asm
 ("
 	.text
@@ -167,6 +197,8 @@ asm
 	iret
 ");
 
+#endif /* APIC_IO */
+
 inthand_t probetrap;
 asm
 ("
@@ -191,8 +223,12 @@ npxprobe(dvp)
 {
 	int	result;
 	u_long	save_eflags;
+#if defined(APIC_IO)
+	u_int	save_apic_mask;
+#else
 	u_char	save_icu1_mask;
 	u_char	save_icu2_mask;
+#endif /* APIC_IO */
 	struct	gate_descriptor save_idt_npxintr;
 	struct	gate_descriptor save_idt_npxtrap;
 	/*
@@ -205,20 +241,32 @@ npxprobe(dvp)
 	npx_intrno = NRSVIDT + ffs(dvp->id_irq) - 1;
 	save_eflags = read_eflags();
 	disable_intr();
+#if defined(APIC_IO)
+	save_apic_mask = INTRGET();
+#else
 	save_icu1_mask = inb(IO_ICU1 + 1);
 	save_icu2_mask = inb(IO_ICU2 + 1);
+#endif /* APIC_IO */
 	save_idt_npxintr = idt[npx_intrno];
 	save_idt_npxtrap = idt[16];
+#if defined(APIC_IO)
+	INTRSET( ~dvp->id_irq );
+#else
 	outb(IO_ICU1 + 1, ~(IRQ_SLAVE | dvp->id_irq));
 	outb(IO_ICU2 + 1, ~(dvp->id_irq >> 8));
+#endif /* APIC_IO */
 	setidt(16, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 	setidt(npx_intrno, probeintr, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 	npx_idt_probeintr = idt[npx_intrno];
 	enable_intr();
 	result = npxprobe1(dvp);
 	disable_intr();
+#if defined(APIC_IO)
+	INTRSET( save_apic_mask );
+#else
 	outb(IO_ICU1 + 1, save_icu1_mask);
 	outb(IO_ICU2 + 1, save_icu2_mask);
+#endif /* APIC_IO */
 	idt[npx_intrno] = save_idt_npxintr;
 	idt[16] = save_idt_npxtrap;
 	write_eflags(save_eflags);
@@ -364,7 +412,8 @@ npxattach(dvp)
 	}
 	npxinit(__INITIAL_NPXCW__);
 
-#ifdef I586_CPU
+#if defined(I586_CPU) && !defined(SMP)
+	/* FPU not working under SMP yet */
 	if (cpu_class == CPUCLASS_586 && npx_ex16) {
 		if (!(dvp->id_flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) {
 			bcopy_vector = i586_bcopy;
@@ -576,18 +625,32 @@ void
 npxsave(addr)
 	struct save87 *addr;
 {
+#if defined(APIC_IO)
+	u_int	apic_mask;
+	u_int	old_apic_mask;
+#else
 	u_char	icu1_mask;
 	u_char	icu2_mask;
 	u_char	old_icu1_mask;
 	u_char	old_icu2_mask;
+#endif /* APIC_IO */
 	struct gate_descriptor	save_idt_npxintr;
 
 	disable_intr();
+#if defined(APIC_IO)
+	old_apic_mask = INTRGET();
+#else
 	old_icu1_mask = inb(IO_ICU1 + 1);
 	old_icu2_mask = inb(IO_ICU2 + 1);
+#endif /* APIC_IO */
 	save_idt_npxintr = idt[npx_intrno];
+#if defined(APIC_IO)
+	/** FIXME: try clrIoApicMaskBit( npx0_imask ); */
+	INTRSET( old_apic_mask & ~(npx0_imask & 0xffff) );
+#else
 	outb(IO_ICU1 + 1, old_icu1_mask & ~(IRQ_SLAVE | npx0_imask));
 	outb(IO_ICU2 + 1, old_icu2_mask & ~(npx0_imask >> 8));
+#endif /* APIC_IO */
 	idt[npx_intrno] = npx_idt_probeintr;
 	enable_intr();
 	stop_emulating();
@@ -596,6 +659,11 @@ npxsave(addr)
 	start_emulating();
 	npxproc = NULL;
 	disable_intr();
+#if defined(APIC_IO)
+	apic_mask = INTRGET();		/* masks may have changed */
+        INTRSET( (apic_mask & ~(npx0_imask & 0xffff)) |
+		 (old_apic_mask & (npx0_imask & 0xffff)));
+#else
 	icu1_mask = inb(IO_ICU1 + 1);	/* masks may have changed */
 	icu2_mask = inb(IO_ICU2 + 1);
 	outb(IO_ICU1 + 1,
@@ -603,6 +671,7 @@ npxsave(addr)
 	outb(IO_ICU2 + 1,
 	     (icu2_mask & ~(npx0_imask >> 8))
 	     | (old_icu2_mask & (npx0_imask >> 8)));
+#endif /* APIC_IO */
 	idt[npx_intrno] = save_idt_npxintr;
 	enable_intr();		/* back to usual state */
 }
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index c1d3b4c1b752..e15535b82786 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)genassym.c	5.11 (Berkeley) 5/10/91
- *	$Id: genassym.c,v 1.43 1997/04/07 06:45:11 peter Exp $
+ *	$Id: genassym.c,v 1.44 1997/04/07 07:15:48 peter Exp $
  */
 
 #include <stdio.h>
@@ -102,6 +102,8 @@ main()
 	printf("#define\tP_WCHAN %p\n", &p->p_wchan);
 	printf("#define\tP_FLAG %p\n", &p->p_flag);
 	printf("#define\tP_PID %p\n", &p->p_pid);
+	printf("#define\tP_ONCPU %p\n", &p->p_oncpu);
+	printf("#define\tP_LASTCPU %p\n", &p->p_lastcpu);
 	printf("#define\tSSLEEP %d\n", SSLEEP);
 	printf("#define\tSRUN %d\n", SRUN);
 	printf("#define\tV_TRAP %p\n", &vm->v_trap);
@@ -131,6 +133,7 @@ main()
 	printf("#define\tPCB_EIP %p\n", &pcb->pcb_eip);
 	printf("#define\tTSS_ESP0 %p\n", &tss->tss_esp0);
 	printf("#define\tPCB_USERLDT %p\n", &pcb->pcb_ldt);
+	printf("#define\tPCB_MPNEST %p\n", &pcb->pcb_mpnest);
 	printf("#define\tU_PROF %p\n", &up->u_stats.p_prof);
 	printf("#define\tU_PROFSCALE %p\n", &up->u_stats.p_prof.pr_scale);
 	printf("#define\tPR_BASE %p\n", &uprof->pr_base);
diff --git a/sys/amd64/amd64/identcpu.c b/sys/amd64/amd64/identcpu.c
index 5defd2aeec00..21a62de7811a 100644
--- a/sys/amd64/amd64/identcpu.c
+++ b/sys/amd64/amd64/identcpu.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	from: Id: machdep.c,v 1.193 1996/06/18 01:22:04 bde Exp
- *	$Id: identcpu.c,v 1.15 1997/04/22 06:55:23 jdp Exp $
+ *	$Id: identcpu.c,v 1.16 1997/04/26 04:08:45 kato Exp $
  */
 
 #include "opt_cpu.h"
@@ -307,17 +307,21 @@ printcpuinfo(void)
 #endif
 #if defined(I586_CPU)
 	case CPUCLASS_586:
+#ifndef SMP
 		printf("%d.%02d-MHz ",
 		       (i586_ctr_freq + 4999) / 1000000,
 		       ((i586_ctr_freq + 4999) / 10000) % 100);
+#endif
 		printf("586");
 		break;
 #endif
 #if defined(I686_CPU)
 	case CPUCLASS_686:
+#ifndef SMP
 		printf("%d.%02d-MHz ",
 		       (i586_ctr_freq + 4999) / 1000000,
 		       ((i586_ctr_freq + 4999) / 10000) % 100);
+#endif
 		printf("686");
 		break;
 #endif
diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S
index 63c11123f222..05ce518c0b7b 100644
--- a/sys/amd64/amd64/locore.S
+++ b/sys/amd64/amd64/locore.S
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)locore.s	7.3 (Berkeley) 5/13/91
- *	$Id: locore.s,v 1.84 1997/04/07 08:38:19 peter Exp $
+ *	$Id: locore.s,v 1.85 1997/04/22 06:55:24 jdp Exp $
  *
  *		originally from: locore.s, by William F. Jolitz
  *
@@ -47,6 +47,9 @@
 #include "opt_cpu.h"
 #include "opt_ddb.h"
 #include "opt_userconfig.h"
+#include "opt_smp.h"
+#include "opt_smp_privpages.h"
+#include "opt_serial.h"
 
 #include <sys/errno.h>
 #include <sys/syscall.h>
@@ -58,6 +61,10 @@
 #include <machine/pmap.h>
 #include <machine/specialreg.h>
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#include <machine/apic.h>
+#endif /* SMP && PRIVPAGES */
+
 #include "assym.s"
 
 /*
@@ -111,6 +118,15 @@ _bootinfo:	.space	BOOTINFO_SIZE		/* bootinfo that we can handle */
 _KERNend:	.long	0			/* phys addr end of kernel (just after bss) */
 physfree:	.long	0			/* phys addr of next free page */
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+cpu0pp:		.long	0			/* phys addr cpu0 private pg */
+cpu0pt:		.long	0			/* phys addr cpu0 private pt */
+
+		.globl	_cpu0prvpage,_cpu0prvpt
+_cpu0prvpage:	.long	0			/* relocated version */
+_cpu0prvpt:	.long	0			/* relocated version */
+#endif /* SMP && PRIVPAGES */
+
 	.globl	_IdlePTD
 _IdlePTD:	.long	0			/* phys addr of kernel PTD */
 
@@ -752,6 +768,20 @@ over_symalloc:
 	addl	$KERNBASE, %esi
 	movl	%esi, R(_proc0paddr)
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+/* Allocate cpu0's private data page */
+	ALLOCPAGES(1)
+	movl	%esi,R(cpu0pp)
+	addl	$KERNBASE, %esi
+	movl	%esi, R(_cpu0prvpage)	/* relocated to KVM space */
+
+/* Allocate cpu0's private page table for mapping priv page, apic, etc */
+	ALLOCPAGES(1)
+	movl	%esi,R(cpu0pt)
+	addl	$KERNBASE, %esi
+	movl	%esi, R(_cpu0prvpt)	/* relocated to KVM space */
+#endif	/* SMP && SMP_PRIVPAGES */
+
 /* Map read-only from zero to the end of the kernel text section */
 	xorl	%eax, %eax
 #ifdef BDE_DEBUGGER
@@ -799,6 +829,42 @@ map_read_write:
 	movl	$ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx
 	fillkptphys($PG_RW)
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+/* Map cpu0's private page into global KVM */
+	movl	R(cpu0pp), %eax
+	movl	$1, %ecx
+	fillkptphys($PG_RW)
+
+/* Map cpu0's private page table into global KVM */
+	movl	R(cpu0pt), %eax
+	movl	$1, %ecx
+	fillkptphys($PG_RW)
+
+/* Map the private page into the private page table (4K @ 0xff80000) */
+	movl	R(cpu0pp), %eax
+	movl	$0, %ebx		/* pte offset = 0 */
+	movl	$1, %ecx		/* one private page coming right up */
+	fillkpt(R(cpu0pt), $PG_RW)
+
+/* Map the default Local APIC address (4K @ 0xff801000) */
+	movl	$DEFAULT_APIC_BASE, %eax	/* XXX just testing.. */
+	movl	$1, %ebx		/* pte offset = 1 */
+	movl	$1, %ecx		/* Bing! Local APIC appears */
+	fillkpt(R(cpu0pt), $PG_RW|PG_N)
+
+/* Map the default IO APIC address (4K @ 0xff802000) */
+	movl	$DEFAULT_IO_APIC_BASE, %eax	/* XXX just testing.. */
+	movl	$2, %ebx		/* pte offset = 2 */
+	movl	$1, %ecx		/* Bing! Local APIC appears */
+	fillkpt(R(cpu0pt), $PG_RW|PG_N)
+
+/* ... and put the page table in the pde. */
+	movl	R(cpu0pt), %eax
+	movl	$MPPTDI, %ebx
+	movl	$1, %ecx
+	fillkpt(R(_IdlePTD), $PG_RW)
+#endif	/* SMP && SMP_PRIVPAGES */
+
 /* install a pde for temporary double map of bottom of VA */
 	movl	R(_KPTphys), %eax
 	xorl	%ebx, %ebx
diff --git a/sys/amd64/amd64/locore.s b/sys/amd64/amd64/locore.s
index 63c11123f222..05ce518c0b7b 100644
--- a/sys/amd64/amd64/locore.s
+++ b/sys/amd64/amd64/locore.s
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)locore.s	7.3 (Berkeley) 5/13/91
- *	$Id: locore.s,v 1.84 1997/04/07 08:38:19 peter Exp $
+ *	$Id: locore.s,v 1.85 1997/04/22 06:55:24 jdp Exp $
  *
  *		originally from: locore.s, by William F. Jolitz
  *
@@ -47,6 +47,9 @@
 #include "opt_cpu.h"
 #include "opt_ddb.h"
 #include "opt_userconfig.h"
+#include "opt_smp.h"
+#include "opt_smp_privpages.h"
+#include "opt_serial.h"
 
 #include <sys/errno.h>
 #include <sys/syscall.h>
@@ -58,6 +61,10 @@
 #include <machine/pmap.h>
 #include <machine/specialreg.h>
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#include <machine/apic.h>
+#endif /* SMP && PRIVPAGES */
+
 #include "assym.s"
 
 /*
@@ -111,6 +118,15 @@ _bootinfo:	.space	BOOTINFO_SIZE		/* bootinfo that we can handle */
 _KERNend:	.long	0			/* phys addr end of kernel (just after bss) */
 physfree:	.long	0			/* phys addr of next free page */
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+cpu0pp:		.long	0			/* phys addr cpu0 private pg */
+cpu0pt:		.long	0			/* phys addr cpu0 private pt */
+
+		.globl	_cpu0prvpage,_cpu0prvpt
+_cpu0prvpage:	.long	0			/* relocated version */
+_cpu0prvpt:	.long	0			/* relocated version */
+#endif /* SMP && PRIVPAGES */
+
 	.globl	_IdlePTD
 _IdlePTD:	.long	0			/* phys addr of kernel PTD */
 
@@ -752,6 +768,20 @@ over_symalloc:
 	addl	$KERNBASE, %esi
 	movl	%esi, R(_proc0paddr)
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+/* Allocate cpu0's private data page */
+	ALLOCPAGES(1)
+	movl	%esi,R(cpu0pp)
+	addl	$KERNBASE, %esi
+	movl	%esi, R(_cpu0prvpage)	/* relocated to KVM space */
+
+/* Allocate cpu0's private page table for mapping priv page, apic, etc */
+	ALLOCPAGES(1)
+	movl	%esi,R(cpu0pt)
+	addl	$KERNBASE, %esi
+	movl	%esi, R(_cpu0prvpt)	/* relocated to KVM space */
+#endif	/* SMP && SMP_PRIVPAGES */
+
 /* Map read-only from zero to the end of the kernel text section */
 	xorl	%eax, %eax
 #ifdef BDE_DEBUGGER
@@ -799,6 +829,42 @@ map_read_write:
 	movl	$ISA_HOLE_LENGTH>>PAGE_SHIFT, %ecx
 	fillkptphys($PG_RW)
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+/* Map cpu0's private page into global KVM */
+	movl	R(cpu0pp), %eax
+	movl	$1, %ecx
+	fillkptphys($PG_RW)
+
+/* Map cpu0's private page table into global KVM */
+	movl	R(cpu0pt), %eax
+	movl	$1, %ecx
+	fillkptphys($PG_RW)
+
+/* Map the private page into the private page table (4K @ 0xff80000) */
+	movl	R(cpu0pp), %eax
+	movl	$0, %ebx		/* pte offset = 0 */
+	movl	$1, %ecx		/* one private page coming right up */
+	fillkpt(R(cpu0pt), $PG_RW)
+
+/* Map the default Local APIC address (4K @ 0xff801000) */
+	movl	$DEFAULT_APIC_BASE, %eax	/* XXX just testing.. */
+	movl	$1, %ebx		/* pte offset = 1 */
+	movl	$1, %ecx		/* Bing! Local APIC appears */
+	fillkpt(R(cpu0pt), $PG_RW|PG_N)
+
+/* Map the default IO APIC address (4K @ 0xff802000) */
+	movl	$DEFAULT_IO_APIC_BASE, %eax	/* XXX just testing.. */
+	movl	$2, %ebx		/* pte offset = 2 */
+	movl	$1, %ecx		/* Bing! Local APIC appears */
+	fillkpt(R(cpu0pt), $PG_RW|PG_N)
+
+/* ... and put the page table in the pde. */
+	movl	R(cpu0pt), %eax
+	movl	$MPPTDI, %ebx
+	movl	$1, %ecx
+	fillkpt(R(_IdlePTD), $PG_RW)
+#endif	/* SMP && SMP_PRIVPAGES */
+
 /* install a pde for temporary double map of bottom of VA */
 	movl	R(_KPTphys), %eax
 	xorl	%ebx, %ebx
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 9ed0b1d42d0f..d3b73299c37f 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)machdep.c	7.4 (Berkeley) 6/3/91
- *	$Id: machdep.c,v 1.237 1997/04/13 04:07:24 dyson Exp $
+ *	$Id: machdep.c,v 1.238 1997/04/22 06:55:26 jdp Exp $
  */
 
 #include "npx.h"
@@ -44,6 +44,7 @@
 #include "opt_bounce.h"
 #include "opt_machdep.h"
 #include "opt_perfmon.h"
+#include "opt_smp.h"
 #include "opt_userconfig.h"
 
 #include <sys/param.h>
@@ -107,6 +108,9 @@
 #include <machine/cons.h>
 #include <machine/bootinfo.h>
 #include <machine/md_var.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
 #ifdef PERFMON
 #include <machine/perfmon.h>
 #endif
@@ -209,6 +213,9 @@ cpu_startup(dummy)
 	 * Good {morning,afternoon,evening,night}.
 	 */
 	printf(version);
+#ifdef SMP
+	mp_announce();
+#endif
 	earlysetcpuclass();
 	startrtclock();
 	printcpuinfo();
@@ -344,6 +351,23 @@ again:
 	u_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr,
 				(maxproc*UPAGES*PAGE_SIZE), FALSE);
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+	/* Per-cpu pages.. (the story so far is... subject to change)
+	 * ========= For the per-cpu data page ========
+	 * 1 private data page
+	 * 1 PDE	(per-cpu PTD entry page)
+	 * 1 PT		(per-cpu page table page)
+	 * ============ For the idle loop =============
+	 * 2 UPAGEs	(per-cpu idle procs)
+	 * 1 PTD	(for per-cpu equiv of IdlePTD)
+	 * ============================================
+	 * = total of 6 pages per cpu.  The BSP reuses the ones allocated
+	 * by locore.s during boot to remove special cases at runtime.
+	 */
+	ppage_map = kmem_suballoc(kernel_map, &minaddr, &maxaddr,
+				(NCPU*6*PAGE_SIZE), FALSE);
+#endif
+
 	/*
 	 * Finally, allocate mbuf pool.  Since mclrefcnt is an off-size
 	 * we use the more space efficient malloc in place of kmem_alloc.
@@ -748,10 +772,24 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock,
 
 int currentldt;
 int _default_ldt;
+#ifdef SMP
+union descriptor gdt[NGDT + NCPU];		/* global descriptor table */
+#else
 union descriptor gdt[NGDT];		/* global descriptor table */
+#endif
 struct gate_descriptor idt[NIDT];	/* interrupt descriptor table */
 union descriptor ldt[NLDT];		/* local descriptor table */
+#ifdef SMP
+/* table descriptors - used to load tables by microp */
+struct region_descriptor r_gdt, r_idt;
+#endif
+
+#ifdef SMP
+struct i386tss SMPcommon_tss[NCPU];	/* One tss per cpu */
+struct i386tss *SMPcommon_tss_ptr[NCPU]; /* for the benefit of asmp code */
+#else
 struct i386tss common_tss;
+#endif
 
 static struct i386tss dblfault_tss;
 static char dblfault_stack[PAGE_SIZE];
@@ -764,7 +802,11 @@ int gsel_tss;
 #endif
 
 /* software prototypes -- in more palatable form */
-struct soft_segment_descriptor gdt_segs[] = {
+struct soft_segment_descriptor gdt_segs[
+#ifdef SMP
+					NGDT + NCPU
+#endif
+						   ] = {
 /* GNULL_SEL	0 Null Descriptor */
 {	0x0,			/* segment base address  */
 	0x0,			/* length */
@@ -820,7 +862,12 @@ struct soft_segment_descriptor gdt_segs[] = {
 	0,			/* unused - default 32 vs 16 bit size */
 	0  			/* limit granularity (byte/page units)*/ },
 /* GPROC0_SEL	6 Proc 0 Tss Descriptor */
-{	(int) &common_tss,	/* segment base address */
+{
+#ifdef SMP
+	(int) &SMPcommon_tss[0],/* segment base address */
+#else
+	(int) &common_tss,	/* segment base address */
+#endif
 	sizeof(struct i386tss)-1,/* length - all address space */
 	SDT_SYS386TSS,		/* segment type */
 	0,			/* segment descriptor priority level */
@@ -968,8 +1015,10 @@ init386(first)
 	int gsel_tss;
 #endif
 	struct isa_device *idp;
+#ifndef SMP
 	/* table descriptors - used to load tables by microp */
 	struct region_descriptor r_gdt, r_idt;
+#endif
 	int	pagesinbase, pagesinext;
 	int	target_page, pa_indx;
 	int	off;
@@ -1002,6 +1051,18 @@ init386(first)
 	for (x = 0; x < NGDT; x++)
 		ssdtosd(&gdt_segs[x], &gdt[x].sd);
 
+#ifdef SMP
+	/*
+	 * Oh puke!
+	 */
+	for (x = 0; x < NCPU; x++) {
+		SMPcommon_tss_ptr[x] = &SMPcommon_tss[x];
+		gdt_segs[NGDT + x] = gdt_segs[GPROC0_SEL];
+		gdt_segs[NGDT + x].ssd_base = (int) SMPcommon_tss_ptr[x];
+		ssdtosd(&gdt_segs[NGDT + x], &gdt[NGDT + x].sd);
+	}
+#endif
+
 	/* make ldt memory segments */
 	/*
 	 * The data segment limit must not cover the user area because we
@@ -1152,7 +1213,13 @@ init386(first)
 			       bootinfo.bi_extmem, biosextmem);
 	}
 
+#ifdef SMP
+	/* make hole for AP bootstrap code */
+	pagesinbase = mp_bootaddress(biosbasemem) / PAGE_SIZE;
+#else
 	pagesinbase = biosbasemem * 1024 / PAGE_SIZE;
+#endif
+
 	pagesinext = biosextmem * 1024 / PAGE_SIZE;
 
 	/*
@@ -1189,6 +1256,11 @@ init386(first)
 	/* call pmap initialization to make new kernel address space */
 	pmap_bootstrap (first, 0);
 
+#ifdef SMP
+	/* fire up the APs and APICs */
+	mp_start();
+#endif
+
 	/*
 	 * Size up each available chunk of physical memory.
 	 */
@@ -1315,12 +1387,23 @@ init386(first)
 			   avail_end + off, VM_PROT_ALL, TRUE);
 	msgbufmapped = 1;
 
+#ifdef SMP
+	for(x = 0; x < NCPU; x++) {
+	/* make an initial tss so cpu can get interrupt stack on syscall! */
+		SMPcommon_tss[x].tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE;
+		SMPcommon_tss[x].tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ;
+		SMPcommon_tss[x].tss_ioopt = (sizeof SMPcommon_tss[x]) << 16;
+	}
+	gsel_tss = GSEL(NGDT + cpunumber(), SEL_KPL);
+	ltr(gsel_tss);
+#else
 	/* make an initial tss so cpu can get interrupt stack on syscall! */
 	common_tss.tss_esp0 = (int) proc0.p_addr + UPAGES*PAGE_SIZE;
 	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL) ;
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
 	ltr(gsel_tss);
+#endif
 
 	dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 =
 	    dblfault_tss.tss_esp2 = (int) &dblfault_stack[sizeof(dblfault_stack)];
@@ -1361,6 +1444,7 @@ init386(first)
 	/* setup proc 0's pcb */
 	proc0.p_addr->u_pcb.pcb_flags = 0;
 	proc0.p_addr->u_pcb.pcb_cr3 = IdlePTD;
+	proc0.p_addr->u_pcb.pcb_mpnest = 1;
 }
 
 /*
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
new file mode 100644
index 000000000000..07fca34efe56
--- /dev/null
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -0,0 +1,1646 @@
+/*
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $
+ */
+
+#include "opt_smp.h"
+#include "opt_smp_invltlb.h"
+#if defined(APIC_IO)
+#if !defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#else				/* APIC_IO */
+#if defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#endif				/* APIC_IO */
+#define FIX_MP_TABLE_WORKS_NOT
+
+#include "opt_serial.h"
+
+#include <sys/param.h>		/* for KERNBASE */
+#include <sys/types.h>
+#include <sys/sysproto.h>
+#include <sys/time.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>		/* for KERNBASE */
+#include <vm/vm_param.h>	/* for KERNBASE */
+#include <vm/pmap.h>		/* for KERNBASE */
+#include <machine/pmap.h>	/* for KERNBASE */
+
+#include <machine/smp.h>
+#include <machine/apic.h>
+#include <machine/mpapic.h>
+#include <machine/cpufunc.h>
+#include <machine/segments.h>
+#include <machine/smptests.h>	/** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */
+
+#include <i386/i386/cons.h>	/* cngetc() */
+
+#if defined(IPI_INTS)
+#include <i386/isa/isa_device.h>
+#include "vector.h"
+#endif				/* IPI_INTS */
+
+#if defined(SMP_INVLTLB)
+#include <i386/isa/icu.h>
+#endif				/* SMP_INVLTLB */
+
+#define WARMBOOT_TARGET	0
+#define WARMBOOT_OFF	(KERNBASE + 0x0467)
+#define WARMBOOT_SEG	(KERNBASE + 0x0469)
+
+#define BIOS_BASE	(0xf0000)
+#define BIOS_SIZE	(0x10000)
+#define BIOS_COUNT	(BIOS_SIZE/4)
+
+#define CMOS_REG	(0x70)
+#define CMOS_DATA	(0x71)
+#define BIOS_RESET	(0x0f)
+#define BIOS_WARM	(0x0a)
+
+/*
+ * this code MUST be enabled here and in mpboot.s.
+ * it follows the very early stages of AP boot by placing values in CMOS ram.
+ * it NORMALLY will never be needed and thus the primitive method for enabling.
+ *
+#define CHECK_POINTS
+ */
+
+#if defined(CHECK_POINTS)
+#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
+#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
+
+#define CHECK_INIT(D);				\
+	CHECK_WRITE(0x34, (D));			\
+	CHECK_WRITE(0x35, (D));			\
+	CHECK_WRITE(0x36, (D));			\
+	CHECK_WRITE(0x37, (D));			\
+	CHECK_WRITE(0x38, (D));			\
+	CHECK_WRITE(0x39, (D));
+
+#define CHECK_PRINT(S);				\
+	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
+	   (S),					\
+	   CHECK_READ(0x34),			\
+	   CHECK_READ(0x35),			\
+	   CHECK_READ(0x36),			\
+	   CHECK_READ(0x37),			\
+	   CHECK_READ(0x38),			\
+	   CHECK_READ(0x39));
+
+#else				/* CHECK_POINTS */
+
+#define CHECK_INIT(D)
+#define CHECK_PRINT(S)
+
+#endif				/* CHECK_POINTS */
+
+
+/** FIXME: what system files declare these??? */
+extern struct region_descriptor r_gdt, r_idt;
+
+/* global data */
+struct proc *SMPcurproc[NCPU];
+struct pcb *SMPcurpcb[NCPU];
+struct timeval SMPruntime[NCPU];
+
+int     mp_ncpus;		/* # of CPUs, including BSP */
+int     mp_naps;		/* # of Applications processors */
+int     mp_nbusses;		/* # of busses */
+int     mp_napics;		/* # of IO APICs */
+int     mpenabled;
+int     boot_cpu_id;		/* designated BSP */
+vm_offset_t cpu_apic_address;
+vm_offset_t io_apic_address[NAPIC];
+
+u_int32_t cpu_apic_versions[NCPU];
+u_int32_t io_apic_versions[NAPIC];
+
+/*
+ * APIC ID logical/physical mapping structures
+ */
+int     cpu_num_to_apic_id[NCPU];
+int     io_num_to_apic_id[NAPIC];
+int     apic_id_to_logical[NAPICID];
+
+/*
+ * look for MP compliant motherboard.
+ */
+
+static u_int boot_address;
+static u_int base_memory;
+
+static int picmode;		/* 0: virtual wire mode, 1: PIC mode */
+static u_int mpfps;
+static int search_for_sig(u_int32_t target, int count);
+static int mp_probe(u_int base_top);
+static void mp_enable(u_int boot_addr);
+
+#if defined(IPI_INTS)
+static void ipi_intr0(void);
+static void ipi_intr1(void);
+static void ipi_intr2(void);
+static void ipi_intr3(void);
+static int 
+ipi_ihandler_attach(int irq, inthand2_t * func,
+    unsigned *maskptr, int unit);
+#endif				/* IPI_INTS */
+
+
+/*
+ * calculate usable address in base memory for AP trampoline code
+ */
+u_int
+mp_bootaddress(u_int basemem)
+{
+	base_memory = basemem * 1024;	/* convert to bytes */
+
+	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
+	if ((base_memory - boot_address) < bootMP_size)
+		boot_address -= 4096;	/* not enough, lower by 4k */
+
+	return boot_address;
+}
+
+
+/*
+ * startup the SMP processors
+ */
+void
+mp_start(void)
+{
+	/* look for MP capable motherboard */
+	if (mp_probe(base_memory))
+		mp_enable(boot_address);
+	else {
+		printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n");
+		panic("can't continue!\n");
+	}
+
+	/* finish pmap initialization - turn off V==P mapping at zero */
+	pmap_bootstrap2();
+}
+
+
+/*
+ * print various information about the SMP system hardware and setup
+ */
+void
+mp_announce(void)
+{
+	int     x;
+
+	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
+	printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0));
+	printf(", version: 0x%08x\n", cpu_apic_versions[0]);
+	for (x = 1; x <= mp_naps; ++x) {
+		printf(" cpu%d (AP):  apic id: %d", x, CPU_TO_ID(x));
+		printf(", version: 0x%08x\n", cpu_apic_versions[x]);
+	}
+
+#if defined(APIC_IO)
+	for (x = 0; x < mp_napics; ++x) {
+		printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x));
+		printf(", version: 0x%08x\n", io_apic_versions[x]);
+	}
+#else
+	printf(" Warning: APIC I/O disabled\n");
+#endif	/* APIC_IO */
+}
+
+
+/*
+ * AP cpu's call this to sync up protected mode.
+ */
+void
+init_secondary(void)
+{
+	int     gsel_tss, slot;
+
+	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
+	r_gdt.rd_base = (int) gdt;
+	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lidt(&r_idt);
+	lldt(_default_ldt);
+
+	slot = NGDT + cpunumber();
+	gsel_tss = GSEL(slot, SEL_KPL);
+	gdt[slot].sd.sd_type = SDT_SYS386TSS;
+	ltr(gsel_tss);
+
+	load_cr0(0x8005003b);	/* XXX! */
+}
+
+
+#if defined(APIC_IO)
+void
+configure_local_apic(void)
+{
+	u_char  byte;
+	u_int32_t temp;
+
+	if (picmode) {
+		outb(0x22, 0x70);	/* select IMCR */
+		byte = inb(0x23);	/* current contents */
+		byte |= 0x01;	/* mask external INTR */
+		outb(0x23, byte);	/* disconnect 8259s/NMI */
+	}
+	/* mask the LVT1 */
+	temp = apic_base[APIC_LVT1];
+	temp |= APIC_LVT_M;
+	apic_base[APIC_LVT1] = temp;
+}
+#endif	/* APIC_IO */
+
+
+/*******************************************************************
+ * local functions and data
+ */
+
+static int
+mp_probe(u_int base_top)
+{
+	int     x;
+	u_long  segment;
+	u_int32_t target;
+
+	/* see if EBDA exists */
+	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
+		/* search first 1K of EBDA */
+		target = (u_int32_t) (segment << 4);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	} else {
+		/*last 1K of base memory, effective 'top of base' is passed in*/
+		target = (u_int32_t) (base_top - 0x400);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	}
+
+	/* search the BIOS */
+	target = (u_int32_t) BIOS_BASE;
+	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
+		goto found;
+
+	/* nothing found */
+	mpfps = mpenabled = 0;
+	return 0;
+
+found:				/* please forgive the 'goto'! */
+	/* flag fact that we are running multiple processors */
+	mpfps = x;
+	mpenabled = 1;
+	return 1;
+}
+
+
+/*
+ * start the SMP system
+ */
+static int parse_mp_table(void);
+static void default_mp_table(int type);
+static int start_all_aps(u_int boot_addr);
+
+#if defined(XFAST_IPI32)
+#include <machine/md_var.h>
+#include <i386/isa/isa_device.h>
+extern void Xfastipi32(u_int, u_int, u_int, u_int);
+#endif				/* XFAST_IPI32 */
+
+static void
+mp_enable(u_int boot_addr)
+{
+	int     x;
+#if defined(APIC_IO)
+	int     apic;
+	u_int   ux;
+#if defined(TEST_UPPERPRIO)
+	u_char  select;		/* the select register is 8 bits */
+	u_int32_t flags;	/* the window register is 32 bits */
+#endif	/* TEST_UPPERPRIO */
+#endif	/* APIC_IO */
+
+	/* examine the MP table for needed info */
+	x = parse_mp_table();
+
+	/* create pages for (address common) cpu APIC and each IO APIC */
+	pmap_bootstrap_apics();
+
+	/* can't process default configs till the CPU APIC is pmapped */
+	if (x)
+		default_mp_table(x);
+
+#if defined(APIC_IO)
+	/* fill the LOGICAL io_apic_versions table */
+	for (apic = 0; apic < mp_napics; ++apic) {
+		ux = io_apic_read(apic, IOAPIC_VER);
+		io_apic_versions[apic] = ux;
+	}
+
+	/*
+	 */
+	for (apic = 0; apic < mp_napics; ++apic)
+		if (io_apic_setup(apic) < 0)
+			panic("IO APIC setup failure\n");
+
+#if defined(IPI_INTS)
+	/* setup IPI INTerrupt mechanism */
+	ipi_ihandler_attach( /* irq */ 24,
+	     /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 25,
+	     /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 26,
+	     /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0);
+#if !defined(SMP_INVLTLB)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+#else
+#if defined(XFAST_IPI32)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+	setidt(ICU_OFFSET + 32,
+	    Xfastipi32,
+	    SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#else
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0);
+#endif	/* XFAST_IPI32 */
+#endif
+
+#endif	/* IPI_INTS */
+
+#if defined(TEST_UPPERPRIO)
+
+#if 1
+	printf("special IRQ10\n");
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~0xff;		/** clear vector */
+	flags |= 64;
+	io_apic_write(0, select, flags);
+#else
+	printf("special IRQ10\n");
+	cngetc();
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~IOART_DELMOD;	/* FIXED mode */
+	io_apic_write(0, select, flags);
+	io_apic_write(0, select + 1, boot_cpu_id << 24);
+#endif	/** 0/1 */
+
+#endif	/* TEST_UPPERPRIO */
+
+#endif	/* APIC_IO */
+
+	/* start each Application Processor */
+	start_all_aps(boot_addr);
+}
+
+
+/*
+ * look for the MP spec signature
+ */
+
+/* string defined by the Intel MP Spec as identifying the MP table */
+#define MP_SIG		0x5f504d5f	/* _MP_ */
+#define NEXT(X)		((X) += 4)
+static int
+search_for_sig(u_int32_t target, int count)
+{
+	int     x;
+	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
+
+	for (x = 0; x < count; NEXT(x))
+		if (addr[x] == MP_SIG)
+			/* make array index a byte index */
+			return (target + (x * sizeof(u_int32_t)));
+
+	return -1;
+}
+
+
+#define PROCENTRY_FLAG_EN	0x01
+#define PROCENTRY_FLAG_BP	0x02
+#define IOAPICENTRY_FLAG_EN	0x01
+
+/* MP Floating Pointer Structure */
+typedef struct MPFPS {
+	char    signature[4];
+	void   *pap;
+	u_char  length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  mpfb1;
+	u_char  mpfb2;
+	u_char  mpfb3;
+	u_char  mpfb4;
+	u_char  mpfb5;
+}      *mpfps_t;
+/* MP Configuration Table Header */
+typedef struct MPCTH {
+	char    signature[4];
+	u_short base_table_length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  oem_id[8];
+	u_char  product_id[12];
+	void   *oem_table_pointer;
+	u_short oem_table_size;
+	u_short entry_count;
+	void   *apic_address;
+	u_short extended_table_length;
+	u_char  extended_table_checksum;
+	u_char  reserved;
+}      *mpcth_t;
+
+
+typedef struct PROCENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  cpu_flags;
+	u_long  cpu_signature;
+	u_long  feature_flags;
+	u_long  reserved1;
+	u_long  reserved2;
+}      *proc_entry_ptr;
+
+typedef struct BUSENTRY {
+	u_char  type;
+	u_char  bus_id;
+	char    bus_type[6];
+}      *bus_entry_ptr;
+
+typedef struct IOAPICENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  apic_flags;
+	void   *apic_address;
+}      *io_apic_entry_ptr;
+
+typedef struct INTENTRY {
+	u_char  type;
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}      *int_entry_ptr;
+/* descriptions of MP basetable entries */
+typedef struct BASETABLE_ENTRY {
+	u_char  type;
+	u_char  length;
+	char    name[16];
+}       basetable_entry;
+
+static basetable_entry basetable_entry_types[] =
+{
+	{0, 20, "Processor"},
+	{1, 8, "Bus"},
+	{2, 8, "I/O APIC"},
+	{3, 8, "I/O INT"},
+	{4, 8, "Local INT"}
+};
+
+typedef struct BUSDATA {
+	u_char  bus_id;
+	enum busTypes bus_type;
+}       bus_datum;
+
+typedef struct INTDATA {
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}       io_int, local_int;
+
+typedef struct BUSTYPENAME {
+	u_char  type;
+	char    name[7];
+}       bus_type_name;
+
+static bus_type_name bus_type_table[] =
+{
+	{CBUS, "CBUS"},
+	{CBUSII, "CBUSII"},
+	{EISA, "EISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{ISA, "ISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{PCI, "PCI"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{XPRESS, "XPRESS"},
+	{UNKNOWN_BUSTYPE, "---"}
+};
+/* from MP spec v1.4, table 5-1 */
+static int default_data[7][5] =
+{
+/*   nbus, id0, type0, id1, type1 */
+	{1, 0, ISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{0, 255, 255, 255, 255},/* MCA not supported */
+	{2, 0, ISA, 1, PCI},
+	{2, 0, EISA, 1, PCI},
+	{0, 255, 255, 255, 255}	/* MCA not supported */
+};
+
+
+/* the bus data */
+bus_datum bus_data[NBUS];
+
+/* the IO INT data, one entry per possible APIC INTerrupt */
+io_int  io_apic_ints[NINTR];
+
+static int nintrs;
+
+#if defined(FIX_MP_TABLE_WORKS)
+static void fix_mp_table __P((void));
+#endif /* FIX_MP_TABLE_WORKS */
+
+static void processor_entry __P((proc_entry_ptr entry, int *cpu));
+static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic));
+static void bus_entry __P((bus_entry_ptr entry, int *bus));
+static void int_entry __P((int_entry_ptr entry, int *intr));
+static int lookup_bus_type __P((char *name));
+
+
+/*
+ * parse an Intel MP specification table
+ */
+static int
+parse_mp_table(void)
+{
+	int     x;
+	mpfps_t fps;
+	mpcth_t cth;
+	int     totalSize;
+	void   *position;
+	int     count;
+	int     type;
+	int     apic, bus, cpu, intr;
+
+	/* clear physical APIC ID to logical CPU/IO table */
+	for (x = 0; x < NAPICID; ++x)
+		ID_TO_IO(x) = -1;
+
+	/* clear logical CPU to APIC ID table */
+	for (x = 0; x < NCPU; ++x)
+		CPU_TO_ID(x) = -1;
+
+	/* clear logical IO to APIC ID table */
+	for (x = 0; x < NAPIC; ++x)
+		IO_TO_ID(x) = -1;
+
+	/* clear IO APIC address table */
+	for (x = 0; x < NAPIC; ++x)
+		io_apic_address[x] = ~0;
+
+	/* clear bus data table */
+	for (x = 0; x < NBUS; ++x)
+		bus_data[x].bus_id = 0xff;
+
+	/* clear IO APIC INT table */
+	for (x = 0; x < NINTR; ++x)
+		io_apic_ints[x].int_type = 0xff;
+	nintrs = 0;
+
+	/* count the BSP */
+	mp_ncpus = 1;
+
+	/* setup the cpu/apic mapping arrays */
+	boot_cpu_id = -1;
+
+	/* local pointer */
+	fps = (mpfps_t) mpfps;
+
+	/* record whether PIC or virtual-wire mode */
+	picmode = (fps->mpfb2 & 0x80) ? 1 : 0;
+
+	/* check for use of 'default' configuration */
+#if defined(TEST_DEFAULT_CONFIG)
+	/* use default addresses */
+	cpu_apic_address = DEFAULT_APIC_BASE;
+	io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+	/* return default configuration type */
+	return TEST_DEFAULT_CONFIG;
+#else
+	if (fps->mpfb1 != 0) {
+		/* use default addresses */
+		cpu_apic_address = DEFAULT_APIC_BASE;
+		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+		/* return default configuration type */
+		return fps->mpfb1;
+	}
+#endif	/* TEST_DEFAULT_CONFIG */
+
+	if ((cth = fps->pap) == 0)
+		panic("MP Configuration Table Header MISSING!\n");
+
+	cpu_apic_address = (vm_offset_t) cth->apic_address;
+
+	totalSize = cth->base_table_length - sizeof(struct MPCTH);
+	position = (u_char *) cth + sizeof(struct MPCTH);
+	count = cth->entry_count;
+
+	apic = 0;		/* logical apic# start @ 0 */
+	bus = 0;		/* logical bus# start @ 0 */
+	cpu = 1;		/* logical cpu# start @ 0, BUT reserve 0 for */
+				/* BSP */
+	intr = 0;		/* unknown */
+
+	/* walk the table, recording info of interest */
+	while (count--) {
+		switch (type = *(u_char *) position) {
+		case 0:
+			processor_entry(position, &cpu);
+			break;
+		case 1:
+			bus_entry(position, &bus);
+			break;
+		case 2:
+			io_apic_entry(position, &apic);
+			break;
+		case 3:
+			int_entry(position, &intr);
+			break;
+		case 4:
+			/* int_entry(position); */
+			break;
+		default:
+			panic("mpfps Base Table HOSED!\n");
+			/* NOTREACHED */
+		}
+
+		totalSize -= basetable_entry_types[type].length;
+		(u_char *) position += basetable_entry_types[type].length;
+	}
+
+	if (boot_cpu_id == -1)
+		panic("NO BSP found!\n");
+
+	/* record # of APs found */
+	mp_naps = (cpu - 1);
+
+	/* record # of busses found */
+	mp_nbusses = bus;
+
+	/* record # of IO APICs found */
+	mp_napics = apic;
+
+	/* record # of IO APICs found */
+	nintrs = intr;
+
+#if defined(FIX_MP_TABLE_WORKS)
+	/* post scan cleanup */
+	fix_mp_table();
+#endif /* FIX_MP_TABLE_WORKS */
+
+	/* report fact that its NOT a default configuration */
+	return 0;
+}
+
+
+/*
+ * parse an Intel MP specification table
+ */
+#if defined(FIX_MP_TABLE_WORKS)
+static void
+fix_mp_table(void)
+{
+	int     x;
+	int     y;
+	int     num_pci_bus;
+	bus_datum bus_record;
+
+	/*
+	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
+	 * did it wrong.  The MP spec says that when more than 1 PCI bus
+	 * exists the BIOS must begin with bus entries for the PCI bus and use
+	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
+	 * exists the BIOS can choose to ignore this ordering, and indeed many
+	 * MP motherboards do ignore it.  This causes a problem when the PCI
+	 * sub-system makes requests of the MP sub-system based on PCI bus
+	 * numbers.	So here we look for the situation and renumber the
+	 * busses and associated INTs in an effort to "make it right".
+	 */
+
+	/* count the number of PCI busses */
+	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
+		if (bus_data[x].bus_type == PCI)
+			++num_pci_bus;
+	}
+
+	/* check the 1 PCI bus case for sanity */
+	if (num_pci_bus == 1) {
+
+		/* if its in the first slot all is well */
+		if (bus_data[0].bus_type == PCI)
+			return;
+
+		/* mis-numbered, swap with whichever bus uses slot 0 */
+
+		/* locate the entry holding the PCI bus */
+		for (x = 1; x < mp_nbusses; ++x) {
+			if (bus_data[x].bus_type == PCI)
+				break;
+		}
+
+		/* swap the bus entry records */
+		bus_record = bus_data[0];
+		bus_data[0] = bus_data[x];
+		bus_data[x] = bus_record;
+
+		/* swap each relavant INTerrupt entry */
+		for (y = 0; y < nintrs; ++y) {
+			if (io_apic_ints[y].src_bus_id == x)
+				io_apic_ints[y].src_bus_id = 0;
+			else
+				if (io_apic_ints[y].src_bus_id == 0)
+					io_apic_ints[y].src_bus_id = x;
+		}
+	}
+	/* sanity check if more than 1 PCI bus */
+	else
+		if (num_pci_bus > 1) {
+			for (x = 0; x < num_pci_bus; ++x) {
+				if (bus_data[x].bus_type != PCI) {
+					printf("bad PCI bus numbering\n");
+					panic("\n");
+				}
+			}
+		}
+}
+#endif /* FIX_MP_TABLE_WORKS */
+
+
+static void
+processor_entry(proc_entry_ptr entry, int *cpu)
+{
+	int     x = *cpu;
+
+	/* check for usability */
+	if (!(entry->cpu_flags & PROCENTRY_FLAG_EN))
+		return;
+
+	/* check for BSP flag */
+	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
+		/* always give boot CPU the logical value of 0 */
+		x = 0;
+		boot_cpu_id = entry->apic_id;
+	} else {
+		/* add another AP to list, if less than max number of CPUs */
+		if (x == NCPU) {
+			printf("Warning: only using %d of the available CPUs!\n", x);
+			return;
+		}
+		++(*cpu);
+	}
+
+	CPU_TO_ID(x) = entry->apic_id;
+	ID_TO_CPU(entry->apic_id) = x;
+}
+
+
+static void
+bus_entry(bus_entry_ptr entry, int *bus)
+{
+	int     x, y;
+	char    name[8];
+	char    c;
+
+	if ((x = (*bus)++) == NBUS)
+		panic("too many busses, increase 'NBUS'\n");
+
+	/* encode the name into an index */
+	for (y = 0; y < 6; ++y) {
+		if ((c = entry->bus_type[y]) == ' ')
+			break;
+		name[y] = c;
+	}
+	name[y] = '\0';
+
+	if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
+		panic("unknown bus type: '%s'\n", name);
+
+	bus_data[x].bus_id = entry->bus_id;
+	bus_data[x].bus_type = y;
+}
+
+
+static void
+io_apic_entry(io_apic_entry_ptr entry, int *apic)
+{
+	int     x;
+
+	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
+		return;
+
+	if ((x = (*apic)++) == NAPIC)
+		panic("too many APICs, increase 'NAPIC'\n");
+
+	IO_TO_ID(x) = entry->apic_id;
+	ID_TO_IO(entry->apic_id) = x;
+
+	io_apic_address[x] = (vm_offset_t) entry->apic_address;
+}
+
+
+static int
+lookup_bus_type(char *name)
+{
+	int     x;
+
+	for (x = 0; x < MAX_BUSTYPE; ++x)
+		if (strcmp(bus_type_table[x].name, name) == 0)
+			return bus_type_table[x].type;
+
+	return UNKNOWN_BUSTYPE;
+}
+
+
+static void
+int_entry(int_entry_ptr entry, int *intr)
+{
+	int     x;
+
+	if ((x = (*intr)++) == NINTR)
+		panic("too many INTs, increase 'NINTR'\n");
+
+	io_apic_ints[x].int_type = entry->int_type;
+	io_apic_ints[x].int_flags = entry->int_flags;
+	io_apic_ints[x].src_bus_id = entry->src_bus_id;
+	io_apic_ints[x].src_bus_irq = entry->src_bus_irq;
+	io_apic_ints[x].dst_apic_id = entry->dst_apic_id;
+	io_apic_ints[x].dst_apic_int = entry->dst_apic_int;
+}
+
+
+static int
+apic_int_is_bus_type(int intr, int bus_type)
+{
+	int     bus;
+
+	for (bus = 0; bus < mp_nbusses; ++bus)
+		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
+		    && ((int) bus_data[bus].bus_type == bus_type))
+			return 1;
+
+	return 0;
+}
+
+
+/*
+ * determine which APIC pin an ISA INT is attached to.
+ */
+#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
+#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
+
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_isa_apic_irq(int isaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (isaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == isaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, ISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin an EISA INT is attached to.
+ */
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_eisa_apic_irq(int eisaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (eisaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == eisaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, EISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin a PCI INT is attached to.
+ */
+#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
+#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
+#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
+int
+get_pci_apic_irq(int pciBus, int pciDevice, int pciInt)
+{
+	int     intr;
+
+	--pciInt;		/* zero based */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each record */
+		if ((INTTYPE(intr) == 0)
+#if defined(FIX_MP_TABLE_WORKS)
+		    && (SRCBUSID(intr) == pciBus)
+#endif /* FIX_MP_TABLE_WORKS */
+		    && (SRCBUSDEVICE(intr) == pciDevice)
+		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, PCI))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSLINE
+#undef SRCBUSDEVICE
+#undef SRCBUSID
+
+#undef INTPIN
+#undef INTTYPE
+
+
+int
+undirect_pci_irq(int rirq)
+{
+#if defined(READY)
+	printf("Freeing irq %d for ISA cards.\n", rirq);
+	/** FIXME: tickle the MB redirector chip */
+	return ???;
+#else
+	printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq);
+	return 0;
+#endif				/* READY */
+}
+
+
+/*
+ * given a bus ID, return:
+ *  the bus type if found
+ *  -1 if NOT found
+ */
+int
+apic_bus_type(int id)
+{
+	int     x;
+
+	for (x = 0; x < mp_nbusses; ++x)
+		if (bus_data[x].bus_id == id)
+			return bus_data[x].bus_type;
+
+	return -1;
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus ID if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_id(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_id);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus IRQ if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_irq(int apic, int pin)
+{
+	int     x;
+
+	for (x = 0; x < nintrs; x++)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_irq);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated INTerrupt type if found
+ *  -1 if NOT found
+ */
+int
+apic_int_type(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_type);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated trigger mode if found
+ *  -1 if NOT found
+ */
+int
+apic_trigger(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated 'active' level if found
+ *  -1 if NOT found
+ */
+int
+apic_polarity(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_flags & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * set data according to MP defaults
+ * FIXME: probably not complete yet...
+ */
+static void
+default_mp_table(int type)
+{
+	int     ap_cpu_id;
+#if defined(APIC_IO)
+	u_int32_t ux;
+	int     io_apic_id;
+	int     pin;
+#endif	/* APIC_IO */
+
+#if 0
+	printf("  MP default config type: %d\n", type);
+	switch (type) {
+	case 1:
+		printf("   bus: ISA, APIC: 82489DX\n");
+		break;
+	case 2:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 3:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 4:
+		printf("   bus: MCA, APIC: 82489DX\n");
+		break;
+	case 5:
+		printf("   bus: ISA+PCI, APIC: Integrated\n");
+		break;
+	case 6:
+		printf("   bus: EISA+PCI, APIC: Integrated\n");
+		break;
+	case 7:
+		printf("   bus: MCA+PCI, APIC: Integrated\n");
+		break;
+	default:
+		printf("   future type\n");
+		break;
+		/* NOTREACHED */
+	}
+#endif	/* 0 */
+
+	boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24;
+	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
+
+	/* BSP */
+	CPU_TO_ID(0) = boot_cpu_id;
+	ID_TO_CPU(boot_cpu_id) = 0;
+
+	/* one and only AP */
+	CPU_TO_ID(1) = ap_cpu_id;
+	ID_TO_CPU(ap_cpu_id) = 1;
+	mp_naps = 1;
+
+	/* one and only IO APIC */
+#if defined(APIC_IO)
+	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
+
+	/*
+	 * sanity check, refer to MP spec section 3.6.6, last paragraph
+	 * necessary as some hardware isn't properly setting up the IO APIC
+	 */
+#if defined(REALLY_ANAL_IOAPICID_VALUE)
+	if (io_apic_id != 2) {
+#else
+	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
+#endif	/* REALLY_ANAL_IOAPICID_VALUE */
+		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
+		ux &= ~APIC_ID_MASK;	/* clear the ID field */
+		ux |= 0x02000000;	/* set it to '2' */
+		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
+		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
+		if ((ux & APIC_ID_MASK) != 0x02000000)
+			panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux);
+		io_apic_id = 2;
+	}
+	IO_TO_ID(0) = io_apic_id;
+	ID_TO_IO(io_apic_id) = 0;
+	mp_napics = 1;
+#else
+	mp_napics = 0;
+#endif	/* APIC_IO */
+
+	/* fill out bus entries */
+	switch (type) {
+	case 1:
+	case 2:
+	case 3:
+	case 5:
+	case 6:
+		mp_nbusses = default_data[type - 1][0];
+		bus_data[0].bus_id = default_data[type - 1][1];
+		bus_data[0].bus_type = default_data[type - 1][2];
+		bus_data[1].bus_id = default_data[type - 1][3];
+		bus_data[1].bus_type = default_data[type - 1][4];
+		break;
+
+	/* case 4: case 7:		   MCA NOT supported */
+	default:		/* illegal/reserved */
+		panic("BAD default MP config: %d\n", type);
+	}
+
+#if defined(APIC_IO)
+	/* general cases from MP v1.4, table 5-2 */
+	for (pin = 0; pin < 16; ++pin) {
+		io_apic_ints[pin].int_type = 0;
+		io_apic_ints[pin].int_flags = 0x05;	/* edge-triggered/active-hi */
+		io_apic_ints[pin].src_bus_id = 0;
+		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 is caught below */
+		io_apic_ints[pin].dst_apic_id = io_apic_id;
+		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 correspondence */
+	}
+
+	/* special cases from MP v1.4, table 5-2 */
+	if (type == 2) {
+		io_apic_ints[2].int_type = 0xff;	/* N/C */
+		io_apic_ints[13].int_type = 0xff;	/* N/C */
+#if !defined(APIC_MIXED_MODE)
+		/** FIXME: ??? */
+		panic("sorry, can't support type 2 default yet\n");
+#endif	/* APIC_MIXED_MODE */
+	} else
+		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
+
+	if (type == 7)
+		io_apic_ints[0].int_type = 0xff;	/* N/C */
+	else
+		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
+
+	nintrs = 16;
+#endif	/* APIC_IO */
+}
+
+
+static void install_ap_tramp(u_int boot_addr);
+static int start_ap(int logicalCpu, u_int boot_addr);
+
+/*
+ * start each AP in our list
+ */
+static int
+start_all_aps(u_int boot_addr)
+{
+	int     x;
+	u_char  mpbiosreason;
+	u_long  mpbioswarmvec;
+
+	/**
+         * NOTE: this needs further thought:
+         *        where does it get released?
+         *        should it be set to empy?
+         *
+         * get the initial mp_lock with a count of 1 for the BSP
+         */
+	mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1;
+
+	/* initialize BSP's local APIC */
+	apic_initialize(1);
+
+	/* install the AP 1st level boot code */
+	install_ap_tramp(boot_addr);
+
+	/* save the current value of the warm-start vector */
+	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
+	outb(CMOS_REG, BIOS_RESET);
+	mpbiosreason = inb(CMOS_DATA);
+
+	/* start each AP */
+	for (x = 1; x <= mp_naps; ++x) {
+
+		/* setup a vector to our boot code */
+		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
+		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
+		outb(CMOS_REG, BIOS_RESET);
+		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
+
+		/* attempt to start the Application Processor */
+		CHECK_INIT(99);	/* setup checkpoints */
+		if (!start_ap(x, boot_addr)) {
+			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
+			CHECK_PRINT("trace");	/* show checkpoints */
+			/*
+			 * better panic as the AP may be running loose
+			 * somewhere
+			 */
+			printf("panic y/n? [n] ");
+			if (cngetc() != 'n')
+				panic("bye-bye\n");
+		}
+		CHECK_PRINT("trace");	/* show checkpoints */
+
+		/* record its version info */
+		cpu_apic_versions[x] = cpu_apic_versions[0];
+	}
+
+	/* fill in our (BSP) APIC version */
+	cpu_apic_versions[0] = apic_base[APIC_VER];
+
+	/* restore the warmstart vector */
+	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, mpbiosreason);
+
+	/* number of APs actually started */
+	return mp_ncpus - 1;
+}
+
+
+/*
+ * load the 1st level AP boot code into base memory.
+ */
+
+/* targets for relocation */
+extern void bigJump(void);
+extern void bootCodeSeg(void);
+extern void bootDataSeg(void);
+extern void MPentry(void);
+extern u_int MP_GDT;
+extern u_int mp_gdtbase;
+
+static void
+install_ap_tramp(u_int boot_addr)
+{
+	int     x;
+	int     size = *(int *) ((u_long) & bootMP_size);
+	u_char *src = (u_char *) ((u_long) bootMP);
+	u_char *dst = (u_char *) boot_addr + KERNBASE;
+	u_int   boot_base = (u_int) bootMP;
+	u_int8_t *dst8;
+	u_int16_t *dst16;
+	u_int32_t *dst32;
+
+	for (x = 0; x < size; ++x)
+		*dst++ = *src++;
+
+	/*
+	 * modify addresses in code we just moved to basemem. unfortunately we
+	 * need fairly detailed info about mpboot.s for this to work.  changes
+	 * to mpboot.s might require changes here.
+	 */
+
+	/* boot code is located in KERNEL space */
+	dst = (u_char *) boot_addr + KERNBASE;
+
+	/* modify the lgdt arg */
+	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
+	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
+
+	/* modify the ljmp target for MPentry() */
+	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
+	*dst32 = ((u_int) MPentry - KERNBASE);
+
+	/* modify the target for boot code segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+
+	/* modify the target for boot data segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+}
+
+
+/*
+ * this function starts the AP (application processor) identified
+ * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
+ * to accomplish this.  This is necessary because of the nuances
+ * of the different hardware we might encounter.  It ain't pretty,
+ * but it seems to work.
+ */
+static int
+start_ap(int logical_cpu, u_int boot_addr)
+{
+	int     physical_cpu;
+	int     vector;
+	int     cpus;
+	u_long  icr_lo, icr_hi;
+
+	/* get the PHYSICAL APIC ID# */
+	physical_cpu = CPU_TO_ID(logical_cpu);
+
+	/* calculate the vector */
+	vector = (boot_addr >> 12) & 0xff;
+
+	/* used as a watchpoint to signal AP startup */
+	cpus = mp_ncpus;
+
+	/*
+	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
+	 * and running the target CPU. OR this INIT IPI might be latched (P5
+	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
+	 * ignored.
+	 */
+
+	/* setup the address for the target AP */
+	icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK;
+	icr_hi |= (physical_cpu << 24);
+	apic_base[APIC_ICR_HI] = icr_hi;
+
+	/* do an INIT IPI: assert RESET */
+	icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000;
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500;
+
+	/* wait for pending status end */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/* do an INIT IPI: deassert RESET */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500;
+
+	/* wait for pending status end */
+	u_sleep(10000);		/* wait ~10mS */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/*
+	 * next we do a STARTUP IPI: the previous INIT IPI might still be
+	 * latched, (P5 bug) this 1st STARTUP would then terminate
+	 * immediately, and the previously started INIT IPI would continue. OR
+	 * the previous INIT IPI has already run. and this STARTUP IPI will
+	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
+	 * will run.
+	 */
+
+	/* do a STARTUP IPI */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/*
+	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
+	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
+	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
+	 * recognized after hardware RESET or INIT IPI.
+	 */
+
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/* wait for it to start */
+	set_apic_timer(5000000);/* == 5 seconds */
+	while (read_apic_timer())
+		if (mp_ncpus > cpus)
+			return 1;	/* return SUCCESS */
+
+	return 0;		/* return FAILURE */
+}
+
+
+#if defined(IPI_INTS)
+
+static void
+ipi_intr0(void)
+{
+	printf("IPI 0\n");
+}
+
+
+static void
+ipi_intr1(void)
+{
+	printf("IPI 1\n");
+}
+
+
+static void
+ipi_intr2(void)
+{
+	printf("IPI 2\n");
+}
+
+
+static void
+ipi_intr3(void)
+{
+	printf("IPI 3\n");
+}
+
+/*-----------------------------------------------------------------------
+**
+**	Register an interupt handler for an IPI.
+**	(Stolen from the PCI<->ISA glue code)
+**
+**-----------------------------------------------------------------------
+*/
+
+static int
+ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit)
+{
+	char    buf[16];
+	char   *cp;
+	int     free_id, id, result;
+
+	sprintf(buf, "ipi irq%d", irq);
+	for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) {
+		if (strcmp(cp, buf) == 0)
+			break;
+		if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0)
+			free_id = id;
+		while (*cp++ != '\0');
+	}
+	if (id == NR_DEVICES) {
+		id = free_id;
+		if (id == 0) {
+			/*
+			 * All ipi irq counters are in use, perhaps because
+			 * config is old so there aren't any.  Abuse the clk0
+			 * counter.
+			 */
+			printf(
+			    "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n",
+			    irq);
+		}
+	}
+	result = register_intr(
+	    irq,		/* isa irq	 */
+	    id,			/* device id    */
+	    0,			/* flags?	 */
+	    func,		/* handler	 */
+	    maskptr,		/* mask pointer */
+	    unit);		/* handler arg  */
+
+	if (result) {
+		printf("WARNING: ipi_ihandler_attach: result=%d\n", result);
+		return (result);
+	};
+
+	return (0);
+}
+#endif	/* IPI_INTS */
+
+
+#ifdef SMP_INVLTLB
+/*
+ * Flush the TLB on all other CPU's
+ *
+ * XXX: Needs to handshake and wait for completion before proceding.
+ */
+
+void
+smp_invltlb()
+{
+	if (smp_active) {
+		if (invldebug & 2)
+#if defined(XFAST_IPI32)
+			all_but_self_ipi(ICU_OFFSET + 32);
+#else
+			all_but_self_ipi(ICU_OFFSET + 27);
+#endif	/* XFAST_IPI32 */
+	}
+}
+
+void
+invlpg(u_int addr)
+{
+	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
+	smp_invltlb();
+}
+
+void
+invltlb(void)
+{
+	u_long  temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
+	 * inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
+	smp_invltlb();
+}
+/*
+ * Handles recieving an "IRQ 27", the invalidate tlb IPI..
+ */
+void
+ipi_invltlb(void)
+{
+	u_long  temp;
+
+	if (invldebug & 4) {
+		/*
+		 * This should be implemented as load_cr3(rcr3()) when
+		 * load_cr3() is inlined.
+		 */
+		__asm   __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp)
+		    ::      "memory");
+
+	}
+}
+#endif	/* SMP_INVLTLB */
diff --git a/sys/amd64/amd64/mpboot.S b/sys/amd64/amd64/mpboot.S
new file mode 100644
index 000000000000..e43b25043b70
--- /dev/null
+++ b/sys/amd64/amd64/mpboot.S
@@ -0,0 +1,317 @@
+/*
+ * Copyright (c) 1995, Jack F. Vogel
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by Jack F. Vogel
+ * 4. The name of the developer may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * mpboot.s:	FreeBSD machine support for the Intel MP Spec
+ *		multiprocessor systems.
+ *
+ *	$Id: mpboot.s,v 1.18 1997/04/25 03:11:27 fsmp Exp $
+ */
+
+
+#include <machine/asmacros.h>		/* miscellaneous asm macros */
+#include <machine/apic.h>
+#include <machine/specialreg.h>
+
+#include "assym.s"
+
+/*
+ * this code MUST be enabled here and in mp_machdep.c
+ * it follows the very early stages of AP boot by placing values in CMOS ram.
+ * it NORMALLY will never be needed and thus the primitive method for enabling.
+ *
+#define CHECK_POINTS
+ */
+
+#if defined(CHECK_POINTS)
+
+#define CMOS_REG	(0x70)
+#define CMOS_DATA	(0x71)
+
+#define CHECKPOINT(A,D)		\
+	movb	$(A),%al ;	\
+	outb	%al,$CMOS_REG ;	\
+	movb	$(D),%al ;	\
+	outb	%al,$CMOS_DATA
+
+#else
+
+#define CHECKPOINT(A,D)
+
+#endif /* CHECK_POINTS */
+
+
+/*
+ * the APs enter here from their trampoline code (bootMP, below)
+ */
+	.align	4
+
+NON_GPROF_ENTRY(MPentry)
+	CHECKPOINT(0x36, 3)
+	movl	$mp_stk-KERNBASE,%esp		/* mp boot stack end loc. */
+	/* Now enable paging mode */
+	movl	_IdlePTD-KERNBASE, %eax
+	movl	%eax,%cr3	
+	movl	%cr0,%eax	
+	orl	$CR0_PE|CR0_PG,%eax		/* enable paging */
+	movl	$0x80000011,%eax
+	movl	%eax,%cr0			/* let the games begin! */
+	movl	$mp_stk,%esp			/* mp boot stack end loc. */
+
+	pushl	$mp_begin			/* jump to high mem */
+	ret
+
+	/*
+	 * Wait for the booting CPU to signal startup
+	 */
+mp_begin:	/* now running relocated at KERNBASE */
+	CHECKPOINT(0x37, 4)
+	call	_init_secondary			/* load i386 tables */
+	CHECKPOINT(0x38, 5)
+
+	/* disable the APIC, just to be SURE */
+	movl	_apic_base, %esi
+	movl	APIC_SVR(%esi), %eax		/* get spurious vector reg. */
+	andl	$~APIC_SVR_SWEN, %eax		/* clear software enable bit */
+	movl	%eax, APIC_SVR(%esi)
+
+	/* signal our startup to the BSP */
+	movl	APIC_VER(%esi), %eax		/* our version reg contents */
+	movl	%eax, _cpu_apic_versions	/* into [ 0 ] */
+	incl	_mp_ncpus			/* signal BSP */
+
+	/* One at a time, we are running on the shared mp_stk */
+	/* This is the Intel reccomended semaphore method */
+#define BL_SET	0xff
+#define BL_CLR	0x00
+	movb	$BL_SET, %al
+1:
+	xchgb	%al, bootlock		/* xchg is implicitly locked */
+	cmpb	$BL_SET, %al		/* was is set? */
+	jz	1b			/* yes, keep trying... */
+	CHECKPOINT(0x39, 6)
+
+	/* Now, let's do some REAL WORK :-) */
+	call	_secondary_main
+/* NOT REACHED */
+2:	hlt
+	jmp 	2b
+
+/*
+ * Let a CPU past the semaphore so it can use mp_stk
+ */
+ENTRY(boot_unlock)
+	movb	$BL_CLR, %al
+	xchgb	%al, bootlock		/* xchg is implicitly locked */
+	ret
+
+/*
+ * This is the embedded trampoline or bootstrap that is
+ * copied into 'real-mode' low memory, it is where the
+ * secondary processor "wakes up". When it is executed
+ * the processor will eventually jump into the routine
+ * MPentry, which resides in normal kernel text above
+ * 1Meg.		-jackv
+ */
+
+#define data32	.byte 0x66
+#define addr32	.byte 0x67
+
+	.data
+	ALIGN_DATA				/* just to be sure */
+
+BOOTMP1:
+
+NON_GPROF_ENTRY(bootMP)
+	cli
+	CHECKPOINT(0x34, 1)
+	/* First guarantee a 'clean slate' */
+	data32
+	xorl	%eax, %eax
+	data32
+	movl	%eax, %ebx
+	data32
+	movl	%eax, %ecx
+	data32
+ 	movl	%eax, %edx
+	data32
+	movl	%eax, %esi
+	data32
+	movl	%eax, %edi
+
+	/* set up data segments */
+	mov	%cs, %ax
+	mov	%ax, %ds
+	mov	%ax, %es
+	mov	%ax, %fs
+	mov	%ax, %gs
+	mov	%ax, %ss
+	mov	$(boot_stk-_bootMP), %sp
+
+	/* Now load the global descriptor table */
+	addr32
+	data32
+	/* XXX: sigh: lgdt	MP_GDTptr-_bootMP GAS BUG! */
+	.byte	0x0f, 0x01, 0x15		/* XXX hand assemble! */
+	.long	MP_GDTptr-_bootMP		/* XXX hand assemble! */
+
+	/* Enable protected mode */
+	data32
+	movl	%cr0, %eax
+	data32
+	orl	$CR0_PE, %eax
+	data32
+	movl	%eax, %cr0 
+
+	/*
+	 * make intrasegment jump to flush the processor pipeline and
+	 * reload CS register
+	 */
+	data32
+	pushl	$0x18
+	data32
+	pushl	$(protmode-_bootMP)
+	data32
+	lret
+
+protmode:
+	CHECKPOINT(0x35, 2)
+
+	/*
+	 * we are NOW running for the first time with %eip
+	 * having the full physical address, BUT we still
+	 * are using a segment descriptor with the origin
+	 * not matching the booting kernel.
+	 *
+ 	 * SO NOW... for the BIG Jump into kernel's segment
+	 * and physical text above 1 Meg.
+	 */
+	mov	$0x10, %ebx
+	movw	%bx, %ds
+	movw	%bx, %es
+	movw	%bx, %fs
+	movw	%bx, %gs
+	movw	%bx, %ss
+
+	.globl	_bigJump
+_bigJump:
+	/* this will be modified by mpInstallTramp() */
+	ljmp	$0x08, $0			/* far jmp to MPentry() */
+	
+dead:	hlt /* We should never get here */
+	jmp	dead
+
+/*
+ * MP boot strap Global Descriptor Table
+ */
+	.align	4
+	.globl	_MP_GDT
+	.globl	_bootCodeSeg
+	.globl	_bootDataSeg
+_MP_GDT:
+
+nulldesc:		/* offset = 0x0 */
+
+	.word	0x0	
+	.word	0x0	
+	.byte	0x0	
+	.byte	0x0	
+	.byte	0x0	
+	.byte	0x0	
+
+kernelcode:		/* offset = 0x08 */
+
+	.word	0xffff	/* segment limit 0..15 */
+	.word	0x0000	/* segment base 0..15 */
+	.byte	0x0	/* segment base 16..23; set for 0K */
+	.byte	0x9f	/* flags; Type	*/
+	.byte	0xcf	/* flags; Limit	*/
+	.byte	0x0	/* segment base 24..32 */
+
+kerneldata:		/* offset = 0x10 */
+
+	.word	0xffff	/* segment limit 0..15 */
+	.word	0x0000	/* segment base 0..15 */
+	.byte	0x0	/* segment base 16..23; set for 0k */
+	.byte	0x93	/* flags; Type  */
+	.byte	0xcf	/* flags; Limit */
+	.byte	0x0	/* segment base 24..32 */
+
+bootcode:		/* offset = 0x18 */
+
+	.word	0xffff	/* segment limit 0..15 */
+_bootCodeSeg:		/* this will be modified by mpInstallTramp() */
+	.word	0x0000	/* segment base 0..15 */
+	.byte	0x00	/* segment base 16...23; set for 0x000xx000 */
+	.byte	0x9e	/* flags; Type  */
+	.byte	0xcf	/* flags; Limit */
+	.byte	0x0	/*segment base 24..32 */
+
+bootdata:		/* offset = 0x20 */
+
+	.word	0xffff	
+_bootDataSeg:		/* this will be modified by mpInstallTramp() */
+	.word	0x0000	/* segment base 0..15 */
+	.byte	0x00	/* segment base 16...23; set for 0x000xx000 */
+	.byte	0x92	
+	.byte	0xcf	
+	.byte	0x0		
+
+/*
+ * GDT pointer for the lgdt call
+ */
+	.globl	_mp_gdtbase
+
+MP_GDTptr:	
+_mp_gdtlimit:
+	.word	0x0028		
+_mp_gdtbase:		/* this will be modified by mpInstallTramp() */
+	.long	0
+
+	.space	0x100	/* space for boot_stk - 1st temporary stack */
+boot_stk:
+
+BOOTMP2:
+	.globl	_bootMP_size
+_bootMP_size:
+	.long	BOOTMP2 - BOOTMP1
+
+	/*
+	 * Temporary stack used while booting AP's
+	 * It is protected by:
+	 * 1: only one cpu is started at a time and it ends up waiting
+	 *    for smp_active before continuing.
+	 * 2: Once smp_active != 0; further access is limited by _bootlock.
+	 */
+	.globl	mp_stk
+	.space	0x2000	/* space for mp_stk - 2nd temporary stack */
+mp_stk:
+
+	.globl	bootlock
+bootlock:	.byte BL_SET
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
new file mode 100644
index 000000000000..07fca34efe56
--- /dev/null
+++ b/sys/amd64/amd64/mptable.c
@@ -0,0 +1,1646 @@
+/*
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $
+ */
+
+#include "opt_smp.h"
+#include "opt_smp_invltlb.h"
+#if defined(APIC_IO)
+#if !defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#else				/* APIC_IO */
+#if defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#endif				/* APIC_IO */
+#define FIX_MP_TABLE_WORKS_NOT
+
+#include "opt_serial.h"
+
+#include <sys/param.h>		/* for KERNBASE */
+#include <sys/types.h>
+#include <sys/sysproto.h>
+#include <sys/time.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>		/* for KERNBASE */
+#include <vm/vm_param.h>	/* for KERNBASE */
+#include <vm/pmap.h>		/* for KERNBASE */
+#include <machine/pmap.h>	/* for KERNBASE */
+
+#include <machine/smp.h>
+#include <machine/apic.h>
+#include <machine/mpapic.h>
+#include <machine/cpufunc.h>
+#include <machine/segments.h>
+#include <machine/smptests.h>	/** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */
+
+#include <i386/i386/cons.h>	/* cngetc() */
+
+#if defined(IPI_INTS)
+#include <i386/isa/isa_device.h>
+#include "vector.h"
+#endif				/* IPI_INTS */
+
+#if defined(SMP_INVLTLB)
+#include <i386/isa/icu.h>
+#endif				/* SMP_INVLTLB */
+
+#define WARMBOOT_TARGET	0
+#define WARMBOOT_OFF	(KERNBASE + 0x0467)
+#define WARMBOOT_SEG	(KERNBASE + 0x0469)
+
+#define BIOS_BASE	(0xf0000)
+#define BIOS_SIZE	(0x10000)
+#define BIOS_COUNT	(BIOS_SIZE/4)
+
+#define CMOS_REG	(0x70)
+#define CMOS_DATA	(0x71)
+#define BIOS_RESET	(0x0f)
+#define BIOS_WARM	(0x0a)
+
+/*
+ * this code MUST be enabled here and in mpboot.s.
+ * it follows the very early stages of AP boot by placing values in CMOS ram.
+ * it NORMALLY will never be needed and thus the primitive method for enabling.
+ *
+#define CHECK_POINTS
+ */
+
+#if defined(CHECK_POINTS)
+#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
+#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
+
+#define CHECK_INIT(D);				\
+	CHECK_WRITE(0x34, (D));			\
+	CHECK_WRITE(0x35, (D));			\
+	CHECK_WRITE(0x36, (D));			\
+	CHECK_WRITE(0x37, (D));			\
+	CHECK_WRITE(0x38, (D));			\
+	CHECK_WRITE(0x39, (D));
+
+#define CHECK_PRINT(S);				\
+	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
+	   (S),					\
+	   CHECK_READ(0x34),			\
+	   CHECK_READ(0x35),			\
+	   CHECK_READ(0x36),			\
+	   CHECK_READ(0x37),			\
+	   CHECK_READ(0x38),			\
+	   CHECK_READ(0x39));
+
+#else				/* CHECK_POINTS */
+
+#define CHECK_INIT(D)
+#define CHECK_PRINT(S)
+
+#endif				/* CHECK_POINTS */
+
+
+/** FIXME: what system files declare these??? */
+extern struct region_descriptor r_gdt, r_idt;
+
+/* global data */
+struct proc *SMPcurproc[NCPU];
+struct pcb *SMPcurpcb[NCPU];
+struct timeval SMPruntime[NCPU];
+
+int     mp_ncpus;		/* # of CPUs, including BSP */
+int     mp_naps;		/* # of Applications processors */
+int     mp_nbusses;		/* # of busses */
+int     mp_napics;		/* # of IO APICs */
+int     mpenabled;
+int     boot_cpu_id;		/* designated BSP */
+vm_offset_t cpu_apic_address;
+vm_offset_t io_apic_address[NAPIC];
+
+u_int32_t cpu_apic_versions[NCPU];
+u_int32_t io_apic_versions[NAPIC];
+
+/*
+ * APIC ID logical/physical mapping structures
+ */
+int     cpu_num_to_apic_id[NCPU];
+int     io_num_to_apic_id[NAPIC];
+int     apic_id_to_logical[NAPICID];
+
+/*
+ * look for MP compliant motherboard.
+ */
+
+static u_int boot_address;
+static u_int base_memory;
+
+static int picmode;		/* 0: virtual wire mode, 1: PIC mode */
+static u_int mpfps;
+static int search_for_sig(u_int32_t target, int count);
+static int mp_probe(u_int base_top);
+static void mp_enable(u_int boot_addr);
+
+#if defined(IPI_INTS)
+static void ipi_intr0(void);
+static void ipi_intr1(void);
+static void ipi_intr2(void);
+static void ipi_intr3(void);
+static int 
+ipi_ihandler_attach(int irq, inthand2_t * func,
+    unsigned *maskptr, int unit);
+#endif				/* IPI_INTS */
+
+
+/*
+ * calculate usable address in base memory for AP trampoline code
+ */
+u_int
+mp_bootaddress(u_int basemem)
+{
+	base_memory = basemem * 1024;	/* convert to bytes */
+
+	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
+	if ((base_memory - boot_address) < bootMP_size)
+		boot_address -= 4096;	/* not enough, lower by 4k */
+
+	return boot_address;
+}
+
+
+/*
+ * startup the SMP processors
+ */
+void
+mp_start(void)
+{
+	/* look for MP capable motherboard */
+	if (mp_probe(base_memory))
+		mp_enable(boot_address);
+	else {
+		printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n");
+		panic("can't continue!\n");
+	}
+
+	/* finish pmap initialization - turn off V==P mapping at zero */
+	pmap_bootstrap2();
+}
+
+
+/*
+ * print various information about the SMP system hardware and setup
+ */
+void
+mp_announce(void)
+{
+	int     x;
+
+	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
+	printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0));
+	printf(", version: 0x%08x\n", cpu_apic_versions[0]);
+	for (x = 1; x <= mp_naps; ++x) {
+		printf(" cpu%d (AP):  apic id: %d", x, CPU_TO_ID(x));
+		printf(", version: 0x%08x\n", cpu_apic_versions[x]);
+	}
+
+#if defined(APIC_IO)
+	for (x = 0; x < mp_napics; ++x) {
+		printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x));
+		printf(", version: 0x%08x\n", io_apic_versions[x]);
+	}
+#else
+	printf(" Warning: APIC I/O disabled\n");
+#endif	/* APIC_IO */
+}
+
+
+/*
+ * AP cpu's call this to sync up protected mode.
+ */
+void
+init_secondary(void)
+{
+	int     gsel_tss, slot;
+
+	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
+	r_gdt.rd_base = (int) gdt;
+	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lidt(&r_idt);
+	lldt(_default_ldt);
+
+	slot = NGDT + cpunumber();
+	gsel_tss = GSEL(slot, SEL_KPL);
+	gdt[slot].sd.sd_type = SDT_SYS386TSS;
+	ltr(gsel_tss);
+
+	load_cr0(0x8005003b);	/* XXX! */
+}
+
+
+#if defined(APIC_IO)
+void
+configure_local_apic(void)
+{
+	u_char  byte;
+	u_int32_t temp;
+
+	if (picmode) {
+		outb(0x22, 0x70);	/* select IMCR */
+		byte = inb(0x23);	/* current contents */
+		byte |= 0x01;	/* mask external INTR */
+		outb(0x23, byte);	/* disconnect 8259s/NMI */
+	}
+	/* mask the LVT1 */
+	temp = apic_base[APIC_LVT1];
+	temp |= APIC_LVT_M;
+	apic_base[APIC_LVT1] = temp;
+}
+#endif	/* APIC_IO */
+
+
+/*******************************************************************
+ * local functions and data
+ */
+
+static int
+mp_probe(u_int base_top)
+{
+	int     x;
+	u_long  segment;
+	u_int32_t target;
+
+	/* see if EBDA exists */
+	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
+		/* search first 1K of EBDA */
+		target = (u_int32_t) (segment << 4);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	} else {
+		/*last 1K of base memory, effective 'top of base' is passed in*/
+		target = (u_int32_t) (base_top - 0x400);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	}
+
+	/* search the BIOS */
+	target = (u_int32_t) BIOS_BASE;
+	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
+		goto found;
+
+	/* nothing found */
+	mpfps = mpenabled = 0;
+	return 0;
+
+found:				/* please forgive the 'goto'! */
+	/* flag fact that we are running multiple processors */
+	mpfps = x;
+	mpenabled = 1;
+	return 1;
+}
+
+
+/*
+ * start the SMP system
+ */
+static int parse_mp_table(void);
+static void default_mp_table(int type);
+static int start_all_aps(u_int boot_addr);
+
+#if defined(XFAST_IPI32)
+#include <machine/md_var.h>
+#include <i386/isa/isa_device.h>
+extern void Xfastipi32(u_int, u_int, u_int, u_int);
+#endif				/* XFAST_IPI32 */
+
+static void
+mp_enable(u_int boot_addr)
+{
+	int     x;
+#if defined(APIC_IO)
+	int     apic;
+	u_int   ux;
+#if defined(TEST_UPPERPRIO)
+	u_char  select;		/* the select register is 8 bits */
+	u_int32_t flags;	/* the window register is 32 bits */
+#endif	/* TEST_UPPERPRIO */
+#endif	/* APIC_IO */
+
+	/* examine the MP table for needed info */
+	x = parse_mp_table();
+
+	/* create pages for (address common) cpu APIC and each IO APIC */
+	pmap_bootstrap_apics();
+
+	/* can't process default configs till the CPU APIC is pmapped */
+	if (x)
+		default_mp_table(x);
+
+#if defined(APIC_IO)
+	/* fill the LOGICAL io_apic_versions table */
+	for (apic = 0; apic < mp_napics; ++apic) {
+		ux = io_apic_read(apic, IOAPIC_VER);
+		io_apic_versions[apic] = ux;
+	}
+
+	/*
+	 */
+	for (apic = 0; apic < mp_napics; ++apic)
+		if (io_apic_setup(apic) < 0)
+			panic("IO APIC setup failure\n");
+
+#if defined(IPI_INTS)
+	/* setup IPI INTerrupt mechanism */
+	ipi_ihandler_attach( /* irq */ 24,
+	     /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 25,
+	     /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 26,
+	     /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0);
+#if !defined(SMP_INVLTLB)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+#else
+#if defined(XFAST_IPI32)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+	setidt(ICU_OFFSET + 32,
+	    Xfastipi32,
+	    SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#else
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0);
+#endif	/* XFAST_IPI32 */
+#endif
+
+#endif	/* IPI_INTS */
+
+#if defined(TEST_UPPERPRIO)
+
+#if 1
+	printf("special IRQ10\n");
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~0xff;		/** clear vector */
+	flags |= 64;
+	io_apic_write(0, select, flags);
+#else
+	printf("special IRQ10\n");
+	cngetc();
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~IOART_DELMOD;	/* FIXED mode */
+	io_apic_write(0, select, flags);
+	io_apic_write(0, select + 1, boot_cpu_id << 24);
+#endif	/** 0/1 */
+
+#endif	/* TEST_UPPERPRIO */
+
+#endif	/* APIC_IO */
+
+	/* start each Application Processor */
+	start_all_aps(boot_addr);
+}
+
+
+/*
+ * look for the MP spec signature
+ */
+
+/* string defined by the Intel MP Spec as identifying the MP table */
+#define MP_SIG		0x5f504d5f	/* _MP_ */
+#define NEXT(X)		((X) += 4)
+static int
+search_for_sig(u_int32_t target, int count)
+{
+	int     x;
+	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
+
+	for (x = 0; x < count; NEXT(x))
+		if (addr[x] == MP_SIG)
+			/* make array index a byte index */
+			return (target + (x * sizeof(u_int32_t)));
+
+	return -1;
+}
+
+
+#define PROCENTRY_FLAG_EN	0x01
+#define PROCENTRY_FLAG_BP	0x02
+#define IOAPICENTRY_FLAG_EN	0x01
+
+/* MP Floating Pointer Structure */
+typedef struct MPFPS {
+	char    signature[4];
+	void   *pap;
+	u_char  length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  mpfb1;
+	u_char  mpfb2;
+	u_char  mpfb3;
+	u_char  mpfb4;
+	u_char  mpfb5;
+}      *mpfps_t;
+/* MP Configuration Table Header */
+typedef struct MPCTH {
+	char    signature[4];
+	u_short base_table_length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  oem_id[8];
+	u_char  product_id[12];
+	void   *oem_table_pointer;
+	u_short oem_table_size;
+	u_short entry_count;
+	void   *apic_address;
+	u_short extended_table_length;
+	u_char  extended_table_checksum;
+	u_char  reserved;
+}      *mpcth_t;
+
+
+typedef struct PROCENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  cpu_flags;
+	u_long  cpu_signature;
+	u_long  feature_flags;
+	u_long  reserved1;
+	u_long  reserved2;
+}      *proc_entry_ptr;
+
+typedef struct BUSENTRY {
+	u_char  type;
+	u_char  bus_id;
+	char    bus_type[6];
+}      *bus_entry_ptr;
+
+typedef struct IOAPICENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  apic_flags;
+	void   *apic_address;
+}      *io_apic_entry_ptr;
+
+typedef struct INTENTRY {
+	u_char  type;
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}      *int_entry_ptr;
+/* descriptions of MP basetable entries */
+typedef struct BASETABLE_ENTRY {
+	u_char  type;
+	u_char  length;
+	char    name[16];
+}       basetable_entry;
+
+static basetable_entry basetable_entry_types[] =
+{
+	{0, 20, "Processor"},
+	{1, 8, "Bus"},
+	{2, 8, "I/O APIC"},
+	{3, 8, "I/O INT"},
+	{4, 8, "Local INT"}
+};
+
+typedef struct BUSDATA {
+	u_char  bus_id;
+	enum busTypes bus_type;
+}       bus_datum;
+
+typedef struct INTDATA {
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}       io_int, local_int;
+
+typedef struct BUSTYPENAME {
+	u_char  type;
+	char    name[7];
+}       bus_type_name;
+
+static bus_type_name bus_type_table[] =
+{
+	{CBUS, "CBUS"},
+	{CBUSII, "CBUSII"},
+	{EISA, "EISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{ISA, "ISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{PCI, "PCI"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{XPRESS, "XPRESS"},
+	{UNKNOWN_BUSTYPE, "---"}
+};
+/* from MP spec v1.4, table 5-1 */
+static int default_data[7][5] =
+{
+/*   nbus, id0, type0, id1, type1 */
+	{1, 0, ISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{0, 255, 255, 255, 255},/* MCA not supported */
+	{2, 0, ISA, 1, PCI},
+	{2, 0, EISA, 1, PCI},
+	{0, 255, 255, 255, 255}	/* MCA not supported */
+};
+
+
+/* the bus data */
+bus_datum bus_data[NBUS];
+
+/* the IO INT data, one entry per possible APIC INTerrupt */
+io_int  io_apic_ints[NINTR];
+
+static int nintrs;
+
+#if defined(FIX_MP_TABLE_WORKS)
+static void fix_mp_table __P((void));
+#endif /* FIX_MP_TABLE_WORKS */
+
+static void processor_entry __P((proc_entry_ptr entry, int *cpu));
+static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic));
+static void bus_entry __P((bus_entry_ptr entry, int *bus));
+static void int_entry __P((int_entry_ptr entry, int *intr));
+static int lookup_bus_type __P((char *name));
+
+
+/*
+ * parse an Intel MP specification table
+ */
+static int
+parse_mp_table(void)
+{
+	int     x;
+	mpfps_t fps;
+	mpcth_t cth;
+	int     totalSize;
+	void   *position;
+	int     count;
+	int     type;
+	int     apic, bus, cpu, intr;
+
+	/* clear physical APIC ID to logical CPU/IO table */
+	for (x = 0; x < NAPICID; ++x)
+		ID_TO_IO(x) = -1;
+
+	/* clear logical CPU to APIC ID table */
+	for (x = 0; x < NCPU; ++x)
+		CPU_TO_ID(x) = -1;
+
+	/* clear logical IO to APIC ID table */
+	for (x = 0; x < NAPIC; ++x)
+		IO_TO_ID(x) = -1;
+
+	/* clear IO APIC address table */
+	for (x = 0; x < NAPIC; ++x)
+		io_apic_address[x] = ~0;
+
+	/* clear bus data table */
+	for (x = 0; x < NBUS; ++x)
+		bus_data[x].bus_id = 0xff;
+
+	/* clear IO APIC INT table */
+	for (x = 0; x < NINTR; ++x)
+		io_apic_ints[x].int_type = 0xff;
+	nintrs = 0;
+
+	/* count the BSP */
+	mp_ncpus = 1;
+
+	/* setup the cpu/apic mapping arrays */
+	boot_cpu_id = -1;
+
+	/* local pointer */
+	fps = (mpfps_t) mpfps;
+
+	/* record whether PIC or virtual-wire mode */
+	picmode = (fps->mpfb2 & 0x80) ? 1 : 0;
+
+	/* check for use of 'default' configuration */
+#if defined(TEST_DEFAULT_CONFIG)
+	/* use default addresses */
+	cpu_apic_address = DEFAULT_APIC_BASE;
+	io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+	/* return default configuration type */
+	return TEST_DEFAULT_CONFIG;
+#else
+	if (fps->mpfb1 != 0) {
+		/* use default addresses */
+		cpu_apic_address = DEFAULT_APIC_BASE;
+		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+		/* return default configuration type */
+		return fps->mpfb1;
+	}
+#endif	/* TEST_DEFAULT_CONFIG */
+
+	if ((cth = fps->pap) == 0)
+		panic("MP Configuration Table Header MISSING!\n");
+
+	cpu_apic_address = (vm_offset_t) cth->apic_address;
+
+	totalSize = cth->base_table_length - sizeof(struct MPCTH);
+	position = (u_char *) cth + sizeof(struct MPCTH);
+	count = cth->entry_count;
+
+	apic = 0;		/* logical apic# start @ 0 */
+	bus = 0;		/* logical bus# start @ 0 */
+	cpu = 1;		/* logical cpu# start @ 0, BUT reserve 0 for */
+				/* BSP */
+	intr = 0;		/* unknown */
+
+	/* walk the table, recording info of interest */
+	while (count--) {
+		switch (type = *(u_char *) position) {
+		case 0:
+			processor_entry(position, &cpu);
+			break;
+		case 1:
+			bus_entry(position, &bus);
+			break;
+		case 2:
+			io_apic_entry(position, &apic);
+			break;
+		case 3:
+			int_entry(position, &intr);
+			break;
+		case 4:
+			/* int_entry(position); */
+			break;
+		default:
+			panic("mpfps Base Table HOSED!\n");
+			/* NOTREACHED */
+		}
+
+		totalSize -= basetable_entry_types[type].length;
+		(u_char *) position += basetable_entry_types[type].length;
+	}
+
+	if (boot_cpu_id == -1)
+		panic("NO BSP found!\n");
+
+	/* record # of APs found */
+	mp_naps = (cpu - 1);
+
+	/* record # of busses found */
+	mp_nbusses = bus;
+
+	/* record # of IO APICs found */
+	mp_napics = apic;
+
+	/* record # of IO APICs found */
+	nintrs = intr;
+
+#if defined(FIX_MP_TABLE_WORKS)
+	/* post scan cleanup */
+	fix_mp_table();
+#endif /* FIX_MP_TABLE_WORKS */
+
+	/* report fact that its NOT a default configuration */
+	return 0;
+}
+
+
+/*
+ * parse an Intel MP specification table
+ */
+#if defined(FIX_MP_TABLE_WORKS)
+static void
+fix_mp_table(void)
+{
+	int     x;
+	int     y;
+	int     num_pci_bus;
+	bus_datum bus_record;
+
+	/*
+	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
+	 * did it wrong.  The MP spec says that when more than 1 PCI bus
+	 * exists the BIOS must begin with bus entries for the PCI bus and use
+	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
+	 * exists the BIOS can choose to ignore this ordering, and indeed many
+	 * MP motherboards do ignore it.  This causes a problem when the PCI
+	 * sub-system makes requests of the MP sub-system based on PCI bus
+	 * numbers.	So here we look for the situation and renumber the
+	 * busses and associated INTs in an effort to "make it right".
+	 */
+
+	/* count the number of PCI busses */
+	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
+		if (bus_data[x].bus_type == PCI)
+			++num_pci_bus;
+	}
+
+	/* check the 1 PCI bus case for sanity */
+	if (num_pci_bus == 1) {
+
+		/* if its in the first slot all is well */
+		if (bus_data[0].bus_type == PCI)
+			return;
+
+		/* mis-numbered, swap with whichever bus uses slot 0 */
+
+		/* locate the entry holding the PCI bus */
+		for (x = 1; x < mp_nbusses; ++x) {
+			if (bus_data[x].bus_type == PCI)
+				break;
+		}
+
+		/* swap the bus entry records */
+		bus_record = bus_data[0];
+		bus_data[0] = bus_data[x];
+		bus_data[x] = bus_record;
+
+		/* swap each relavant INTerrupt entry */
+		for (y = 0; y < nintrs; ++y) {
+			if (io_apic_ints[y].src_bus_id == x)
+				io_apic_ints[y].src_bus_id = 0;
+			else
+				if (io_apic_ints[y].src_bus_id == 0)
+					io_apic_ints[y].src_bus_id = x;
+		}
+	}
+	/* sanity check if more than 1 PCI bus */
+	else
+		if (num_pci_bus > 1) {
+			for (x = 0; x < num_pci_bus; ++x) {
+				if (bus_data[x].bus_type != PCI) {
+					printf("bad PCI bus numbering\n");
+					panic("\n");
+				}
+			}
+		}
+}
+#endif /* FIX_MP_TABLE_WORKS */
+
+
+static void
+processor_entry(proc_entry_ptr entry, int *cpu)
+{
+	int     x = *cpu;
+
+	/* check for usability */
+	if (!(entry->cpu_flags & PROCENTRY_FLAG_EN))
+		return;
+
+	/* check for BSP flag */
+	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
+		/* always give boot CPU the logical value of 0 */
+		x = 0;
+		boot_cpu_id = entry->apic_id;
+	} else {
+		/* add another AP to list, if less than max number of CPUs */
+		if (x == NCPU) {
+			printf("Warning: only using %d of the available CPUs!\n", x);
+			return;
+		}
+		++(*cpu);
+	}
+
+	CPU_TO_ID(x) = entry->apic_id;
+	ID_TO_CPU(entry->apic_id) = x;
+}
+
+
+static void
+bus_entry(bus_entry_ptr entry, int *bus)
+{
+	int     x, y;
+	char    name[8];
+	char    c;
+
+	if ((x = (*bus)++) == NBUS)
+		panic("too many busses, increase 'NBUS'\n");
+
+	/* encode the name into an index */
+	for (y = 0; y < 6; ++y) {
+		if ((c = entry->bus_type[y]) == ' ')
+			break;
+		name[y] = c;
+	}
+	name[y] = '\0';
+
+	if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
+		panic("unknown bus type: '%s'\n", name);
+
+	bus_data[x].bus_id = entry->bus_id;
+	bus_data[x].bus_type = y;
+}
+
+
+static void
+io_apic_entry(io_apic_entry_ptr entry, int *apic)
+{
+	int     x;
+
+	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
+		return;
+
+	if ((x = (*apic)++) == NAPIC)
+		panic("too many APICs, increase 'NAPIC'\n");
+
+	IO_TO_ID(x) = entry->apic_id;
+	ID_TO_IO(entry->apic_id) = x;
+
+	io_apic_address[x] = (vm_offset_t) entry->apic_address;
+}
+
+
+static int
+lookup_bus_type(char *name)
+{
+	int     x;
+
+	for (x = 0; x < MAX_BUSTYPE; ++x)
+		if (strcmp(bus_type_table[x].name, name) == 0)
+			return bus_type_table[x].type;
+
+	return UNKNOWN_BUSTYPE;
+}
+
+
+static void
+int_entry(int_entry_ptr entry, int *intr)
+{
+	int     x;
+
+	if ((x = (*intr)++) == NINTR)
+		panic("too many INTs, increase 'NINTR'\n");
+
+	io_apic_ints[x].int_type = entry->int_type;
+	io_apic_ints[x].int_flags = entry->int_flags;
+	io_apic_ints[x].src_bus_id = entry->src_bus_id;
+	io_apic_ints[x].src_bus_irq = entry->src_bus_irq;
+	io_apic_ints[x].dst_apic_id = entry->dst_apic_id;
+	io_apic_ints[x].dst_apic_int = entry->dst_apic_int;
+}
+
+
+static int
+apic_int_is_bus_type(int intr, int bus_type)
+{
+	int     bus;
+
+	for (bus = 0; bus < mp_nbusses; ++bus)
+		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
+		    && ((int) bus_data[bus].bus_type == bus_type))
+			return 1;
+
+	return 0;
+}
+
+
+/*
+ * determine which APIC pin an ISA INT is attached to.
+ */
+#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
+#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
+
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_isa_apic_irq(int isaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (isaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == isaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, ISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin an EISA INT is attached to.
+ */
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_eisa_apic_irq(int eisaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (eisaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == eisaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, EISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin a PCI INT is attached to.
+ */
+#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
+#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
+#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
+int
+get_pci_apic_irq(int pciBus, int pciDevice, int pciInt)
+{
+	int     intr;
+
+	--pciInt;		/* zero based */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each record */
+		if ((INTTYPE(intr) == 0)
+#if defined(FIX_MP_TABLE_WORKS)
+		    && (SRCBUSID(intr) == pciBus)
+#endif /* FIX_MP_TABLE_WORKS */
+		    && (SRCBUSDEVICE(intr) == pciDevice)
+		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, PCI))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSLINE
+#undef SRCBUSDEVICE
+#undef SRCBUSID
+
+#undef INTPIN
+#undef INTTYPE
+
+
+int
+undirect_pci_irq(int rirq)
+{
+#if defined(READY)
+	printf("Freeing irq %d for ISA cards.\n", rirq);
+	/** FIXME: tickle the MB redirector chip */
+	return ???;
+#else
+	printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq);
+	return 0;
+#endif				/* READY */
+}
+
+
+/*
+ * given a bus ID, return:
+ *  the bus type if found
+ *  -1 if NOT found
+ */
+int
+apic_bus_type(int id)
+{
+	int     x;
+
+	for (x = 0; x < mp_nbusses; ++x)
+		if (bus_data[x].bus_id == id)
+			return bus_data[x].bus_type;
+
+	return -1;
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus ID if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_id(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_id);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus IRQ if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_irq(int apic, int pin)
+{
+	int     x;
+
+	for (x = 0; x < nintrs; x++)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_irq);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated INTerrupt type if found
+ *  -1 if NOT found
+ */
+int
+apic_int_type(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_type);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated trigger mode if found
+ *  -1 if NOT found
+ */
+int
+apic_trigger(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated 'active' level if found
+ *  -1 if NOT found
+ */
+int
+apic_polarity(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_flags & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * set data according to MP defaults
+ * FIXME: probably not complete yet...
+ */
+static void
+default_mp_table(int type)
+{
+	int     ap_cpu_id;
+#if defined(APIC_IO)
+	u_int32_t ux;
+	int     io_apic_id;
+	int     pin;
+#endif	/* APIC_IO */
+
+#if 0
+	printf("  MP default config type: %d\n", type);
+	switch (type) {
+	case 1:
+		printf("   bus: ISA, APIC: 82489DX\n");
+		break;
+	case 2:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 3:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 4:
+		printf("   bus: MCA, APIC: 82489DX\n");
+		break;
+	case 5:
+		printf("   bus: ISA+PCI, APIC: Integrated\n");
+		break;
+	case 6:
+		printf("   bus: EISA+PCI, APIC: Integrated\n");
+		break;
+	case 7:
+		printf("   bus: MCA+PCI, APIC: Integrated\n");
+		break;
+	default:
+		printf("   future type\n");
+		break;
+		/* NOTREACHED */
+	}
+#endif	/* 0 */
+
+	boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24;
+	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
+
+	/* BSP */
+	CPU_TO_ID(0) = boot_cpu_id;
+	ID_TO_CPU(boot_cpu_id) = 0;
+
+	/* one and only AP */
+	CPU_TO_ID(1) = ap_cpu_id;
+	ID_TO_CPU(ap_cpu_id) = 1;
+	mp_naps = 1;
+
+	/* one and only IO APIC */
+#if defined(APIC_IO)
+	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
+
+	/*
+	 * sanity check, refer to MP spec section 3.6.6, last paragraph
+	 * necessary as some hardware isn't properly setting up the IO APIC
+	 */
+#if defined(REALLY_ANAL_IOAPICID_VALUE)
+	if (io_apic_id != 2) {
+#else
+	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
+#endif	/* REALLY_ANAL_IOAPICID_VALUE */
+		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
+		ux &= ~APIC_ID_MASK;	/* clear the ID field */
+		ux |= 0x02000000;	/* set it to '2' */
+		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
+		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
+		if ((ux & APIC_ID_MASK) != 0x02000000)
+			panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux);
+		io_apic_id = 2;
+	}
+	IO_TO_ID(0) = io_apic_id;
+	ID_TO_IO(io_apic_id) = 0;
+	mp_napics = 1;
+#else
+	mp_napics = 0;
+#endif	/* APIC_IO */
+
+	/* fill out bus entries */
+	switch (type) {
+	case 1:
+	case 2:
+	case 3:
+	case 5:
+	case 6:
+		mp_nbusses = default_data[type - 1][0];
+		bus_data[0].bus_id = default_data[type - 1][1];
+		bus_data[0].bus_type = default_data[type - 1][2];
+		bus_data[1].bus_id = default_data[type - 1][3];
+		bus_data[1].bus_type = default_data[type - 1][4];
+		break;
+
+	/* case 4: case 7:		   MCA NOT supported */
+	default:		/* illegal/reserved */
+		panic("BAD default MP config: %d\n", type);
+	}
+
+#if defined(APIC_IO)
+	/* general cases from MP v1.4, table 5-2 */
+	for (pin = 0; pin < 16; ++pin) {
+		io_apic_ints[pin].int_type = 0;
+		io_apic_ints[pin].int_flags = 0x05;	/* edge-triggered/active-hi */
+		io_apic_ints[pin].src_bus_id = 0;
+		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 is caught below */
+		io_apic_ints[pin].dst_apic_id = io_apic_id;
+		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 correspondence */
+	}
+
+	/* special cases from MP v1.4, table 5-2 */
+	if (type == 2) {
+		io_apic_ints[2].int_type = 0xff;	/* N/C */
+		io_apic_ints[13].int_type = 0xff;	/* N/C */
+#if !defined(APIC_MIXED_MODE)
+		/** FIXME: ??? */
+		panic("sorry, can't support type 2 default yet\n");
+#endif	/* APIC_MIXED_MODE */
+	} else
+		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
+
+	if (type == 7)
+		io_apic_ints[0].int_type = 0xff;	/* N/C */
+	else
+		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
+
+	nintrs = 16;
+#endif	/* APIC_IO */
+}
+
+
+static void install_ap_tramp(u_int boot_addr);
+static int start_ap(int logicalCpu, u_int boot_addr);
+
+/*
+ * start each AP in our list
+ */
+static int
+start_all_aps(u_int boot_addr)
+{
+	int     x;
+	u_char  mpbiosreason;
+	u_long  mpbioswarmvec;
+
+	/**
+         * NOTE: this needs further thought:
+         *        where does it get released?
+         *        should it be set to empy?
+         *
+         * get the initial mp_lock with a count of 1 for the BSP
+         */
+	mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1;
+
+	/* initialize BSP's local APIC */
+	apic_initialize(1);
+
+	/* install the AP 1st level boot code */
+	install_ap_tramp(boot_addr);
+
+	/* save the current value of the warm-start vector */
+	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
+	outb(CMOS_REG, BIOS_RESET);
+	mpbiosreason = inb(CMOS_DATA);
+
+	/* start each AP */
+	for (x = 1; x <= mp_naps; ++x) {
+
+		/* setup a vector to our boot code */
+		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
+		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
+		outb(CMOS_REG, BIOS_RESET);
+		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
+
+		/* attempt to start the Application Processor */
+		CHECK_INIT(99);	/* setup checkpoints */
+		if (!start_ap(x, boot_addr)) {
+			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
+			CHECK_PRINT("trace");	/* show checkpoints */
+			/*
+			 * better panic as the AP may be running loose
+			 * somewhere
+			 */
+			printf("panic y/n? [n] ");
+			if (cngetc() != 'n')
+				panic("bye-bye\n");
+		}
+		CHECK_PRINT("trace");	/* show checkpoints */
+
+		/* record its version info */
+		cpu_apic_versions[x] = cpu_apic_versions[0];
+	}
+
+	/* fill in our (BSP) APIC version */
+	cpu_apic_versions[0] = apic_base[APIC_VER];
+
+	/* restore the warmstart vector */
+	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, mpbiosreason);
+
+	/* number of APs actually started */
+	return mp_ncpus - 1;
+}
+
+
+/*
+ * load the 1st level AP boot code into base memory.
+ */
+
+/* targets for relocation */
+extern void bigJump(void);
+extern void bootCodeSeg(void);
+extern void bootDataSeg(void);
+extern void MPentry(void);
+extern u_int MP_GDT;
+extern u_int mp_gdtbase;
+
+static void
+install_ap_tramp(u_int boot_addr)
+{
+	int     x;
+	int     size = *(int *) ((u_long) & bootMP_size);
+	u_char *src = (u_char *) ((u_long) bootMP);
+	u_char *dst = (u_char *) boot_addr + KERNBASE;
+	u_int   boot_base = (u_int) bootMP;
+	u_int8_t *dst8;
+	u_int16_t *dst16;
+	u_int32_t *dst32;
+
+	for (x = 0; x < size; ++x)
+		*dst++ = *src++;
+
+	/*
+	 * modify addresses in code we just moved to basemem. unfortunately we
+	 * need fairly detailed info about mpboot.s for this to work.  changes
+	 * to mpboot.s might require changes here.
+	 */
+
+	/* boot code is located in KERNEL space */
+	dst = (u_char *) boot_addr + KERNBASE;
+
+	/* modify the lgdt arg */
+	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
+	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
+
+	/* modify the ljmp target for MPentry() */
+	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
+	*dst32 = ((u_int) MPentry - KERNBASE);
+
+	/* modify the target for boot code segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+
+	/* modify the target for boot data segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+}
+
+
+/*
+ * this function starts the AP (application processor) identified
+ * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
+ * to accomplish this.  This is necessary because of the nuances
+ * of the different hardware we might encounter.  It ain't pretty,
+ * but it seems to work.
+ */
+static int
+start_ap(int logical_cpu, u_int boot_addr)
+{
+	int     physical_cpu;
+	int     vector;
+	int     cpus;
+	u_long  icr_lo, icr_hi;
+
+	/* get the PHYSICAL APIC ID# */
+	physical_cpu = CPU_TO_ID(logical_cpu);
+
+	/* calculate the vector */
+	vector = (boot_addr >> 12) & 0xff;
+
+	/* used as a watchpoint to signal AP startup */
+	cpus = mp_ncpus;
+
+	/*
+	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
+	 * and running the target CPU. OR this INIT IPI might be latched (P5
+	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
+	 * ignored.
+	 */
+
+	/* setup the address for the target AP */
+	icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK;
+	icr_hi |= (physical_cpu << 24);
+	apic_base[APIC_ICR_HI] = icr_hi;
+
+	/* do an INIT IPI: assert RESET */
+	icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000;
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500;
+
+	/* wait for pending status end */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/* do an INIT IPI: deassert RESET */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500;
+
+	/* wait for pending status end */
+	u_sleep(10000);		/* wait ~10mS */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/*
+	 * next we do a STARTUP IPI: the previous INIT IPI might still be
+	 * latched, (P5 bug) this 1st STARTUP would then terminate
+	 * immediately, and the previously started INIT IPI would continue. OR
+	 * the previous INIT IPI has already run. and this STARTUP IPI will
+	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
+	 * will run.
+	 */
+
+	/* do a STARTUP IPI */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/*
+	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
+	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
+	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
+	 * recognized after hardware RESET or INIT IPI.
+	 */
+
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/* wait for it to start */
+	set_apic_timer(5000000);/* == 5 seconds */
+	while (read_apic_timer())
+		if (mp_ncpus > cpus)
+			return 1;	/* return SUCCESS */
+
+	return 0;		/* return FAILURE */
+}
+
+
+#if defined(IPI_INTS)
+
+static void
+ipi_intr0(void)
+{
+	printf("IPI 0\n");
+}
+
+
+static void
+ipi_intr1(void)
+{
+	printf("IPI 1\n");
+}
+
+
+static void
+ipi_intr2(void)
+{
+	printf("IPI 2\n");
+}
+
+
+static void
+ipi_intr3(void)
+{
+	printf("IPI 3\n");
+}
+
+/*-----------------------------------------------------------------------
+**
+**	Register an interupt handler for an IPI.
+**	(Stolen from the PCI<->ISA glue code)
+**
+**-----------------------------------------------------------------------
+*/
+
+static int
+ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit)
+{
+	char    buf[16];
+	char   *cp;
+	int     free_id, id, result;
+
+	sprintf(buf, "ipi irq%d", irq);
+	for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) {
+		if (strcmp(cp, buf) == 0)
+			break;
+		if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0)
+			free_id = id;
+		while (*cp++ != '\0');
+	}
+	if (id == NR_DEVICES) {
+		id = free_id;
+		if (id == 0) {
+			/*
+			 * All ipi irq counters are in use, perhaps because
+			 * config is old so there aren't any.  Abuse the clk0
+			 * counter.
+			 */
+			printf(
+			    "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n",
+			    irq);
+		}
+	}
+	result = register_intr(
+	    irq,		/* isa irq	 */
+	    id,			/* device id    */
+	    0,			/* flags?	 */
+	    func,		/* handler	 */
+	    maskptr,		/* mask pointer */
+	    unit);		/* handler arg  */
+
+	if (result) {
+		printf("WARNING: ipi_ihandler_attach: result=%d\n", result);
+		return (result);
+	};
+
+	return (0);
+}
+#endif	/* IPI_INTS */
+
+
+#ifdef SMP_INVLTLB
+/*
+ * Flush the TLB on all other CPU's
+ *
+ * XXX: Needs to handshake and wait for completion before proceding.
+ */
+
+void
+smp_invltlb()
+{
+	if (smp_active) {
+		if (invldebug & 2)
+#if defined(XFAST_IPI32)
+			all_but_self_ipi(ICU_OFFSET + 32);
+#else
+			all_but_self_ipi(ICU_OFFSET + 27);
+#endif	/* XFAST_IPI32 */
+	}
+}
+
+void
+invlpg(u_int addr)
+{
+	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
+	smp_invltlb();
+}
+
+void
+invltlb(void)
+{
+	u_long  temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
+	 * inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
+	smp_invltlb();
+}
+/*
+ * Handles recieving an "IRQ 27", the invalidate tlb IPI..
+ */
+void
+ipi_invltlb(void)
+{
+	u_long  temp;
+
+	if (invldebug & 4) {
+		/*
+		 * This should be implemented as load_cr3(rcr3()) when
+		 * load_cr3() is inlined.
+		 */
+		__asm   __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp)
+		    ::      "memory");
+
+	}
+}
+#endif	/* SMP_INVLTLB */
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 26089cc4fdbc..3f218646416e 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.140 1997/04/13 01:48:08 dyson Exp $
+ *	$Id: pmap.c,v 1.141 1997/04/13 03:35:30 dyson Exp $
  */
 
 /*
@@ -68,6 +68,7 @@
  *	and to when physical maps must be made correct.
  */
 
+#include "opt_smp.h"
 #include "opt_cpu.h"
 
 #define PMAP_LOCK 1
@@ -96,10 +97,16 @@
 
 #include <sys/user.h>
 
+#include <machine/cpu.h>
 #include <machine/pcb.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
+#if defined(SMP) || defined(APIC_IO)
+#include <machine/smp.h>
+#include <machine/apic.h>
+#include <machine/smptests.h>
+#endif /* SMP || APIC_IO */
 
 #define PMAP_KEEP_PDIRS
 #define PMAP_SHPGPERPROC 200
@@ -159,7 +166,6 @@ static vm_page_t nkpg;
 vm_offset_t kernel_vm_end;
 
 extern vm_offset_t clean_sva, clean_eva;
-extern int cpu_class;
 
 #define PV_FREELIST_MIN ((PAGE_SIZE / sizeof (struct pv_entry)) / 2)
 
@@ -182,6 +188,14 @@ static caddr_t CADDR2;
 static pt_entry_t *msgbufmap;
 struct msgbuf *msgbufp=0;
 
+#if defined(SMP) || defined(APIC_IO)
+static pt_entry_t *apic_map;
+#endif /* SMP || APIC_IO */
+
+#if defined(APIC_IO)
+static pt_entry_t *io_apic_map;
+#endif /* APIC_IO */
+
 pt_entry_t *PMAP1 = 0;
 unsigned *PADDR1 = 0;
 
@@ -297,6 +311,20 @@ pmap_bootstrap(firstaddr, loadaddr)
 	va = virtual_avail;
 	pte = (pt_entry_t *) pmap_pte(kernel_pmap, va);
 
+#if defined(SMP) || defined(APIC_IO)
+	/*
+	 * apic_map is the pt for where the local (CPU) apic is mapped in.
+	 */
+	SYSMAP(unsigned int *, apic_map, apic_base, 1)
+#endif /* SMP || APIC_IO */
+
+#if defined(APIC_IO)
+	/*
+	 * io_apic_map is the pt for where the I/O apic is mapped in.
+	 */
+	SYSMAP(unsigned int *, io_apic_map, io_apic_base, 1)
+#endif /* APIC_IO */
+
 	/*
 	 * CMAP1/CMAP2 are used for zeroing and copying pages.
 	 */
@@ -323,14 +351,50 @@ pmap_bootstrap(firstaddr, loadaddr)
 
 	virtual_avail = va;
 
-	*(int *) CMAP1 = *(int *) CMAP2 = *(int *) PTD = 0;
+	*(int *) CMAP1 = *(int *) CMAP2 = 0;
+
 	invltlb();
+
+#if !defined(SMP)
 	if (cpu_feature & CPUID_PGE)
 		pgeflag = PG_G;
 	else
+#endif /* !SMP */
 		pgeflag = 0;
+}
 
+#if defined(SMP) || defined(APIC_IO)
+void
+pmap_bootstrap_apics()
+{
+	if (cpu_apic_address == 0) {
+		printf("pmap: BSP APIC address NOT found!\n");
+		panic("pmap_bootstrap_apics: no apic");
+	}
+
+	*(int*)apic_map = PG_V | PG_RW | ((u_long)cpu_apic_address & PG_FRAME);
+
+#if defined(APIC_IO)
+#if defined(MULTIPLE_IOAPICS)
+#error MULTIPLE_IOAPICSXXX
+#else
+	*(int*)io_apic_map = PG_V | PG_RW
+	    | ((u_long)io_apic_address[0] & PG_FRAME);
+#endif /* MULTIPLE_IOAPICS */
+#endif /* APIC_IO */
+
+	invltlb();
 }
+#endif /* SMP || APIC_IO */
+
+#ifdef SMP
+void
+pmap_bootstrap2()
+{
+	*(int *) PTD = 0;
+	invltlb();
+}
+#endif
 
 /*
  *	Initialize the pmap module.
diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S
index 0bfef221a9e1..5197f7ea1dc5 100644
--- a/sys/amd64/amd64/support.S
+++ b/sys/amd64/amd64/support.S
@@ -30,9 +30,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: support.s,v 1.50 1997/02/22 09:32:46 peter Exp $
+ *	$Id: support.s,v 1.51 1997/03/05 16:30:55 bde Exp $
  */
 
+#include "opt_smp.h"
 #include "npx.h"
 #include "opt_cpu.h"
 
@@ -40,10 +41,12 @@
 #include <machine/cputypes.h>
 #include <machine/pmap.h>
 #include <machine/specialreg.h>
+#include <machine/smpasm.h>
 
 #include "assym.s"
 
 #define KDSEL		0x10			/* kernel data selector */
+#define KCSEL		0x8			/* kernel code selector */
 #define IDXSHIFT	10
 
 	.data
@@ -193,7 +196,7 @@ do0:
 	ret
 #endif
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_bzero)
 	movl	4(%esp),%edx
 	movl	8(%esp),%ecx
@@ -436,7 +439,7 @@ ENTRY(generic_bcopy)
 	cld
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_bcopy)
 	pushl	%esi
 	pushl	%edi
@@ -621,7 +624,7 @@ ENTRY(copyout)
 	jmp	*_copyout_vector
 
 ENTRY(generic_copyout)
-	movl	_curpcb,%eax
+	GETCURPCB(%eax)
 	movl	$copyout_fault,PCB_ONFAULT(%eax)
 	pushl	%esi
 	pushl	%edi
@@ -714,7 +717,7 @@ ENTRY(generic_copyout)
 3:
 	movl	%ebx,%ecx
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 slow_copyout:
 #endif
@@ -732,7 +735,7 @@ done_copyout:
 	popl	%edi
 	popl	%esi
 	xorl	%eax,%eax
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	%eax,PCB_ONFAULT(%edx)
 	ret
 
@@ -741,12 +744,12 @@ copyout_fault:
 	popl	%ebx
 	popl	%edi
 	popl	%esi
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	$0,PCB_ONFAULT(%edx)
 	movl	$EFAULT,%eax
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_copyout)
 	/*
 	 * Duplicated from generic_copyout.  Could be done a bit better.
@@ -806,7 +809,7 @@ ENTRY(copyin)
 	jmp	*_copyin_vector
 
 ENTRY(generic_copyin)
-	movl	_curpcb,%eax
+	GETCURPCB(%eax)
 	movl	$copyin_fault,PCB_ONFAULT(%eax)
 	pushl	%esi
 	pushl	%edi
@@ -823,7 +826,7 @@ ENTRY(generic_copyin)
 	cmpl	$VM_MAXUSER_ADDRESS,%edx
 	ja	copyin_fault
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 slow_copyin:
 #endif
@@ -837,14 +840,14 @@ slow_copyin:
 	rep
 	movsb
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 done_copyin:
 #endif
 	popl	%edi
 	popl	%esi
 	xorl	%eax,%eax
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	%eax,PCB_ONFAULT(%edx)
 	ret
 
@@ -852,12 +855,12 @@ done_copyin:
 copyin_fault:
 	popl	%edi
 	popl	%esi
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	$0,PCB_ONFAULT(%edx)
 	movl	$EFAULT,%eax
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_copyin)
 	/*
 	 * Duplicated from generic_copyin.  Could be done a bit better.
@@ -892,7 +895,7 @@ ENTRY(i586_copyin)
 	jmp	done_copyin
 #endif /* I586_CPU && NNPX > 0 */
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 /* fastmove(src, dst, len)
 	src in %esi
 	dst in %edi
@@ -1084,7 +1087,7 @@ fastmove_tail_fault:
  * fu{byte,sword,word} : fetch a byte (sword, word) from user memory
  */
 ENTRY(fuword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx			/* from */
 
@@ -1107,7 +1110,7 @@ ENTRY(fuswintr)
 	ret
 
 ENTRY(fusword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1119,7 +1122,7 @@ ENTRY(fusword)
 	ret
 
 ENTRY(fubyte)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1132,7 +1135,7 @@ ENTRY(fubyte)
 
 	ALIGN_TEXT
 fusufault:
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	xorl	%eax,%eax
 	movl	%eax,PCB_ONFAULT(%ecx)
 	decl	%eax
@@ -1142,7 +1145,7 @@ fusufault:
  * su{byte,sword,word}: write a byte (word, longword) to user memory
  */
 ENTRY(suword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1186,12 +1189,12 @@ ENTRY(suword)
 	movl	8(%esp),%eax
 	movl	%eax,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
 ENTRY(susword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1235,13 +1238,13 @@ ENTRY(susword)
 	movw	8(%esp),%ax
 	movw	%ax,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx			/* restore trashed register */
+	GETCURPCB(%ecx)				/* restore trashed register */
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
 ALTENTRY(suibyte)
 ENTRY(subyte)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1284,7 +1287,7 @@ ENTRY(subyte)
 	movb	8(%esp),%al
 	movb	%al,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx			/* restore trashed register */
+	GETCURPCB(%ecx)				/* restore trashed register */
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
@@ -1298,7 +1301,7 @@ ENTRY(subyte)
 ENTRY(copyinstr)
 	pushl	%esi
 	pushl	%edi
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$cpystrflt,PCB_ONFAULT(%ecx)
 
 	movl	12(%esp),%esi			/* %esi = from */
@@ -1346,7 +1349,7 @@ cpystrflt:
 
 cpystrflt_x:
 	/* set *lencopied and return %eax */
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$0,PCB_ONFAULT(%ecx)
 	movl	20(%esp),%ecx
 	subl	%edx,%ecx
@@ -1446,13 +1449,14 @@ ENTRY(lgdt)
 	movl	$KDSEL,%eax
 	movl	%ax,%ds
 	movl	%ax,%es
+	movl	%ax,%fs
+	movl	%ax,%gs
 	movl	%ax,%ss
 
 	/* reload code selector by turning return into intersegmental return */
 	movl	(%esp),%eax
 	pushl	%eax
-#	movl	$KCSEL,4(%esp)
-	movl	$8,4(%esp)
+	movl	$KCSEL,4(%esp)
 	lret
 
 /*
diff --git a/sys/amd64/amd64/support.s b/sys/amd64/amd64/support.s
index 0bfef221a9e1..5197f7ea1dc5 100644
--- a/sys/amd64/amd64/support.s
+++ b/sys/amd64/amd64/support.s
@@ -30,9 +30,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: support.s,v 1.50 1997/02/22 09:32:46 peter Exp $
+ *	$Id: support.s,v 1.51 1997/03/05 16:30:55 bde Exp $
  */
 
+#include "opt_smp.h"
 #include "npx.h"
 #include "opt_cpu.h"
 
@@ -40,10 +41,12 @@
 #include <machine/cputypes.h>
 #include <machine/pmap.h>
 #include <machine/specialreg.h>
+#include <machine/smpasm.h>
 
 #include "assym.s"
 
 #define KDSEL		0x10			/* kernel data selector */
+#define KCSEL		0x8			/* kernel code selector */
 #define IDXSHIFT	10
 
 	.data
@@ -193,7 +196,7 @@ do0:
 	ret
 #endif
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_bzero)
 	movl	4(%esp),%edx
 	movl	8(%esp),%ecx
@@ -436,7 +439,7 @@ ENTRY(generic_bcopy)
 	cld
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_bcopy)
 	pushl	%esi
 	pushl	%edi
@@ -621,7 +624,7 @@ ENTRY(copyout)
 	jmp	*_copyout_vector
 
 ENTRY(generic_copyout)
-	movl	_curpcb,%eax
+	GETCURPCB(%eax)
 	movl	$copyout_fault,PCB_ONFAULT(%eax)
 	pushl	%esi
 	pushl	%edi
@@ -714,7 +717,7 @@ ENTRY(generic_copyout)
 3:
 	movl	%ebx,%ecx
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 slow_copyout:
 #endif
@@ -732,7 +735,7 @@ done_copyout:
 	popl	%edi
 	popl	%esi
 	xorl	%eax,%eax
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	%eax,PCB_ONFAULT(%edx)
 	ret
 
@@ -741,12 +744,12 @@ copyout_fault:
 	popl	%ebx
 	popl	%edi
 	popl	%esi
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	$0,PCB_ONFAULT(%edx)
 	movl	$EFAULT,%eax
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_copyout)
 	/*
 	 * Duplicated from generic_copyout.  Could be done a bit better.
@@ -806,7 +809,7 @@ ENTRY(copyin)
 	jmp	*_copyin_vector
 
 ENTRY(generic_copyin)
-	movl	_curpcb,%eax
+	GETCURPCB(%eax)
 	movl	$copyin_fault,PCB_ONFAULT(%eax)
 	pushl	%esi
 	pushl	%edi
@@ -823,7 +826,7 @@ ENTRY(generic_copyin)
 	cmpl	$VM_MAXUSER_ADDRESS,%edx
 	ja	copyin_fault
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 slow_copyin:
 #endif
@@ -837,14 +840,14 @@ slow_copyin:
 	rep
 	movsb
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 	ALIGN_TEXT
 done_copyin:
 #endif
 	popl	%edi
 	popl	%esi
 	xorl	%eax,%eax
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	%eax,PCB_ONFAULT(%edx)
 	ret
 
@@ -852,12 +855,12 @@ done_copyin:
 copyin_fault:
 	popl	%edi
 	popl	%esi
-	movl	_curpcb,%edx
+	GETCURPCB(%edx)
 	movl	$0,PCB_ONFAULT(%edx)
 	movl	$EFAULT,%eax
 	ret
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 ENTRY(i586_copyin)
 	/*
 	 * Duplicated from generic_copyin.  Could be done a bit better.
@@ -892,7 +895,7 @@ ENTRY(i586_copyin)
 	jmp	done_copyin
 #endif /* I586_CPU && NNPX > 0 */
 
-#if defined(I586_CPU) && NNPX > 0
+#if defined(I586_CPU) && NNPX > 0 && !defined(SMP)
 /* fastmove(src, dst, len)
 	src in %esi
 	dst in %edi
@@ -1084,7 +1087,7 @@ fastmove_tail_fault:
  * fu{byte,sword,word} : fetch a byte (sword, word) from user memory
  */
 ENTRY(fuword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx			/* from */
 
@@ -1107,7 +1110,7 @@ ENTRY(fuswintr)
 	ret
 
 ENTRY(fusword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1119,7 +1122,7 @@ ENTRY(fusword)
 	ret
 
 ENTRY(fubyte)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1132,7 +1135,7 @@ ENTRY(fubyte)
 
 	ALIGN_TEXT
 fusufault:
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	xorl	%eax,%eax
 	movl	%eax,PCB_ONFAULT(%ecx)
 	decl	%eax
@@ -1142,7 +1145,7 @@ fusufault:
  * su{byte,sword,word}: write a byte (word, longword) to user memory
  */
 ENTRY(suword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1186,12 +1189,12 @@ ENTRY(suword)
 	movl	8(%esp),%eax
 	movl	%eax,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
 ENTRY(susword)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1235,13 +1238,13 @@ ENTRY(susword)
 	movw	8(%esp),%ax
 	movw	%ax,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx			/* restore trashed register */
+	GETCURPCB(%ecx)				/* restore trashed register */
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
 ALTENTRY(suibyte)
 ENTRY(subyte)
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$fusufault,PCB_ONFAULT(%ecx)
 	movl	4(%esp),%edx
 
@@ -1284,7 +1287,7 @@ ENTRY(subyte)
 	movb	8(%esp),%al
 	movb	%al,(%edx)
 	xorl	%eax,%eax
-	movl	_curpcb,%ecx			/* restore trashed register */
+	GETCURPCB(%ecx)				/* restore trashed register */
 	movl	%eax,PCB_ONFAULT(%ecx)
 	ret
 
@@ -1298,7 +1301,7 @@ ENTRY(subyte)
 ENTRY(copyinstr)
 	pushl	%esi
 	pushl	%edi
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$cpystrflt,PCB_ONFAULT(%ecx)
 
 	movl	12(%esp),%esi			/* %esi = from */
@@ -1346,7 +1349,7 @@ cpystrflt:
 
 cpystrflt_x:
 	/* set *lencopied and return %eax */
-	movl	_curpcb,%ecx
+	GETCURPCB(%ecx)
 	movl	$0,PCB_ONFAULT(%ecx)
 	movl	20(%esp),%ecx
 	subl	%edx,%ecx
@@ -1446,13 +1449,14 @@ ENTRY(lgdt)
 	movl	$KDSEL,%eax
 	movl	%ax,%ds
 	movl	%ax,%es
+	movl	%ax,%fs
+	movl	%ax,%gs
 	movl	%ax,%ss
 
 	/* reload code selector by turning return into intersegmental return */
 	movl	(%esp),%eax
 	pushl	%eax
-#	movl	$KCSEL,4(%esp)
-	movl	$8,4(%esp)
+	movl	$KCSEL,4(%esp)
 	lret
 
 /*
diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s
index a74cd09b5a6c..014ddd2f2c57 100644
--- a/sys/amd64/amd64/swtch.s
+++ b/sys/amd64/amd64/swtch.s
@@ -33,16 +33,24 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.46 1997/04/20 06:41:26 phk Exp $
+ *	$Id: swtch.s,v 1.47 1997/04/22 06:55:29 jdp Exp $
  */
 
 #include "npx.h"
 #include "opt_user_ldt.h"
+#include "opt_smp.h"
+#include "opt_smp_privpages.h"
 
 #include <sys/rtprio.h>
 
 #include <machine/asmacros.h>
 #include <machine/spl.h>
+#include <machine/smpasm.h>
+#include <machine/smptests.h>		/** TEST_LOPRIO */
+
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#include <machine/pmap.h>
+#endif
 
 #include "assym.s"
 
@@ -61,8 +69,12 @@
  * queues.
  */
 	.data
+#ifndef SMP
 	.globl	_curpcb
 _curpcb:	.long	0			/* pointer to curproc's PCB area */
+#endif
+	.globl	_whichqs, _whichrtqs, _whichidqs
+
 _whichqs:	.long	0			/* which run queues have data */
 _whichrtqs:	.long	0			/* which realtime run queues have data */
 _whichidqs:	.long	0			/* which idletime run queues have data */
@@ -229,16 +241,30 @@ rem3id:	.asciz	"remrq.id"
 /*
  * When no processes are on the runq, cpu_switch() branches to _idle
  * to wait for something to come ready.
+ *
+ * NOTE: on an SMP system this routine is a startup-only code path.
+ * once initialization is over, meaning the idle procs have been
+ * created, we should NEVER branch here.
  */
 	ALIGN_TEXT
 _idle:
+#ifdef SMP
+	movl	_smp_active, %eax
+	cmpl	$0, %eax
+	jnz	badsw
+#endif /* SMP */
 	xorl	%ebp,%ebp
 	movl	$HIDENAME(tmpstk),%esp
 	movl	_IdlePTD,%ecx
 	movl	%ecx,%cr3
 
 	/* update common_tss.tss_esp0 pointer */
+#ifdef SMP
+	GETCPUID(%eax)
+	movl	_SMPcommon_tss_ptr(,%eax,4), %eax
+#else
 	movl	$_common_tss, %eax
+#endif
 	movl	%esp, TSS_ESP0(%eax)
 
 #ifdef TSS_IS_CACHED				/* example only */
@@ -288,13 +314,20 @@ ENTRY(default_halt)
  * cpu_switch()
  */
 ENTRY(cpu_switch)
+	
 	/* switch to new process. first, save context as needed */
-	movl	_curproc,%ecx
+	GETCURPROC(%ecx)
 
 	/* if no process to save, don't bother */
 	testl	%ecx,%ecx
 	je	sw1
 
+#ifdef SMP
+	movb	P_ONCPU(%ecx), %al		/* save "last" cpu */
+	movb	%al, P_LASTCPU(%ecx)
+	movb	$0xff, P_ONCPU(%ecx)		/* "leave" the cpu */
+#endif
+
 	movl	P_ADDR(%ecx),%ecx
 
 	movl	(%esp),%eax			/* Hardware registers */
@@ -305,10 +338,19 @@ ENTRY(cpu_switch)
 	movl	%esi,PCB_ESI(%ecx)
 	movl	%edi,PCB_EDI(%ecx)
 
+#ifdef SMP
+	movl	_mp_lock, %eax
+	cmpl	$0xffffffff, %eax		/* is it free? */
+	je	badsw				/* yes, bad medicine! */
+	andl	$0x00ffffff, %eax		/* clear CPU portion */
+	movl	%eax,PCB_MPNEST(%ecx)		/* store it */
+#endif /* SMP */
+
 #if NNPX > 0
 	/* have we used fp, and need a save? */
-	mov	_curproc,%eax
-	cmp	%eax,_npxproc
+	GETCURPROC(%eax)
+	GETNPXPROC(%ebx)
+	cmp	%eax,%ebx
 	jne	1f
 	addl	$PCB_SAVEFPU,%ecx		/* h/w bugs make saving complicated */
 	pushl	%ecx
@@ -319,7 +361,7 @@ ENTRY(cpu_switch)
 
 	movb	$1,_intr_nesting_level		/* charge Intr, not Sys/Idle */
 
-	movl	$0,_curproc			/* out of process */
+	SETCURPROC($0, %edi)
 
 	/* save is done, now choose a new process or idle */
 sw1:
@@ -417,9 +459,30 @@ swtch_com:
 	movl	P_ADDR(%ecx),%edx
 	movl	PCB_CR3(%edx),%ebx
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+	/* Grab the private PT pointer from the outgoing process's PTD */
+	movl	$_PTD,%esi
+	movl	4*MPPTDI(%esi), %eax		/* fetch cpu's prv pt */
+#endif
+
 	/* switch address space */
 	movl	%ebx,%cr3
 
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+	/* Copy the private PT to the new process's PTD */
+	/* XXX yuck, the _PTD changes when we switch, so we have to
+	 * reload %cr3 after changing the address space.
+	 * We need to fix this by storing a pointer to the virtual
+	 * location of the per-process PTD in the PCB or something quick.
+	 * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm.
+	 */
+	movl	$_PTD,%esi
+	movl	%eax, 4*MPPTDI(%esi)		/* restore cpu's prv page */
+
+	/* XXX: we have just changed the page tables.. reload.. */
+	movl	%ebx,%cr3
+#endif
+
 #ifdef HOW_TO_SWITCH_TSS			/* example only */
 	/* Fix up tss pointer to floating pcb/stack structure */
 	/* XXX probably lots faster to store the 64 bits of tss entry
@@ -435,7 +498,12 @@ swtch_com:
 #endif
 
 	/* update common_tss.tss_esp0 pointer */
+#ifdef SMP
+	GETCPUID(%eax)
+	movl	_SMPcommon_tss_ptr(,%eax,4), %eax
+#else
 	movl	$_common_tss, %eax
+#endif
 	movl	%edx, %ebx			/* pcb */
 	addl	$(UPAGES * PAGE_SIZE), %ebx
 	movl	%ebx, TSS_ESP0(%eax)
@@ -457,9 +525,32 @@ swtch_com:
 	movl	PCB_EIP(%edx),%eax
 	movl	%eax,(%esp)
 
-	movl	%edx,_curpcb
-	movl	%ecx,_curproc			/* into next process */
+#ifdef SMP
+	GETCPUID(%eax)
+	movb	%al, P_ONCPU(%ecx)
+#endif
+	SETCURPCB(%edx, %eax)
+	SETCURPROC(%ecx, %eax)
+
 	movb	$0,_intr_nesting_level
+#ifdef SMP
+	movl	_apic_base, %eax		/* base addr of LOCAL APIC */
+#if defined(TEST_LOPRIO)
+	/**
+	 * FIXME: the belief here is that we ALWAYS leave here
+	 *         holding a the lock, is this TRUE???
+	 */
+	pushl	%edx
+	movl	APIC_TPR(%eax), %edx		/* get TPR register contents */
+	andl	$~0xff, %edx			/* clear the prio field */
+	movl	%edx, APIC_TPR(%eax)		/* now hold loprio for INTs */
+	popl	%edx
+#endif /* TEST_LOPRIO */
+	movl	APIC_ID(%eax), %eax		/* APIC ID register */
+	andl	$APIC_ID_MASK, %eax		/* extract ID portion */
+	orl	PCB_MPNEST(%edx), %eax		/* add count from PROC */
+	movl	%eax, _mp_lock			/* load the mp_lock */
+#endif /* SMP */
 
 #ifdef	USER_LDT
 	cmpl	$0, PCB_USERLDT(%edx)
@@ -520,7 +611,7 @@ ENTRY(savectx)
 	 * have to handle h/w bugs for reloading.  We used to lose the
 	 * parent's npx state for forks by forgetting to reload.
 	 */
-	mov	_npxproc,%eax
+	GETNPXPROC(%eax)
 	testl	%eax,%eax
 	je	1f
 
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index 8a86965196d5..aa4e3c5d39d5 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
- *	$Id: trap.c,v 1.91 1997/04/07 07:15:55 peter Exp $
+ *	$Id: trap.c,v 1.92 1997/04/14 13:52:52 bde Exp $
  */
 
 /*
@@ -44,6 +44,7 @@
 
 #include "opt_ktrace.h"
 #include "opt_ddb.h"
+#include "opt_smp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -76,6 +77,7 @@
 #include <machine/reg.h>
 #include <machine/trap.h>
 #include <machine/../isa/isa_device.h>
+#include <machine/smp.h>
 
 #ifdef POWERFAIL_NMI
 #include <sys/syslog.h>
@@ -85,7 +87,11 @@
 #include "isa.h"
 #include "npx.h"
 
+#ifdef SMP
+extern struct i386tss *SMPcommon_tss_ptr[NCPU];
+#else
 extern struct i386tss common_tss;
+#endif
 
 int (*pmath_emulate) __P((struct trapframe *));
 
@@ -678,6 +684,9 @@ trap_fatal(frame)
 		printf("\n\nFatal trap %d: %s while in %s mode\n",
 			type, trap_msg[type],
 			ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
+#ifdef SMP
+	printf("cpunumber = %d\n", cpunumber());
+#endif
 	if (type == T_PAGEFLT) {
 		printf("fault virtual address	= 0x%x\n", eva);
 		printf("fault code		= %s %s, %s\n",
@@ -740,6 +749,7 @@ trap_fatal(frame)
 	if (kdb_trap (type, 0, frame))
 		return;
 #endif
+	printf("trap number		= %d\n", type);
 	if (type <= MAX_TRAP_MSG)
 		panic(trap_msg[type]);
 	else
@@ -761,11 +771,20 @@ trap_fatal(frame)
 void
 dblfault_handler()
 {
+#ifdef SMP
+	int x = cpunumber();
+#endif
 
 	printf("\nFatal double fault:\n");
+#ifdef SMP
+	printf("eip = 0x%x\n", SMPcommon_tss_ptr[x]->tss_eip);
+	printf("esp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_esp);
+	printf("ebp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_ebp);
+#else
 	printf("eip = 0x%x\n", common_tss.tss_eip);
 	printf("esp = 0x%x\n", common_tss.tss_esp);
 	printf("ebp = 0x%x\n", common_tss.tss_ebp);
+#endif
 	panic("double fault");
 }
 
diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c
index b9425536d67a..d46f982806bd 100644
--- a/sys/amd64/amd64/tsc.c
+++ b/sys/amd64/amd64/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $
+ *	$Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $
  */
 
 /*
@@ -49,6 +49,7 @@
  */
 
 #include "opt_clock.h"
+#include "opt_smp.h"
 #include "opt_cpu.h"
 
 #include <sys/param.h>
@@ -100,7 +101,7 @@
 int	adjkerntz;		/* local offset	from GMT in seconds */
 int	disable_rtc_set;	/* disable resettodr() if != 0 */
 u_int	idelayed;
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 u_int	i586_ctr_bias;
 u_int	i586_ctr_comultiplier;
 u_int	i586_ctr_freq;
@@ -143,7 +144,7 @@ static	u_char	timer0_state;
 static	u_char	timer2_state;
 static	void	(*timer_func) __P((struct clockframe *frame)) = hardclock;
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 static	void	set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq);
 #endif
 static	void	set_timer_freq(u_int freq, int intr_freq);
@@ -553,7 +554,7 @@ calibrate_clocks(void)
 		goto fail;
 	tot_count = 0;
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686)
 		wrmsr(0x10, 0LL);	/* XXX 0x10 is the MSR for the TSC */
 #endif
@@ -586,7 +587,7 @@ calibrate_clocks(void)
 			goto fail;
 	}
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	/*
 	 * Read the cpu cycle counter.  The timing considerations are
 	 * similar to those for the i8254 clock.
@@ -667,14 +668,14 @@ startrtclock()
 			printf(
 		    "%d Hz differs from default of %d Hz by more than 1%%\n",
 			       freq, timer_freq);
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 		i586_ctr_freq = 0;
 #endif
 	}
 
 	set_timer_freq(timer_freq, hz);
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 #ifndef CLK_USE_I586_CALIBRATION
 	if (i586_ctr_freq != 0) {
 		if (bootverbose)
@@ -821,6 +822,20 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
+#if defined(APIC_IO)
+
+/* from icu.s: */
+extern u_int	hwisrs[];
+extern void	vec8254	__P((void));
+extern void	vecRTC	__P((void));
+extern u_int	ivectors[];
+extern u_int	Xintr8254;
+extern u_int	XintrRTC;
+extern u_int	mask8254;
+extern u_int	maskRTC;
+
+#endif /* APIC_IO */
+
 /*
  * Start both clocks running.
  */
@@ -828,6 +843,9 @@ void
 cpu_initclocks()
 {
 	int diag;
+#if defined(APIC_IO)
+	int x;
+#endif /* APIC_IO */
 
 	if (statclock_disable) {
 		/*
@@ -845,11 +863,47 @@ cpu_initclocks()
         }
 
 	/* Finish initializing 8253 timer 0. */
+#if defined(APIC_IO)
+	/* 8254 is traditionally on ISA IRQ0 */
+	if ((x = get_isa_apic_irq(0)) < 0) {
+		/*
+		 * bummer, this mb doesn't have the 8254 on ISA irq0,
+		 *  perhaps it's on the EISA bus...
+		 */
+		if ((x = get_eisa_apic_irq(0)) < 0) {
+			/* double bummer, attempt to redirect thru the 8259 */
+			if (bootverbose)
+				printf("APIC missing 8254 connection\n");
+
+			/* allow 8254 timer to INTerrupt 8259 */
+#if !defined(IO_ICU1)
+#define IO_ICU1 0x20
+#endif
+			x = inb(IO_ICU1 + 1);	/* current mask in 8259 */
+			x &= ~1;		/* clear 8254 timer mask */
+			outb(IO_ICU1 + 1, x);	/* write new mask */
+
+			/* program IO APIC for type 3 INT on INT0 */
+			if (ext_int_setup(0, 0) < 0)
+				panic("8254 redirect impossible!");
+			x = 0;			/* 8259 is on 0 */
+		}
+	}
+
+	hwisrs[x] = (u_int)vec8254;
+	Xintr8254 = (u_int)ivectors[x];	/* XXX might need Xfastintr# */
+	mask8254 = (1 << x);
+	register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0,
+		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
+		      /* unit */ 0);
+	INTREN(mask8254);
+#else
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
-#if defined(I586_CPU) || defined(I686_CPU)
+#endif /* APIC_IO */
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	/*
 	 * Finish setting up anti-jitter measures.
 	 */
@@ -867,11 +921,36 @@ cpu_initclocks()
 	diag = rtcin(RTC_DIAG);
 	if (diag != 0)
 		printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS);
+#if defined(APIC_IO)
+	/* RTC is traditionally on ISA IRQ8 */
+	if ((x = get_isa_apic_irq(8)) < 0) {
+		if ((x = get_eisa_apic_irq(8)) < 0) {
+			panic("APIC missing RTC connection");
+	    }
+	}
+
+	hwisrs[x] = (u_int)vecRTC;
+	XintrRTC = (u_int)ivectors[x];	/* XXX might need Xfastintr# */
+	maskRTC = (1 << x);
+	register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0,
+		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
+		      /* unit */ 0);
+	INTREN(maskRTC);
+#else
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 	writertc(RTC_STATUSB, rtc_statusb);
+
+#if defined(APIC_IO)
+	printf("Enabled INTs: ");
+	for (x = 0; x < 24; ++x)
+		if ((imen & (1 << x)) == 0)
+	        	printf("%d, ", x);
+	printf("imen: 0x%08x\n", imen);
+#endif /* APIC_IO */
 }
 
 void
@@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS
 		if (timer0_state != 0)
 			return (EBUSY);	/* too much trouble to handle */
 		set_timer_freq(freq, hz);
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 		set_i586_ctr_freq(i586_ctr_freq, timer_freq);
 #endif
 	}
@@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS
 SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW,
 	    0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", "");
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 static void
 set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq)
 {
@@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS
 
 SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW,
 	    0, sizeof(u_int), sysctl_machdep_i586_freq, "I", "");
-#endif /* defined(I586_CPU) || defined(I686_CPU) */
+#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */
diff --git a/sys/amd64/include/apicreg.h b/sys/amd64/include/apicreg.h
new file mode 100644
index 000000000000..a585efba626d
--- /dev/null
+++ b/sys/amd64/include/apicreg.h
@@ -0,0 +1,409 @@
+/*
+ * Copyright (c) 1996, by Peter Wemm and Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id: apic.h,v 1.17 1997/04/26 06:41:36 peter Exp $
+ */
+
+#ifndef _MACHINE_APIC_H_
+#define _MACHINE_APIC_H_
+
+/*
+ * Local && I/O APIC definitions.
+ */
+
+/*
+ * Pentium P54C+ Build-in APIC
+ * (Advanced programmable Interrupt Controller)
+ * 
+ * Base Address of Build-in APIC in memory location
+ * is 0xfee00000.
+ * 
+ * Map of APIC REgisters:
+ * 
+ * Offset (hex)    Description                     Read/Write state
+ * 000             Reserved
+ * 010             Reserved
+ * 020 ID          Local APIC ID                   R/W
+ * 030 VER         Local APIC Version              R
+ * 040             Reserved
+ * 050             Reserved
+ * 060             Reserved
+ * 070             Reserved
+ * 080             Task Priority Register          R/W
+ * 090             Arbitration Priority Register   R
+ * 0A0             Processor Priority Register     R
+ * 0B0             EOI Register                    W
+ * 0C0 RRR         Remote read                     R
+ * 0D0             Logical Destination             R/W
+ * 0E0             Destination Format Register     0..27 R;  28..31 R/W
+ * 0F0 SVR         Spurious Interrupt Vector Reg.  0..3  R;  4..9   R/W
+ * 100             ISR  000-031                    R
+ * 110             ISR  032-063                    R
+ * 120             ISR  064-095                    R
+ * 130             ISR  095-128                    R
+ * 140             ISR  128-159                    R
+ * 150             ISR  160-191                    R
+ * 160             ISR  192-223                    R
+ * 170             ISR  224-255                    R
+ * 180             TMR  000-031                    R
+ * 190             TMR  032-063                    R
+ * 1A0             TMR  064-095                    R
+ * 1B0             TMR  095-128                    R
+ * 1C0             TMR  128-159                    R
+ * 1D0             TMR  160-191                    R
+ * 1E0             TMR  192-223                    R
+ * 1F0             TMR  224-255                    R
+ * 200             IRR  000-031                    R
+ * 210             IRR  032-063                    R
+ * 220             IRR  064-095                    R
+ * 230             IRR  095-128                    R
+ * 240             IRR  128-159                    R
+ * 250             IRR  160-191                    R
+ * 260             IRR  192-223                    R
+ * 270             IRR  224-255                    R
+ * 280             Error Status Register           R
+ * 290             Reserved
+ * 2A0             Reserved
+ * 2B0             Reserved
+ * 2C0             Reserved
+ * 2D0             Reserved
+ * 2E0             Reserved
+ * 2F0             Reserved
+ * 300 ICR_LOW     Interrupt Command Reg. (0-31)   R/W
+ * 310 ICR_HI      Interrupt Command Reg. (32-63)  R/W
+ * 320             Local Vector Table (Timer)      R/W
+ * 330             Reserved
+ * 340             Reserved
+ * 350 LVT1        Local Vector Table (LINT0)      R/W
+ * 360 LVT2        Local Vector Table (LINT1)      R/W
+ * 370 LVT3        Local Vector Table (ERROR)      R/W
+ * 380             Initial Count Reg. for Timer    R/W
+ * 390             Current Count of Timer          R
+ * 3A0             Reserved
+ * 3B0             Reserved
+ * 3C0             Reserved
+ * 3D0             Reserved
+ * 3E0             Timer Divide Configuration Reg. R/W
+ * 3F0             Reserved
+ */
+
+
+/******************************************************************************
+ * global defines, etc.
+ */
+
+/* enable the InterProcessor Interrupt code, FIXME: temporary marker */
+#define IPI_INTS
+
+/**
+ * this enables code concerned with handling more than one IO APIC.
+ * Note: this is NOT READY for use!
+ *
+#define MULTIPLE_IOAPICS
+ */
+
+/******************************************************************************
+ * LOCAL APIC defines
+ */
+
+/* default physical locations of LOCAL (CPU) APICs */
+#define DEFAULT_APIC_BASE	0xfee00000
+
+# if defined(LOCORE)
+
+#define APIC_ID			0x020
+#define APIC_VER		0x030
+#define APIC_TPR		0x080
+#define APIC_APR		0x090
+#define APIC_PPR		0x0a0
+#define APIC_EOI		0x0b0
+#define APIC_RR			0x0c0
+#define APIC_LDR		0x0d0
+#define APIC_DFR		0x0e0
+#define APIC_SVR		0x0f0
+#define APIC_ISR		0x100
+#define APIC_ISR0		0x100
+#define APIC_ISR1		0x110
+#define APIC_ISR2		0x120
+#define APIC_TMR		0x180
+#define APIC_IRR		0x200
+#define APIC_IRR0		0x200
+#define APIC_IRR1		0x210
+#define APIC_IRR2		0x220
+#define APIC_ESR		0x280
+#define APIC_ICR_LOW		0x300
+#define APIC_ICR_HI		0x310
+#define APIC_LVTT		0x320
+#define APIC_LVT1		0x350
+#define APIC_LVT2		0x360
+#define APIC_LVT3		0x370
+#define APIC_TICR		0x380
+#define APIC_TCCR		0x390
+#define APIC_TDCR		0x3e0
+
+# else /* !LOCORE */
+
+/* offsets in apic_base[] */
+#define APIC_ID			(0x020/4)
+#define APIC_VER		(0x030/4)
+#define APIC_TPR		(0x080/4)
+#define APIC_APR		(0x090/4)
+#define APIC_PPR		(0x0a0/4)
+#define APIC_EOI		(0x0b0/4)
+#define APIC_RR			(0x0c0/4)
+#define APIC_LDR		(0x0d0/4)
+#define APIC_DFR		(0x0e0/4)
+#define APIC_SVR		(0x0f0/4)
+#define APIC_ISR		(0x100/4)
+#define APIC_ISR0		(0x100/4)
+#define APIC_ISR1		(0x110/4)
+#define APIC_ISR2		(0x120/4)
+#define APIC_TMR		(0x180/4)
+#define APIC_IRR		(0x200/4)
+#define APIC_IRR0		(0x200/4)
+#define APIC_IRR1		(0x210/4)
+#define APIC_IRR2		(0x220/4)
+#define APIC_ESR		(0x280/4)
+#define APIC_ICR_LOW		(0x300/4)
+#define APIC_ICR_HI		(0x310/4)
+#define APIC_LVTT		(0x320/4)
+#define APIC_LVT1		(0x350/4)
+#define APIC_LVT2		(0x360/4)
+#define APIC_LVT3		(0x370/4)
+#define APIC_TICR		(0x380/4)
+#define APIC_TCCR		(0x390/4)
+#define APIC_TDCR		(0x3e0/4)
+
+# endif /* LOCORE */
+
+
+/* fields in VER */
+#define APIC_VER_VERSION	0x000000ff
+#define APIC_VER_MAXLVT		0x00ff0000
+#define MAXLVTSHIFT		16
+
+/* fields in SVR */
+#define APIC_SVR_ENABLE		0x00000100
+# define APIC_SVR_SWDIS		0x00000000
+# define APIC_SVR_SWEN		0x00000100
+#define APIC_SVR_FOCUS		0x00000200
+# define APIC_SVR_FEN		0x00000000
+# define APIC_SVR_FDIS		0x00000200
+#define APIC_TPR_PRIO		0x000000ff
+# define APIC_TPR_INT		0x000000f0
+# define APIC_TPR_SUB		0x0000000f
+
+
+/* fields in ICR_LOW */
+#define APIC_VECTOR_MASK	0x000000ff
+
+#define APIC_DELMODE_MASK	0x00000700
+# define APIC_DELMODE_FIXED	0x00000000
+# define APIC_DELMODE_LOWPRIO	0x00000100
+# define APIC_DELMODE_SMI	0x00000200
+# define APIC_DELMODE_RR	0x00000300
+# define APIC_DELMODE_NMI	0x00000400
+# define APIC_DELMODE_INIT	0x00000500
+# define APIC_DELMODE_STARTUP	0x00000600
+# define APIC_DELMODE_RESV	0x00000700
+
+#define APIC_DESTMODE_MASK	0x00000800
+# define APIC_DESTMODE_PHY	0x00000000
+# define APIC_DESTMODE_LOG	0x00000800
+
+#define APIC_DELSTAT_MASK	0x00001000
+# define APIC_DELSTAT_IDLE	0x00000000
+# define APIC_DELSTAT_PEND	0x00001000
+
+#define APIC_RESV1_MASK		0x00002000
+
+#define APIC_LEVEL_MASK		0x00004000
+# define APIC_LEVEL_DEASSERT	0x00000000
+# define APIC_LEVEL_ASSERT	0x00004000
+
+#define APIC_TRIGMOD_MASK	0x00008000
+# define APIC_TRIGMOD_EDGE	0x00000000
+# define APIC_TRIGMOD_LEVEL	0x00008000
+
+#define APIC_RRSTAT_MASK	0x00030000
+# define APIC_RRSTAT_INVALID	0x00000000
+# define APIC_RRSTAT_INPROG	0x00010000
+# define APIC_RRSTAT_VALID	0x00020000
+# define APIC_RRSTAT_RESV	0x00030000
+
+#define APIC_DEST_MASK		0x000c0000
+# define APIC_DEST_DESTFLD	0x00000000
+# define APIC_DEST_SELF		0x00040000
+# define APIC_DEST_ALLISELF	0x00080000
+# define APIC_DEST_ALLESELF	0x000c0000
+
+#define APIC_RESV2_MASK		0xfff00000
+
+
+/* fields in ICR_HIGH */
+#define APIC_ID_MASK		0x0f000000
+
+
+/* fields in LVT1/2 */
+#define APIC_LVT_VECTOR		0x000000ff
+#define APIC_LVT_DS		0x00001000
+#define APIC_LVT_M		0x00010000
+
+
+/* fields in LVT Timer */
+#define APIC_LVTT_VECTOR	0x000000ff
+#define APIC_LVTT_DS		0x00001000
+#define APIC_LVTT_M		0x00010000
+#define APIC_LVTT_TM		0x00020000
+
+
+/* fields in TDCR */
+#define APIC_TDCR_2		0x00
+#define APIC_TDCR_4		0x01
+#define APIC_TDCR_8		0x02
+#define APIC_TDCR_16		0x03
+#define APIC_TDCR_32		0x08
+#define APIC_TDCR_64		0x09
+#define APIC_TDCR_128		0x0a
+#define APIC_TDCR_1		0x0b
+
+
+/*
+ * fields in IRR
+ * ISA INTerrupts are in bits 16-31 of the 1st IRR register.
+ * these masks DON'T EQUAL the isa IRQs of the same name.
+ * FIXME: how do we make this portable for MP table configurations???
+ *        look for "HARD_VECTORXXX" marking places with this problem.
+ */
+#define APIC_IRQ0		0x00000001
+#define APIC_IRQ1		0x00000002
+#define APIC_IRQ2		0x00000004
+#define APIC_IRQ3		0x00000008
+#define APIC_IRQ4		0x00000010
+#define APIC_IRQ5		0x00000020
+#define APIC_IRQ6		0x00000040
+#define APIC_IRQ7		0x00000080
+#define APIC_IRQ8		0x00000100
+#define APIC_IRQ9		0x00000200
+#define APIC_IRQ10		0x00000400
+#define APIC_IRQ11		0x00000800
+#define APIC_IRQ12		0x00001000
+#define APIC_IRQ13		0x00002000
+#define APIC_IRQ14		0x00004000
+#define APIC_IRQ15		0x00008000
+#define APIC_IRQ16		0x00010000
+#define APIC_IRQ17		0x00020000
+#define APIC_IRQ18		0x00040000
+#define APIC_IRQ19		0x00080000
+#define APIC_IRQ20		0x00100000
+#define APIC_IRQ21		0x00200000
+#define APIC_IRQ22		0x00400000
+#define APIC_IRQ23		0x00800000
+
+
+/******************************************************************************
+ * I/O APIC defines
+ */
+
+/* default physical locations of an IO APIC */
+#define DEFAULT_IO_APIC_BASE	0xfec00000
+
+/* window register offset */
+#define IOAPIC_WINDOW		0x10
+
+/* indexes into IO APIC */
+#define IOAPIC_ID		0x00
+#define IOAPIC_VER		0x01
+#define IOAPIC_ARB		0x02
+#define IOAPIC_REDTBL		0x10
+#define IOAPIC_REDTBL0		IOAPIC_REDTBL
+#define IOAPIC_REDTBL1		(IOAPIC_REDTBL+0x02)
+#define IOAPIC_REDTBL2		(IOAPIC_REDTBL+0x04)
+#define IOAPIC_REDTBL3		(IOAPIC_REDTBL+0x06)
+#define IOAPIC_REDTBL4		(IOAPIC_REDTBL+0x08)
+#define IOAPIC_REDTBL5		(IOAPIC_REDTBL+0x0a)
+#define IOAPIC_REDTBL6		(IOAPIC_REDTBL+0x0c)
+#define IOAPIC_REDTBL7		(IOAPIC_REDTBL+0x0e)
+#define IOAPIC_REDTBL8		(IOAPIC_REDTBL+0x10)
+#define IOAPIC_REDTBL9		(IOAPIC_REDTBL+0x12)
+#define IOAPIC_REDTBL10		(IOAPIC_REDTBL+0x14)
+#define IOAPIC_REDTBL11		(IOAPIC_REDTBL+0x16)
+#define IOAPIC_REDTBL12		(IOAPIC_REDTBL+0x18)
+#define IOAPIC_REDTBL13		(IOAPIC_REDTBL+0x1a)
+#define IOAPIC_REDTBL14		(IOAPIC_REDTBL+0x1c)
+#define IOAPIC_REDTBL15		(IOAPIC_REDTBL+0x1e)
+#define IOAPIC_REDTBL16		(IOAPIC_REDTBL+0x20)
+#define IOAPIC_REDTBL17		(IOAPIC_REDTBL+0x22)
+#define IOAPIC_REDTBL18		(IOAPIC_REDTBL+0x24)
+#define IOAPIC_REDTBL19		(IOAPIC_REDTBL+0x26)
+#define IOAPIC_REDTBL20		(IOAPIC_REDTBL+0x28)
+#define IOAPIC_REDTBL21		(IOAPIC_REDTBL+0x2a)
+#define IOAPIC_REDTBL22		(IOAPIC_REDTBL+0x2c)
+#define IOAPIC_REDTBL23		(IOAPIC_REDTBL+0x2e)
+
+/* fields in VER */
+#define IOART_VER_VERSION	0x000000ff
+#define IOART_VER_MAXREDIR	0x00ff0000
+#define MAXREDIRSHIFT		16
+
+/*
+ * fields in the IO APIC's redirection table entries
+ */
+#define IOART_DEST	APIC_ID_MASK	/* broadcast addr: all APICs */
+
+#define IOART_RESV	0x00fe0000	/* reserved */
+
+#define IOART_INTMASK	0x00010000	/* R/W: INTerrupt mask */
+# define IOART_INTMCLR	0x00000000	/*       clear, allow INTs */
+# define IOART_INTMSET	0x00010000	/*       set, inhibit INTs */
+
+#define IOART_TRGRMOD	0x00008000	/* R/W: trigger mode */
+# define IOART_TRGREDG	0x00000000	/*       edge */
+# define IOART_TRGRLVL	0x00008000	/*       level */
+
+#define IOART_REM_IRR	0x00004000	/* RO: remote IRR */
+
+#define IOART_INTPOL	0x00002000	/* R/W: INT input pin polarity */
+# define IOART_INTAHI	0x00000000	/*      active high */
+# define IOART_INTALO	0x00002000	/*      active low */
+
+#define IOART_DELIVS	0x00001000	/* RO: delivery status */
+
+#define IOART_DESTMOD	0x00000800	/* R/W: destination mode */
+# define IOART_DESTPHY	0x00000000	/*      physical */
+# define IOART_DESTLOG	0x00000800	/*      logical */
+
+#define IOART_DELMOD	0x00000700	/* R/W: delivery mode */
+# define IOART_DELFIXED	0x00000000	/*       fixed */
+# define IOART_DELLOPRI	0x00000100	/*       lowest priority */
+# define IOART_DELSMI	0x00000200	/*       System Management INT */
+# define IOART_DELRSV1	0x00000300	/*       reserved */
+# define IOART_DELNMI	0x00000400	/*       NMI signal */
+# define IOART_DELINIT	0x00000500	/*       INIT signal */
+# define IOART_DELRSV2	0x00000600	/*       reserved */
+# define IOART_DELEXINT	0x00000700	/*       External INTerrupt */
+
+#define IOART_INTVEC	0x000000ff	/* R/W: INTerrupt vector field */
+
+#endif /* _MACHINE_APIC_H_ */
diff --git a/sys/amd64/include/clock.h b/sys/amd64/include/clock.h
index 30ff97a7f80f..df8a20c5e4e8 100644
--- a/sys/amd64/include/clock.h
+++ b/sys/amd64/include/clock.h
@@ -3,13 +3,13 @@
  * Garrett Wollman, September 1994.
  * This file is in the public domain.
  *
- *	$Id$
+ *	$Id: clock.h,v 1.24 1997/02/22 09:33:59 peter Exp $
  */
 
 #ifndef _MACHINE_CLOCK_H_
 #define	_MACHINE_CLOCK_H_
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 #define CPU_CLOCKUPDATE(otime, ntime)	cpu_clockupdate((otime), (ntime))
 #else
 #define CPU_CLOCKUPDATE(otime, ntime)	(*(otime) = *(ntime))
@@ -28,7 +28,7 @@
  */
 extern int	adjkerntz;
 extern int	disable_rtc_set;
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 extern u_int	i586_ctr_bias;
 extern u_int	i586_ctr_comultiplier;
 extern u_int	i586_ctr_freq;
@@ -81,7 +81,7 @@ clock_latency(void)
 		- ((high << 8) | low));
 }
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 /*
  * When we update `time', on i586's we also update `i586_ctr_bias'
  * atomically.  `i586_ctr_bias' is the best available approximation to
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index 9cd03517e3a1..740b7b23765f 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -30,7 +30,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: cpufunc.h,v 1.61 1997/02/22 09:34:08 peter Exp $
+ *	$Id: cpufunc.h,v 1.62 1997/03/22 18:52:57 kato Exp $
  */
 
 /*
@@ -42,6 +42,10 @@
 
 #include <sys/cdefs.h>
 #include <sys/types.h>
+#include <machine/smp.h>
+
+#include "opt_smp.h"
+#include "opt_smp_invltlb.h"
 
 #ifdef	__GNUC__
 
@@ -201,6 +205,16 @@ invd(void)
 	__asm __volatile("invd");
 }
 
+#if defined(SMP) && defined(SMP_INVLTLB)
+
+/*
+ * When using APIC IPI's, the inlining cost is prohibitive..
+ */
+void	invlpg		__P((u_int addr));
+void	invltlb		__P((void));
+
+#else
+
 static __inline void
 invlpg(u_int addr)
 {
@@ -218,6 +232,7 @@ invltlb(void)
 	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
 			 : : "memory");
 }
+#endif	/* SMP && SMP_INVLTLB */
 
 static __inline u_short
 inw(u_int port)
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
new file mode 100644
index 000000000000..07fca34efe56
--- /dev/null
+++ b/sys/amd64/include/mptable.h
@@ -0,0 +1,1646 @@
+/*
+ * Copyright (c) 1996, by Steve Passe
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. The name of the developer may NOT be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id: mp_machdep.c,v 1.47 1997/04/26 08:11:49 peter Exp $
+ */
+
+#include "opt_smp.h"
+#include "opt_smp_invltlb.h"
+#if defined(APIC_IO)
+#if !defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#else				/* APIC_IO */
+#if defined(SMP_INVLTLB)
+#error you must define BOTH APIC_IO and SMP_INVLTLB or NEITHER
+#endif
+#endif				/* APIC_IO */
+#define FIX_MP_TABLE_WORKS_NOT
+
+#include "opt_serial.h"
+
+#include <sys/param.h>		/* for KERNBASE */
+#include <sys/types.h>
+#include <sys/sysproto.h>
+#include <sys/time.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>		/* for KERNBASE */
+#include <vm/vm_param.h>	/* for KERNBASE */
+#include <vm/pmap.h>		/* for KERNBASE */
+#include <machine/pmap.h>	/* for KERNBASE */
+
+#include <machine/smp.h>
+#include <machine/apic.h>
+#include <machine/mpapic.h>
+#include <machine/cpufunc.h>
+#include <machine/segments.h>
+#include <machine/smptests.h>	/** TEST_UPPERPRIO, TEST_DEFAULT_CONFIG */
+
+#include <i386/i386/cons.h>	/* cngetc() */
+
+#if defined(IPI_INTS)
+#include <i386/isa/isa_device.h>
+#include "vector.h"
+#endif				/* IPI_INTS */
+
+#if defined(SMP_INVLTLB)
+#include <i386/isa/icu.h>
+#endif				/* SMP_INVLTLB */
+
+#define WARMBOOT_TARGET	0
+#define WARMBOOT_OFF	(KERNBASE + 0x0467)
+#define WARMBOOT_SEG	(KERNBASE + 0x0469)
+
+#define BIOS_BASE	(0xf0000)
+#define BIOS_SIZE	(0x10000)
+#define BIOS_COUNT	(BIOS_SIZE/4)
+
+#define CMOS_REG	(0x70)
+#define CMOS_DATA	(0x71)
+#define BIOS_RESET	(0x0f)
+#define BIOS_WARM	(0x0a)
+
+/*
+ * this code MUST be enabled here and in mpboot.s.
+ * it follows the very early stages of AP boot by placing values in CMOS ram.
+ * it NORMALLY will never be needed and thus the primitive method for enabling.
+ *
+#define CHECK_POINTS
+ */
+
+#if defined(CHECK_POINTS)
+#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
+#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
+
+#define CHECK_INIT(D);				\
+	CHECK_WRITE(0x34, (D));			\
+	CHECK_WRITE(0x35, (D));			\
+	CHECK_WRITE(0x36, (D));			\
+	CHECK_WRITE(0x37, (D));			\
+	CHECK_WRITE(0x38, (D));			\
+	CHECK_WRITE(0x39, (D));
+
+#define CHECK_PRINT(S);				\
+	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
+	   (S),					\
+	   CHECK_READ(0x34),			\
+	   CHECK_READ(0x35),			\
+	   CHECK_READ(0x36),			\
+	   CHECK_READ(0x37),			\
+	   CHECK_READ(0x38),			\
+	   CHECK_READ(0x39));
+
+#else				/* CHECK_POINTS */
+
+#define CHECK_INIT(D)
+#define CHECK_PRINT(S)
+
+#endif				/* CHECK_POINTS */
+
+
+/** FIXME: what system files declare these??? */
+extern struct region_descriptor r_gdt, r_idt;
+
+/* global data */
+struct proc *SMPcurproc[NCPU];
+struct pcb *SMPcurpcb[NCPU];
+struct timeval SMPruntime[NCPU];
+
+int     mp_ncpus;		/* # of CPUs, including BSP */
+int     mp_naps;		/* # of Applications processors */
+int     mp_nbusses;		/* # of busses */
+int     mp_napics;		/* # of IO APICs */
+int     mpenabled;
+int     boot_cpu_id;		/* designated BSP */
+vm_offset_t cpu_apic_address;
+vm_offset_t io_apic_address[NAPIC];
+
+u_int32_t cpu_apic_versions[NCPU];
+u_int32_t io_apic_versions[NAPIC];
+
+/*
+ * APIC ID logical/physical mapping structures
+ */
+int     cpu_num_to_apic_id[NCPU];
+int     io_num_to_apic_id[NAPIC];
+int     apic_id_to_logical[NAPICID];
+
+/*
+ * look for MP compliant motherboard.
+ */
+
+static u_int boot_address;
+static u_int base_memory;
+
+static int picmode;		/* 0: virtual wire mode, 1: PIC mode */
+static u_int mpfps;
+static int search_for_sig(u_int32_t target, int count);
+static int mp_probe(u_int base_top);
+static void mp_enable(u_int boot_addr);
+
+#if defined(IPI_INTS)
+static void ipi_intr0(void);
+static void ipi_intr1(void);
+static void ipi_intr2(void);
+static void ipi_intr3(void);
+static int 
+ipi_ihandler_attach(int irq, inthand2_t * func,
+    unsigned *maskptr, int unit);
+#endif				/* IPI_INTS */
+
+
+/*
+ * calculate usable address in base memory for AP trampoline code
+ */
+u_int
+mp_bootaddress(u_int basemem)
+{
+	base_memory = basemem * 1024;	/* convert to bytes */
+
+	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
+	if ((base_memory - boot_address) < bootMP_size)
+		boot_address -= 4096;	/* not enough, lower by 4k */
+
+	return boot_address;
+}
+
+
+/*
+ * startup the SMP processors
+ */
+void
+mp_start(void)
+{
+	/* look for MP capable motherboard */
+	if (mp_probe(base_memory))
+		mp_enable(boot_address);
+	else {
+		printf("MP FPS NOT FOUND, suggest use of 'mptable' program\n");
+		panic("can't continue!\n");
+	}
+
+	/* finish pmap initialization - turn off V==P mapping at zero */
+	pmap_bootstrap2();
+}
+
+
+/*
+ * print various information about the SMP system hardware and setup
+ */
+void
+mp_announce(void)
+{
+	int     x;
+
+	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
+	printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0));
+	printf(", version: 0x%08x\n", cpu_apic_versions[0]);
+	for (x = 1; x <= mp_naps; ++x) {
+		printf(" cpu%d (AP):  apic id: %d", x, CPU_TO_ID(x));
+		printf(", version: 0x%08x\n", cpu_apic_versions[x]);
+	}
+
+#if defined(APIC_IO)
+	for (x = 0; x < mp_napics; ++x) {
+		printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x));
+		printf(", version: 0x%08x\n", io_apic_versions[x]);
+	}
+#else
+	printf(" Warning: APIC I/O disabled\n");
+#endif	/* APIC_IO */
+}
+
+
+/*
+ * AP cpu's call this to sync up protected mode.
+ */
+void
+init_secondary(void)
+{
+	int     gsel_tss, slot;
+
+	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
+	r_gdt.rd_base = (int) gdt;
+	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lidt(&r_idt);
+	lldt(_default_ldt);
+
+	slot = NGDT + cpunumber();
+	gsel_tss = GSEL(slot, SEL_KPL);
+	gdt[slot].sd.sd_type = SDT_SYS386TSS;
+	ltr(gsel_tss);
+
+	load_cr0(0x8005003b);	/* XXX! */
+}
+
+
+#if defined(APIC_IO)
+void
+configure_local_apic(void)
+{
+	u_char  byte;
+	u_int32_t temp;
+
+	if (picmode) {
+		outb(0x22, 0x70);	/* select IMCR */
+		byte = inb(0x23);	/* current contents */
+		byte |= 0x01;	/* mask external INTR */
+		outb(0x23, byte);	/* disconnect 8259s/NMI */
+	}
+	/* mask the LVT1 */
+	temp = apic_base[APIC_LVT1];
+	temp |= APIC_LVT_M;
+	apic_base[APIC_LVT1] = temp;
+}
+#endif	/* APIC_IO */
+
+
+/*******************************************************************
+ * local functions and data
+ */
+
+static int
+mp_probe(u_int base_top)
+{
+	int     x;
+	u_long  segment;
+	u_int32_t target;
+
+	/* see if EBDA exists */
+	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
+		/* search first 1K of EBDA */
+		target = (u_int32_t) (segment << 4);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	} else {
+		/*last 1K of base memory, effective 'top of base' is passed in*/
+		target = (u_int32_t) (base_top - 0x400);
+		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
+			goto found;
+	}
+
+	/* search the BIOS */
+	target = (u_int32_t) BIOS_BASE;
+	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
+		goto found;
+
+	/* nothing found */
+	mpfps = mpenabled = 0;
+	return 0;
+
+found:				/* please forgive the 'goto'! */
+	/* flag fact that we are running multiple processors */
+	mpfps = x;
+	mpenabled = 1;
+	return 1;
+}
+
+
+/*
+ * start the SMP system
+ */
+static int parse_mp_table(void);
+static void default_mp_table(int type);
+static int start_all_aps(u_int boot_addr);
+
+#if defined(XFAST_IPI32)
+#include <machine/md_var.h>
+#include <i386/isa/isa_device.h>
+extern void Xfastipi32(u_int, u_int, u_int, u_int);
+#endif				/* XFAST_IPI32 */
+
+static void
+mp_enable(u_int boot_addr)
+{
+	int     x;
+#if defined(APIC_IO)
+	int     apic;
+	u_int   ux;
+#if defined(TEST_UPPERPRIO)
+	u_char  select;		/* the select register is 8 bits */
+	u_int32_t flags;	/* the window register is 32 bits */
+#endif	/* TEST_UPPERPRIO */
+#endif	/* APIC_IO */
+
+	/* examine the MP table for needed info */
+	x = parse_mp_table();
+
+	/* create pages for (address common) cpu APIC and each IO APIC */
+	pmap_bootstrap_apics();
+
+	/* can't process default configs till the CPU APIC is pmapped */
+	if (x)
+		default_mp_table(x);
+
+#if defined(APIC_IO)
+	/* fill the LOGICAL io_apic_versions table */
+	for (apic = 0; apic < mp_napics; ++apic) {
+		ux = io_apic_read(apic, IOAPIC_VER);
+		io_apic_versions[apic] = ux;
+	}
+
+	/*
+	 */
+	for (apic = 0; apic < mp_napics; ++apic)
+		if (io_apic_setup(apic) < 0)
+			panic("IO APIC setup failure\n");
+
+#if defined(IPI_INTS)
+	/* setup IPI INTerrupt mechanism */
+	ipi_ihandler_attach( /* irq */ 24,
+	     /* XXX */ (inthand2_t *) ipi_intr0, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 25,
+	     /* XXX */ (inthand2_t *) ipi_intr1, NULL, /* unit */ 0);
+	ipi_ihandler_attach( /* irq */ 26,
+	     /* XXX */ (inthand2_t *) ipi_intr2, NULL, /* unit */ 0);
+#if !defined(SMP_INVLTLB)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+#else
+#if defined(XFAST_IPI32)
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_intr3, NULL, /* unit */ 0);
+	setidt(ICU_OFFSET + 32,
+	    Xfastipi32,
+	    SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#else
+	ipi_ihandler_attach( /* irq */ 27,
+	     /* XXX */ (inthand2_t *) ipi_invltlb, NULL, /* unit */ 0);
+#endif	/* XFAST_IPI32 */
+#endif
+
+#endif	/* IPI_INTS */
+
+#if defined(TEST_UPPERPRIO)
+
+#if 1
+	printf("special IRQ10\n");
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~0xff;		/** clear vector */
+	flags |= 64;
+	io_apic_write(0, select, flags);
+#else
+	printf("special IRQ10\n");
+	cngetc();
+	select = IOAPIC_REDTBL10;	/** HARD_VECTORXXX:  */
+	flags = io_apic_read(0, select);
+	flags &= ~IOART_DELMOD;	/* FIXED mode */
+	io_apic_write(0, select, flags);
+	io_apic_write(0, select + 1, boot_cpu_id << 24);
+#endif	/** 0/1 */
+
+#endif	/* TEST_UPPERPRIO */
+
+#endif	/* APIC_IO */
+
+	/* start each Application Processor */
+	start_all_aps(boot_addr);
+}
+
+
+/*
+ * look for the MP spec signature
+ */
+
+/* string defined by the Intel MP Spec as identifying the MP table */
+#define MP_SIG		0x5f504d5f	/* _MP_ */
+#define NEXT(X)		((X) += 4)
+static int
+search_for_sig(u_int32_t target, int count)
+{
+	int     x;
+	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
+
+	for (x = 0; x < count; NEXT(x))
+		if (addr[x] == MP_SIG)
+			/* make array index a byte index */
+			return (target + (x * sizeof(u_int32_t)));
+
+	return -1;
+}
+
+
+#define PROCENTRY_FLAG_EN	0x01
+#define PROCENTRY_FLAG_BP	0x02
+#define IOAPICENTRY_FLAG_EN	0x01
+
+/* MP Floating Pointer Structure */
+typedef struct MPFPS {
+	char    signature[4];
+	void   *pap;
+	u_char  length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  mpfb1;
+	u_char  mpfb2;
+	u_char  mpfb3;
+	u_char  mpfb4;
+	u_char  mpfb5;
+}      *mpfps_t;
+/* MP Configuration Table Header */
+typedef struct MPCTH {
+	char    signature[4];
+	u_short base_table_length;
+	u_char  spec_rev;
+	u_char  checksum;
+	u_char  oem_id[8];
+	u_char  product_id[12];
+	void   *oem_table_pointer;
+	u_short oem_table_size;
+	u_short entry_count;
+	void   *apic_address;
+	u_short extended_table_length;
+	u_char  extended_table_checksum;
+	u_char  reserved;
+}      *mpcth_t;
+
+
+typedef struct PROCENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  cpu_flags;
+	u_long  cpu_signature;
+	u_long  feature_flags;
+	u_long  reserved1;
+	u_long  reserved2;
+}      *proc_entry_ptr;
+
+typedef struct BUSENTRY {
+	u_char  type;
+	u_char  bus_id;
+	char    bus_type[6];
+}      *bus_entry_ptr;
+
+typedef struct IOAPICENTRY {
+	u_char  type;
+	u_char  apic_id;
+	u_char  apic_version;
+	u_char  apic_flags;
+	void   *apic_address;
+}      *io_apic_entry_ptr;
+
+typedef struct INTENTRY {
+	u_char  type;
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}      *int_entry_ptr;
+/* descriptions of MP basetable entries */
+typedef struct BASETABLE_ENTRY {
+	u_char  type;
+	u_char  length;
+	char    name[16];
+}       basetable_entry;
+
+static basetable_entry basetable_entry_types[] =
+{
+	{0, 20, "Processor"},
+	{1, 8, "Bus"},
+	{2, 8, "I/O APIC"},
+	{3, 8, "I/O INT"},
+	{4, 8, "Local INT"}
+};
+
+typedef struct BUSDATA {
+	u_char  bus_id;
+	enum busTypes bus_type;
+}       bus_datum;
+
+typedef struct INTDATA {
+	u_char  int_type;
+	u_short int_flags;
+	u_char  src_bus_id;
+	u_char  src_bus_irq;
+	u_char  dst_apic_id;
+	u_char  dst_apic_int;
+}       io_int, local_int;
+
+typedef struct BUSTYPENAME {
+	u_char  type;
+	char    name[7];
+}       bus_type_name;
+
+static bus_type_name bus_type_table[] =
+{
+	{CBUS, "CBUS"},
+	{CBUSII, "CBUSII"},
+	{EISA, "EISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{ISA, "ISA"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{PCI, "PCI"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{UNKNOWN_BUSTYPE, "---"},
+	{XPRESS, "XPRESS"},
+	{UNKNOWN_BUSTYPE, "---"}
+};
+/* from MP spec v1.4, table 5-1 */
+static int default_data[7][5] =
+{
+/*   nbus, id0, type0, id1, type1 */
+	{1, 0, ISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{1, 0, EISA, 255, 255},
+	{0, 255, 255, 255, 255},/* MCA not supported */
+	{2, 0, ISA, 1, PCI},
+	{2, 0, EISA, 1, PCI},
+	{0, 255, 255, 255, 255}	/* MCA not supported */
+};
+
+
+/* the bus data */
+bus_datum bus_data[NBUS];
+
+/* the IO INT data, one entry per possible APIC INTerrupt */
+io_int  io_apic_ints[NINTR];
+
+static int nintrs;
+
+#if defined(FIX_MP_TABLE_WORKS)
+static void fix_mp_table __P((void));
+#endif /* FIX_MP_TABLE_WORKS */
+
+static void processor_entry __P((proc_entry_ptr entry, int *cpu));
+static void io_apic_entry __P((io_apic_entry_ptr entry, int *apic));
+static void bus_entry __P((bus_entry_ptr entry, int *bus));
+static void int_entry __P((int_entry_ptr entry, int *intr));
+static int lookup_bus_type __P((char *name));
+
+
+/*
+ * parse an Intel MP specification table
+ */
+static int
+parse_mp_table(void)
+{
+	int     x;
+	mpfps_t fps;
+	mpcth_t cth;
+	int     totalSize;
+	void   *position;
+	int     count;
+	int     type;
+	int     apic, bus, cpu, intr;
+
+	/* clear physical APIC ID to logical CPU/IO table */
+	for (x = 0; x < NAPICID; ++x)
+		ID_TO_IO(x) = -1;
+
+	/* clear logical CPU to APIC ID table */
+	for (x = 0; x < NCPU; ++x)
+		CPU_TO_ID(x) = -1;
+
+	/* clear logical IO to APIC ID table */
+	for (x = 0; x < NAPIC; ++x)
+		IO_TO_ID(x) = -1;
+
+	/* clear IO APIC address table */
+	for (x = 0; x < NAPIC; ++x)
+		io_apic_address[x] = ~0;
+
+	/* clear bus data table */
+	for (x = 0; x < NBUS; ++x)
+		bus_data[x].bus_id = 0xff;
+
+	/* clear IO APIC INT table */
+	for (x = 0; x < NINTR; ++x)
+		io_apic_ints[x].int_type = 0xff;
+	nintrs = 0;
+
+	/* count the BSP */
+	mp_ncpus = 1;
+
+	/* setup the cpu/apic mapping arrays */
+	boot_cpu_id = -1;
+
+	/* local pointer */
+	fps = (mpfps_t) mpfps;
+
+	/* record whether PIC or virtual-wire mode */
+	picmode = (fps->mpfb2 & 0x80) ? 1 : 0;
+
+	/* check for use of 'default' configuration */
+#if defined(TEST_DEFAULT_CONFIG)
+	/* use default addresses */
+	cpu_apic_address = DEFAULT_APIC_BASE;
+	io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+	/* return default configuration type */
+	return TEST_DEFAULT_CONFIG;
+#else
+	if (fps->mpfb1 != 0) {
+		/* use default addresses */
+		cpu_apic_address = DEFAULT_APIC_BASE;
+		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
+
+		/* return default configuration type */
+		return fps->mpfb1;
+	}
+#endif	/* TEST_DEFAULT_CONFIG */
+
+	if ((cth = fps->pap) == 0)
+		panic("MP Configuration Table Header MISSING!\n");
+
+	cpu_apic_address = (vm_offset_t) cth->apic_address;
+
+	totalSize = cth->base_table_length - sizeof(struct MPCTH);
+	position = (u_char *) cth + sizeof(struct MPCTH);
+	count = cth->entry_count;
+
+	apic = 0;		/* logical apic# start @ 0 */
+	bus = 0;		/* logical bus# start @ 0 */
+	cpu = 1;		/* logical cpu# start @ 0, BUT reserve 0 for */
+				/* BSP */
+	intr = 0;		/* unknown */
+
+	/* walk the table, recording info of interest */
+	while (count--) {
+		switch (type = *(u_char *) position) {
+		case 0:
+			processor_entry(position, &cpu);
+			break;
+		case 1:
+			bus_entry(position, &bus);
+			break;
+		case 2:
+			io_apic_entry(position, &apic);
+			break;
+		case 3:
+			int_entry(position, &intr);
+			break;
+		case 4:
+			/* int_entry(position); */
+			break;
+		default:
+			panic("mpfps Base Table HOSED!\n");
+			/* NOTREACHED */
+		}
+
+		totalSize -= basetable_entry_types[type].length;
+		(u_char *) position += basetable_entry_types[type].length;
+	}
+
+	if (boot_cpu_id == -1)
+		panic("NO BSP found!\n");
+
+	/* record # of APs found */
+	mp_naps = (cpu - 1);
+
+	/* record # of busses found */
+	mp_nbusses = bus;
+
+	/* record # of IO APICs found */
+	mp_napics = apic;
+
+	/* record # of IO APICs found */
+	nintrs = intr;
+
+#if defined(FIX_MP_TABLE_WORKS)
+	/* post scan cleanup */
+	fix_mp_table();
+#endif /* FIX_MP_TABLE_WORKS */
+
+	/* report fact that its NOT a default configuration */
+	return 0;
+}
+
+
+/*
+ * parse an Intel MP specification table
+ */
+#if defined(FIX_MP_TABLE_WORKS)
+static void
+fix_mp_table(void)
+{
+	int     x;
+	int     y;
+	int     num_pci_bus;
+	bus_datum bus_record;
+
+	/*
+	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
+	 * did it wrong.  The MP spec says that when more than 1 PCI bus
+	 * exists the BIOS must begin with bus entries for the PCI bus and use
+	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
+	 * exists the BIOS can choose to ignore this ordering, and indeed many
+	 * MP motherboards do ignore it.  This causes a problem when the PCI
+	 * sub-system makes requests of the MP sub-system based on PCI bus
+	 * numbers.	So here we look for the situation and renumber the
+	 * busses and associated INTs in an effort to "make it right".
+	 */
+
+	/* count the number of PCI busses */
+	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
+		if (bus_data[x].bus_type == PCI)
+			++num_pci_bus;
+	}
+
+	/* check the 1 PCI bus case for sanity */
+	if (num_pci_bus == 1) {
+
+		/* if its in the first slot all is well */
+		if (bus_data[0].bus_type == PCI)
+			return;
+
+		/* mis-numbered, swap with whichever bus uses slot 0 */
+
+		/* locate the entry holding the PCI bus */
+		for (x = 1; x < mp_nbusses; ++x) {
+			if (bus_data[x].bus_type == PCI)
+				break;
+		}
+
+		/* swap the bus entry records */
+		bus_record = bus_data[0];
+		bus_data[0] = bus_data[x];
+		bus_data[x] = bus_record;
+
+		/* swap each relavant INTerrupt entry */
+		for (y = 0; y < nintrs; ++y) {
+			if (io_apic_ints[y].src_bus_id == x)
+				io_apic_ints[y].src_bus_id = 0;
+			else
+				if (io_apic_ints[y].src_bus_id == 0)
+					io_apic_ints[y].src_bus_id = x;
+		}
+	}
+	/* sanity check if more than 1 PCI bus */
+	else
+		if (num_pci_bus > 1) {
+			for (x = 0; x < num_pci_bus; ++x) {
+				if (bus_data[x].bus_type != PCI) {
+					printf("bad PCI bus numbering\n");
+					panic("\n");
+				}
+			}
+		}
+}
+#endif /* FIX_MP_TABLE_WORKS */
+
+
+static void
+processor_entry(proc_entry_ptr entry, int *cpu)
+{
+	int     x = *cpu;
+
+	/* check for usability */
+	if (!(entry->cpu_flags & PROCENTRY_FLAG_EN))
+		return;
+
+	/* check for BSP flag */
+	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
+		/* always give boot CPU the logical value of 0 */
+		x = 0;
+		boot_cpu_id = entry->apic_id;
+	} else {
+		/* add another AP to list, if less than max number of CPUs */
+		if (x == NCPU) {
+			printf("Warning: only using %d of the available CPUs!\n", x);
+			return;
+		}
+		++(*cpu);
+	}
+
+	CPU_TO_ID(x) = entry->apic_id;
+	ID_TO_CPU(entry->apic_id) = x;
+}
+
+
+static void
+bus_entry(bus_entry_ptr entry, int *bus)
+{
+	int     x, y;
+	char    name[8];
+	char    c;
+
+	if ((x = (*bus)++) == NBUS)
+		panic("too many busses, increase 'NBUS'\n");
+
+	/* encode the name into an index */
+	for (y = 0; y < 6; ++y) {
+		if ((c = entry->bus_type[y]) == ' ')
+			break;
+		name[y] = c;
+	}
+	name[y] = '\0';
+
+	if ((y = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
+		panic("unknown bus type: '%s'\n", name);
+
+	bus_data[x].bus_id = entry->bus_id;
+	bus_data[x].bus_type = y;
+}
+
+
+static void
+io_apic_entry(io_apic_entry_ptr entry, int *apic)
+{
+	int     x;
+
+	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
+		return;
+
+	if ((x = (*apic)++) == NAPIC)
+		panic("too many APICs, increase 'NAPIC'\n");
+
+	IO_TO_ID(x) = entry->apic_id;
+	ID_TO_IO(entry->apic_id) = x;
+
+	io_apic_address[x] = (vm_offset_t) entry->apic_address;
+}
+
+
+static int
+lookup_bus_type(char *name)
+{
+	int     x;
+
+	for (x = 0; x < MAX_BUSTYPE; ++x)
+		if (strcmp(bus_type_table[x].name, name) == 0)
+			return bus_type_table[x].type;
+
+	return UNKNOWN_BUSTYPE;
+}
+
+
+static void
+int_entry(int_entry_ptr entry, int *intr)
+{
+	int     x;
+
+	if ((x = (*intr)++) == NINTR)
+		panic("too many INTs, increase 'NINTR'\n");
+
+	io_apic_ints[x].int_type = entry->int_type;
+	io_apic_ints[x].int_flags = entry->int_flags;
+	io_apic_ints[x].src_bus_id = entry->src_bus_id;
+	io_apic_ints[x].src_bus_irq = entry->src_bus_irq;
+	io_apic_ints[x].dst_apic_id = entry->dst_apic_id;
+	io_apic_ints[x].dst_apic_int = entry->dst_apic_int;
+}
+
+
+static int
+apic_int_is_bus_type(int intr, int bus_type)
+{
+	int     bus;
+
+	for (bus = 0; bus < mp_nbusses; ++bus)
+		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
+		    && ((int) bus_data[bus].bus_type == bus_type))
+			return 1;
+
+	return 0;
+}
+
+
+/*
+ * determine which APIC pin an ISA INT is attached to.
+ */
+#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
+#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
+
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_isa_apic_irq(int isaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (isaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == isaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, ISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin an EISA INT is attached to.
+ */
+#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
+int
+get_eisa_apic_irq(int eisaIRQ)
+{
+	int     intr;
+
+#if defined(SMP_TIMER_NC)
+	if (eisaIRQ == 0)
+		return -1;
+#endif				/* SMP_TIMER_NC */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each INT record */
+		if ((INTTYPE(intr) == 0)
+		    && (SRCBUSIRQ(intr) == eisaIRQ))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, EISA))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSIRQ
+
+
+/*
+ * determine which APIC pin a PCI INT is attached to.
+ */
+#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
+#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
+#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
+int
+get_pci_apic_irq(int pciBus, int pciDevice, int pciInt)
+{
+	int     intr;
+
+	--pciInt;		/* zero based */
+
+	for (intr = 0; intr < nintrs; ++intr)	/* search each record */
+		if ((INTTYPE(intr) == 0)
+#if defined(FIX_MP_TABLE_WORKS)
+		    && (SRCBUSID(intr) == pciBus)
+#endif /* FIX_MP_TABLE_WORKS */
+		    && (SRCBUSDEVICE(intr) == pciDevice)
+		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
+			if (apic_int_is_bus_type(intr, PCI))	/* check bus match */
+				return INTPIN(intr);	/* exact match */
+
+	return -1;		/* NOT found */
+}
+#undef SRCBUSLINE
+#undef SRCBUSDEVICE
+#undef SRCBUSID
+
+#undef INTPIN
+#undef INTTYPE
+
+
+int
+undirect_pci_irq(int rirq)
+{
+#if defined(READY)
+	printf("Freeing irq %d for ISA cards.\n", rirq);
+	/** FIXME: tickle the MB redirector chip */
+	return ???;
+#else
+	printf("Freeing (NOT implimented) irq %d for ISA cards.\n", rirq);
+	return 0;
+#endif				/* READY */
+}
+
+
+/*
+ * given a bus ID, return:
+ *  the bus type if found
+ *  -1 if NOT found
+ */
+int
+apic_bus_type(int id)
+{
+	int     x;
+
+	for (x = 0; x < mp_nbusses; ++x)
+		if (bus_data[x].bus_id == id)
+			return bus_data[x].bus_type;
+
+	return -1;
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus ID if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_id(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_id);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated src bus IRQ if found
+ *  -1 if NOT found
+ */
+int
+apic_src_bus_irq(int apic, int pin)
+{
+	int     x;
+
+	for (x = 0; x < nintrs; x++)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].src_bus_irq);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated INTerrupt type if found
+ *  -1 if NOT found
+ */
+int
+apic_int_type(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_type);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated trigger mode if found
+ *  -1 if NOT found
+ */
+int
+apic_trigger(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * given a LOGICAL APIC# and pin#, return:
+ *  the associated 'active' level if found
+ *  -1 if NOT found
+ */
+int
+apic_polarity(int apic, int pin)
+{
+	int     x;
+
+	/* search each of the possible INTerrupt sources */
+	for (x = 0; x < nintrs; ++x)
+		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
+		    (pin == io_apic_ints[x].dst_apic_int))
+			return (io_apic_ints[x].int_flags & 0x03);
+
+	return -1;		/* NOT found */
+}
+
+
+/*
+ * set data according to MP defaults
+ * FIXME: probably not complete yet...
+ */
+static void
+default_mp_table(int type)
+{
+	int     ap_cpu_id;
+#if defined(APIC_IO)
+	u_int32_t ux;
+	int     io_apic_id;
+	int     pin;
+#endif	/* APIC_IO */
+
+#if 0
+	printf("  MP default config type: %d\n", type);
+	switch (type) {
+	case 1:
+		printf("   bus: ISA, APIC: 82489DX\n");
+		break;
+	case 2:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 3:
+		printf("   bus: EISA, APIC: 82489DX\n");
+		break;
+	case 4:
+		printf("   bus: MCA, APIC: 82489DX\n");
+		break;
+	case 5:
+		printf("   bus: ISA+PCI, APIC: Integrated\n");
+		break;
+	case 6:
+		printf("   bus: EISA+PCI, APIC: Integrated\n");
+		break;
+	case 7:
+		printf("   bus: MCA+PCI, APIC: Integrated\n");
+		break;
+	default:
+		printf("   future type\n");
+		break;
+		/* NOTREACHED */
+	}
+#endif	/* 0 */
+
+	boot_cpu_id = (apic_base[APIC_ID] & APIC_ID_MASK) >> 24;
+	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
+
+	/* BSP */
+	CPU_TO_ID(0) = boot_cpu_id;
+	ID_TO_CPU(boot_cpu_id) = 0;
+
+	/* one and only AP */
+	CPU_TO_ID(1) = ap_cpu_id;
+	ID_TO_CPU(ap_cpu_id) = 1;
+	mp_naps = 1;
+
+	/* one and only IO APIC */
+#if defined(APIC_IO)
+	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
+
+	/*
+	 * sanity check, refer to MP spec section 3.6.6, last paragraph
+	 * necessary as some hardware isn't properly setting up the IO APIC
+	 */
+#if defined(REALLY_ANAL_IOAPICID_VALUE)
+	if (io_apic_id != 2) {
+#else
+	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
+#endif	/* REALLY_ANAL_IOAPICID_VALUE */
+		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
+		ux &= ~APIC_ID_MASK;	/* clear the ID field */
+		ux |= 0x02000000;	/* set it to '2' */
+		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
+		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
+		if ((ux & APIC_ID_MASK) != 0x02000000)
+			panic("Problem: can't control IO APIC ID, reg: 0x%08x\n", ux);
+		io_apic_id = 2;
+	}
+	IO_TO_ID(0) = io_apic_id;
+	ID_TO_IO(io_apic_id) = 0;
+	mp_napics = 1;
+#else
+	mp_napics = 0;
+#endif	/* APIC_IO */
+
+	/* fill out bus entries */
+	switch (type) {
+	case 1:
+	case 2:
+	case 3:
+	case 5:
+	case 6:
+		mp_nbusses = default_data[type - 1][0];
+		bus_data[0].bus_id = default_data[type - 1][1];
+		bus_data[0].bus_type = default_data[type - 1][2];
+		bus_data[1].bus_id = default_data[type - 1][3];
+		bus_data[1].bus_type = default_data[type - 1][4];
+		break;
+
+	/* case 4: case 7:		   MCA NOT supported */
+	default:		/* illegal/reserved */
+		panic("BAD default MP config: %d\n", type);
+	}
+
+#if defined(APIC_IO)
+	/* general cases from MP v1.4, table 5-2 */
+	for (pin = 0; pin < 16; ++pin) {
+		io_apic_ints[pin].int_type = 0;
+		io_apic_ints[pin].int_flags = 0x05;	/* edge-triggered/active-hi */
+		io_apic_ints[pin].src_bus_id = 0;
+		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 is caught below */
+		io_apic_ints[pin].dst_apic_id = io_apic_id;
+		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 correspondence */
+	}
+
+	/* special cases from MP v1.4, table 5-2 */
+	if (type == 2) {
+		io_apic_ints[2].int_type = 0xff;	/* N/C */
+		io_apic_ints[13].int_type = 0xff;	/* N/C */
+#if !defined(APIC_MIXED_MODE)
+		/** FIXME: ??? */
+		panic("sorry, can't support type 2 default yet\n");
+#endif	/* APIC_MIXED_MODE */
+	} else
+		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
+
+	if (type == 7)
+		io_apic_ints[0].int_type = 0xff;	/* N/C */
+	else
+		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
+
+	nintrs = 16;
+#endif	/* APIC_IO */
+}
+
+
+static void install_ap_tramp(u_int boot_addr);
+static int start_ap(int logicalCpu, u_int boot_addr);
+
+/*
+ * start each AP in our list
+ */
+static int
+start_all_aps(u_int boot_addr)
+{
+	int     x;
+	u_char  mpbiosreason;
+	u_long  mpbioswarmvec;
+
+	/**
+         * NOTE: this needs further thought:
+         *        where does it get released?
+         *        should it be set to empy?
+         *
+         * get the initial mp_lock with a count of 1 for the BSP
+         */
+	mp_lock = (apic_base[APIC_ID] & APIC_ID_MASK) + 1;
+
+	/* initialize BSP's local APIC */
+	apic_initialize(1);
+
+	/* install the AP 1st level boot code */
+	install_ap_tramp(boot_addr);
+
+	/* save the current value of the warm-start vector */
+	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
+	outb(CMOS_REG, BIOS_RESET);
+	mpbiosreason = inb(CMOS_DATA);
+
+	/* start each AP */
+	for (x = 1; x <= mp_naps; ++x) {
+
+		/* setup a vector to our boot code */
+		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
+		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
+		outb(CMOS_REG, BIOS_RESET);
+		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
+
+		/* attempt to start the Application Processor */
+		CHECK_INIT(99);	/* setup checkpoints */
+		if (!start_ap(x, boot_addr)) {
+			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
+			CHECK_PRINT("trace");	/* show checkpoints */
+			/*
+			 * better panic as the AP may be running loose
+			 * somewhere
+			 */
+			printf("panic y/n? [n] ");
+			if (cngetc() != 'n')
+				panic("bye-bye\n");
+		}
+		CHECK_PRINT("trace");	/* show checkpoints */
+
+		/* record its version info */
+		cpu_apic_versions[x] = cpu_apic_versions[0];
+	}
+
+	/* fill in our (BSP) APIC version */
+	cpu_apic_versions[0] = apic_base[APIC_VER];
+
+	/* restore the warmstart vector */
+	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, mpbiosreason);
+
+	/* number of APs actually started */
+	return mp_ncpus - 1;
+}
+
+
+/*
+ * load the 1st level AP boot code into base memory.
+ */
+
+/* targets for relocation */
+extern void bigJump(void);
+extern void bootCodeSeg(void);
+extern void bootDataSeg(void);
+extern void MPentry(void);
+extern u_int MP_GDT;
+extern u_int mp_gdtbase;
+
+static void
+install_ap_tramp(u_int boot_addr)
+{
+	int     x;
+	int     size = *(int *) ((u_long) & bootMP_size);
+	u_char *src = (u_char *) ((u_long) bootMP);
+	u_char *dst = (u_char *) boot_addr + KERNBASE;
+	u_int   boot_base = (u_int) bootMP;
+	u_int8_t *dst8;
+	u_int16_t *dst16;
+	u_int32_t *dst32;
+
+	for (x = 0; x < size; ++x)
+		*dst++ = *src++;
+
+	/*
+	 * modify addresses in code we just moved to basemem. unfortunately we
+	 * need fairly detailed info about mpboot.s for this to work.  changes
+	 * to mpboot.s might require changes here.
+	 */
+
+	/* boot code is located in KERNEL space */
+	dst = (u_char *) boot_addr + KERNBASE;
+
+	/* modify the lgdt arg */
+	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
+	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
+
+	/* modify the ljmp target for MPentry() */
+	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
+	*dst32 = ((u_int) MPentry - KERNBASE);
+
+	/* modify the target for boot code segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+
+	/* modify the target for boot data segment */
+	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
+	dst8 = (u_int8_t *) (dst16 + 1);
+	*dst16 = (u_int) boot_addr & 0xffff;
+	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
+}
+
+
+/*
+ * this function starts the AP (application processor) identified
+ * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
+ * to accomplish this.  This is necessary because of the nuances
+ * of the different hardware we might encounter.  It ain't pretty,
+ * but it seems to work.
+ */
+static int
+start_ap(int logical_cpu, u_int boot_addr)
+{
+	int     physical_cpu;
+	int     vector;
+	int     cpus;
+	u_long  icr_lo, icr_hi;
+
+	/* get the PHYSICAL APIC ID# */
+	physical_cpu = CPU_TO_ID(logical_cpu);
+
+	/* calculate the vector */
+	vector = (boot_addr >> 12) & 0xff;
+
+	/* used as a watchpoint to signal AP startup */
+	cpus = mp_ncpus;
+
+	/*
+	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
+	 * and running the target CPU. OR this INIT IPI might be latched (P5
+	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
+	 * ignored.
+	 */
+
+	/* setup the address for the target AP */
+	icr_hi = apic_base[APIC_ICR_HI] & ~APIC_ID_MASK;
+	icr_hi |= (physical_cpu << 24);
+	apic_base[APIC_ICR_HI] = icr_hi;
+
+	/* do an INIT IPI: assert RESET */
+	icr_lo = apic_base[APIC_ICR_LOW] & 0xfff00000;
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x0000c500;
+
+	/* wait for pending status end */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/* do an INIT IPI: deassert RESET */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00008500;
+
+	/* wait for pending status end */
+	u_sleep(10000);		/* wait ~10mS */
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+
+	/*
+	 * next we do a STARTUP IPI: the previous INIT IPI might still be
+	 * latched, (P5 bug) this 1st STARTUP would then terminate
+	 * immediately, and the previously started INIT IPI would continue. OR
+	 * the previous INIT IPI has already run. and this STARTUP IPI will
+	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
+	 * will run.
+	 */
+
+	/* do a STARTUP IPI */
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/*
+	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
+	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
+	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
+	 * recognized after hardware RESET or INIT IPI.
+	 */
+
+	apic_base[APIC_ICR_LOW] = icr_lo | 0x00000600 | vector;
+	while (apic_base[APIC_ICR_LOW] & APIC_DELSTAT_MASK)
+		 /* spin */ ;
+	u_sleep(200);		/* wait ~200uS */
+
+	/* wait for it to start */
+	set_apic_timer(5000000);/* == 5 seconds */
+	while (read_apic_timer())
+		if (mp_ncpus > cpus)
+			return 1;	/* return SUCCESS */
+
+	return 0;		/* return FAILURE */
+}
+
+
+#if defined(IPI_INTS)
+
+static void
+ipi_intr0(void)
+{
+	printf("IPI 0\n");
+}
+
+
+static void
+ipi_intr1(void)
+{
+	printf("IPI 1\n");
+}
+
+
+static void
+ipi_intr2(void)
+{
+	printf("IPI 2\n");
+}
+
+
+static void
+ipi_intr3(void)
+{
+	printf("IPI 3\n");
+}
+
+/*-----------------------------------------------------------------------
+**
+**	Register an interupt handler for an IPI.
+**	(Stolen from the PCI<->ISA glue code)
+**
+**-----------------------------------------------------------------------
+*/
+
+static int
+ipi_ihandler_attach(int irq, inthand2_t * func, unsigned *maskptr, int unit)
+{
+	char    buf[16];
+	char   *cp;
+	int     free_id, id, result;
+
+	sprintf(buf, "ipi irq%d", irq);
+	for (cp = intrnames, free_id = 0, id = 0; id < NR_DEVICES; id++) {
+		if (strcmp(cp, buf) == 0)
+			break;
+		if (free_id <= 0 && strcmp(cp, "ipi irqnn") == 0)
+			free_id = id;
+		while (*cp++ != '\0');
+	}
+	if (id == NR_DEVICES) {
+		id = free_id;
+		if (id == 0) {
+			/*
+			 * All ipi irq counters are in use, perhaps because
+			 * config is old so there aren't any.  Abuse the clk0
+			 * counter.
+			 */
+			printf(
+			    "ipi_ihandler_attach: counting ipi irq%d's as clk0 irqs\n",
+			    irq);
+		}
+	}
+	result = register_intr(
+	    irq,		/* isa irq	 */
+	    id,			/* device id    */
+	    0,			/* flags?	 */
+	    func,		/* handler	 */
+	    maskptr,		/* mask pointer */
+	    unit);		/* handler arg  */
+
+	if (result) {
+		printf("WARNING: ipi_ihandler_attach: result=%d\n", result);
+		return (result);
+	};
+
+	return (0);
+}
+#endif	/* IPI_INTS */
+
+
+#ifdef SMP_INVLTLB
+/*
+ * Flush the TLB on all other CPU's
+ *
+ * XXX: Needs to handshake and wait for completion before proceding.
+ */
+
+void
+smp_invltlb()
+{
+	if (smp_active) {
+		if (invldebug & 2)
+#if defined(XFAST_IPI32)
+			all_but_self_ipi(ICU_OFFSET + 32);
+#else
+			all_but_self_ipi(ICU_OFFSET + 27);
+#endif	/* XFAST_IPI32 */
+	}
+}
+
+void
+invlpg(u_int addr)
+{
+	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
+	smp_invltlb();
+}
+
+void
+invltlb(void)
+{
+	u_long  temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
+	 * inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
+	smp_invltlb();
+}
+/*
+ * Handles recieving an "IRQ 27", the invalidate tlb IPI..
+ */
+void
+ipi_invltlb(void)
+{
+	u_long  temp;
+
+	if (invldebug & 4) {
+		/*
+		 * This should be implemented as load_cr3(rcr3()) when
+		 * load_cr3() is inlined.
+		 */
+		__asm   __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp)
+		    ::      "memory");
+
+	}
+}
+#endif	/* SMP_INVLTLB */
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
index ecfd8b2c315d..73ee223404cc 100644
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)pcb.h	5.10 (Berkeley) 5/12/91
- *	$Id: pcb.h,v 1.18 1997/02/22 09:34:56 peter Exp $
+ *	$Id: pcb.h,v 1.19 1997/04/07 06:45:18 peter Exp $
  */
 
 #ifndef _I386_PCB_H_
@@ -60,6 +60,8 @@ struct pcb {
 	u_char	pcb_flags;
 #define	FP_SOFTFP	0x01	/* process using software fltng pnt emulator */
 	caddr_t	pcb_onfault;	/* copyin/out fault recovery */
+	u_long	pcb_mpnest;
+	u_long	__pcb_spare[7];	/* adjust to avoid core dump size changes */
 #if 0	/* some day we may switch between procs that have their own i386tss */
 	struct	i386tss pcb_tss;
 	u_char	pcb_iomap[NPORT/sizeof(u_char)]; /* i/o port bitmap */
@@ -74,7 +76,15 @@ struct md_coredump {
 };
 
 #ifdef KERNEL
+
+#include "opt_smp.h"
+#include <machine/smp.h>
+#ifdef SMP
+#define curpcb (SMPcurpcb[cpunumber()])
+#else /* !SMP */
 extern struct pcb *curpcb;		/* our current running pcb */
+#endif /* SMP */
+
 void	savectx __P((struct pcb*));
 #endif
 
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index 6b27cd28ac34..f0d4c017178b 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -42,7 +42,7 @@
  *
  *	from: hp300: @(#)pmap.h	7.2 (Berkeley) 12/16/90
  *	from: @(#)pmap.h	7.4 (Berkeley) 5/12/91
- * 	$Id: pmap.h,v 1.48 1997/02/22 09:34:58 peter Exp $
+ * 	$Id: pmap.h,v 1.49 1997/04/07 09:30:20 peter Exp $
  */
 
 #ifndef _MACHINE_PMAP_H_
@@ -91,7 +91,11 @@
 #define	NKPT			9	/* actual number of kernel page tables */
 #endif
 #ifndef NKPDE
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#define NKPDE			62	/* addressable number of page tables/pde's */
+#else	/* SMP && SMP_PRIVPAGES */
 #define NKPDE			63	/* addressable number of page tables/pde's */
+#endif	/* SMP && SMP_PRIVPAGES */
 #endif
 
 /*
@@ -99,9 +103,16 @@
  *
  * XXX This works for now, but I am not real happy with it, I'll fix it
  * right after I fix locore.s and the magic 28K hole
+ *
+ * SMP_PRIVPAGES: The per-cpu address space is 0xff80000 -> 0xffbfffff
  */
 #define	APTDPTDI	(NPDEPG-1)	/* alt ptd entry that points to APTD */
+#if defined(SMP) && defined(SMP_PRIVPAGES)
+#define MPPTDI		(APTDPTDI-1)	/* per cpu ptd entry */
+#define	KPTDI		(MPPTDI-NKPDE)	/* start of kernel virtual pde's */
+#else	/* SMP && SMP_PRIVPAGES */
 #define	KPTDI		(APTDPTDI-NKPDE)/* start of kernel virtual pde's */
+#endif	/* SMP && SMP_PRIVPAGES */
 #define	PTDPTDI		(KPTDI-1)	/* ptd entry that points to ptd! */
 #define	UMAXPTDI	(PTDPTDI-1)	/* ptd entry for user space end */
 #define	UMAXPTEOFF	(NPTEPG-UPAGES_HOLE) /* pte entry for user space end */
diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h
index 27525889053d..e1e6b4684c91 100644
--- a/sys/amd64/include/segments.h
+++ b/sys/amd64/include/segments.h
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)segments.h	7.1 (Berkeley) 5/9/91
- *	$Id$
+ *	$Id: segments.h,v 1.13 1997/02/22 09:35:09 peter Exp $
  */
 
 #ifndef _MACHINE_SEGMENTS_H_
@@ -234,7 +234,7 @@ struct region_descriptor {
 #ifdef KERNEL
 extern int	currentldt;
 extern int	_default_ldt;
-extern union descriptor gdt[NGDT];
+extern union descriptor gdt[];
 extern struct soft_segment_descriptor gdt_segs[];
 extern struct gate_descriptor idt[NIDT];
 extern union descriptor ldt[NLDT];
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
new file mode 100644
index 000000000000..905ca3529327
--- /dev/null
+++ b/sys/amd64/include/smp.h
@@ -0,0 +1,135 @@
+/*
+ * ----------------------------------------------------------------------------
+ * "THE BEER-WARE LICENSE" (Revision 42):
+ * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
+ * can do whatever you want with this stuff. If we meet some day, and you think
+ * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
+ * ----------------------------------------------------------------------------
+ *
+ * $Id: smp.h,v 1.29 1997/04/26 08:11:50 peter Exp $
+ *
+ */
+
+#ifndef _MACHINE_SMP_H_
+#define _MACHINE_SMP_H_
+
+#ifdef KERNEL
+
+#include "opt_smp.h"
+#include "opt_smp_invltlb.h"	/* aiee! (for cpufunc.h!) */
+
+#if defined(SMP) && !defined(NCPU)
+# define NCPU			2
+#endif /* SMP && NCPU */
+
+#if defined(SMP) || defined(APIC_IO)
+
+/* global data in mpboot.s */
+extern int			bootMP_size;
+
+/* functions in mpboot.s */
+void	bootMP			__P((void));
+
+/* global data in mplock.s */
+extern u_int			mp_lock;
+
+/* functions in mplock.s */
+void	get_mplock		__P((void));
+void	rel_mplock		__P((void));
+void	expect_mplock		__P((void));
+
+/* global data in mp_machdep.c */
+extern struct proc*		SMPcurproc[NCPU];
+extern struct pcb*		SMPcurpcb[NCPU];
+extern struct timeval		SMPruntime[NCPU];
+extern int			mp_ncpus;
+extern int			mp_naps;
+extern int			mp_nbusses;
+extern int			mp_napics;
+extern int			mp_picmode;
+extern int			mpenabled;
+extern int			boot_cpu_id;
+extern vm_offset_t		cpu_apic_address;
+extern vm_offset_t		io_apic_address[];
+extern u_int32_t		cpu_apic_versions[];
+extern u_int32_t		io_apic_versions[];
+extern int			cpu_num_to_apic_id[];
+extern int			io_num_to_apic_id[];
+extern int			apic_id_to_logical[];
+
+/* functions in mp_machdep.c */
+u_int	mp_bootaddress		__P((u_int));
+void	mp_start		__P((void));
+void	mp_announce		__P((void));
+int	get_isa_apic_irq	__P((int));
+int	get_eisa_apic_irq	__P((int));
+int	get_pci_apic_irq	__P((int, int, int));
+int	undirect_pci_irq	__P((int));
+int	apic_bus_type		__P((int));
+int	apic_src_bus_id		__P((int, int));
+int	apic_src_bus_irq	__P((int, int));
+int	apic_int_type		__P((int, int));
+int	apic_trigger		__P((int, int));
+int	apic_polarity		__P((int, int));
+void	configure_local_apic	__P((void));
+void	init_secondary		__P((void));
+#ifdef SMP_INVLTLB
+void	ipi_invltlb		__P((void));
+void	smp_invltlb		__P((void));
+#endif
+
+/* global data in mpapic.c */
+extern volatile u_int*		apic_base;
+
+#if defined(MULTIPLE_IOAPICS)
+#error MULTIPLE_IOAPICSXXX
+#else
+extern volatile u_int*		io_apic_base;
+#endif /* MULTIPLE_IOAPICS */
+
+/* functions in mpapic.c */
+void	apic_initialize		__P((int));
+int	selected_apic_ipi	__P((u_int, int, int));
+int	io_apic_setup		__P((int));
+int	ext_int_setup		__P((int, int));
+void	write_io_apic_mask24	__P((int, u_int32_t));
+
+#if defined(READY)
+void	clr_io_apic_mask24	__P((int, u_int32_t));
+void	set_io_apic_mask24	__P((int, u_int32_t));
+#endif /* READY */
+
+void	set_apic_timer		__P((int));
+int	read_apic_timer		__P((void));
+void	u_sleep			__P((int));
+
+/* global data in init_smp.c */
+extern int			smp_active;
+extern int			invldebug;
+
+/* in pmap.c FIXME: belongs in pmap.h??? */
+void	pmap_bootstrap_apics	__P((void));
+void	pmap_bootstrap2		__P((void));
+
+#if 0
+/* chicken and egg problem... */
+static __inline unsigned
+cpunumber(void)
+{
+	return (unsigned)ID_TO_CPU((apic_base[APIC_ID] & APIC_ID_MASK) >> 24);
+}
+#else
+/*
+ * we 'borrow' this info from apic.h
+ * this will go away soon...
+ */
+static __inline unsigned
+cpunumber(void)
+{
+	return (unsigned)(apic_id_to_logical[(apic_base[8] & 0x0f000000) >> 24]);
+}
+#endif /* 0 */
+
+#endif /* SMP || APIC_IO */
+#endif /* KERNEL */
+#endif /* _MACHINE_SMP_H_ */
diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c
index b9425536d67a..d46f982806bd 100644
--- a/sys/amd64/isa/clock.c
+++ b/sys/amd64/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.79 1997/03/05 08:08:48 bde Exp $
+ *	$Id: clock.c,v 1.80 1997/04/06 13:25:48 mckay Exp $
  */
 
 /*
@@ -49,6 +49,7 @@
  */
 
 #include "opt_clock.h"
+#include "opt_smp.h"
 #include "opt_cpu.h"
 
 #include <sys/param.h>
@@ -100,7 +101,7 @@
 int	adjkerntz;		/* local offset	from GMT in seconds */
 int	disable_rtc_set;	/* disable resettodr() if != 0 */
 u_int	idelayed;
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 u_int	i586_ctr_bias;
 u_int	i586_ctr_comultiplier;
 u_int	i586_ctr_freq;
@@ -143,7 +144,7 @@ static	u_char	timer0_state;
 static	u_char	timer2_state;
 static	void	(*timer_func) __P((struct clockframe *frame)) = hardclock;
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 static	void	set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq);
 #endif
 static	void	set_timer_freq(u_int freq, int intr_freq);
@@ -553,7 +554,7 @@ calibrate_clocks(void)
 		goto fail;
 	tot_count = 0;
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	if (cpu_class == CPUCLASS_586 || cpu_class == CPUCLASS_686)
 		wrmsr(0x10, 0LL);	/* XXX 0x10 is the MSR for the TSC */
 #endif
@@ -586,7 +587,7 @@ calibrate_clocks(void)
 			goto fail;
 	}
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	/*
 	 * Read the cpu cycle counter.  The timing considerations are
 	 * similar to those for the i8254 clock.
@@ -667,14 +668,14 @@ startrtclock()
 			printf(
 		    "%d Hz differs from default of %d Hz by more than 1%%\n",
 			       freq, timer_freq);
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 		i586_ctr_freq = 0;
 #endif
 	}
 
 	set_timer_freq(timer_freq, hz);
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 #ifndef CLK_USE_I586_CALIBRATION
 	if (i586_ctr_freq != 0) {
 		if (bootverbose)
@@ -821,6 +822,20 @@ resettodr()
 	writertc(RTC_STATUSB, rtc_statusb);
 }
 
+#if defined(APIC_IO)
+
+/* from icu.s: */
+extern u_int	hwisrs[];
+extern void	vec8254	__P((void));
+extern void	vecRTC	__P((void));
+extern u_int	ivectors[];
+extern u_int	Xintr8254;
+extern u_int	XintrRTC;
+extern u_int	mask8254;
+extern u_int	maskRTC;
+
+#endif /* APIC_IO */
+
 /*
  * Start both clocks running.
  */
@@ -828,6 +843,9 @@ void
 cpu_initclocks()
 {
 	int diag;
+#if defined(APIC_IO)
+	int x;
+#endif /* APIC_IO */
 
 	if (statclock_disable) {
 		/*
@@ -845,11 +863,47 @@ cpu_initclocks()
         }
 
 	/* Finish initializing 8253 timer 0. */
+#if defined(APIC_IO)
+	/* 8254 is traditionally on ISA IRQ0 */
+	if ((x = get_isa_apic_irq(0)) < 0) {
+		/*
+		 * bummer, this mb doesn't have the 8254 on ISA irq0,
+		 *  perhaps it's on the EISA bus...
+		 */
+		if ((x = get_eisa_apic_irq(0)) < 0) {
+			/* double bummer, attempt to redirect thru the 8259 */
+			if (bootverbose)
+				printf("APIC missing 8254 connection\n");
+
+			/* allow 8254 timer to INTerrupt 8259 */
+#if !defined(IO_ICU1)
+#define IO_ICU1 0x20
+#endif
+			x = inb(IO_ICU1 + 1);	/* current mask in 8259 */
+			x &= ~1;		/* clear 8254 timer mask */
+			outb(IO_ICU1 + 1, x);	/* write new mask */
+
+			/* program IO APIC for type 3 INT on INT0 */
+			if (ext_int_setup(0, 0) < 0)
+				panic("8254 redirect impossible!");
+			x = 0;			/* 8259 is on 0 */
+		}
+	}
+
+	hwisrs[x] = (u_int)vec8254;
+	Xintr8254 = (u_int)ivectors[x];	/* XXX might need Xfastintr# */
+	mask8254 = (1 << x);
+	register_intr(/* irq */ x, /* XXX id */ 0, /* flags */ 0,
+		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
+		      /* unit */ 0);
+	INTREN(mask8254);
+#else
 	register_intr(/* irq */ 0, /* XXX id */ 0, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)clkintr, &clk_imask,
 		      /* unit */ 0);
 	INTREN(IRQ0);
-#if defined(I586_CPU) || defined(I686_CPU)
+#endif /* APIC_IO */
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 	/*
 	 * Finish setting up anti-jitter measures.
 	 */
@@ -867,11 +921,36 @@ cpu_initclocks()
 	diag = rtcin(RTC_DIAG);
 	if (diag != 0)
 		printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS);
+#if defined(APIC_IO)
+	/* RTC is traditionally on ISA IRQ8 */
+	if ((x = get_isa_apic_irq(8)) < 0) {
+		if ((x = get_eisa_apic_irq(8)) < 0) {
+			panic("APIC missing RTC connection");
+	    }
+	}
+
+	hwisrs[x] = (u_int)vecRTC;
+	XintrRTC = (u_int)ivectors[x];	/* XXX might need Xfastintr# */
+	maskRTC = (1 << x);
+	register_intr(/* irq */ x, /* XXX id */ 1, /* flags */ 0,
+		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
+		      /* unit */ 0);
+	INTREN(maskRTC);
+#else
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
 	INTREN(IRQ8);
+#endif /* APIC_IO */
 	writertc(RTC_STATUSB, rtc_statusb);
+
+#if defined(APIC_IO)
+	printf("Enabled INTs: ");
+	for (x = 0; x < 24; ++x)
+		if ((imen & (1 << x)) == 0)
+	        	printf("%d, ", x);
+	printf("imen: 0x%08x\n", imen);
+#endif /* APIC_IO */
 }
 
 void
@@ -900,7 +979,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS
 		if (timer0_state != 0)
 			return (EBUSY);	/* too much trouble to handle */
 		set_timer_freq(freq, hz);
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 		set_i586_ctr_freq(i586_ctr_freq, timer_freq);
 #endif
 	}
@@ -910,7 +989,7 @@ sysctl_machdep_i8254_freq SYSCTL_HANDLER_ARGS
 SYSCTL_PROC(_machdep, OID_AUTO, i8254_freq, CTLTYPE_INT | CTLFLAG_RW,
 	    0, sizeof(u_int), sysctl_machdep_i8254_freq, "I", "");
 
-#if defined(I586_CPU) || defined(I686_CPU)
+#if (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP)
 static void
 set_i586_ctr_freq(u_int i586_freq, u_int i8254_freq)
 {
@@ -949,4 +1028,4 @@ sysctl_machdep_i586_freq SYSCTL_HANDLER_ARGS
 
 SYSCTL_PROC(_machdep, OID_AUTO, i586_freq, CTLTYPE_INT | CTLFLAG_RW,
 	    0, sizeof(u_int), sysctl_machdep_i586_freq, "I", "");
-#endif /* defined(I586_CPU) || defined(I686_CPU) */
+#endif /* (defined(I586_CPU) || defined(I686_CPU)) && !defined(SMP) */
diff --git a/sys/amd64/isa/icu.h b/sys/amd64/isa/icu.h
index ba721727f340..d0f148289dc3 100644
--- a/sys/amd64/isa/icu.h
+++ b/sys/amd64/isa/icu.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)icu.h	5.6 (Berkeley) 5/9/91
- *	$Id$
+ *	$Id: icu.h,v 1.10 1997/02/22 09:36:13 peter Exp $
  */
 
 /*
@@ -47,15 +47,62 @@
 
 #ifndef	LOCORE
 
+#include "opt_smp.h" /* so we dont have to change EVERY file including icu.h */
+
 /*
  * Interrupt "level" mechanism variables, masks, and macros
  */
 extern	unsigned imen;		/* interrupt mask enable */
 
+#if defined(APIC_IO)
+
+# if !defined(_MACHINE_SMP_H_)
+/** XXX what a hack, its this or include <machine/smp.h>! */
+void write_io_apic_mask24 __P((int, u_int32_t)); /* i386/i386/mpapic.c */
+# endif /* _MACHINE_SMP_H_ */
+
+#if defined(MULTIPLE_IOAPICS)
+#error MULTIPLE_IOAPICSXXX: cannot assume apic #0 in the following functions.
+#endif /* MULTIPLE_IOAPICS */
+
+static __inline u_int32_t
+INTRGET( void )
+{
+	return (imen & 0x00ffffff);		/* return our global copy */
+}
+
+static __inline void
+INTRSET( unsigned s )
+{
+	write_io_apic_mask24( 0, s );
+	imen = s;
+}
+
+static __inline void
+INTREN( unsigned s )
+{
+	write_io_apic_mask24( 0, imen & ~s );
+	imen &= ~s;
+}
+
+static __inline void
+INTRDIS( unsigned s )
+{
+	write_io_apic_mask24( 0, imen | s );
+	imen |= s;
+}
+
+
+#define	INTRMASK(msk,s)		(msk |= (s))
+#define INTRUNMASK(msk,s)	(msk &= ~(s))
+
+#else /* APIC_IO */
+
 #define	INTREN(s)		(imen &= ~(s), SET_ICUS())
 #define	INTRDIS(s)		(imen |= (s), SET_ICUS())
 #define	INTRMASK(msk,s)		(msk |= (s))
 #define INTRUNMASK(msk,s)	(msk &= ~(s))
+
 #if 0
 #ifdef PC98
 #define	SET_ICUS()	(outb(IO_ICU1 + 2, imen), outb(IU_ICU2 + 2, imen >> 8))
@@ -74,6 +121,8 @@ extern	unsigned imen;		/* interrupt mask enable */
 #endif
 #endif
 
+#endif /* APIC_IO */
+
 #endif /* LOCORE */
 
 /*
@@ -108,6 +157,22 @@ extern	unsigned imen;		/* interrupt mask enable */
  * Interrupt Control offset into Interrupt descriptor table (IDT)
  */
 #define	ICU_OFFSET	32		/* 0-31 are processor exceptions */
+
+#if defined(APIC_IO)
+
+#include <machine/apic.h>
+#if defined(IPI_INTS)
+/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23, 56-59: LOCAL APIC IPI */
+#define	ICU_LEN		28
+#else
+/* 32-47: ISA IRQ0-IRQ15, 48-55: IO APIC IRQ16-IRQ23 */
+#define	ICU_LEN		24
+#endif /* IPI_INTS */
+
+#else
+
 #define	ICU_LEN		16		/* 32-47 are ISA interrupts */
 
+#endif /* APIC_IO */
+
 #endif /* !_I386_ISA_ICU_H_ */
diff --git a/sys/amd64/isa/isa.c b/sys/amd64/isa/isa.c
index b4fa75b6665a..8b809e257c8a 100644
--- a/sys/amd64/isa/isa.c
+++ b/sys/amd64/isa/isa.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)isa.c	7.2 (Berkeley) 5/13/91
- *	$Id: isa.c,v 1.79 1997/03/25 03:29:40 ache Exp $
+ *	$Id: isa.c,v 1.80 1997/03/28 01:02:17 ache Exp $
  */
 
 /*
@@ -48,6 +48,7 @@
  */
 
 #include "opt_auto_eoi.h"
+#include "opt_smp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -56,6 +57,10 @@
 #include <sys/malloc.h>
 #include <machine/md_var.h>
 #include <machine/segments.h>
+#if defined(APIC_IO)
+#include <machine/smp.h>
+#include <machine/apic.h>
+#endif /* APIC_IO */
 #include <vm/vm.h>
 #include <vm/vm_param.h>
 #include <vm/pmap.h>
@@ -65,6 +70,14 @@
 #include <i386/isa/ic/i8237.h>
 #include "vector.h"
 
+#ifdef APIC_IO
+/*
+ * This is to accommodate "mixed-mode" programming for 
+ * motherboards that don't connect the 8254 to the IO APIC.
+ */
+#define AUTO_EOI_1
+#endif
+
 /*
 **  Register definitions for DMA controller 1 (channels 0..3):
 */
@@ -96,6 +109,16 @@ static inthand_t *fastintr[ICU_LEN] = {
 	&IDTVEC(fastintr10), &IDTVEC(fastintr11),
 	&IDTVEC(fastintr12), &IDTVEC(fastintr13),
 	&IDTVEC(fastintr14), &IDTVEC(fastintr15)
+#if defined(APIC_IO)
+	, &IDTVEC(fastintr16), &IDTVEC(fastintr17),
+	&IDTVEC(fastintr18), &IDTVEC(fastintr19),
+	&IDTVEC(fastintr20), &IDTVEC(fastintr21),
+	&IDTVEC(fastintr22), &IDTVEC(fastintr23)
+#if defined(IPI_INTS)
+/* XXX probably NOT needed, we register_intr(slowintr[I]) */ 
+	, &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27)
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 };
 
 static inthand_t *slowintr[ICU_LEN] = {
@@ -103,6 +126,13 @@ static inthand_t *slowintr[ICU_LEN] = {
 	&IDTVEC(intr4), &IDTVEC(intr5), &IDTVEC(intr6), &IDTVEC(intr7),
 	&IDTVEC(intr8), &IDTVEC(intr9), &IDTVEC(intr10), &IDTVEC(intr11),
 	&IDTVEC(intr12), &IDTVEC(intr13), &IDTVEC(intr14), &IDTVEC(intr15)
+#if defined(APIC_IO)
+	, &IDTVEC(intr16), &IDTVEC(intr17), &IDTVEC(intr18), &IDTVEC(intr19),
+	&IDTVEC(intr20), &IDTVEC(intr21), &IDTVEC(intr22), &IDTVEC(intr23)
+#if defined(IPI_INTS)
+	, &IDTVEC(ipi24), &IDTVEC(ipi25), &IDTVEC(ipi26), &IDTVEC(ipi27)
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 };
 
 static void config_isadev __P((struct isa_device *isdp, u_int *mp));
@@ -920,18 +950,52 @@ struct isa_device *find_isadev(table, driverp, unit)
 /*
  * Return nonzero if a (masked) irq is pending for a given device.
  */
+#if defined(APIC_IO)
+
 int
 isa_irq_pending(dvp)
 	struct isa_device *dvp;
 {
+	/* read APIC IRR containing the 16 ISA INTerrupts */
+#if defined(TEST_UPPERPRIO)
+	if ((u_int32_t)dvp->id_irq == APIC_IRQ10)
+		return (int)(apic_base[APIC_IRR2] & 1);
+	else
+#endif /** TEST_UPPERPRIO */
+		return ((apic_base[APIC_IRR1] & 0x00ffffff)
+		    & (u_int32_t)dvp->id_irq) ? 1 : 0;
+}
+
+/*
+ * an 8259 specific routine,
+ * for use by boot probes in certain device drivers.
+ */
+int
+icu_irq_pending(dvp)
+	struct isa_device *dvp;
+{
 	unsigned id_irq;
+	id_irq = dvp->id_irq;
+	if (id_irq & 0xff)
+		return (inb(IO_ICU1) & id_irq);
+	return (inb(IO_ICU2) & (id_irq >> 8));
+}
+
+#else /* APIC_IO */
 
+int
+isa_irq_pending(dvp)
+	struct isa_device *dvp;
+{
+	unsigned id_irq;
 	id_irq = dvp->id_irq;
 	if (id_irq & 0xff)
 		return (inb(IO_ICU1) & id_irq);
 	return (inb(IO_ICU2) & (id_irq >> 8));
 }
 
+#endif /* APIC_IO */
+
 int
 update_intr_masks(void)
 {
@@ -939,7 +1003,11 @@ update_intr_masks(void)
 	u_int mask,*maskptr;
 
 	for (intr=0; intr < ICU_LEN; intr ++) {
-		if (intr==2) continue;
+#if defined(APIC_IO)
+		/* no 8259 SLAVE to ignore */
+#else
+		if (intr==2) continue;	/* ignore 8259 SLAVE output */
+#endif /* APIC_IO */
 		maskptr = intr_mptr[intr];
 		if (!maskptr) continue;
 		*maskptr |= 1 << intr;
@@ -971,7 +1039,11 @@ register_intr(intr, device_id, flags, handler, maskptr, unit)
 	int	id;
 	u_int	mask = (maskptr ? *maskptr : 0);
 
+#if defined(APIC_IO)
+	if ((u_int)intr >= ICU_LEN	/* no 8259 SLAVE to ignore */
+#else
 	if ((u_int)intr >= ICU_LEN || intr == 2
+#endif /* APIC_IO */
 	    || (u_int)device_id >= NR_DEVICES)
 		return (EINVAL);
 	if (intr_handler[intr] != isa_strayintr)
@@ -983,9 +1055,24 @@ register_intr(intr, device_id, flags, handler, maskptr, unit)
 	intr_mptr[intr] = maskptr;
 	intr_mask[intr] = mask | (1 << intr);
 	intr_unit[intr] = unit;
+#if defined(TEST_UPPERPRIO)
+	if (intr == 10) {
+	    printf("--- setting IRQ10 to IDT64\n");
+	    setidt(64,
+	       flags & RI_FAST ? fastintr[intr] : slowintr[intr],
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	}
+	else {
+	    printf("setting IRQ%02d to IDT%02d\n", intr, ICU_OFFSET+intr);
+	    setidt(ICU_OFFSET + intr,
+	       flags & RI_FAST ? fastintr[intr] : slowintr[intr],
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+	}
+#else
 	setidt(ICU_OFFSET + intr,
 	       flags & RI_FAST ? fastintr[intr] : slowintr[intr],
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif /** TEST_UPPERPRIO */
 	write_eflags(ef);
 	for (cp = intrnames, id = 0; id <= device_id; id++)
 		while (*cp++ != '\0')
@@ -995,9 +1082,12 @@ register_intr(intr, device_id, flags, handler, maskptr, unit)
 	if (intr < 10) {
 		cp[-3] = intr + '0';
 		cp[-2] = ' ';
-	} else {
+	} else if (intr < 20) {
 		cp[-3] = '1';
 		cp[-2] = intr - 10 + '0';
+	} else {
+		cp[-3] = '2';
+		cp[-2] = intr - 20 + '0';
 	}
 	return (0);
 }
diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c
index 6e3f5c77e077..402b1cebb76f 100644
--- a/sys/amd64/isa/npx.c
+++ b/sys/amd64/isa/npx.c
@@ -32,7 +32,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)npx.c	7.2 (Berkeley) 5/12/91
- *	$Id: npx.c,v 1.40 1997/03/24 11:23:58 bde Exp $
+ *	$Id: npx.c,v 1.41 1997/04/22 06:55:38 jdp Exp $
  */
 
 #include "npx.h"
@@ -40,6 +40,7 @@
 
 #include "opt_cpu.h"
 #include "opt_math_emulate.h"
+#include "opt_smp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -60,6 +61,10 @@
 #include <machine/trap.h>
 #include <machine/clock.h>
 #include <machine/specialreg.h>
+#if defined(APIC_IO)
+#include <machine/apic.h>
+#include <machine/mpapic.h>
+#endif /* APIC_IO */
 
 #include <i386/isa/icu.h>
 #include <i386/isa/isa_device.h>
@@ -133,7 +138,12 @@ SYSCTL_INT(_hw,HW_FLOATINGPT, floatingpoint,
 	"Floatingpoint instructions executed in hardware");
 
 static u_int	npx0_imask = SWI_CLOCK_MASK;
+#ifdef SMP
+#define npxproc	(SMPnpxproc[cpunumber()])
+struct proc	*SMPnpxproc[NCPU];
+#else
 struct proc	*npxproc;
+#endif
 
 static	bool_t			npx_ex16;
 static	bool_t			npx_exists;
@@ -148,8 +158,28 @@ static	volatile u_int		npx_traps_while_probing;
  * interrupts.  We'll still need a special exception 16 handler.  The busy
  * latch stuff in probeintr() can be moved to npxprobe().
  */
-
 inthand_t probeintr;
+
+#if defined(APIC_IO)
+
+asm
+("
+	.text
+	.p2align 2,0x90
+" __XSTRING(CNAME(probeintr)) ":
+	ss
+	incl	" __XSTRING(CNAME(npx_intrs_while_probing)) "
+	pushl	%eax
+	movl	" __XSTRING(CNAME(apic_base)) ",%eax	# EOI to local APIC
+	movl	$0,0xb0(,%eax,1)	# movl $0, APIC_EOI(%eax)
+	movb	$0,%al
+	outb	%al,$0xf0		# clear BUSY# latch
+	popl	%eax
+	iret
+");
+
+#else
+
 asm
 ("
 	.text
@@ -167,6 +197,8 @@ asm
 	iret
 ");
 
+#endif /* APIC_IO */
+
 inthand_t probetrap;
 asm
 ("
@@ -191,8 +223,12 @@ npxprobe(dvp)
 {
 	int	result;
 	u_long	save_eflags;
+#if defined(APIC_IO)
+	u_int	save_apic_mask;
+#else
 	u_char	save_icu1_mask;
 	u_char	save_icu2_mask;
+#endif /* APIC_IO */
 	struct	gate_descriptor save_idt_npxintr;
 	struct	gate_descriptor save_idt_npxtrap;
 	/*
@@ -205,20 +241,32 @@ npxprobe(dvp)
 	npx_intrno = NRSVIDT + ffs(dvp->id_irq) - 1;
 	save_eflags = read_eflags();
 	disable_intr();
+#if defined(APIC_IO)
+	save_apic_mask = INTRGET();
+#else
 	save_icu1_mask = inb(IO_ICU1 + 1);
 	save_icu2_mask = inb(IO_ICU2 + 1);
+#endif /* APIC_IO */
 	save_idt_npxintr = idt[npx_intrno];
 	save_idt_npxtrap = idt[16];
+#if defined(APIC_IO)
+	INTRSET( ~dvp->id_irq );
+#else
 	outb(IO_ICU1 + 1, ~(IRQ_SLAVE | dvp->id_irq));
 	outb(IO_ICU2 + 1, ~(dvp->id_irq >> 8));
+#endif /* APIC_IO */
 	setidt(16, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 	setidt(npx_intrno, probeintr, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 	npx_idt_probeintr = idt[npx_intrno];
 	enable_intr();
 	result = npxprobe1(dvp);
 	disable_intr();
+#if defined(APIC_IO)
+	INTRSET( save_apic_mask );
+#else
 	outb(IO_ICU1 + 1, save_icu1_mask);
 	outb(IO_ICU2 + 1, save_icu2_mask);
+#endif /* APIC_IO */
 	idt[npx_intrno] = save_idt_npxintr;
 	idt[16] = save_idt_npxtrap;
 	write_eflags(save_eflags);
@@ -364,7 +412,8 @@ npxattach(dvp)
 	}
 	npxinit(__INITIAL_NPXCW__);
 
-#ifdef I586_CPU
+#if defined(I586_CPU) && !defined(SMP)
+	/* FPU not working under SMP yet */
 	if (cpu_class == CPUCLASS_586 && npx_ex16) {
 		if (!(dvp->id_flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) {
 			bcopy_vector = i586_bcopy;
@@ -576,18 +625,32 @@ void
 npxsave(addr)
 	struct save87 *addr;
 {
+#if defined(APIC_IO)
+	u_int	apic_mask;
+	u_int	old_apic_mask;
+#else
 	u_char	icu1_mask;
 	u_char	icu2_mask;
 	u_char	old_icu1_mask;
 	u_char	old_icu2_mask;
+#endif /* APIC_IO */
 	struct gate_descriptor	save_idt_npxintr;
 
 	disable_intr();
+#if defined(APIC_IO)
+	old_apic_mask = INTRGET();
+#else
 	old_icu1_mask = inb(IO_ICU1 + 1);
 	old_icu2_mask = inb(IO_ICU2 + 1);
+#endif /* APIC_IO */
 	save_idt_npxintr = idt[npx_intrno];
+#if defined(APIC_IO)
+	/** FIXME: try clrIoApicMaskBit( npx0_imask ); */
+	INTRSET( old_apic_mask & ~(npx0_imask & 0xffff) );
+#else
 	outb(IO_ICU1 + 1, old_icu1_mask & ~(IRQ_SLAVE | npx0_imask));
 	outb(IO_ICU2 + 1, old_icu2_mask & ~(npx0_imask >> 8));
+#endif /* APIC_IO */
 	idt[npx_intrno] = npx_idt_probeintr;
 	enable_intr();
 	stop_emulating();
@@ -596,6 +659,11 @@ npxsave(addr)
 	start_emulating();
 	npxproc = NULL;
 	disable_intr();
+#if defined(APIC_IO)
+	apic_mask = INTRGET();		/* masks may have changed */
+        INTRSET( (apic_mask & ~(npx0_imask & 0xffff)) |
+		 (old_apic_mask & (npx0_imask & 0xffff)));
+#else
 	icu1_mask = inb(IO_ICU1 + 1);	/* masks may have changed */
 	icu2_mask = inb(IO_ICU2 + 1);
 	outb(IO_ICU1 + 1,
@@ -603,6 +671,7 @@ npxsave(addr)
 	outb(IO_ICU2 + 1,
 	     (icu2_mask & ~(npx0_imask >> 8))
 	     | (old_icu2_mask & (npx0_imask >> 8)));
+#endif /* APIC_IO */
 	idt[npx_intrno] = save_idt_npxintr;
 	enable_intr();		/* back to usual state */
 }
diff --git a/sys/amd64/isa/vector.S b/sys/amd64/isa/vector.S
index fa2fd42afb54..270ae208cc29 100644
--- a/sys/amd64/isa/vector.S
+++ b/sys/amd64/isa/vector.S
@@ -1,6 +1,6 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id$
+ *	$Id: vector.s,v 1.25 1997/02/22 09:37:23 peter Exp $
  */
 
 /*
@@ -8,6 +8,12 @@
  */
 
 #include "opt_auto_eoi.h"
+#include "opt_smp.h"
+
+#if defined(SMP)
+#include <machine/smpasm.h>	/* this includes <machine/apic.h> */
+#include <machine/smptests.h>	/** TEST_CPUHITS */
+#endif /* SMP */
 
 #include <i386/isa/icu.h>
 #ifdef PC98
@@ -22,6 +28,111 @@
 #define ICU_IMR_OFFSET		1	/* IO_ICU{1,2} + 1 */
 #endif
 
+
+#if defined(SMP)
+
+#define GET_MPLOCK	call _get_mplock
+#define REL_MPLOCK	call _rel_mplock
+
+#else
+
+#define GET_MPLOCK	/* NOP get Kernel Mutex */
+#define REL_MPLOCK	/* NOP release mutex */
+
+#endif /* SMP */
+
+
+#if defined(APIC_IO)
+
+#if defined(SMP) && defined(TEST_CPUHITS)
+
+#undef GET_MPLOCK
+#define GET_MPLOCK		\
+	call	_get_mplock ; 	\
+	GETCPUID(%eax) ; 	\
+	incl	_cpuhits(,%eax,4)
+
+#endif /* SMP && TEST_CPUHITS */
+
+#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
+#define IRQ_BIT(irq_num)	(1 << (irq_num))
+
+#define ENABLE_APIC			\
+	movl	_apic_base, %eax ;	\
+	movl	$0, APIC_EOI(%eax)
+
+#define ENABLE_ICU1		ENABLE_APIC
+#define ENABLE_ICU1_AND_2	ENABLE_APIC
+
+#define MASK_IRQ(irq_num,icu)						\
+	orl	$IRQ_BIT(irq_num),_imen ;	/* set the mask bit */	\
+	movl	_io_apic_base,%ecx ;		/* io apic addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */
+
+#define UNMASK_IRQ(irq_num,icu)						\
+	andl	$~IRQ_BIT(irq_num),_imen ;	/* clear mask bit */	\
+	movl	_io_apic_base,%ecx ;		/* io apic addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */
+
+#define TEST_IRQ(irq_num,reg)						\
+	testl	$IRQ_BIT(irq_num),%eax
+
+#define SET_IPENDING(irq_num)						\
+	orl	$IRQ_BIT(irq_num),_ipending
+
+/*
+ * 'lazy masking' code submitted by: Bruce Evans <bde@zeta.org.au>
+ */
+#define MAYBE_MASK_IRQ(irq_num,icu)					\
+	testl	$IRQ_BIT(irq_num),iactive ;	/* lazy masking */	\
+	je	1f ;			/* NOT currently active */	\
+  	MASK_IRQ(irq_num,icu) ;						\
+  	ENABLE_APIC ;							\
+	SET_IPENDING(irq_num) ;						\
+	REL_MPLOCK ;			/* SMP release global lock */	\
+	popl	%es ;							\
+	popl	%ds ;							\
+	popal ;								\
+	addl	$4+4,%esp ;						\
+	iret ;								\
+;									\
+	ALIGN_TEXT ;							\
+1: ;									\
+	orl	$IRQ_BIT(irq_num),iactive
+
+#define MAYBE_UNMASK_IRQ(irq_num,icu)					\
+	andl	$~IRQ_BIT(irq_num),iactive ;				\
+	testl	$IRQ_BIT(irq_num),_imen ;				\
+	je	3f ;							\
+	UNMASK_IRQ(irq_num,icu) ;					\
+3:
+
+#else /* APIC_IO */
+
+#define MASK_IRQ(irq_num,icu)						\
+	movb	_imen + IRQ_BYTE(irq_num),%al ;				\
+	orb	$IRQ_BIT(irq_num),%al ;					\
+	movb	%al,_imen + IRQ_BYTE(irq_num) ;				\
+	outb	%al,$icu+ICU_IMR_OFFSET
+
+#define UNMASK_IRQ(irq_num,icu)						\
+	movb	_imen + IRQ_BYTE(irq_num),%al ;				\
+	andb	$~IRQ_BIT(irq_num),%al ;				\
+	movb	%al,_imen + IRQ_BYTE(irq_num) ;				\
+	outb	%al,$icu+ICU_IMR_OFFSET
+
+#define TEST_IRQ(irq_num,reg)						\
+	testb	$IRQ_BIT(irq_num),%reg
+	
+#define SET_IPENDING(irq_num)						\
+	orb	$IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num)
+
 #define	ICU_EOI			0x20	/* XXX - define elsewhere */
 
 #define	IRQ_BIT(irq_num)	(1 << ((irq_num) % 8))
@@ -50,6 +161,15 @@
 	OUTB_ICU1		/* ... then first icu (if !AUTO_EOI_1) */
 #endif
 
+#define MAYBE_MASK_IRQ(irq_num,icu)					\
+  	MASK_IRQ(irq_num,icu)
+
+#define MAYBE_UNMASK_IRQ(irq_num,icu)					\
+	UNMASK_IRQ(irq_num,icu)
+
+#endif /* APIC_IO */
+
+
 #ifdef FAST_INTR_HANDLER_USES_ES
 #define	ACTUALLY_PUSHED		1
 #define	MAYBE_MOVW_AX_ES	movl	%ax,%es
@@ -125,6 +245,7 @@ IDTVEC(vec_name) ; \
 	movl	%ax,%ds ; \
 	MAYBE_MOVW_AX_ES ; \
 	FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \
+	GET_MPLOCK ;		/* SMP Spin lock */ \
 	pushl	_intr_unit + (irq_num) * 4 ; \
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	enable_icus ;		/* (re)enable ASAP (helps edge trigger?) */ \
@@ -138,6 +259,7 @@ IDTVEC(vec_name) ; \
 	jne	2f ; 		/* yes, maybe handle them */ \
 1: ; \
 	MEXITCOUNT ; \
+	REL_MPLOCK ;		/* SMP release global lock */ \
 	MAYBE_POPL_ES ; \
 	popl	%ds ; \
 	popl	%edx ; \
@@ -183,13 +305,11 @@ IDTVEC(vec_name) ; \
 	movl	$KDSEL,%eax ;	/* ... and reload with kernel's own ... */ \
 	movl	%ax,%ds ;	/* ... early for obsolete reasons */ \
 	movl	%ax,%es ; \
-	movb	_imen + IRQ_BYTE(irq_num),%al ; \
-	orb	$IRQ_BIT(irq_num),%al ; \
-	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
-	outb	%al,$icu+ICU_IMR_OFFSET ; \
+	GET_MPLOCK ;		/* SMP Spin lock */ \
+	MAYBE_MASK_IRQ(irq_num,icu) ; \
 	enable_icus ; \
 	movl	_cpl,%eax ; \
-	testb	$IRQ_BIT(irq_num),%reg ; \
+	TEST_IRQ(irq_num,reg) ; \
 	jne	2f ; \
 	incb	_intr_nesting_level ; \
 __CONCAT(Xresume,irq_num): ; \
@@ -205,10 +325,7 @@ __CONCAT(Xresume,irq_num): ; \
 	sti ; \
 	call	*_intr_handler + (irq_num) * 4 ; \
 	cli ;			/* must unmask _imen and icu atomically */ \
-	movb	_imen + IRQ_BYTE(irq_num),%al ; \
-	andb	$~IRQ_BIT(irq_num),%al ; \
-	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
-	outb	%al,$icu+ICU_IMR_OFFSET ; \
+	MAYBE_UNMASK_IRQ(irq_num,icu) ; \
 	sti ;			/* XXX _doreti repeats the cli/sti */ \
 	MEXITCOUNT ; \
 	/* We could usually avoid the following jmp by inlining some of */ \
@@ -218,13 +335,66 @@ __CONCAT(Xresume,irq_num): ; \
 	ALIGN_TEXT ; \
 2: ; \
 	/* XXX skip mcounting here to avoid double count */ \
-	orb	$IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \
+	SET_IPENDING(irq_num) ; \
+	REL_MPLOCK ;		/* SMP release global lock */  \
 	popl	%es ; \
 	popl	%ds ; \
 	popal ; \
 	addl	$4+4,%esp ; \
 	iret
 
+
+#if defined(APIC_IO) && defined(IPI_INTS)
+/*
+ * A simple IPI_INTR() macro based on a heavily cut down FAST_INTR().
+ *  call it's handler, EOI and return.
+ */
+#define	IPI_INTR(irq_num, vec_name) \
+	.text ; \
+	SUPERALIGN_TEXT ; \
+IDTVEC(vec_name) ; \
+	pushl	%eax ;		/* save only call-used registers */ \
+	pushl	%ecx ; \
+	pushl	%edx ; \
+	pushl	%ds ; \
+	MAYBE_PUSHL_ES ; \
+	movl	$KDSEL,%eax ; \
+	movl	%ax,%ds ; \
+	MAYBE_MOVW_AX_ES ; \
+	pushl	_intr_unit + (irq_num) * 4 ; \
+	call	*_intr_handler + (irq_num) * 4 ; \
+	ENABLE_APIC ; \
+	addl	$4,%esp ; \
+	incl	_cnt+V_INTR ;	/* book-keeping can wait */ \
+	movl	_intr_countp + (irq_num) * 4,%eax ; \
+	incl	(%eax) ; \
+	MAYBE_POPL_ES ; \
+	popl	%ds ; \
+	popl	%edx ; \
+	popl	%ecx ; \
+	popl	%eax ; \
+	iret
+#endif /* APIC_IO && IPI_INTS */
+
+#if defined(XFAST_IPI32)
+	.text
+	SUPERALIGN_TEXT
+	.globl	_Xfastipi32
+_Xfastipi32:
+	pushl	%eax
+	movl	%cr3, %eax
+	movl	%eax, %cr3
+	pushl	%ds
+	movl	$KDSEL,%eax
+	movl	%ax,%ds
+	incl	_ipihits
+	movl	_apic_base, %eax
+	movl	$0, APIC_EOI(%eax)
+	popl	%ds
+	popl	%eax
+	iret
+#endif /* XFAST_IPI32 */
+
 MCOUNT_LABEL(bintr)
 	FAST_INTR(0,fastintr0, ENABLE_ICU1)
 	FAST_INTR(1,fastintr1, ENABLE_ICU1)
@@ -242,6 +412,16 @@ MCOUNT_LABEL(bintr)
 	FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2)
 	FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2)
 	FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2)
+#if defined(APIC_IO)
+	FAST_INTR(16,fastintr16, ENABLE_ICU1_AND_2)
+	FAST_INTR(17,fastintr17, ENABLE_ICU1_AND_2)
+	FAST_INTR(18,fastintr18, ENABLE_ICU1_AND_2)
+	FAST_INTR(19,fastintr19, ENABLE_ICU1_AND_2)
+	FAST_INTR(20,fastintr20, ENABLE_ICU1_AND_2)
+	FAST_INTR(21,fastintr21, ENABLE_ICU1_AND_2)
+	FAST_INTR(22,fastintr22, ENABLE_ICU1_AND_2)
+	FAST_INTR(23,fastintr23, ENABLE_ICU1_AND_2)
+#endif /* APIC_IO */
 	INTR(0,intr0, IO_ICU1, ENABLE_ICU1, al)
 	INTR(1,intr1, IO_ICU1, ENABLE_ICU1, al)
 	INTR(2,intr2, IO_ICU1, ENABLE_ICU1, al)
@@ -258,6 +438,22 @@ MCOUNT_LABEL(bintr)
 	INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2, ah)
 	INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2, ah)
 	INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+#if defined(APIC_IO)
+	INTR(16,intr16, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(17,intr17, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(18,intr18, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(19,intr19, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(20,intr20, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(21,intr21, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(22,intr22, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(23,intr23, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+#if defined(IPI_INTS)
+	IPI_INTR(24, ipi24)
+	IPI_INTR(25, ipi25)
+	IPI_INTR(26, ipi26)
+	IPI_INTR(27, ipi27)
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 MCOUNT_LABEL(eintr)
 
 	.data
@@ -267,17 +463,74 @@ ihandlers:			/* addresses of interrupt handlers */
 	.long	Xresume4, Xresume5, Xresume6, Xresume7
 	.long	Xresume8, Xresume9, Xresume10, Xresume11
 	.long	Xresume12, Xresume13, Xresume14, Xresume15 
+#if defined(APIC_IO)
+	.long	Xresume16, Xresume17, Xresume18, Xresume19
+	.long	Xresume20, Xresume21, Xresume22, Xresume23
+	.long	0, 0, 0, 0, swi_tty, swi_net, _softclock, swi_ast
+#else
 	.long	swi_tty, swi_net, 0, 0, 0, 0, 0, 0
 	.long	0, 0, 0, 0, 0, 0, _softclock, swi_ast
+#endif /* APIC_IO */
+
 imasks:				/* masks for interrupt handlers */
 	.space	NHWI*4		/* padding; HWI masks are elsewhere */
+
+#if defined(APIC_IO)
+#if defined(IPI_INTS)
+	/* these 4 IPI slots are counted as HARDWARE INTs, ie NHWI, above */
+#else
+	.long	0, 0, 0, 0	/* padding */
+#endif /* IPI_INTS */
+	.long	SWI_TTY_MASK, SWI_NET_MASK, SWI_CLOCK_MASK, SWI_AST_MASK
+#else
 	.long	SWI_TTY_MASK, SWI_NET_MASK, 0, 0, 0, 0, 0, 0
 	.long	0, 0, 0, 0, 0, 0, SWI_CLOCK_MASK, SWI_AST_MASK
+#endif /* APIC_IO */
+
 	.globl	_intr_nesting_level
 _intr_nesting_level:
 	.byte	0
 	.space	3
 
+#if defined(APIC_IO)
+
+	.globl _ivectors
+_ivectors:
+	.long	_Xintr0,  _Xintr1,  _Xintr2,  _Xintr3 
+	.long	_Xintr4,  _Xintr5,  _Xintr6,  _Xintr7
+	.long	_Xintr8,  _Xintr9,  _Xintr10, _Xintr11
+	.long	_Xintr12, _Xintr13, _Xintr14, _Xintr15 
+	.long	_Xintr16, _Xintr17, _Xintr18, _Xintr19
+	.long	_Xintr20, _Xintr21, _Xintr22, _Xintr23
+#if defined(IPI_INTS)
+	.long	_Xipi24, _Xipi25, _Xipi26, _Xipi27
+#endif /* IPI_INTS */
+
+/* active flag for lazy masking */
+iactive:
+	.long	0
+
+#if defined(XFAST_IPI32)
+	.globl _ipihits
+_ipihits:
+	.long	0
+#endif /* XFAST_IPI32 */
+
+#if defined(TEST_CPUHITS)
+	.globl _cpuhits
+_cpuhits:
+#if !defined(NCPU)
+/**
+ * FIXME: need a way to pass NCPU to .s files.
+ *        NCPU currently defined in smp.h IF NOT defined in opt_smp.h.
+ */
+#define NCPU	4
+#endif /* NCPU */
+	.space	NCPU*4
+#endif /* TEST_CPUHITS */
+
+#endif /* APIC_IO */
+
 /*
  * Interrupt counters and names.  The format of these and the label names
  * must agree with what vmstat expects.  The tables are indexed by device
@@ -309,6 +562,22 @@ _intrnames:
 	.asciz	"stray irq13"
 	.asciz	"stray irq14"
 	.asciz	"stray irq15"
+#if defined(APIC_IO)
+	.asciz	"stray irq16"
+	.asciz	"stray irq17"
+	.asciz	"stray irq18"
+	.asciz	"stray irq19"
+	.asciz	"stray irq20"
+	.asciz	"stray irq21"
+	.asciz	"stray irq22"
+	.asciz	"stray irq23"
+#if defined(IPI_INTS)
+	.asciz	"stray irq24"
+	.asciz	"stray irq25"
+	.asciz	"stray irq26"
+	.asciz	"stray irq27"
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 _eintrnames:
 
 	.text
diff --git a/sys/amd64/isa/vector.s b/sys/amd64/isa/vector.s
index fa2fd42afb54..270ae208cc29 100644
--- a/sys/amd64/isa/vector.s
+++ b/sys/amd64/isa/vector.s
@@ -1,6 +1,6 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id$
+ *	$Id: vector.s,v 1.25 1997/02/22 09:37:23 peter Exp $
  */
 
 /*
@@ -8,6 +8,12 @@
  */
 
 #include "opt_auto_eoi.h"
+#include "opt_smp.h"
+
+#if defined(SMP)
+#include <machine/smpasm.h>	/* this includes <machine/apic.h> */
+#include <machine/smptests.h>	/** TEST_CPUHITS */
+#endif /* SMP */
 
 #include <i386/isa/icu.h>
 #ifdef PC98
@@ -22,6 +28,111 @@
 #define ICU_IMR_OFFSET		1	/* IO_ICU{1,2} + 1 */
 #endif
 
+
+#if defined(SMP)
+
+#define GET_MPLOCK	call _get_mplock
+#define REL_MPLOCK	call _rel_mplock
+
+#else
+
+#define GET_MPLOCK	/* NOP get Kernel Mutex */
+#define REL_MPLOCK	/* NOP release mutex */
+
+#endif /* SMP */
+
+
+#if defined(APIC_IO)
+
+#if defined(SMP) && defined(TEST_CPUHITS)
+
+#undef GET_MPLOCK
+#define GET_MPLOCK		\
+	call	_get_mplock ; 	\
+	GETCPUID(%eax) ; 	\
+	incl	_cpuhits(,%eax,4)
+
+#endif /* SMP && TEST_CPUHITS */
+
+#define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
+#define IRQ_BIT(irq_num)	(1 << (irq_num))
+
+#define ENABLE_APIC			\
+	movl	_apic_base, %eax ;	\
+	movl	$0, APIC_EOI(%eax)
+
+#define ENABLE_ICU1		ENABLE_APIC
+#define ENABLE_ICU1_AND_2	ENABLE_APIC
+
+#define MASK_IRQ(irq_num,icu)						\
+	orl	$IRQ_BIT(irq_num),_imen ;	/* set the mask bit */	\
+	movl	_io_apic_base,%ecx ;		/* io apic addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */
+
+#define UNMASK_IRQ(irq_num,icu)						\
+	andl	$~IRQ_BIT(irq_num),_imen ;	/* clear mask bit */	\
+	movl	_io_apic_base,%ecx ;		/* io apic addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */
+
+#define TEST_IRQ(irq_num,reg)						\
+	testl	$IRQ_BIT(irq_num),%eax
+
+#define SET_IPENDING(irq_num)						\
+	orl	$IRQ_BIT(irq_num),_ipending
+
+/*
+ * 'lazy masking' code submitted by: Bruce Evans <bde@zeta.org.au>
+ */
+#define MAYBE_MASK_IRQ(irq_num,icu)					\
+	testl	$IRQ_BIT(irq_num),iactive ;	/* lazy masking */	\
+	je	1f ;			/* NOT currently active */	\
+  	MASK_IRQ(irq_num,icu) ;						\
+  	ENABLE_APIC ;							\
+	SET_IPENDING(irq_num) ;						\
+	REL_MPLOCK ;			/* SMP release global lock */	\
+	popl	%es ;							\
+	popl	%ds ;							\
+	popal ;								\
+	addl	$4+4,%esp ;						\
+	iret ;								\
+;									\
+	ALIGN_TEXT ;							\
+1: ;									\
+	orl	$IRQ_BIT(irq_num),iactive
+
+#define MAYBE_UNMASK_IRQ(irq_num,icu)					\
+	andl	$~IRQ_BIT(irq_num),iactive ;				\
+	testl	$IRQ_BIT(irq_num),_imen ;				\
+	je	3f ;							\
+	UNMASK_IRQ(irq_num,icu) ;					\
+3:
+
+#else /* APIC_IO */
+
+#define MASK_IRQ(irq_num,icu)						\
+	movb	_imen + IRQ_BYTE(irq_num),%al ;				\
+	orb	$IRQ_BIT(irq_num),%al ;					\
+	movb	%al,_imen + IRQ_BYTE(irq_num) ;				\
+	outb	%al,$icu+ICU_IMR_OFFSET
+
+#define UNMASK_IRQ(irq_num,icu)						\
+	movb	_imen + IRQ_BYTE(irq_num),%al ;				\
+	andb	$~IRQ_BIT(irq_num),%al ;				\
+	movb	%al,_imen + IRQ_BYTE(irq_num) ;				\
+	outb	%al,$icu+ICU_IMR_OFFSET
+
+#define TEST_IRQ(irq_num,reg)						\
+	testb	$IRQ_BIT(irq_num),%reg
+	
+#define SET_IPENDING(irq_num)						\
+	orb	$IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num)
+
 #define	ICU_EOI			0x20	/* XXX - define elsewhere */
 
 #define	IRQ_BIT(irq_num)	(1 << ((irq_num) % 8))
@@ -50,6 +161,15 @@
 	OUTB_ICU1		/* ... then first icu (if !AUTO_EOI_1) */
 #endif
 
+#define MAYBE_MASK_IRQ(irq_num,icu)					\
+  	MASK_IRQ(irq_num,icu)
+
+#define MAYBE_UNMASK_IRQ(irq_num,icu)					\
+	UNMASK_IRQ(irq_num,icu)
+
+#endif /* APIC_IO */
+
+
 #ifdef FAST_INTR_HANDLER_USES_ES
 #define	ACTUALLY_PUSHED		1
 #define	MAYBE_MOVW_AX_ES	movl	%ax,%es
@@ -125,6 +245,7 @@ IDTVEC(vec_name) ; \
 	movl	%ax,%ds ; \
 	MAYBE_MOVW_AX_ES ; \
 	FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \
+	GET_MPLOCK ;		/* SMP Spin lock */ \
 	pushl	_intr_unit + (irq_num) * 4 ; \
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	enable_icus ;		/* (re)enable ASAP (helps edge trigger?) */ \
@@ -138,6 +259,7 @@ IDTVEC(vec_name) ; \
 	jne	2f ; 		/* yes, maybe handle them */ \
 1: ; \
 	MEXITCOUNT ; \
+	REL_MPLOCK ;		/* SMP release global lock */ \
 	MAYBE_POPL_ES ; \
 	popl	%ds ; \
 	popl	%edx ; \
@@ -183,13 +305,11 @@ IDTVEC(vec_name) ; \
 	movl	$KDSEL,%eax ;	/* ... and reload with kernel's own ... */ \
 	movl	%ax,%ds ;	/* ... early for obsolete reasons */ \
 	movl	%ax,%es ; \
-	movb	_imen + IRQ_BYTE(irq_num),%al ; \
-	orb	$IRQ_BIT(irq_num),%al ; \
-	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
-	outb	%al,$icu+ICU_IMR_OFFSET ; \
+	GET_MPLOCK ;		/* SMP Spin lock */ \
+	MAYBE_MASK_IRQ(irq_num,icu) ; \
 	enable_icus ; \
 	movl	_cpl,%eax ; \
-	testb	$IRQ_BIT(irq_num),%reg ; \
+	TEST_IRQ(irq_num,reg) ; \
 	jne	2f ; \
 	incb	_intr_nesting_level ; \
 __CONCAT(Xresume,irq_num): ; \
@@ -205,10 +325,7 @@ __CONCAT(Xresume,irq_num): ; \
 	sti ; \
 	call	*_intr_handler + (irq_num) * 4 ; \
 	cli ;			/* must unmask _imen and icu atomically */ \
-	movb	_imen + IRQ_BYTE(irq_num),%al ; \
-	andb	$~IRQ_BIT(irq_num),%al ; \
-	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
-	outb	%al,$icu+ICU_IMR_OFFSET ; \
+	MAYBE_UNMASK_IRQ(irq_num,icu) ; \
 	sti ;			/* XXX _doreti repeats the cli/sti */ \
 	MEXITCOUNT ; \
 	/* We could usually avoid the following jmp by inlining some of */ \
@@ -218,13 +335,66 @@ __CONCAT(Xresume,irq_num): ; \
 	ALIGN_TEXT ; \
 2: ; \
 	/* XXX skip mcounting here to avoid double count */ \
-	orb	$IRQ_BIT(irq_num),_ipending + IRQ_BYTE(irq_num) ; \
+	SET_IPENDING(irq_num) ; \
+	REL_MPLOCK ;		/* SMP release global lock */  \
 	popl	%es ; \
 	popl	%ds ; \
 	popal ; \
 	addl	$4+4,%esp ; \
 	iret
 
+
+#if defined(APIC_IO) && defined(IPI_INTS)
+/*
+ * A simple IPI_INTR() macro based on a heavily cut down FAST_INTR().
+ *  call it's handler, EOI and return.
+ */
+#define	IPI_INTR(irq_num, vec_name) \
+	.text ; \
+	SUPERALIGN_TEXT ; \
+IDTVEC(vec_name) ; \
+	pushl	%eax ;		/* save only call-used registers */ \
+	pushl	%ecx ; \
+	pushl	%edx ; \
+	pushl	%ds ; \
+	MAYBE_PUSHL_ES ; \
+	movl	$KDSEL,%eax ; \
+	movl	%ax,%ds ; \
+	MAYBE_MOVW_AX_ES ; \
+	pushl	_intr_unit + (irq_num) * 4 ; \
+	call	*_intr_handler + (irq_num) * 4 ; \
+	ENABLE_APIC ; \
+	addl	$4,%esp ; \
+	incl	_cnt+V_INTR ;	/* book-keeping can wait */ \
+	movl	_intr_countp + (irq_num) * 4,%eax ; \
+	incl	(%eax) ; \
+	MAYBE_POPL_ES ; \
+	popl	%ds ; \
+	popl	%edx ; \
+	popl	%ecx ; \
+	popl	%eax ; \
+	iret
+#endif /* APIC_IO && IPI_INTS */
+
+#if defined(XFAST_IPI32)
+	.text
+	SUPERALIGN_TEXT
+	.globl	_Xfastipi32
+_Xfastipi32:
+	pushl	%eax
+	movl	%cr3, %eax
+	movl	%eax, %cr3
+	pushl	%ds
+	movl	$KDSEL,%eax
+	movl	%ax,%ds
+	incl	_ipihits
+	movl	_apic_base, %eax
+	movl	$0, APIC_EOI(%eax)
+	popl	%ds
+	popl	%eax
+	iret
+#endif /* XFAST_IPI32 */
+
 MCOUNT_LABEL(bintr)
 	FAST_INTR(0,fastintr0, ENABLE_ICU1)
 	FAST_INTR(1,fastintr1, ENABLE_ICU1)
@@ -242,6 +412,16 @@ MCOUNT_LABEL(bintr)
 	FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2)
 	FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2)
 	FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2)
+#if defined(APIC_IO)
+	FAST_INTR(16,fastintr16, ENABLE_ICU1_AND_2)
+	FAST_INTR(17,fastintr17, ENABLE_ICU1_AND_2)
+	FAST_INTR(18,fastintr18, ENABLE_ICU1_AND_2)
+	FAST_INTR(19,fastintr19, ENABLE_ICU1_AND_2)
+	FAST_INTR(20,fastintr20, ENABLE_ICU1_AND_2)
+	FAST_INTR(21,fastintr21, ENABLE_ICU1_AND_2)
+	FAST_INTR(22,fastintr22, ENABLE_ICU1_AND_2)
+	FAST_INTR(23,fastintr23, ENABLE_ICU1_AND_2)
+#endif /* APIC_IO */
 	INTR(0,intr0, IO_ICU1, ENABLE_ICU1, al)
 	INTR(1,intr1, IO_ICU1, ENABLE_ICU1, al)
 	INTR(2,intr2, IO_ICU1, ENABLE_ICU1, al)
@@ -258,6 +438,22 @@ MCOUNT_LABEL(bintr)
 	INTR(13,intr13, IO_ICU2, ENABLE_ICU1_AND_2, ah)
 	INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2, ah)
 	INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+#if defined(APIC_IO)
+	INTR(16,intr16, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(17,intr17, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(18,intr18, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(19,intr19, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(20,intr20, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(21,intr21, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(22,intr22, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+	INTR(23,intr23, IO_ICU2, ENABLE_ICU1_AND_2, ah)
+#if defined(IPI_INTS)
+	IPI_INTR(24, ipi24)
+	IPI_INTR(25, ipi25)
+	IPI_INTR(26, ipi26)
+	IPI_INTR(27, ipi27)
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 MCOUNT_LABEL(eintr)
 
 	.data
@@ -267,17 +463,74 @@ ihandlers:			/* addresses of interrupt handlers */
 	.long	Xresume4, Xresume5, Xresume6, Xresume7
 	.long	Xresume8, Xresume9, Xresume10, Xresume11
 	.long	Xresume12, Xresume13, Xresume14, Xresume15 
+#if defined(APIC_IO)
+	.long	Xresume16, Xresume17, Xresume18, Xresume19
+	.long	Xresume20, Xresume21, Xresume22, Xresume23
+	.long	0, 0, 0, 0, swi_tty, swi_net, _softclock, swi_ast
+#else
 	.long	swi_tty, swi_net, 0, 0, 0, 0, 0, 0
 	.long	0, 0, 0, 0, 0, 0, _softclock, swi_ast
+#endif /* APIC_IO */
+
 imasks:				/* masks for interrupt handlers */
 	.space	NHWI*4		/* padding; HWI masks are elsewhere */
+
+#if defined(APIC_IO)
+#if defined(IPI_INTS)
+	/* these 4 IPI slots are counted as HARDWARE INTs, ie NHWI, above */
+#else
+	.long	0, 0, 0, 0	/* padding */
+#endif /* IPI_INTS */
+	.long	SWI_TTY_MASK, SWI_NET_MASK, SWI_CLOCK_MASK, SWI_AST_MASK
+#else
 	.long	SWI_TTY_MASK, SWI_NET_MASK, 0, 0, 0, 0, 0, 0
 	.long	0, 0, 0, 0, 0, 0, SWI_CLOCK_MASK, SWI_AST_MASK
+#endif /* APIC_IO */
+
 	.globl	_intr_nesting_level
 _intr_nesting_level:
 	.byte	0
 	.space	3
 
+#if defined(APIC_IO)
+
+	.globl _ivectors
+_ivectors:
+	.long	_Xintr0,  _Xintr1,  _Xintr2,  _Xintr3 
+	.long	_Xintr4,  _Xintr5,  _Xintr6,  _Xintr7
+	.long	_Xintr8,  _Xintr9,  _Xintr10, _Xintr11
+	.long	_Xintr12, _Xintr13, _Xintr14, _Xintr15 
+	.long	_Xintr16, _Xintr17, _Xintr18, _Xintr19
+	.long	_Xintr20, _Xintr21, _Xintr22, _Xintr23
+#if defined(IPI_INTS)
+	.long	_Xipi24, _Xipi25, _Xipi26, _Xipi27
+#endif /* IPI_INTS */
+
+/* active flag for lazy masking */
+iactive:
+	.long	0
+
+#if defined(XFAST_IPI32)
+	.globl _ipihits
+_ipihits:
+	.long	0
+#endif /* XFAST_IPI32 */
+
+#if defined(TEST_CPUHITS)
+	.globl _cpuhits
+_cpuhits:
+#if !defined(NCPU)
+/**
+ * FIXME: need a way to pass NCPU to .s files.
+ *        NCPU currently defined in smp.h IF NOT defined in opt_smp.h.
+ */
+#define NCPU	4
+#endif /* NCPU */
+	.space	NCPU*4
+#endif /* TEST_CPUHITS */
+
+#endif /* APIC_IO */
+
 /*
  * Interrupt counters and names.  The format of these and the label names
  * must agree with what vmstat expects.  The tables are indexed by device
@@ -309,6 +562,22 @@ _intrnames:
 	.asciz	"stray irq13"
 	.asciz	"stray irq14"
 	.asciz	"stray irq15"
+#if defined(APIC_IO)
+	.asciz	"stray irq16"
+	.asciz	"stray irq17"
+	.asciz	"stray irq18"
+	.asciz	"stray irq19"
+	.asciz	"stray irq20"
+	.asciz	"stray irq21"
+	.asciz	"stray irq22"
+	.asciz	"stray irq23"
+#if defined(IPI_INTS)
+	.asciz	"stray irq24"
+	.asciz	"stray irq25"
+	.asciz	"stray irq26"
+	.asciz	"stray irq27"
+#endif /* IPI_INTS */
+#endif /* APIC_IO */
 _eintrnames:
 
 	.text